From 7a2ef0ae79bcb8afd0e66e45082c7db98cf63b09 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 6 Dec 2024 16:25:01 +0800 Subject: [PATCH 001/114] refactor: translator --- pdf2zh/translator.py | 251 ++++++++++++------------------------------- pyproject.toml | 1 + 2 files changed, 68 insertions(+), 184 deletions(-) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index a45e753c..2e1b2a41 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -1,13 +1,7 @@ -import hashlib -import hmac import html import logging import os import re -import time -from datetime import timezone, datetime - -from json import dumps, loads import unicodedata import deepl @@ -16,6 +10,10 @@ import requests from azure.ai.translation.text import TextTranslationClient from azure.core.credentials import AzureKeyCredential +from tencentcloud.common import credential +from tencentcloud.tmt.v20180321.tmt_client import TmtClient +from tencentcloud.tmt.v20180321.models import TextTranslateRequest +from tencentcloud.tmt.v20180321.models import TextTranslateResponse def remove_control_characters(s): @@ -23,13 +21,16 @@ def remove_control_characters(s): class BaseTranslator: + envs = {} + def __init__(self, service, lang_out, lang_in, model): self.service = service self.lang_out = lang_out self.lang_in = lang_in self.model = model - def translate(self, text) -> str: ... # noqa: E704 + def translate(self, text): + pass def __str__(self): return f"{self.service} {self.lang_out} {self.lang_in}" @@ -41,7 +42,7 @@ def __init__(self, service, lang_out, lang_in, model): lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) self.session = requests.Session() - self.base_link = "http://translate.google.com/m" + self.endpoint = "http://translate.google.com/m" self.headers = { "User-Agent": "Mozilla/4.0 (compatible;MSIE 6.0;Windows NT 5.1;SV1;.NET CLR 1.1.4322;.NET CLR 2.0.50727;.NET CLR 3.0.04506.30)" # noqa: E501 } @@ -49,7 +50,7 @@ def __init__(self, service, lang_out, lang_in, model): def translate(self, text): text = text[:5000] # google translate max length response = self.session.get( - self.base_link, + self.endpoint, params={"tl": self.lang_out, "sl": self.lang_in, "q": text}, headers=self.headers, ) @@ -58,195 +59,74 @@ def translate(self, text): ) if response.status_code == 400: result = "IRREPARABLE TRANSLATION ERROR" - elif len(re_result) == 0: - raise ValueError("Empty translation result") else: result = html.unescape(re_result[0]) return remove_control_characters(result) class TencentTranslator(BaseTranslator): - def sign(self, key, msg): - return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() + # https://github.com/TencentCloud/tencentcloud-sdk-python + envs = { + "TENCENTCLOUD_SECRET_ID": None, + "TENCENTCLOUD_SECRET_KEY": None, + } def __init__(self, service, lang_out, lang_in, model): lang_out = "zh" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) - try: - server_url = "tmt.tencentcloudapi.com" - self.secret_id = os.getenv("TENCENT_SECRET_ID") - self.secret_key = os.getenv("TENCENT_SECRET_KEY") - - except KeyError as e: - missing_var = e.args[0] - raise ValueError( - f"The environment variable '{missing_var}' is required but not set." - ) from e - - self.session = requests.Session() - self.base_link = f"{server_url}" + cred = credential.DefaultCredentialProvider().get_credential() + self.client = TmtClient(cred, "ap-beijing") + self.req = TextTranslateRequest() + self.req.Source = self.lang_in + self.req.Target = self.lang_out + self.req.ProjectId = 0 def translate(self, text): - text = text[:5000] - data = { - "SourceText": text, - "Source": self.lang_in, - "Target": self.lang_out, - "ProjectId": 0, - } - payloadx = dumps(data) - hashed_request_payload = hashlib.sha256(payloadx.encode("utf-8")).hexdigest() - canonical_request = ( - "POST" - + "\n" - + "/" - + "\n" - + "" - + "\n" - + "content-type:application/json; charset=utf-8\nhost:tmt.tencentcloudapi.com\nx-tc-action:texttranslate\n" - + "\n" - + "content-type;host;x-tc-action" - + "\n" - + hashed_request_payload - ) - - timestamp = int(time.time()) - date = datetime.fromtimestamp(timestamp, timezone.utc).strftime("%Y-%m-%d") - credential_scope = date + "/tmt/tc3_request" - hashed_canonical_request = hashlib.sha256( - canonical_request.encode("utf-8") - ).hexdigest() - algorithm = "TC3-HMAC-SHA256" - string_to_sign = ( - algorithm - + "\n" - + str(timestamp) - + "\n" - + credential_scope - + "\n" - + hashed_canonical_request - ) - secret_date = self.sign(("TC3" + str(self.secret_key)).encode("utf-8"), date) - secret_service = self.sign(secret_date, "tmt") - secret_signing = self.sign(secret_service, "tc3_request") - signed_headers = "content-type;host;x-tc-action" - signature = hmac.new( - secret_signing, string_to_sign.encode("utf-8"), hashlib.sha256 - ).hexdigest() - authorization = ( - algorithm - + " " - + "Credential=" - + str(self.secret_id) - + "/" - + credential_scope - + ", " - + "SignedHeaders=" - + signed_headers - + ", " - + "Signature=" - + signature - ) - self.headers = { - "Authorization": authorization, - "Content-Type": "application/json; charset=utf-8", - "Host": "tmt.tencentcloudapi.com", - "X-TC-Action": "TextTranslate", - "X-TC-Region": "ap-beijing", - "X-TC-Timestamp": str(timestamp), - "X-TC-Version": "2018-03-21", - } - - response = self.session.post( - "https://" + self.base_link, - json=data, - headers=self.headers, - ) - # 1. Status code test - if response.status_code == 200: - result = loads(response.text) - else: - raise ValueError("HTTP error: " + str(response.status_code)) - # 2. Result test - try: - result = result["Response"]["TargetText"] - # return result - except KeyError: - result = "" - # raise ValueError("No valid key in Tencent's response") - # # 3. Result length check - # if len(result) == 0: - # raise ValueError("Empty translation result") - return result + self.req.SourceText = text + resp: TextTranslateResponse = self.client.TextTranslate(self.req) + return resp.TargetText class DeepLXTranslator(BaseTranslator): + # https://deeplx.owo.network/endpoints/free.html + envs = { + "DEEPLX_ENDPOINT": "https://api.deepl.com/v2/translate", + } + def __init__(self, service, lang_out, lang_in, model): lang_out = "zh" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) - try: - auth_key = os.getenv("DEEPLX_AUTH_KEY") - server_url = ( - "https://api.deeplx.org" - if not os.getenv("DEEPLX_SERVER_URL") - else os.getenv("DEEPLX_SERVER_URL") - ) - except KeyError as e: - missing_var = e.args[0] - raise ValueError( - f"The environment variable '{missing_var}' is required but not set." - ) from e - + self.endpoint = os.getenv("DEEPLX_ENDPOINT") self.session = requests.Session() - server_url = str(server_url).rstrip("/") - if auth_key: - self.base_link = f"{server_url}/{auth_key}/translate" - else: - self.base_link = f"{server_url}/translate" - self.headers = { - "User-Agent": "Mozilla/4.0 (compatible;MSIE 6.0;Windows NT 5.1;SV1;.NET CLR 1.1.4322;.NET CLR 2.0.50727;.NET CLR 3.0.04506.30)" # noqa: E501 - } def translate(self, text): - text = text[:5000] # google translate max length - response = self.session.post( - self.base_link, - dumps( - { - "target_lang": self.lang_out, - "text": text, - } - ), - headers=self.headers, + resp = self.session.post( + self.endpoint, + json={ + "source_lang": self.lang_in, + "target_lang": self.lang_out, + "text": text, + }, ) - # 1. Status code test - if response.status_code == 200: - result = loads(response.text) - else: - raise ValueError("HTTP error: " + str(response.status_code)) - # 2. Result test - try: - result = result["data"] - return result - except KeyError: - result = "" - raise ValueError("No valid key in DeepLX's response") - # 3. Result length check - if len(result) == 0: - raise ValueError("Empty translation result") - return result + return resp.json()["data"] class DeepLTranslator(BaseTranslator): + # https://github.com/DeepLcom/deepl-python + envs = { + "DEEPL_SERVER_URL": "https://api.deepl.com", + "DEEPL_AUTH_KEY": None, + } + def __init__(self, service, lang_out, lang_in, model): - lang_out = "ZH" if lang_out == "auto" else lang_out - lang_in = "EN" if lang_in == "auto" else lang_in + lang_out = "zh" if lang_out == "auto" else lang_out + lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) self.session = requests.Session() - auth_key = os.getenv("DEEPL_AUTH_KEY") server_url = os.getenv("DEEPL_SERVER_URL") + auth_key = os.getenv("DEEPL_AUTH_KEY") self.client = deepl.Translator(auth_key, server_url=server_url) def translate(self, text): @@ -257,12 +137,16 @@ def translate(self, text): class OllamaTranslator(BaseTranslator): + # https://github.com/ollama/ollama-python + envs = { + "OLLAMA_HOST": "http://127.0.0.1:11434", + } + def __init__(self, service, lang_out, lang_in, model): lang_out = "zh-CN" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 - # OLLAMA_HOST self.client = ollama.Client() def translate(self, text): @@ -284,13 +168,17 @@ def translate(self, text): class OpenAITranslator(BaseTranslator): + # https://github.com/openai/openai-python + envs = { + "OPENAI_BASE_URL": "https://api.openai.com/v1", + "OPENAI_API_KEY": None, + } + def __init__(self, service, lang_out, lang_in, model): lang_out = "zh-CN" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 - # OPENAI_BASE_URL - # OPENAI_API_KEY self.client = openai.OpenAI() def translate(self, text) -> str: @@ -312,26 +200,22 @@ def translate(self, text) -> str: class AzureTranslator(BaseTranslator): + # https://github.com/Azure/azure-sdk-for-python + envs = { + "AZURE_ENDPOINT": "https://api.translator.azure.cn", + "AZURE_APIKEY": None, + } + def __init__(self, service, lang_out, lang_in, model): lang_out = "zh-Hans" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) - - try: - api_key = os.environ["AZURE_APIKEY"] - endpoint = os.environ["AZURE_ENDPOINT"] - region = os.environ["AZURE_REGION"] - except KeyError as e: - missing_var = e.args[0] - raise ValueError( - f"The environment variable '{missing_var}' is required but not set." - ) from e - + endpoint = os.environ["AZURE_ENDPOINT"] + api_key = os.environ["AZURE_APIKEY"] credential = AzureKeyCredential(api_key) self.client = TextTranslationClient( - endpoint=endpoint, credential=credential, region=region + endpoint=endpoint, credential=credential, region="chinaeast2" ) - # https://github.com/Azure/azure-sdk-for-python/issues/9422 logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy") logger.setLevel(logging.WARNING) @@ -342,6 +226,5 @@ def translate(self, text) -> str: from_language=self.lang_in, to_language=[self.lang_out], ) - translated_text = response[0].translations[0].text return translated_text diff --git a/pyproject.toml b/pyproject.toml index 6a13913b..79e2a806 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "onnx", "onnxruntime", "opencv-python-headless", + "tencentcloud-sdk-python", "pdfminer.six>=20240706", ] From 5612c41379dfd4f17d141c37ad83321bc509b27e Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 6 Dec 2024 16:48:35 +0800 Subject: [PATCH 002/114] feat: translator default model --- pdf2zh/converter.py | 20 +++++++++++--------- pdf2zh/translator.py | 40 ++++++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 6cdf23a0..b6ba4784 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -136,19 +136,21 @@ def __init__( self.noto = noto self.translator: BaseTranslator = None param = service.split(":", 1) - if param[0] == "google": + service_id = param[0] + service_model = param[1] if len(param) > 1 else None + if service_id == "google": self.translator = GoogleTranslator(service, lang_out, lang_in, None) - elif param[0] == "deepl": + elif service_id == "deepl": self.translator = DeepLTranslator(service, lang_out, lang_in, None) - elif param[0] == "deeplx": + elif service_id == "deeplx": self.translator = DeepLXTranslator(service, lang_out, lang_in, None) - elif param[0] == "ollama": - self.translator = OllamaTranslator(service, lang_out, lang_in, param[1]) - elif param[0] == "openai": - self.translator = OpenAITranslator(service, lang_out, lang_in, param[1]) - elif param[0] == "azure": + elif service_id == "ollama": + self.translator = OllamaTranslator(service, lang_out, lang_in, service_model) + elif service_id == "openai": + self.translator = OpenAITranslator(service, lang_out, lang_in, service_model) + elif service_id == "azure": self.translator = AzureTranslator(service, lang_out, lang_in, None) - elif param[0] == "tencent": + elif service_id == "tencent": self.translator = TencentTranslator(service, lang_out, lang_in, None) else: raise ValueError("Unsupported translation service") diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 2e1b2a41..5041924c 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -32,6 +32,18 @@ def __init__(self, service, lang_out, lang_in, model): def translate(self, text): pass + def prompt(self, text): + return [ + { + "role": "system", + "content": "You are a professional,authentic machine translation engine.", + }, + { + "role": "user", + "content": f"Translate the following markdown source text to {self.lang_out}. Keep the formula notation $v*$ unchanged. Output translation directly without any additional text.\nSource Text: {text}\nTranslated Text:", # noqa: E501 + }, + ] + def __str__(self): return f"{self.service} {self.lang_out} {self.lang_in}" @@ -140,11 +152,14 @@ class OllamaTranslator(BaseTranslator): # https://github.com/ollama/ollama-python envs = { "OLLAMA_HOST": "http://127.0.0.1:11434", + "OLLAMA_MODEL": "gemma2", } def __init__(self, service, lang_out, lang_in, model): lang_out = "zh-CN" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in + if not model: + model = os.getenv("OLLAMA_MODEL", self.envs["OLLAMA_MODEL"]) super().__init__(service, lang_out, lang_in, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = ollama.Client() @@ -153,16 +168,7 @@ def translate(self, text): response = self.client.chat( model=self.model, options=self.options, - messages=[ - { - "role": "system", - "content": "You are a professional,authentic machine translation engine.", - }, - { - "role": "user", - "content": f"Translate the following markdown source text to {self.lang_out}. Keep the formula notation $v*$ unchanged. Output translation directly without any additional text.\nSource Text: {text}\nTranslated Text:", # noqa: E501 - }, - ], + messages=self.prompt(text), ) return response["message"]["content"].strip() @@ -172,11 +178,14 @@ class OpenAITranslator(BaseTranslator): envs = { "OPENAI_BASE_URL": "https://api.openai.com/v1", "OPENAI_API_KEY": None, + "OPENAI_MODEL": "gpt-4o", } def __init__(self, service, lang_out, lang_in, model): lang_out = "zh-CN" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in + if not model: + model = os.getenv("OPENAI_MODEL", self.envs["OPENAI_MODEL"]) super().__init__(service, lang_out, lang_in, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = openai.OpenAI() @@ -185,16 +194,7 @@ def translate(self, text) -> str: response = self.client.chat.completions.create( model=self.model, **self.options, - messages=[ - { - "role": "system", - "content": "You are a professional,authentic machine translation engine.", - }, - { - "role": "user", - "content": f"Translate the following markdown source text to {self.lang_out}. Keep the formula notation $v*$ unchanged. Output translation directly without any additional text.\nSource Text: {text}\nTranslated Text:", # noqa: E501 - }, - ], + messages=self.prompt(text), ) return response.choices[0].message.content.strip() From 6519ce5079ae72d64884a80ebea8b5110d2826db Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 6 Dec 2024 18:16:50 +0800 Subject: [PATCH 003/114] feat: bing --- pdf2zh/converter.py | 21 +--- pdf2zh/gui.py | 221 ++++++++++++------------------------------- pdf2zh/translator.py | 89 ++++++++++++----- 3 files changed, 135 insertions(+), 196 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index b6ba4784..dc6e8a7a 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -136,23 +136,12 @@ def __init__( self.noto = noto self.translator: BaseTranslator = None param = service.split(":", 1) - service_id = param[0] + service_name = param[0] service_model = param[1] if len(param) > 1 else None - if service_id == "google": - self.translator = GoogleTranslator(service, lang_out, lang_in, None) - elif service_id == "deepl": - self.translator = DeepLTranslator(service, lang_out, lang_in, None) - elif service_id == "deeplx": - self.translator = DeepLXTranslator(service, lang_out, lang_in, None) - elif service_id == "ollama": - self.translator = OllamaTranslator(service, lang_out, lang_in, service_model) - elif service_id == "openai": - self.translator = OpenAITranslator(service, lang_out, lang_in, service_model) - elif service_id == "azure": - self.translator = AzureTranslator(service, lang_out, lang_in, None) - elif service_id == "tencent": - self.translator = TencentTranslator(service, lang_out, lang_in, None) - else: + for translator in [GoogleTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, OpenAITranslator, AzureTranslator, TencentTranslator]: + if service_name == translator.name: + self.translator = translator(service, lang_out, lang_in, service_model) + if not self.translator: raise ValueError("Unsupported translation service") def receive_layout(self, ltpage: LTPage): diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 4d334dfc..78956131 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -3,6 +3,16 @@ from pathlib import Path from pdf2zh import __version__ from pdf2zh.pdf2zh import extract_text +from pdf2zh.translator import ( + BaseTranslator, + GoogleTranslator, + DeepLTranslator, + DeepLXTranslator, + OllamaTranslator, + OpenAITranslator, + AzureTranslator, + TencentTranslator, +) import gradio as gr import numpy as np @@ -13,106 +23,14 @@ # Map service names to pdf2zh service options # five value, padding with None -service_map = { - "Google": (None, None, None), - "DeepL": ("DEEPL_SERVER_URL", "DEEPL_AUTH_KEY", None), - "DeepLX": ("DEEPLX_SERVER_URL", "DEEPLX_AUTH_KEY", None), - "Ollama": ("OLLAMA_HOST", None, None), - "OpenAI": ("OPENAI_BASE_URL", None, "OPENAI_API_KEY"), - "Azure": ("AZURE_APIKEY", "AZURE_ENDPOINT", "AZURE_REGION"), - "Tencent": ("TENCENT_SECRET_KEY", "TENCENT_SECRET_ID", None), -} -service_config = { - "Google": { - "apikey_content": {"visible": False}, - "apikey2_visibility": {"visible": False}, - "model_visibility": {"visible": False}, - "apikey3_visibility": {"visible": False}, - }, - "DeepL": { - "apikey_content": lambda s: { - "visible": True, - "value": os.environ.get(s[0]), - "label": s[0], - }, - "apikey2_visibility": lambda s: { - "visible": True, - "value": os.environ.get(s[1]), - "label": s[1], - }, - "model_visibility": {"visible": False}, - "apikey3_visibility": {"visible": False}, - }, - "DeepLX": { - "apikey_content": lambda s: { - "visible": True, - "value": os.environ.get(s[0]), - "label": s[0], - }, - "apikey2_visibility": lambda s: { - "visible": True, - "value": os.environ.get(s[1]), - "label": s[1], - }, - "model_visibility": {"visible": False}, - "apikey3_visibility": {"visible": False}, - }, - "Ollama": { - "apikey_content": lambda s: { - "visible": True, - "value": os.environ.get(s[0]), - "label": s[0], - }, - "apikey2_visibility": {"visible": False}, - "model_visibility": lambda s: {"visible": True, "value": s[1]}, - "apikey3_visibility": {"visible": False}, - }, - "OpenAI": { - "apikey_content": lambda s: { - "visible": True, - "value": os.environ.get(s[2]), - "label": s[2], - }, - "apikey2_visibility": lambda s: { - "visible": True, - "value": os.environ.get(s[0]), - "label": s[0], - }, - "model_visibility": {"visible": True, "value": "gpt-4o"}, - "apikey3_visibility": {"visible": False}, - }, - "Azure": { - "apikey_content": lambda s: { - "visible": True, - "value": os.environ.get(s[0]), - "label": s[0], - }, - "apikey2_visibility": lambda s: { - "visible": True, - "value": os.environ.get(s[1]), - "label": s[1], - }, - "model_visibility": {"visible": False}, - "apikey3_visibility": lambda s: { - "visible": True, - "value": os.environ.get(s[2]), - "label": s[2], - }, - }, - "Tencent": { - "apikey_content": lambda s: { - "visible": True, - "value": os.environ.get(s[0]), - "label": s[0], - }, - "apikey2_visibility": lambda s: { - "visible": True, - "value": os.environ.get(s[1]), - "label": s[1], - }, - "model_visibility": {"visible": False}, - "apikey3_visibility": {"visible": False}, - }, +service_map: dict[str, BaseTranslator] = { + "Google": GoogleTranslator, + "DeepL": DeepLTranslator, + "DeepLX": DeepLXTranslator, + "Ollama": OllamaTranslator, + "OpenAI": OpenAITranslator, + "Azure": AzureTranslator, + "Tencent": TencentTranslator, } lang_map = { "Chinese": "zh", @@ -135,7 +53,7 @@ if os.environ.get("PDF2ZH_DEMO"): flag_demo = True service_map = { - "Google": ("google", None, None), + "Google": GoogleTranslator, } page_map = { "First": [0], @@ -147,14 +65,10 @@ def verify_recaptcha(response): recaptcha_url = "https://www.google.com/recaptcha/api/siteverify" - print("reCAPTCHA", server_key, response) - data = {"secret": server_key, "response": response} result = requests.post(recaptcha_url, data=data).json() - print("reCAPTCHA", result.get("success")) - return result.get("success") @@ -167,18 +81,8 @@ def pdf_preview(file): def upload_file(file, service, progress=gr.Progress()): - """Handle file upload, validation, and initial preview.""" - if not file or not os.path.exists(file): - return None, None - - try: - # Convert first page for preview - preview_image = pdf_preview(file) - - return file, preview_image - except Exception as e: - print(f"Error converting PDF: {e}") - return None, None + preview_image = pdf_preview(file) + return file, preview_image def download_with_limit(url, save_path, size_limit): @@ -187,10 +91,10 @@ def download_with_limit(url, save_path, size_limit): with requests.get(url, stream=True, timeout=10) as response: response.raise_for_status() content = response.headers.get("Content-Disposition") - try: + try: # filename from header _, params = cgi.parse_header(content) filename = params["filename"] - except Exception: + except Exception: # filename from url filename = os.path.basename(url) with open(save_path / filename, "wb") as file: for chunk in response.iter_content(chunk_size=chunk_size): @@ -508,45 +412,46 @@ def env_var_checker(env_var_name: str) -> str: return details_wrapper(envs_status) def on_select_service(service, evt: gr.EventData): - if service in service_config: - config = service_config[service] - apikey_content = gr.update( - **( - config["apikey_content"](service_map[service]) - if callable(config["apikey_content"]) - else config["apikey_content"] - ) - ) - apikey2_visibility = gr.update( - **( - config["apikey2_visibility"](service_map[service]) - if callable(config["apikey2_visibility"]) - else config["apikey2_visibility"] - ) - ) - model_visibility = gr.update( - **( - config["model_visibility"](service_map[service]) - if callable(config["model_visibility"]) - else config["model_visibility"] - ) - ) - apikey3_visibility = gr.update( - **( - config["apikey3_visibility"](service_map[service]) - if callable(config["apikey3_visibility"]) - else config["apikey3_visibility"] - ) - ) - else: - raise gr.Error("Strange Service") - return ( - env_var_checker(service_map[service]), - model_visibility, - apikey_content, - apikey2_visibility, - apikey3_visibility, - ) + # if service in service_config: + # config = service_config[service] + # apikey_content = gr.update( + # **( + # config["apikey_content"](service_map[service]) + # if callable(config["apikey_content"]) + # else config["apikey_content"] + # ) + # ) + # apikey2_visibility = gr.update( + # **( + # config["apikey2_visibility"](service_map[service]) + # if callable(config["apikey2_visibility"]) + # else config["apikey2_visibility"] + # ) + # ) + # model_visibility = gr.update( + # **( + # config["model_visibility"](service_map[service]) + # if callable(config["model_visibility"]) + # else config["model_visibility"] + # ) + # ) + # apikey3_visibility = gr.update( + # **( + # config["apikey3_visibility"](service_map[service]) + # if callable(config["apikey3_visibility"]) + # else config["apikey3_visibility"] + # ) + # ) + # else: + # raise gr.Error("Strange Service") + # return ( + # env_var_checker(service_map[service]), + # model_visibility, + # apikey_content, + # apikey2_visibility, + # apikey3_visibility, + # ) + pass def on_select_filetype(file_type): return ( diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 5041924c..7386bdd6 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -21,6 +21,7 @@ def remove_control_characters(s): class BaseTranslator: + name = "base" envs = {} def __init__(self, service, lang_out, lang_in, model): @@ -49,6 +50,8 @@ def __str__(self): class GoogleTranslator(BaseTranslator): + name = "google" + def __init__(self, service, lang_out, lang_in, model): lang_out = "zh-CN" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in @@ -76,8 +79,45 @@ def translate(self, text): return remove_control_characters(result) +class BingTranslator(BaseTranslator): + # https://github.com/immersive-translate/old-immersive-translate/blob/6df13da22664bea2f51efe5db64c63aca59c4e79/src/background/translationService.js + # TODO: IID & IG + name = "bing" + + def __init__(self, service, lang_out, lang_in, model): + lang_out = "zh" if lang_out == "auto" else lang_out + lang_in = "en" if lang_in == "auto" else lang_in + super().__init__(service, lang_out, lang_in, model) + self.session = requests.Session() + self.endpoint = "https://www.bing.com/ttranslatev3?isVertical=1" + + def fineSID(self): + resp = self.session.get("https://www.bing.com/translator") + result = re.findall( + r"params_AbusePreventionHelper\s=\s\[(.*?),\"(.*?)\",", resp.text + )[0] + return result + + def translate(self, text): + sid = self.fineSID() + resp = self.session.post( + self.endpoint, + data={ + "fromLang": self.lang_in, + "text": text, + "to": self.lang_out, + "tryFetchingGenderDebiasedTranslations": True, + "token": sid[1], + "key": sid[0], + }, + ) + print(resp.json()) + return resp.json()[0]["translations"][0]["text"] + + class TencentTranslator(BaseTranslator): # https://github.com/TencentCloud/tencentcloud-sdk-python + name = "tencent" envs = { "TENCENTCLOUD_SECRET_ID": None, "TENCENTCLOUD_SECRET_KEY": None, @@ -100,56 +140,59 @@ def translate(self, text): return resp.TargetText -class DeepLXTranslator(BaseTranslator): - # https://deeplx.owo.network/endpoints/free.html +class DeepLTranslator(BaseTranslator): + # https://github.com/DeepLcom/deepl-python + name = "deepl" envs = { - "DEEPLX_ENDPOINT": "https://api.deepl.com/v2/translate", + "DEEPL_SERVER_URL": "https://api.deepl.com", + "DEEPL_AUTH_KEY": None, } def __init__(self, service, lang_out, lang_in, model): lang_out = "zh" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) - self.endpoint = os.getenv("DEEPLX_ENDPOINT") self.session = requests.Session() + server_url = os.getenv("DEEPL_SERVER_URL") + auth_key = os.getenv("DEEPL_AUTH_KEY") + self.client = deepl.Translator(auth_key, server_url=server_url) def translate(self, text): - resp = self.session.post( - self.endpoint, - json={ - "source_lang": self.lang_in, - "target_lang": self.lang_out, - "text": text, - }, + response = self.client.translate_text( + text, target_lang=self.lang_out, source_lang=self.lang_in ) - return resp.json()["data"] + return response.text -class DeepLTranslator(BaseTranslator): - # https://github.com/DeepLcom/deepl-python +class DeepLXTranslator(BaseTranslator): + # https://deeplx.owo.network/endpoints/free.html + name = "deeplx" envs = { - "DEEPL_SERVER_URL": "https://api.deepl.com", - "DEEPL_AUTH_KEY": None, + "DEEPLX_ENDPOINT": "https://api.deepl.com/translate", } def __init__(self, service, lang_out, lang_in, model): lang_out = "zh" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) + self.endpoint = os.getenv("DEEPLX_ENDPOINT") self.session = requests.Session() - server_url = os.getenv("DEEPL_SERVER_URL") - auth_key = os.getenv("DEEPL_AUTH_KEY") - self.client = deepl.Translator(auth_key, server_url=server_url) def translate(self, text): - response = self.client.translate_text( - text, target_lang=self.lang_out, source_lang=self.lang_in + resp = self.session.post( + self.endpoint, + json={ + "source_lang": self.lang_in, + "target_lang": self.lang_out, + "text": text, + }, ) - return response.text + return resp.json()["data"] class OllamaTranslator(BaseTranslator): # https://github.com/ollama/ollama-python + name = "ollama" envs = { "OLLAMA_HOST": "http://127.0.0.1:11434", "OLLAMA_MODEL": "gemma2", @@ -175,6 +218,7 @@ def translate(self, text): class OpenAITranslator(BaseTranslator): # https://github.com/openai/openai-python + name = "openai" envs = { "OPENAI_BASE_URL": "https://api.openai.com/v1", "OPENAI_API_KEY": None, @@ -201,6 +245,7 @@ def translate(self, text) -> str: class AzureTranslator(BaseTranslator): # https://github.com/Azure/azure-sdk-for-python + name = "azure" envs = { "AZURE_ENDPOINT": "https://api.translator.azure.cn", "AZURE_APIKEY": None, From f631a5014df569de7ce599a0a3e90ba4c04461f6 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 6 Dec 2024 21:31:27 +0800 Subject: [PATCH 004/114] fix: bing --- pdf2zh/converter.py | 5 ++++- pdf2zh/gui.py | 2 ++ pdf2zh/translator.py | 28 +++++++++++++--------------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index dc6e8a7a..3288a76c 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -19,6 +19,7 @@ from pdf2zh.translator import ( BaseTranslator, GoogleTranslator, + BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, @@ -138,7 +139,7 @@ def __init__( param = service.split(":", 1) service_name = param[0] service_model = param[1] if len(param) > 1 else None - for translator in [GoogleTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, OpenAITranslator, AzureTranslator, TencentTranslator]: + for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, OpenAITranslator, AzureTranslator, TencentTranslator]: if service_name == translator.name: self.translator = translator(service, lang_out, lang_in, service_model) if not self.translator: @@ -320,6 +321,8 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 @retry(wait=wait_fixed(1)) def worker(s: str): # 多线程翻译 + if re.match(r"^\$v\d+\$$", s): # 公式不翻译 + return s try: hash_key_paragraph = cache.deterministic_hash( (s, str(self.translator)) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 78956131..3d6c72ab 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -6,6 +6,7 @@ from pdf2zh.translator import ( BaseTranslator, GoogleTranslator, + BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, @@ -25,6 +26,7 @@ # five value, padding with None service_map: dict[str, BaseTranslator] = { "Google": GoogleTranslator, + "Bing": BingTranslator, "DeepL": DeepLTranslator, "DeepLX": DeepLXTranslator, "Ollama": OllamaTranslator, diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 7386bdd6..50e63f15 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -85,33 +85,31 @@ class BingTranslator(BaseTranslator): name = "bing" def __init__(self, service, lang_out, lang_in, model): - lang_out = "zh" if lang_out == "auto" else lang_out + lang_out = "zh-Hans" if lang_out == "auto" else lang_out lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) self.session = requests.Session() - self.endpoint = "https://www.bing.com/ttranslatev3?isVertical=1" + self.endpoint = "https://www.bing.com/ttranslatev3" + self.headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0", # noqa: E501 + } def fineSID(self): resp = self.session.get("https://www.bing.com/translator") - result = re.findall( + ig = re.findall(r"\"ig\":\"(.*?)\"", resp.text)[0] + iid = re.findall(r"data-iid=\"(.*?)\"", resp.text)[-1] + key, token = re.findall( r"params_AbusePreventionHelper\s=\s\[(.*?),\"(.*?)\",", resp.text )[0] - return result + return ig, iid, key, token def translate(self, text): - sid = self.fineSID() + ig, iid, key, token = self.fineSID() resp = self.session.post( - self.endpoint, - data={ - "fromLang": self.lang_in, - "text": text, - "to": self.lang_out, - "tryFetchingGenderDebiasedTranslations": True, - "token": sid[1], - "key": sid[0], - }, + f"{self.endpoint}?IG={ig}&IID={iid}", + data={"fromLang": self.lang_in, "to": self.lang_out, "text": text, "token": token, "key": key}, + headers=self.headers, ) - print(resp.json()) return resp.json()[0]["translations"][0]["text"] From 6d1591bf273100666c485ea5945def2c463f12e4 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 6 Dec 2024 21:33:00 +0800 Subject: [PATCH 005/114] chore: format --- pdf2zh/translator.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 50e63f15..bcedea36 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -107,7 +107,13 @@ def translate(self, text): ig, iid, key, token = self.fineSID() resp = self.session.post( f"{self.endpoint}?IG={ig}&IID={iid}", - data={"fromLang": self.lang_in, "to": self.lang_out, "text": text, "token": token, "key": key}, + data={ + "fromLang": self.lang_in, + "to": self.lang_out, + "text": text, + "token": token, + "key": key, + }, headers=self.headers, ) return resp.json()[0]["translations"][0]["text"] From 4e9a99d3a72227755847b18263eb0fbd8cd8cd3e Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 6 Dec 2024 22:43:47 +0800 Subject: [PATCH 006/114] feat: bind envs --- pdf2zh/gui.py | 200 ++++++++----------------------------------- pdf2zh/translator.py | 11 +++ 2 files changed, 45 insertions(+), 166 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 3d6c72ab..72b0dba6 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -22,8 +22,6 @@ import requests import cgi -# Map service names to pdf2zh service options -# five value, padding with None service_map: dict[str, BaseTranslator] = { "Google": GoogleTranslator, "Bing": BingTranslator, @@ -112,15 +110,12 @@ def translate( file_input, link_input, service, - apikey, - apikey2, - apikey3, - model_id, lang_from, lang_to, page_range, recaptcha_response, progress=gr.Progress(), + *envs, ): """Translate PDF content using selected service.""" if flag_demo and not verify_recaptcha(recaptcha_response): @@ -149,13 +144,13 @@ def translate( file_zh = output / f"{filename}-zh.pdf" file_dual = output / f"{filename}-dual.pdf" - selected_service = service + translator = service_map[service] selected_page = page_map[page_range] lang_from = lang_map[lang_from] lang_to = lang_map[lang_to] - VariablesSetter = TranslationVariables(service_map, apikey, apikey2, apikey3) - VariablesSetter.process_service(lang_from, lang_to, selected_service) + for i, env in enumerate(translator.envs.items()): + os.environ.setdefault(env[0], envs[i]) print(f"Files before translation: {os.listdir(output)}") @@ -167,7 +162,7 @@ def progress_bar(t: tqdm.tqdm): "pages": selected_page, "lang_in": lang_from, "lang_out": lang_to, - "service": f"{selected_service}:{model_id}", + "service": f"{translator.name}", "output": output, "thread": 4, "callback": progress_bar, @@ -196,44 +191,6 @@ def progress_bar(t: tqdm.tqdm): ) -class TranslationVariables: - def __init__(self, service_map, apikey, apikey2=None, apikey3=None): - self.service_map = service_map - self.apikey = apikey - self.apikey2 = apikey2 - self.apikey3 = apikey3 - - def set_language(self, lang_from, lang_to, selected_service): - """Sets the language parameters based on the selected service.""" - if selected_service == "google": - lang_from = "zh-CN" if lang_from == "zh" else lang_from - lang_to = "zh-CN" if lang_to == "zh" else lang_to - return lang_from, lang_to - - def set_environment_variables(self, selected_service): - """Sets the environment variables based on the selected service.""" - print(self.service_map, selected_service) - if selected_service in self.service_map: - service_info = self.service_map[selected_service] - if service_info[0]: - os.environ.setdefault(service_info[0], self.apikey) - print(service_info[0], self.apikey) - if service_info[1]: - os.environ.setdefault(service_info[1], self.apikey2) - print(service_info[1], self.apikey2) - if service_info[2]: - os.environ.setdefault(service_info[2], self.apikey3) - print(service_info[2], self.apikey3) - else: - raise gr.Error("Strange Service") - - def process_service(self, lang_from, lang_to, selected_service): - """Main processing method for the selected service.""" - lang_from, lang_to = self.set_language(lang_from, lang_to, selected_service) - self.set_environment_variables(selected_service) - return lang_from, lang_to - - # Global setup custom_blue = gr.themes.Color( c50="#E8F3FF", @@ -282,19 +239,6 @@ def process_service(self, lang_from, lang_to, selected_service): .progress-bar { border-radius: 8px !important; } - - # .input-file label { - # color: #165DFF !important; - # border: 1.2px dashed #165DFF !important; - # border-left: none !important; - # border-top: none !important; - # } - # .input-file .wrap { - # color: #165DFF !important; - # } - # .input-file .or { - # color: #165DFF !important; - # } """, head=( """ @@ -336,16 +280,18 @@ def process_service(self, lang_from, lang_to, selected_service): interactive=True, ) gr.Markdown("## Option") - with gr.Row(): - service = gr.Dropdown( - label="Service", - choices=service_map.keys(), - value="Google", - ) - apikey = gr.Textbox( - label="API Key", - max_lines=1, - visible=False, + service = gr.Dropdown( + label="Service", + choices=service_map.keys(), + value="Google", + ) + envs = [] + for i in range(3): + envs.append( + gr.Textbox( + visible=False, + interactive=True, + ) ) with gr.Row(): lang_from = gr.Dropdown( @@ -363,97 +309,17 @@ def process_service(self, lang_from, lang_to, selected_service): label="Pages", value=list(page_map.keys())[0], ) - model_id = gr.Textbox( - label="Model ID", - visible=False, - interactive=True, - ) - apikey2 = gr.Textbox( - label="API Key 2", - max_lines=1, - visible=False, - ) - apikey3 = gr.Textbox( - label="API Key 3", - max_lines=1, - visible=False, - ) - envs_status = "- Properly configured.
" - - def details_wrapper(text_markdown): - text = f""" - Technical details - {text_markdown} - - GitHub: Byaidu/PDFMathTranslate
- - GUI by: Rongxin
- - Version: {__version__} - """ - return text - - def env_var_checker(env_var_name: str) -> str: - envvarflag = True - envs_status = "" - for envvar in env_var_name: - if envvar: - if not os.environ.get(envvar): - envs_status += f"- Warning: environmental not found or error ({envvar}).
" - envvarflag = False - else: - value = str(os.environ.get(envvar)) - envs_status += ( - f"- {envvar}: {value[:13]}***
" - ) - - if envvarflag: - envs_status = ( - "- Properly configured.
" - ) - else: - envs_status += "- Please make sure that the environment variables are properly configured " - envs_status += "(guide).
" - return details_wrapper(envs_status) def on_select_service(service, evt: gr.EventData): - # if service in service_config: - # config = service_config[service] - # apikey_content = gr.update( - # **( - # config["apikey_content"](service_map[service]) - # if callable(config["apikey_content"]) - # else config["apikey_content"] - # ) - # ) - # apikey2_visibility = gr.update( - # **( - # config["apikey2_visibility"](service_map[service]) - # if callable(config["apikey2_visibility"]) - # else config["apikey2_visibility"] - # ) - # ) - # model_visibility = gr.update( - # **( - # config["model_visibility"](service_map[service]) - # if callable(config["model_visibility"]) - # else config["model_visibility"] - # ) - # ) - # apikey3_visibility = gr.update( - # **( - # config["apikey3_visibility"](service_map[service]) - # if callable(config["apikey3_visibility"]) - # else config["apikey3_visibility"] - # ) - # ) - # else: - # raise gr.Error("Strange Service") - # return ( - # env_var_checker(service_map[service]), - # model_visibility, - # apikey_content, - # apikey2_visibility, - # apikey3_visibility, - # ) - pass + translator = service_map[service] + _envs = [] + for i in range(3): + _envs.append(gr.update(visible=False, value="")) + for i, env in enumerate(translator.envs.items()): + _envs[i] = gr.update( + visible=True, label=env[0], value=os.environ.get(env[0], env[1]) + ) + return _envs def on_select_filetype(file_type): return ( @@ -472,13 +338,18 @@ def on_select_filetype(file_type): recaptcha_box = gr.HTML('
') translate_btn = gr.Button("Translate", variant="primary") tech_details_tog = gr.Markdown( - details_wrapper(envs_status), + f""" + Technical details + - GitHub: Byaidu/PDFMathTranslate
+ - GUI by: Rongxin
+ - Version: {__version__} + """, elem_classes=["secondary-text"], ) service.select( on_select_service, service, - [tech_details_tog, model_id, apikey, apikey2, apikey3], + envs, ) file_type.select( on_select_filetype, @@ -534,14 +405,11 @@ def on_select_filetype(file_type): file_input, link_input, service, - apikey, - apikey2, - apikey3, - model_id, lang_from, lang_to, page_range, recaptcha_response, + *envs, ], outputs=[ output_file, diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index bcedea36..5e04805e 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -23,8 +23,13 @@ def remove_control_characters(s): class BaseTranslator: name = "base" envs = {} + lang_map = {} def __init__(self, service, lang_out, lang_in, model): + lang_out = "zh" if lang_out == "auto" else lang_out + lang_in = "en" if lang_in == "auto" else lang_in + lang_out = self.lang_map.get(lang_out, lang_out) + lang_in = self.lang_map.get(lang_in, lang_in) self.service = service self.lang_out = lang_out self.lang_in = lang_in @@ -51,6 +56,7 @@ def __str__(self): class GoogleTranslator(BaseTranslator): name = "google" + lang_map = {"zh": "zh-CN"} def __init__(self, service, lang_out, lang_in, model): lang_out = "zh-CN" if lang_out == "auto" else lang_out @@ -83,6 +89,7 @@ class BingTranslator(BaseTranslator): # https://github.com/immersive-translate/old-immersive-translate/blob/6df13da22664bea2f51efe5db64c63aca59c4e79/src/background/translationService.js # TODO: IID & IG name = "bing" + lang_map = {"zh": "zh-Hans"} def __init__(self, service, lang_out, lang_in, model): lang_out = "zh-Hans" if lang_out == "auto" else lang_out @@ -104,6 +111,7 @@ def fineSID(self): return ig, iid, key, token def translate(self, text): + text = text[:1000] # bing translate max length ig, iid, key, token = self.fineSID() resp = self.session.post( f"{self.endpoint}?IG={ig}&IID={iid}", @@ -151,6 +159,7 @@ class DeepLTranslator(BaseTranslator): "DEEPL_SERVER_URL": "https://api.deepl.com", "DEEPL_AUTH_KEY": None, } + lang_map = {"zh": "zh-Hans"} def __init__(self, service, lang_out, lang_in, model): lang_out = "zh" if lang_out == "auto" else lang_out @@ -174,6 +183,7 @@ class DeepLXTranslator(BaseTranslator): envs = { "DEEPLX_ENDPOINT": "https://api.deepl.com/translate", } + lang_map = {"zh": "zh-Hans"} def __init__(self, service, lang_out, lang_in, model): lang_out = "zh" if lang_out == "auto" else lang_out @@ -254,6 +264,7 @@ class AzureTranslator(BaseTranslator): "AZURE_ENDPOINT": "https://api.translator.azure.cn", "AZURE_APIKEY": None, } + lang_map = {"zh": "zh-Hans"} def __init__(self, service, lang_out, lang_in, model): lang_out = "zh-Hans" if lang_out == "auto" else lang_out From 6981951ef5eebadd02a6e49527e07e03ee9a0d00 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 6 Dec 2024 23:01:03 +0800 Subject: [PATCH 007/114] fix: zh codes --- pdf2zh/converter.py | 4 ++-- pdf2zh/pdf2zh.py | 23 +++++++++++++---------- pdf2zh/translator.py | 24 +++--------------------- 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 3288a76c..4d648381 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -414,8 +414,8 @@ def raw_string(fcur: str, cstk: str): # 编码字符串 cstk = "" if brk and x + adv > x1 + 0.1 * size: # 到达右边界且原文段落存在换行 x = x0 - lang_space = {"zh-CN": 1.4, "zh-TW": 1.4, "ja": 1.1, "ko": 1.2, "en": 1.2, "ar": 1.0, "ru": 0.8, "uk": 0.8, "ta": 0.8} - y -= size * lang_space.get(self.translator.lang_out, 1.1) # 小语种大多适配 1.1 + lang_space = {"zh-cn": 1.4, "zh-tw": 1.4, "zh-hans": 1.4, "zh-hant": 1.4, "zh": 1.4, "ja": 1.1, "ko": 1.2, "en": 1.2, "ar": 1.0, "ru": 0.8, "uk": 0.8, "ta": 0.8} + y -= size * lang_space.get(self.translator.lang_out.lower(), 1.1) # 小语种大多适配 1.1 if vy_regex: # 插入公式 fix = 0 if fcur is not None: # 段落内公式修正纵向偏移 diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index 91adcf00..1ed37e59 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -27,8 +27,11 @@ model = DocLayoutModel.load_available() resfont_map = { - "zh-CN": "china-ss", - "zh-TW": "china-ts", + "zh-cn": "china-ss", + "zh-tw": "china-ts", + "zh-hans": "china-ss", + "zh-hant": "china-ts", + "zh": "china-ss", "ja": "japan-s", "ko": "korea-s", } @@ -49,11 +52,11 @@ "mr", # Marathi "ru", # Russian "sr", # Serbian - # "zh-CN",# Chinese (PRC) + # "zh-cn",# SC "ta", # Tamil "te", # Telugu "th", # Thai - # "zh-TW",# Chinese (Taiwan) + # "zh-tw",# TC "ur", # Urdu "uk", # Ukrainian ] @@ -114,10 +117,10 @@ def extract_text( font_list = [("tiro", None)] noto = None - if lang_out in resfont_map: # CJK - resfont = resfont_map[lang_out] + if lang_out.lower() in resfont_map: # CJK + resfont = resfont_map[lang_out.lower()] font_list.append((resfont, None)) - elif lang_out in noto_list: # noto + elif lang_out.lower() in noto_list: # noto resfont = "noto" ttf_path = os.path.join(tempfile.gettempdir(), "GoNotoKurrent-Regular.ttf") if not os.path.exists(ttf_path): @@ -128,7 +131,7 @@ def extract_text( ) font_list.append(("noto", ttf_path)) noto = pymupdf.Font("noto", ttf_path) - else: # auto + else: # fallback resfont = "china-ss" font_list.append(("china-ss", None)) @@ -240,14 +243,14 @@ def create_parser() -> argparse.ArgumentParser: "--lang-in", "-li", type=str, - default="auto", + default="en", help="The code of source language.", ) parse_params.add_argument( "--lang-out", "-lo", type=str, - default="auto", + default="zh", help="The code of target language.", ) parse_params.add_argument( diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 5e04805e..37de05c6 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -25,11 +25,9 @@ class BaseTranslator: envs = {} lang_map = {} - def __init__(self, service, lang_out, lang_in, model): - lang_out = "zh" if lang_out == "auto" else lang_out - lang_in = "en" if lang_in == "auto" else lang_in - lang_out = self.lang_map.get(lang_out, lang_out) - lang_in = self.lang_map.get(lang_in, lang_in) + def __init__(self, service, lang_out: str, lang_in: str, model): + lang_out = self.lang_map.get(lang_out.lower(), lang_out) + lang_in = self.lang_map.get(lang_in.lower(), lang_in) self.service = service self.lang_out = lang_out self.lang_in = lang_in @@ -59,8 +57,6 @@ class GoogleTranslator(BaseTranslator): lang_map = {"zh": "zh-CN"} def __init__(self, service, lang_out, lang_in, model): - lang_out = "zh-CN" if lang_out == "auto" else lang_out - lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) self.session = requests.Session() self.endpoint = "http://translate.google.com/m" @@ -92,8 +88,6 @@ class BingTranslator(BaseTranslator): lang_map = {"zh": "zh-Hans"} def __init__(self, service, lang_out, lang_in, model): - lang_out = "zh-Hans" if lang_out == "auto" else lang_out - lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) self.session = requests.Session() self.endpoint = "https://www.bing.com/ttranslatev3" @@ -136,8 +130,6 @@ class TencentTranslator(BaseTranslator): } def __init__(self, service, lang_out, lang_in, model): - lang_out = "zh" if lang_out == "auto" else lang_out - lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) cred = credential.DefaultCredentialProvider().get_credential() self.client = TmtClient(cred, "ap-beijing") @@ -162,8 +154,6 @@ class DeepLTranslator(BaseTranslator): lang_map = {"zh": "zh-Hans"} def __init__(self, service, lang_out, lang_in, model): - lang_out = "zh" if lang_out == "auto" else lang_out - lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) self.session = requests.Session() server_url = os.getenv("DEEPL_SERVER_URL") @@ -186,8 +176,6 @@ class DeepLXTranslator(BaseTranslator): lang_map = {"zh": "zh-Hans"} def __init__(self, service, lang_out, lang_in, model): - lang_out = "zh" if lang_out == "auto" else lang_out - lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) self.endpoint = os.getenv("DEEPLX_ENDPOINT") self.session = requests.Session() @@ -213,8 +201,6 @@ class OllamaTranslator(BaseTranslator): } def __init__(self, service, lang_out, lang_in, model): - lang_out = "zh-CN" if lang_out == "auto" else lang_out - lang_in = "en" if lang_in == "auto" else lang_in if not model: model = os.getenv("OLLAMA_MODEL", self.envs["OLLAMA_MODEL"]) super().__init__(service, lang_out, lang_in, model) @@ -240,8 +226,6 @@ class OpenAITranslator(BaseTranslator): } def __init__(self, service, lang_out, lang_in, model): - lang_out = "zh-CN" if lang_out == "auto" else lang_out - lang_in = "en" if lang_in == "auto" else lang_in if not model: model = os.getenv("OPENAI_MODEL", self.envs["OPENAI_MODEL"]) super().__init__(service, lang_out, lang_in, model) @@ -267,8 +251,6 @@ class AzureTranslator(BaseTranslator): lang_map = {"zh": "zh-Hans"} def __init__(self, service, lang_out, lang_in, model): - lang_out = "zh-Hans" if lang_out == "auto" else lang_out - lang_in = "en" if lang_in == "auto" else lang_in super().__init__(service, lang_out, lang_in, model) endpoint = os.environ["AZURE_ENDPOINT"] api_key = os.environ["AZURE_APIKEY"] From a48b46498350f8368386b94e4c311cd3688e4f95 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 6 Dec 2024 23:36:48 +0800 Subject: [PATCH 008/114] chore: env --- pdf2zh/gui.py | 10 +++++----- pdf2zh/translator.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 72b0dba6..5d635c16 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -50,7 +50,7 @@ } flag_demo = False -if os.environ.get("PDF2ZH_DEMO"): +if os.getenv("PDF2ZH_DEMO"): flag_demo = True service_map = { "Google": GoogleTranslator, @@ -59,8 +59,8 @@ "First": [0], "First 20 pages": list(range(0, 20)), } - client_key = os.environ.get("PDF2ZH_CLIENT_KEY") - server_key = os.environ.get("PDF2ZH_SERVER_KEY") + client_key = os.getenv("PDF2ZH_CLIENT_KEY") + server_key = os.getenv("PDF2ZH_SERVER_KEY") def verify_recaptcha(response): @@ -150,7 +150,7 @@ def translate( lang_to = lang_map[lang_to] for i, env in enumerate(translator.envs.items()): - os.environ.setdefault(env[0], envs[i]) + os.putenv(env[0], envs[i]) print(f"Files before translation: {os.listdir(output)}") @@ -317,7 +317,7 @@ def on_select_service(service, evt: gr.EventData): _envs.append(gr.update(visible=False, value="")) for i, env in enumerate(translator.envs.items()): _envs[i] = gr.update( - visible=True, label=env[0], value=os.environ.get(env[0], env[1]) + visible=True, label=env[0], value=os.getenv(env[0], env[1]) ) return _envs diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 37de05c6..dc6a65f0 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -156,7 +156,7 @@ class DeepLTranslator(BaseTranslator): def __init__(self, service, lang_out, lang_in, model): super().__init__(service, lang_out, lang_in, model) self.session = requests.Session() - server_url = os.getenv("DEEPL_SERVER_URL") + server_url = os.getenv("DEEPL_SERVER_URL", self.envs["DEEPL_SERVER_URL"]) auth_key = os.getenv("DEEPL_AUTH_KEY") self.client = deepl.Translator(auth_key, server_url=server_url) @@ -177,7 +177,7 @@ class DeepLXTranslator(BaseTranslator): def __init__(self, service, lang_out, lang_in, model): super().__init__(service, lang_out, lang_in, model) - self.endpoint = os.getenv("DEEPLX_ENDPOINT") + self.endpoint = os.getenv("DEEPLX_ENDPOINT", self.envs["DEEPLX_ENDPOINT"]) self.session = requests.Session() def translate(self, text): @@ -252,8 +252,8 @@ class AzureTranslator(BaseTranslator): def __init__(self, service, lang_out, lang_in, model): super().__init__(service, lang_out, lang_in, model) - endpoint = os.environ["AZURE_ENDPOINT"] - api_key = os.environ["AZURE_APIKEY"] + endpoint = os.getenv("AZURE_ENDPOINT", self.envs["AZURE_ENDPOINT"]) + api_key = os.getenv("AZURE_APIKEY") credential = AzureKeyCredential(api_key) self.client = TextTranslationClient( endpoint=endpoint, credential=credential, region="chinaeast2" From df326bc26648f046bbbcf2b9a844126ddfd2f3a9 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 7 Dec 2024 00:48:05 +0800 Subject: [PATCH 009/114] feat: add zhipu and silicon --- README.md | 2 +- README_zh-CN.md | 2 +- pdf2zh/translator.py | 46 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3c80ff4e..5988bbc2 100644 --- a/README.md +++ b/README.md @@ -234,7 +234,7 @@ pdf2zh example.pdf -li en -lo ja - `OPENAI_API_KEY`, e.g., `export OPENAI_API_KEY=xxx` ```bash - pdf2zh example.pdf -s openai:gpt-4o + pdf2zh example.pdf -s openai:gpt-4o-mini ``` - **Azure** diff --git a/README_zh-CN.md b/README_zh-CN.md index e94ce68b..0229e7eb 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -238,7 +238,7 @@ pdf2zh example.pdf -s ollama:gemma2 - `OPENAI_API_KEY`, e.g., `export OPENAI_API_KEY=xxx` ```bash -pdf2zh example.pdf -s openai:gpt-4o +pdf2zh example.pdf -s openai:gpt-4o-mini ``` - **Azure** diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index dc6a65f0..c510d33f 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -222,15 +222,15 @@ class OpenAITranslator(BaseTranslator): envs = { "OPENAI_BASE_URL": "https://api.openai.com/v1", "OPENAI_API_KEY": None, - "OPENAI_MODEL": "gpt-4o", + "OPENAI_MODEL": "gpt-4o-mini", } - def __init__(self, service, lang_out, lang_in, model): + def __init__(self, service, lang_out, lang_in, model, base_url=None, api_key=None): if not model: model = os.getenv("OPENAI_MODEL", self.envs["OPENAI_MODEL"]) super().__init__(service, lang_out, lang_in, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 - self.client = openai.OpenAI() + self.client = openai.OpenAI(base_url=base_url, api_key=api_key) def translate(self, text) -> str: response = self.client.chat.completions.create( @@ -241,19 +241,55 @@ def translate(self, text) -> str: return response.choices[0].message.content.strip() +class ZhipuTranslator(OpenAITranslator): + # https://bigmodel.cn/dev/api/thirdparty-frame/openai-sdk + name = "zhipu" + envs = { + "ZHIPU_API_KEY": None, + "ZHIPU_MODEL": "glm-4-flash", + } + + def __init__(self, service, lang_out, lang_in, model): + base_url = "https://open.bigmodel.cn/api/paas/v4" + api_key = os.getenv("ZHIPU_API_KEY") + if not model: + model = os.getenv("ZHIPU_MODEL", self.envs["ZHIPU_MODEL"]) + super().__init__( + service, lang_out, lang_in, model, base_url=base_url, api_key=api_key + ) + + +class SiliconTranslator(OpenAITranslator): + # https://docs.siliconflow.cn/quickstart + name = "silicon" + envs = { + "SILICON_API_KEY": None, + "SILICON_MODEL": "Qwen/Qwen2.5-7B-Instruct", + } + + def __init__(self, service, lang_out, lang_in, model): + base_url = "https://api.siliconflow.cn/v1" + api_key = os.getenv("SILICON_API_KEY") + if not model: + model = os.getenv("SILICON_MODEL", self.envs["SILICON_MODEL"]) + super().__init__( + service, lang_out, lang_in, model, base_url=base_url, api_key=api_key + ) + + class AzureTranslator(BaseTranslator): # https://github.com/Azure/azure-sdk-for-python name = "azure" envs = { "AZURE_ENDPOINT": "https://api.translator.azure.cn", - "AZURE_APIKEY": None, + "AZURE_API_KEY": None, } lang_map = {"zh": "zh-Hans"} def __init__(self, service, lang_out, lang_in, model): super().__init__(service, lang_out, lang_in, model) endpoint = os.getenv("AZURE_ENDPOINT", self.envs["AZURE_ENDPOINT"]) - api_key = os.getenv("AZURE_APIKEY") + api_key = os.getenv("AZURE_API_KEY") credential = AzureKeyCredential(api_key) self.client = TextTranslationClient( endpoint=endpoint, credential=credential, region="chinaeast2" From 306578d01f6f0055d24927172aacb783e8c82b72 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 7 Dec 2024 00:52:53 +0800 Subject: [PATCH 010/114] chore: translator entry --- pdf2zh/converter.py | 4 +++- pdf2zh/gui.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 4d648381..0f9f3f8b 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -24,6 +24,8 @@ DeepLXTranslator, OllamaTranslator, OpenAITranslator, + ZhipuTranslator, + SiliconTranslator, AzureTranslator, TencentTranslator, ) @@ -139,7 +141,7 @@ def __init__( param = service.split(":", 1) service_name = param[0] service_model = param[1] if len(param) > 1 else None - for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, OpenAITranslator, AzureTranslator, TencentTranslator]: + for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, OpenAITranslator, ZhipuTranslator, SiliconTranslator, AzureTranslator, TencentTranslator]: if service_name == translator.name: self.translator = translator(service, lang_out, lang_in, service_model) if not self.translator: diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 5d635c16..d363338a 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -11,6 +11,8 @@ DeepLXTranslator, OllamaTranslator, OpenAITranslator, + ZhipuTranslator, + SiliconTranslator, AzureTranslator, TencentTranslator, ) @@ -29,6 +31,8 @@ "DeepLX": DeepLXTranslator, "Ollama": OllamaTranslator, "OpenAI": OpenAITranslator, + "Zhipu": ZhipuTranslator, + "Silicon": SiliconTranslator, "Azure": AzureTranslator, "Tencent": TencentTranslator, } From c2b07b947b2ab860a1597421b6bd8b1e10e60a9a Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 7 Dec 2024 01:48:28 +0800 Subject: [PATCH 011/114] doc: envs table --- README.md | 102 ++++++++++++------------------------------- README_zh-CN.md | 97 ++++++++++------------------------------ pdf2zh/translator.py | 46 +++++++++---------- 3 files changed, 75 insertions(+), 170 deletions(-) diff --git a/README.md b/README.md index 5988bbc2..ff69f661 100644 --- a/README.md +++ b/README.md @@ -9,20 +9,20 @@ English | [简体中文](README_zh-CN.md)

- + - + - + - + - +

@@ -131,17 +131,16 @@ See [documentation for GUI](./docs/README_GUI.md) for more details. For docker deployment on cloud service: +
Deploy - Deploy to Koyeb - Deploy on Zeabur - Deploy to Koyeb +

Advanced Options

@@ -190,76 +189,33 @@ pdf2zh example.pdf -li en -lo ja

Translate with Different Services

-- **DeepL** - - See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API) - - Set ENVs to construct an endpoint like: `{DEEPL_SERVER_URL}/translate` - - `DEEPL_SERVER_URL` (Optional), e.g., `export DEEPL_SERVER_URL=https://api.deepl.com` - - `DEEPL_AUTH_KEY`, e.g., `export DEEPL_AUTH_KEY=xxx` - - ```bash - pdf2zh example.pdf -s deepl - ``` - -- **DeepLX** +The table below outlines the required environment variables for each translation service. Make sure to set them before using the respective service. - See [DeepLX](https://github.com/OwO-Network/DeepLX) +|**Translator**|**Service**|**Environment Variables**|**Default Values**|**Notes**| +|-|-|-|-|-| +|**Google (Default)**|`google`|None|N/A|None| +|**Bing**|`bing`|None|N/A|None| +|**DeepL**|`deepl`|`DEEPL_SERVER_URL`,`DEEPL_AUTH_KEY`|`https://api.deepl.com`, `[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| +|**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| +|**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| +|**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| +|**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| +|**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| +|**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| +|**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| - Set ENVs to construct an endpoint like: `{DEEPLX_SERVER_URL}/{DEEPLX_AUTH_KEY}/translate` - - `DEEPLX_SERVER_URL` (Optional), e.g., `export DEEPLX_SERVER_URL=https://api.deeplx.org` - - `DEEPLX_AUTH_KEY`, e.g., `export DEEPLX_AUTH_KEY=xxx` - - ```bash - pdf2zh example.pdf -s deeplx - ``` - -- **Ollama** - - See [Ollama](https://github.com/ollama/ollama) - - Set ENVs to construct an endpoint like: `{OLLAMA_HOST}/api/chat` - - `OLLAMA_HOST` (Optional), e.g., `export OLLAMA_HOST=https://localhost:11434` - - ```bash - pdf2zh example.pdf -s ollama:gemma2 - ``` +Use `-s service` or `-s service:model` to specify service: -- **LLM with OpenAI compatible schemas (OpenAI / SiliconCloud / Zhipu)** - - See [SiliconCloud](https://docs.siliconflow.cn/quickstart), [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk) - - Set ENVs to construct an endpoint like: `{OPENAI_BASE_URL}/chat/completions` - - `OPENAI_BASE_URL` (Optional), e.g., `export OPENAI_BASE_URL=https://api.openai.com/v1` - - `OPENAI_API_KEY`, e.g., `export OPENAI_API_KEY=xxx` - - ```bash - pdf2zh example.pdf -s openai:gpt-4o-mini - ``` - -- **Azure** - - See [Azure Text Translation](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview) - - Following ENVs are required: - - `AZURE_APIKEY`, e.g., `export AZURE_APIKEY=xxx` - - `AZURE_ENDPOINT`, e.g, `export AZURE_ENDPOINT=https://api.translator.azure.cn/` - - `AZURE_REGION`, e.g., `export AZURE_REGION=chinaeast2` - - ```bash - pdf2zh example.pdf -s azure - ``` -- **Tencent Machine Translation** - - See [Tencent Machine Translation](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104) +```bash +pdf2zh example.pdf -s openai:gpt-4o-mini +``` - Following ENVs are required: - - `TENCENT_SECRET_ID`, e.g., `export TENCENT_SECRET_ID=AKIDxxx` - - `TENCENT_SECRET_KEY`, e.g, `export TENCENT_SECRET_KEY=xxx` +Or specify model with environment variables: - ```bash - pdf2zh example.pdf -s tencent - ``` +```bash +set OPENAI_MODEL=gpt-4o-mini +pdf2zh example.pdf -s openai +```

Translate wih exceptions

diff --git a/README_zh-CN.md b/README_zh-CN.md index 0229e7eb..d3314ddb 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -33,7 +33,7 @@ - 🌐 支持 [多种语言](#language) 和 [诸多翻译服务](#services) - 🤖 提供 [命令行工具](#usage)，[图形交互界面](#gui)，以及 [容器化部署](#docker) -欢迎在 [GitHub Issues](https://github.com/Byaidu/PDFMathTranslate/issues)、[Telegram 用户群](https://t.me/+Z9_SgnxmsmA5NzBl) 或 [QQ 用户群](https://qm.qq.com/q/DixZCxQej0) 中提供反馈。 +欢迎在 [GitHub Issues](https://github.com/Byaidu/PDFMathTranslate/issues)、[Telegram 用户群](https://t.me/+Z9_SgnxmsmA5NzBl) 或 [QQ 用户群](https://qm.qq.com/q/DixZCxQej0) 中提供反馈

近期更新

@@ -57,12 +57,12 @@ ### 免费服务 () -你可以立即尝试 [免费公共服务](https://pdf2zh.com/) 而无需安装。 +你可以立即尝试 [免费公共服务](https://pdf2zh.com/) 而无需安装 ### Hugging Face 在线演示 -你可以立即尝试 [在 HuggingFace 上的在线演示](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker) 而无需安装。 -请注意，演示的计算资源有限，因此请避免滥用。 +你可以立即尝试 [在 HuggingFace 上的在线演示](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker) 而无需安装 +请注意，演示的计算资源有限，因此请避免滥用

安装和使用

@@ -131,17 +131,16 @@ 用于在云服务上部署容器镜像： +
Deploy - Deploy to Koyeb - Deploy on Zeabur - Deploy to Koyeb +

高级选项

@@ -190,82 +189,32 @@ pdf2zh example.pdf -li en -lo ja

使用不同的翻译服务

-- **DeepL** - -参考 [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API) - -设置环境变量构建接入点：`{DEEPL_SERVER_URL}/translate` - -- `DEEPL_SERVER_URL`（可选）, e.g., `export DEEPL_SERVER_URL=https://api.deepl.com` -- `DEEPL_AUTH_KEY`, e.g., `export DEEPL_AUTH_KEY=xxx` - -```bash -pdf2zh example.pdf -s deepl -``` - -- **DeepLX** - -参考 [DeepLX](https://github.com/OwO-Network/DeepLX) - -设置环境变量构建接入点：`{DEEPLX_SERVER_URL}/{DEEPLX_AUTH_KEY}/translate` - -- `DEEPLX_SERVER_URL`（可选）, e.g., `export DEEPLX_SERVER_URL=https://api.deeplx.org` -- `DEEPLX_AUTH_KEY`, e.g., `export DEEPLX_AUTH_KEY=xxx` - -```bash -pdf2zh example.pdf -s deeplx -``` - -- **Ollama** +下表列出了每个翻译服务所需的环境变量，在使用相应服务之前，请确保已设置这些变量 -参考 [Ollama](https://github.com/ollama/ollama) +|**Translator**|**Service**|**Environment Variables**|**Default Values**|**Notes**| +|-|-|-|-|-| +|**Google (Default)**|`google`|None|N/A|None| +|**Bing**|`bing`|None|N/A|None| +|**DeepL**|`deepl`|`DEEPL_SERVER_URL`,`DEEPL_AUTH_KEY`|`https://api.deepl.com`, `[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| +|**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| +|**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| +|**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| +|**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| +|**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| +|**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| +|**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| -设置环境变量构建接入点：`{OLLAMA_HOST}/api/chat` - -- `OLLAMA_HOST`（可选）, e.g., `export OLLAMA_HOST=https://localhost:11434` - -```bash -pdf2zh example.pdf -s ollama:gemma2 -``` - -- **支持 OpenAI 协议的 LLM（如 OpenAI、SiliconCloud、Zhipu）** - -参考 [SiliconCloud](https://docs.siliconflow.cn/quickstart), [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk) - -设置环境变量构建接入点：`{OPENAI_BASE_URL}/chat/completions` - -- `OPENAI_BASE_URL`（可选）, e.g., `export OPENAI_BASE_URL=https://api.openai.com/v1` -- `OPENAI_API_KEY`, e.g., `export OPENAI_API_KEY=xxx` +使用 `-s service` 或 `-s service:model` 指定翻译服务: ```bash pdf2zh example.pdf -s openai:gpt-4o-mini ``` -- **Azure** - -参考 [Azure Text Translation](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview) - -需设置以下环境变量： - -- `AZURE_APIKEY`, e.g., `export AZURE_APIKEY=xxx` -- `AZURE_ENDPOINT`, e.g., `export AZURE_ENDPOINT=https://api.translator.azure.cn/` -- `AZURE_REGION`, e.g., `export AZURE_REGION=chinaeast2` - -```bash -pdf2zh example.pdf -s azure -``` - -- **腾讯机器翻译** - -参考 [腾讯机器翻译](https://cloud.tencent.com/product/tmt) - -需设置以下环境变量： - -- `TENCENT_SECRET_ID`, e.g., `export TENCENT_SECRET_ID=AKIDxxx` -- `TENCENT_SECRET_KEY`, e.g., `export TENCENT_SECRET_KEY=xxx` +或者使用环境变量指定模型： ```bash -pdf2zh example.pdf -s tencent +set OPENAI_MODEL=gpt-4o-mini +pdf2zh example.pdf -s openai ```

指定例外规则

diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index c510d33f..ca885c25 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -121,29 +121,6 @@ def translate(self, text): return resp.json()[0]["translations"][0]["text"] -class TencentTranslator(BaseTranslator): - # https://github.com/TencentCloud/tencentcloud-sdk-python - name = "tencent" - envs = { - "TENCENTCLOUD_SECRET_ID": None, - "TENCENTCLOUD_SECRET_KEY": None, - } - - def __init__(self, service, lang_out, lang_in, model): - super().__init__(service, lang_out, lang_in, model) - cred = credential.DefaultCredentialProvider().get_credential() - self.client = TmtClient(cred, "ap-beijing") - self.req = TextTranslateRequest() - self.req.Source = self.lang_in - self.req.Target = self.lang_out - self.req.ProjectId = 0 - - def translate(self, text): - self.req.SourceText = text - resp: TextTranslateResponse = self.client.TextTranslate(self.req) - return resp.TargetText - - class DeepLTranslator(BaseTranslator): # https://github.com/DeepLcom/deepl-python name = "deepl" @@ -306,3 +283,26 @@ def translate(self, text) -> str: ) translated_text = response[0].translations[0].text return translated_text + + +class TencentTranslator(BaseTranslator): + # https://github.com/TencentCloud/tencentcloud-sdk-python + name = "tencent" + envs = { + "TENCENTCLOUD_SECRET_ID": None, + "TENCENTCLOUD_SECRET_KEY": None, + } + + def __init__(self, service, lang_out, lang_in, model): + super().__init__(service, lang_out, lang_in, model) + cred = credential.DefaultCredentialProvider().get_credential() + self.client = TmtClient(cred, "ap-beijing") + self.req = TextTranslateRequest() + self.req.Source = self.lang_in + self.req.Target = self.lang_out + self.req.ProjectId = 0 + + def translate(self, text): + self.req.SourceText = text + resp: TextTranslateResponse = self.client.TextTranslate(self.req) + return resp.TargetText From b2c8d48f2c707daad364aa9c8ac08324f519f247 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 7 Dec 2024 01:49:01 +0800 Subject: [PATCH 012/114] release: 1.8.5 --- pdf2zh/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pdf2zh/__init__.py b/pdf2zh/__init__.py index 8267a28b..9ddfcf99 100644 --- a/pdf2zh/__init__.py +++ b/pdf2zh/__init__.py @@ -2,5 +2,5 @@ log = logging.getLogger(__name__) -__version__ = "1.8.4" +__version__ = "1.8.5" __author__ = "Byaidu" diff --git a/pyproject.toml b/pyproject.toml index 79e2a806..3f67fb80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pdf2zh" -version = "1.8.4" +version = "1.8.5" description = "Latex PDF Translator" authors = [{ name = "Byaidu", email = "byaidux@gmail.com" }] license = "AGPL-3.0" From 83843a0c18f38a63c4531d4fe2a0922e267bc06d Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 7 Dec 2024 02:26:50 +0800 Subject: [PATCH 013/114] doc: envs --- README.md | 4 +--- README_zh-CN.md | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ff69f661..8227053e 100644 --- a/README.md +++ b/README.md @@ -163,8 +163,6 @@ In the following table, we list all advanced options for reference: | `-o` | Output dir | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [Exceptions](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | -Some services require setting [environmental variables](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4). -

Full / partial document translation

- Entire document @@ -189,7 +187,7 @@ pdf2zh example.pdf -li en -lo ja

Translate with Different Services

-The table below outlines the required environment variables for each translation service. Make sure to set them before using the respective service. +The table below outlines the required [environment variables](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4) for each translation service. Make sure to set them before using the respective service. |**Translator**|**Service**|**Environment Variables**|**Default Values**|**Notes**| |-|-|-|-|-| diff --git a/README_zh-CN.md b/README_zh-CN.md index d3314ddb..efadf812 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -163,8 +163,6 @@ | `-o` | 输出目录 | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [例外规则](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | -某些服务需要 [设置环境变量](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4) -

全文或部分文档翻译

- **全文翻译** @@ -189,7 +187,7 @@ pdf2zh example.pdf -li en -lo ja

使用不同的翻译服务

-下表列出了每个翻译服务所需的环境变量，在使用相应服务之前，请确保已设置这些变量 +下表列出了每个翻译服务所需的 [环境变量](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4)，在使用相应服务之前，请确保已设置这些变量 |**Translator**|**Service**|**Environment Variables**|**Default Values**|**Notes**| |-|-|-|-|-| From 8245904afdd81d2a1d50a82d8031f31aca75b3f4 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 7 Dec 2024 02:53:22 +0800 Subject: [PATCH 014/114] fix: set env --- pdf2zh/gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index d363338a..cdde175b 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -154,7 +154,7 @@ def translate( lang_to = lang_map[lang_to] for i, env in enumerate(translator.envs.items()): - os.putenv(env[0], envs[i]) + os.environ[env[0]]=envs[i] print(f"Files before translation: {os.listdir(output)}") From 5741bee2515b5f333dcaef968954f1cae0da6600 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 7 Dec 2024 02:53:51 +0800 Subject: [PATCH 015/114] release: 1.8.6 --- pdf2zh/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pdf2zh/__init__.py b/pdf2zh/__init__.py index 9ddfcf99..40c7b6db 100644 --- a/pdf2zh/__init__.py +++ b/pdf2zh/__init__.py @@ -2,5 +2,5 @@ log = logging.getLogger(__name__) -__version__ = "1.8.5" +__version__ = "1.8.6" __author__ = "Byaidu" diff --git a/pyproject.toml b/pyproject.toml index 3f67fb80..e2e639bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pdf2zh" -version = "1.8.5" +version = "1.8.6" description = "Latex PDF Translator" authors = [{ name = "Byaidu", email = "byaidux@gmail.com" }] license = "AGPL-3.0" From 62b33441aef42936a9c2af8a8393d62e7e707d3a Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 7 Dec 2024 02:54:44 +0800 Subject: [PATCH 016/114] chore: format --- pdf2zh/gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index cdde175b..7e5d88d8 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -154,7 +154,7 @@ def translate( lang_to = lang_map[lang_to] for i, env in enumerate(translator.envs.items()): - os.environ[env[0]]=envs[i] + os.environ[env[0]] = envs[i] print(f"Files before translation: {os.listdir(output)}") From c886982bcc077b9dffcf4bd317d122a91c459765 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 7 Dec 2024 12:23:43 +0800 Subject: [PATCH 017/114] fix: bat --- setup.bat | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.bat b/setup.bat index 5a07879e..25b0a740 100644 --- a/setup.bat +++ b/setup.bat @@ -5,13 +5,13 @@ set PYTHON_URL=https://www.python.org/ftp/python/3.12.7/python-3.12.7-embed-amd6 set PIP_URL=https://bootstrap.pypa.io/get-pip.py set HF_ENDPOINT=https://hf-mirror.com -if not exist pdf2zh/python.exe ( +if not exist pdf2zh_dist/python.exe ( powershell -Command "& {Invoke-WebRequest -Uri !PYTHON_URL! -OutFile python.zip}" - powershell -Command "& {Expand-Archive -Path python.zip -DestinationPath pdf2zh -Force}" + powershell -Command "& {Expand-Archive -Path python.zip -DestinationPath pdf2zh_dist -Force}" del python.zip - echo import site >> pdf2zh/python312._pth + echo import site >> pdf2zh_dist/python312._pth ) -cd pdf2zh +cd pdf2zh_dist if not exist Scripts/pip.exe ( powershell -Command "& {Invoke-WebRequest -Uri !PIP_URL! -OutFile get-pip.py}" From 8bc01370494d243cde501ea1f2865cf582538ac0 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 7 Dec 2024 13:02:12 +0800 Subject: [PATCH 018/114] fix: vflag --- pdf2zh/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 0f9f3f8b..8d188929 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -178,7 +178,7 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 return True else: if re.match( # latex 字体 - r"(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|TeX-|rsfs|txsy|wasy|stmary|.*Mono|.*Code|.*Ital|.*Sym|.*Math)", + r"(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-|rsfs|txsy|wasy|stmary|.*Mono|.*Code|.*Ital|.*Sym|.*Math)", font, ): return True From 4caf3df43f74c55383a32929bffa4a99cecef6c1 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 8 Dec 2024 00:31:06 +0800 Subject: [PATCH 019/114] chore: rm original font --- pdf2zh/converter.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 8d188929..19079659 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -1,5 +1,5 @@ from pdfminer.pdfinterp import PDFGraphicState, PDFResourceManager -from pdfminer.pdffont import PDFFont, PDFCIDFont +from pdfminer.pdffont import PDFCIDFont from pdfminer.converter import PDFConverter from pdfminer.pdffont import PDFUnicodeNotDefined from pdfminer.utils import apply_matrix_pt, mult_matrix @@ -105,13 +105,12 @@ def render_char( class Paragraph: - def __init__(self, y, x, x0, x1, size, font, brk): + def __init__(self, y, x, x0, x1, size, brk): self.y: float = y # 初始纵坐标 self.x: float = x # 初始横坐标 self.x0: float = x0 # 左边界 self.x1: float = x1 # 右边界 self.size: float = size # 字体大小 - self.font: PDFFont = font # 字体 self.brk: bool = brk # 换行标记 @@ -258,21 +257,14 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 pstk[-1].brk = True else: # 根据当前字符构建一个新的段落 sstk.append("") - pstk.append(Paragraph(child.y0, child.x0, child.x0, child.x0, child.size, child.font, False)) + pstk.append(Paragraph(child.y0, child.x0, child.x0, child.x0, child.size, False)) if not cur_v: # 文字入栈 if ( # 根据当前字符修正段落属性 child.size > pstk[-1].size / 0.79 # 1. 当前字符显著比段落字体大 or len(sstk[-1].strip()) == 1 # 2. 当前字符为段落第二个文字（考虑首字母放大的情况） - or vflag(pstk[-1].font.fontname, "") # 3. 段落字体为公式字体 - or re.match( # 4. 段落字体为粗体 - r"(.*Medi|.*Bold)", - pstk[-1].font.fontname, - re.IGNORECASE, - ) ): pstk[-1].y -= child.size - pstk[-1].size # hack 这个段落纵向位置的修正有问题，不过先凑合用吧 pstk[-1].size = child.size - pstk[-1].font = child.font sstk[-1] += child.get_text() else: # 公式入栈 if ( # 根据公式左侧的文字修正公式的纵向偏移 @@ -358,18 +350,17 @@ def raw_string(fcur: str, cstk: str): # 编码字符串 _x, _y = 0, 0 for id, new in enumerate(news): x: float = pstk[id].x # 段落初始横坐标 - y: float = pstk[id].y # 段落上边界 + y: float = pstk[id].y # 段落初始纵坐标 x0: float = pstk[id].x0 # 段落左边界 x1: float = pstk[id].x1 # 段落右边界 size: float = pstk[id].size # 段落字体大小 - font: PDFFont = pstk[id].font # 段落字体 - brk: bool = pstk[id].brk # 段落属性 + brk: bool = pstk[id].brk # 段落换行标记 cstk: str = "" # 当前文字栈 - fcur: str = None # 当前字体ID + fcur: str = None # 当前字体 ID tx = x fcur_ = fcur ptr = 0 - log.debug(f"< {y} {x} {x0} {x1} {size} {font.fontname} {brk} > {sstk[id]} | {new}") + log.debug(f"< {y} {x} {x0} {x1} {size} {brk} > {sstk[id]} | {new}") while ptr < len(new): vy_regex = re.match( r"\$?\s*v([\d\s]+)\$", new[ptr:], re.IGNORECASE @@ -387,12 +378,6 @@ def raw_string(fcur: str, cstk: str): # 编码字符串 else: # 加载文字 ch = new[ptr] fcur_ = None - # 原字体编码容易出问题，这里直接放弃掉 - # try: - # if font.widths.get(ord(ch)) and font.to_unichr(ord(ch))==ch: - # fcur_=self.fontid[font] # 原字体 - # except: - # pass try: if fcur_ is None and self.fontmap["tiro"].to_unichr(ord(ch)) == ch: fcur_ = "tiro" # 默认拉丁字体 @@ -400,7 +385,6 @@ def raw_string(fcur: str, cstk: str): # 编码字符串 pass if fcur_ is None: fcur_ = self.resfont # 默认非拉丁字体 - # print(self.fontid[font],fcur_,ch,font.char_width(ord(ch))) if fcur_ == 'noto': adv = self.noto.char_lengths(ch, size)[0] else: From 5685b05c3d0ed907de2c9db4ed2905304cb68357 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 8 Dec 2024 14:05:01 +0800 Subject: [PATCH 020/114] doc: portable --- README.md | 2 +- README_zh-CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8227053e..50d8157d 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ We provide three methods for using this project: [Commandline](#cmd), [Portable] No need to pre-install Python environment -Download and double-click to run [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/setup.bat) +Download [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/setup.bat) and double-click to run

Method III. GUI

diff --git a/README_zh-CN.md b/README_zh-CN.md index efadf812..77359bbe 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -87,7 +87,7 @@ 无需预先安装 Python 环境 -下载并双击运行 [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/setup.bat) +下载 [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/setup.bat) 并双击运行

方法三、图形交互界面

From e9386aa667ba0737352429dfe054de18e69ebebb Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 8 Dec 2024 18:14:13 +0800 Subject: [PATCH 021/114] fix: code overlap --- pdf2zh/converter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 19079659..6538d08b 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -231,7 +231,11 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 if ( # 判定当前公式是否结束 not cur_v # 1. 当前字符不属于公式 or cls != xt_cls # 2. 当前字符与前一个字符不属于同一段落 - or (abs(child.x0 - xt.x0) > vmax and cls != 0) # 3. 段落内换行，可能是一长串斜体的段落，也可能是段内分式换行，这里设个阈值进行区分 + # or (abs(child.x0 - xt.x0) > vmax and cls != 0) # 3. 段落内换行，可能是一长串斜体的段落，也可能是段内分式换行，这里设个阈值进行区分 + # 禁止纯公式（代码）段落换行，直到文字开始再重开文字段落，保证只存在两种情况 + # A. 纯公式（代码）段落（锚定绝对位置）sstk[-1]=="" -> sstk[-1]=="$v*$" + # B. 文字开头段落（排版相对位置）sstk[-1]!="" + or (sstk[-1] != "" and abs(child.x0 - xt.x0) > vmax) # 因为 cls==xt_cls==0 一定有 sstk[-1]==""，所以这里不需要再判定 cls!=0 ): if vstk: if ( # 根据公式右侧的文字修正公式的纵向偏移 @@ -240,6 +244,8 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 and child.x0 > max([vch.x0 for vch in vstk]) # 3. 当前字符在公式右侧 ): vfix = vstk[0].y0 - child.y0 + if sstk[-1] == "": + xt_cls = -1 # 禁止纯公式段落（sstk[-1]=="$v*$"）的后续连接，但是要考虑新字符和后续字符的连接，所以这里修改的是上个字符的类别 sstk[-1] += f"$v{len(var)}$" var.append(vstk) varl.append(vlstk) @@ -263,7 +269,7 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 child.size > pstk[-1].size / 0.79 # 1. 当前字符显著比段落字体大 or len(sstk[-1].strip()) == 1 # 2. 当前字符为段落第二个文字（考虑首字母放大的情况） ): - pstk[-1].y -= child.size - pstk[-1].size # hack 这个段落纵向位置的修正有问题，不过先凑合用吧 + pstk[-1].y -= child.size - pstk[-1].size # 修正段落初始纵坐标，假设两个不同大小字符的上边界对齐 pstk[-1].size = child.size sstk[-1] += child.get_text() else: # 公式入栈 From efe48bfa40e6837545fe85e73661d3b9b618fb97 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 8 Dec 2024 22:40:18 +0800 Subject: [PATCH 022/114] fix: tqdm --- pdf2zh/high_level.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 47bc16ba..510798bf 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -48,13 +48,11 @@ def extract_text_to_fp( parser = PDFParser(inf) doc = PDFDocument(parser, password=password) - with tqdm.tqdm( - enumerate(PDFPage.create_pages(doc)), - total=total_pages, - ) as progress: - for pageno, page in progress: + with tqdm.tqdm(total=total_pages) as progress: + for pageno, page in enumerate(PDFPage.create_pages(doc)): if pages and (pageno not in pages): continue + progress.update() if callback: callback(progress) page.pageno = pageno From 628d75a03cfaeb70ee7ca14397faa72b856a811b Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 8 Dec 2024 22:53:57 +0800 Subject: [PATCH 023/114] release: 1.8.7 --- pdf2zh/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pdf2zh/__init__.py b/pdf2zh/__init__.py index 40c7b6db..cd0a125f 100644 --- a/pdf2zh/__init__.py +++ b/pdf2zh/__init__.py @@ -2,5 +2,5 @@ log = logging.getLogger(__name__) -__version__ = "1.8.6" +__version__ = "1.8.7" __author__ = "Byaidu" diff --git a/pyproject.toml b/pyproject.toml index e2e639bd..594b33c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pdf2zh" -version = "1.8.6" +version = "1.8.7" description = "Latex PDF Translator" authors = [{ name = "Byaidu", email = "byaidux@gmail.com" }] license = "AGPL-3.0" From adfc4e5b023c2ed5b130a4a38f3afdf5fbb85a1f Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 8 Dec 2024 23:40:22 +0800 Subject: [PATCH 024/114] feat: pip mirror --- setup.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.bat b/setup.bat index 25b0a740..3f86d25f 100644 --- a/setup.bat +++ b/setup.bat @@ -4,6 +4,7 @@ setlocal enabledelayedexpansion set PYTHON_URL=https://www.python.org/ftp/python/3.12.7/python-3.12.7-embed-amd64.zip set PIP_URL=https://bootstrap.pypa.io/get-pip.py set HF_ENDPOINT=https://hf-mirror.com +set PIP_MIRROR=https://mirrors.aliyun.com/pypi/simple if not exist pdf2zh_dist/python.exe ( powershell -Command "& {Invoke-WebRequest -Uri !PYTHON_URL! -OutFile python.zip}" @@ -19,7 +20,7 @@ if not exist Scripts/pip.exe ( ) path Scripts -pip install --no-warn-script-location --upgrade pdf2zh +pip install --no-warn-script-location --upgrade pdf2zh -i !PIP_MIRROR! pdf2zh -i pause From 26f1d7a496f4f92c21eb08ffb8c9754192607f68 Mon Sep 17 00:00:00 2001 From: yidasanqian Date: Tue, 10 Dec 2024 14:26:50 +0800 Subject: [PATCH 025/114] feat(translator): add AzureOpenAITranslator --- pdf2zh/converter.py | 4 +++- pdf2zh/gui.py | 2 ++ pdf2zh/translator.py | 27 +++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 6538d08b..53d11d9b 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -17,6 +17,7 @@ from tenacity import retry, wait_fixed from pdf2zh import cache from pdf2zh.translator import ( + AzureOpenAITranslator, BaseTranslator, GoogleTranslator, BingTranslator, @@ -140,7 +141,8 @@ def __init__( param = service.split(":", 1) service_name = param[0] service_model = param[1] if len(param) > 1 else None - for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, OpenAITranslator, ZhipuTranslator, SiliconTranslator, AzureTranslator, TencentTranslator]: + for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator, + OpenAITranslator, ZhipuTranslator, SiliconTranslator, AzureTranslator, TencentTranslator]: if service_name == translator.name: self.translator = translator(service, lang_out, lang_in, service_model) if not self.translator: diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 7e5d88d8..2236bde0 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -10,6 +10,7 @@ DeepLTranslator, DeepLXTranslator, OllamaTranslator, + AzureOpenAITranslator, OpenAITranslator, ZhipuTranslator, SiliconTranslator, @@ -30,6 +31,7 @@ "DeepL": DeepLTranslator, "DeepLX": DeepLXTranslator, "Ollama": OllamaTranslator, + "AzureOpenAI": AzureOpenAITranslator, "OpenAI": OpenAITranslator, "Zhipu": ZhipuTranslator, "Silicon": SiliconTranslator, diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index ca885c25..17a11026 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -216,6 +216,33 @@ def translate(self, text) -> str: messages=self.prompt(text), ) return response.choices[0].message.content.strip() + + +class AzureOpenAITranslator(BaseTranslator): + name = "azure-openai" + envs = { + "AZURE_OPENAI_BASE_URL": None, # e.g. "https://xxx.openai.azure.com" + "AZURE_OPENAI_API_KEY": None, + "AZURE_OPENAI_MODEL": "gpt-4o-mini", + "AZURE_OPENAI_API_VERSION": "2024-06-01", + } + + def __init__(self, service, lang_out, lang_in, model, base_url=None, api_key=None): + base_url = os.getenv("AZURE_OPENAI_BASE_URL", self.envs["AZURE_OPENAI_BASE_URL"]) + api_version = os.getenv("AZURE_OPENAI_API_VERSION", self.envs["AZURE_OPENAI_API_VERSION"]) + if not model: + model = os.getenv("AZURE_OPENAI_MODEL", self.envs["AZURE_OPENAI_MODEL"]) + super().__init__(service, lang_out, lang_in, model) + self.options = {"temperature": 0} + self.client = openai.AzureOpenAI(azure_endpoint=base_url, azure_deployment=model, api_version=api_version, api_key=api_key) + + def translate(self, text) -> str: + response = self.client.chat.completions.create( + model=self.model, + **self.options, + messages=self.prompt(text), + ) + return response.choices[0].message.content.strip() class ZhipuTranslator(OpenAITranslator): From c03217476010d5910ff300b5e1beed275b2e2a8f Mon Sep 17 00:00:00 2001 From: yidasanqian Date: Tue, 10 Dec 2024 17:04:59 +0800 Subject: [PATCH 026/114] feat: add AzureOpenAI configuration to README files --- README.md | 1 + README_zh-CN.md | 1 + pdf2zh/translator.py | 5 ++--- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 50d8157d..83d27c22 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ The table below outlines the required [environment variables](https://chatgpt.co |**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| |**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| |**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| +|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`,`AZURE_OPENAI_API_VERSION`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`，`2024-06-01`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| |**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| |**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| diff --git a/README_zh-CN.md b/README_zh-CN.md index 77359bbe..3800cf26 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -197,6 +197,7 @@ pdf2zh example.pdf -li en -lo ja |**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| |**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| |**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| +|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`,`AZURE_OPENAI_API_VERSION`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`，`2024-06-01`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| |**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| |**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 17a11026..fc8aebec 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -223,13 +223,12 @@ class AzureOpenAITranslator(BaseTranslator): envs = { "AZURE_OPENAI_BASE_URL": None, # e.g. "https://xxx.openai.azure.com" "AZURE_OPENAI_API_KEY": None, - "AZURE_OPENAI_MODEL": "gpt-4o-mini", - "AZURE_OPENAI_API_VERSION": "2024-06-01", + "AZURE_OPENAI_MODEL": "gpt-4o-mini" } def __init__(self, service, lang_out, lang_in, model, base_url=None, api_key=None): base_url = os.getenv("AZURE_OPENAI_BASE_URL", self.envs["AZURE_OPENAI_BASE_URL"]) - api_version = os.getenv("AZURE_OPENAI_API_VERSION", self.envs["AZURE_OPENAI_API_VERSION"]) + api_version = os.getenv("AZURE_OPENAI_API_VERSION", "2024-06-01") if not model: model = os.getenv("AZURE_OPENAI_MODEL", self.envs["AZURE_OPENAI_MODEL"]) super().__init__(service, lang_out, lang_in, model) From 1697ee3312116e571bffcbdcd2ea4e1ca30232c2 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Tue, 10 Dec 2024 23:49:00 +0800 Subject: [PATCH 027/114] doc: todo --- README.md | 4 ++++ pdf2zh/translator.py | 21 ++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 83d27c22..1e037cab 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,10 @@ pdf2zh example.pdf -t 1 - [ ] Support non-PDF/A files +- [ ] Provide API interface + +- [ ] Plugins of [Zotero](https://github.com/zotero/zotero) and [Obsidian](https://github.com/obsidianmd/obsidian-releases) +

Acknowledgements

- Document merging: [PyMuPDF](https://github.com/pymupdf/PyMuPDF) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index fc8aebec..2c80f9f8 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -216,24 +216,31 @@ def translate(self, text) -> str: messages=self.prompt(text), ) return response.choices[0].message.content.strip() - + class AzureOpenAITranslator(BaseTranslator): name = "azure-openai" envs = { - "AZURE_OPENAI_BASE_URL": None, # e.g. "https://xxx.openai.azure.com" + "AZURE_OPENAI_BASE_URL": None, # e.g. "https://xxx.openai.azure.com" "AZURE_OPENAI_API_KEY": None, - "AZURE_OPENAI_MODEL": "gpt-4o-mini" + "AZURE_OPENAI_MODEL": "gpt-4o-mini", } def __init__(self, service, lang_out, lang_in, model, base_url=None, api_key=None): - base_url = os.getenv("AZURE_OPENAI_BASE_URL", self.envs["AZURE_OPENAI_BASE_URL"]) + base_url = os.getenv( + "AZURE_OPENAI_BASE_URL", self.envs["AZURE_OPENAI_BASE_URL"] + ) api_version = os.getenv("AZURE_OPENAI_API_VERSION", "2024-06-01") if not model: model = os.getenv("AZURE_OPENAI_MODEL", self.envs["AZURE_OPENAI_MODEL"]) super().__init__(service, lang_out, lang_in, model) - self.options = {"temperature": 0} - self.client = openai.AzureOpenAI(azure_endpoint=base_url, azure_deployment=model, api_version=api_version, api_key=api_key) + self.options = {"temperature": 0} + self.client = openai.AzureOpenAI( + azure_endpoint=base_url, + azure_deployment=model, + api_version=api_version, + api_key=api_key, + ) def translate(self, text) -> str: response = self.client.chat.completions.create( @@ -241,7 +248,7 @@ def translate(self, text) -> str: **self.options, messages=self.prompt(text), ) - return response.choices[0].message.content.strip() + return response.choices[0].message.content.strip() class ZhipuTranslator(OpenAITranslator): From 778b91437343be22be5d6ce97919d7b179f575f3 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Wed, 11 Dec 2024 16:58:50 +0800 Subject: [PATCH 028/114] =?UTF-8?q?AzureOpenAI=E7=8E=AF=E5=A2=83=E5=8F=98?= =?UTF-8?q?=E9=87=8F=E6=9B=B4=E6=96=B0=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_zh-CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1e037cab..c566d3d5 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ The table below outlines the required [environment variables](https://chatgpt.co |**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| |**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| |**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| -|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`,`AZURE_OPENAI_API_VERSION`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`，`2024-06-01`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| +|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| |**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| |**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| diff --git a/README_zh-CN.md b/README_zh-CN.md index 3800cf26..aff0e336 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -197,7 +197,7 @@ pdf2zh example.pdf -li en -lo ja |**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| |**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| |**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| -|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`,`AZURE_OPENAI_API_VERSION`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`，`2024-06-01`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| +|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| |**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| |**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| From 95e19e3ec4a8a4df38f566008042bbbb8e29d609 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Wed, 11 Dec 2024 23:53:43 +0800 Subject: [PATCH 029/114] refactor(main): add translate --- pdf2zh/gui.py | 8 +- pdf2zh/high_level.py | 181 ++++++++++++++++++++++++++++++++++++++++- pdf2zh/pdf2zh.py | 187 ++----------------------------------------- 3 files changed, 187 insertions(+), 189 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 2236bde0..130c2579 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -2,7 +2,7 @@ import shutil from pathlib import Path from pdf2zh import __version__ -from pdf2zh.pdf2zh import extract_text +from pdf2zh.high_level import translate from pdf2zh.translator import ( BaseTranslator, GoogleTranslator, @@ -111,7 +111,7 @@ def download_with_limit(url, save_path, size_limit): return save_path / filename -def translate( +def translate_file( file_type, file_input, link_input, @@ -174,7 +174,7 @@ def progress_bar(t: tqdm.tqdm): "callback": progress_bar, } print(param) - extract_text(**param) + translate(**param) print(f"Files after translation: {os.listdir(output)}") if not file_zh.exists() or not file_dual.exists(): @@ -405,7 +405,7 @@ def on_select_filetype(file_type): ) translate_btn.click( - translate, + translate_file, inputs=[ file_type, file_input, diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 510798bf..4c736fc1 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -3,21 +3,77 @@ from typing import BinaryIO import numpy as np import tqdm -from pymupdf import Document +import sys +from pymupdf import Font, Document from pdfminer.pdfpage import PDFPage from pdfminer.pdfinterp import PDFResourceManager from pdfminer.pdfdocument import PDFDocument from pdfminer.pdfparser import PDFParser +from pdfminer.pdfexceptions import PDFValueError from pdf2zh.converter import TranslateConverter from pdf2zh.pdfinterp import PDFPageInterpreterEx -from pymupdf import Font +from pdf2zh.doclayout import DocLayoutModel +from pathlib import Path +from typing import Any, Container, Iterable, List, Optional +import urllib.request +import requests +import tempfile +import os +model = DocLayoutModel.load_available() -def extract_text_to_fp( +resfont_map = { + "zh-cn": "china-ss", + "zh-tw": "china-ts", + "zh-hans": "china-ss", + "zh-hant": "china-ts", + "zh": "china-ss", + "ja": "japan-s", + "ko": "korea-s", +} + +noto_list = [ + "am", # Amharic + "ar", # Arabic + "bn", # Bengali + "bg", # Bulgarian + "chr", # Cherokee + "el", # Greek + "gu", # Gujarati + "iw", # Hebrew + "hi", # Hindi + # "ja", # Japanese + "kn", # Kannada + # "ko", # Korean + "ml", # Malayalam + "mr", # Marathi + "ru", # Russian + "sr", # Serbian + # "zh-cn",# SC + "ta", # Tamil + "te", # Telugu + "th", # Thai + # "zh-tw",# TC + "ur", # Urdu + "uk", # Ukrainian +] + + +def check_files(files: List[str]) -> List[str]: + files = [ + f for f in files if not f.startswith("http://") + ] # exclude online files, http + files = [ + f for f in files if not f.startswith("https://") + ] # exclude online files, https + missing_files = [file for file in files if not os.path.exists(file)] + return missing_files + + +def translate_patch( inf: BinaryIO, pages=None, password: str = "", - debug: bool = False, page_count: int = 0, vfont: str = "", vchar: str = "", @@ -95,3 +151,120 @@ def extract_text_to_fp( device.close() return obj_patch + + +def translate( + files: Iterable[str] = [], + pages: Optional[Container[int]] = None, + password: str = "", + vfont: str = "", + vchar: str = "", + thread: int = 0, + lang_in: str = "", + lang_out: str = "", + service: str = "", + callback: object = None, + output: str = "", + **kwargs: Any, +): + if not files: + raise PDFValueError("No files to process.") + + missing_files = check_files(files) + + if missing_files: + print("The following files do not exist:", file=sys.stderr) + for file in missing_files: + print(f" {file}", file=sys.stderr) + raise PDFValueError("Some files do not exist.") + + for file in files: + if file is str and (file.startswith("http://") or file.startswith("https://")): + print("Online files detected, downloading...") + try: + r = requests.get(file, allow_redirects=True) + if r.status_code == 200: + if not os.path.exists("./pdf2zh_files"): + print("Making a temporary dir for downloading PDF files...") + os.mkdir(os.path.dirname("./pdf2zh_files")) + with open("./pdf2zh_files/tmp_download.pdf", "wb") as f: + print(f"Writing the file: {file}...") + f.write(r.content) + file = "./pdf2zh_files/tmp_download.pdf" + else: + r.raise_for_status() + except Exception as e: + raise PDFValueError( + f"Errors occur in downloading the PDF file. Please check the link(s).\nError:\n{e}" + ) + filename = os.path.splitext(os.path.basename(file))[0] + + font_list = [("tiro", None)] + noto = None + if lang_out.lower() in resfont_map: # CJK + resfont = resfont_map[lang_out.lower()] + font_list.append((resfont, None)) + elif lang_out.lower() in noto_list: # noto + resfont = "noto" + ttf_path = os.path.join(tempfile.gettempdir(), "GoNotoKurrent-Regular.ttf") + if not os.path.exists(ttf_path): + print("Downloading Noto font...") + urllib.request.urlretrieve( + "https://github.com/satbyy/go-noto-universal/releases/download/v7.0/GoNotoKurrent-Regular.ttf", + ttf_path, + ) + font_list.append(("noto", ttf_path)) + noto = Font("noto", ttf_path) + else: # fallback + resfont = "china-ss" + font_list.append(("china-ss", None)) + + doc_en = Document(file) + page_count = doc_en.page_count + # font_list = [("china-ss", None), ("tiro", None)] + font_id = {} + for page in doc_en: + for font in font_list: + font_id[font[0]] = page.insert_font(font[0], font[1]) + xreflen = doc_en.xref_length() + for xref in range(1, xreflen): + for label in ["Resources/", ""]: # 可能是基于 xobj 的 res + try: # xref 读写可能出错 + font_res = doc_en.xref_get_key(xref, f"{label}Font") + if font_res[0] == "dict": + for font in font_list: + font_exist = doc_en.xref_get_key( + xref, f"{label}Font/{font[0]}" + ) + if font_exist[0] == "null": + doc_en.xref_set_key( + xref, + f"{label}Font/{font[0]}", + f"{font_id[font[0]]} 0 R", + ) + except Exception: + pass + doc_en.save(Path(output) / f"{filename}-en.pdf") + + with open(Path(output) / f"{filename}-en.pdf", "rb") as fp: + obj_patch: dict = translate_patch(fp, model=model, **locals()) + + for obj_id, ops_new in obj_patch.items(): + # ops_old=doc_en.xref_stream(obj_id) + # print(obj_id) + # print(ops_old) + # print(ops_new.encode()) + doc_en.update_stream(obj_id, ops_new.encode()) + + doc_zh = doc_en + doc_dual = Document(Path(output) / f"{filename}-en.pdf") + doc_dual.insert_file(doc_zh) + for id in range(page_count): + doc_dual.move_page(page_count + id, id * 2 + 1) + doc_zh.save(Path(output) / f"{filename}-zh.pdf", deflate=1) + doc_dual.save(Path(output) / f"{filename}-dual.pdf", deflate=1) + doc_zh.close() + doc_dual.close() + os.remove(Path(output) / f"{filename}-en.pdf") + + return diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index 1ed37e59..d0a06733 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -6,185 +6,14 @@ from __future__ import annotations import argparse -import os import sys import logging -from pathlib import Path -from typing import Any, Container, Iterable, List, Optional -import urllib.request -from pdfminer.pdfexceptions import PDFValueError - -import pymupdf -import requests -import tempfile - +from typing import List, Optional from pdf2zh import __version__, log -from pdf2zh.high_level import extract_text_to_fp -from pdf2zh.doclayout import DocLayoutModel +from pdf2zh.high_level import translate logging.basicConfig() -model = DocLayoutModel.load_available() - -resfont_map = { - "zh-cn": "china-ss", - "zh-tw": "china-ts", - "zh-hans": "china-ss", - "zh-hant": "china-ts", - "zh": "china-ss", - "ja": "japan-s", - "ko": "korea-s", -} -noto_list = [ - "am", # Amharic - "ar", # Arabic - "bn", # Bengali - "bg", # Bulgarian - "chr", # Cherokee - "el", # Greek - "gu", # Gujarati - "iw", # Hebrew - "hi", # Hindi - # "ja", # Japanese - "kn", # Kannada - # "ko", # Korean - "ml", # Malayalam - "mr", # Marathi - "ru", # Russian - "sr", # Serbian - # "zh-cn",# SC - "ta", # Tamil - "te", # Telugu - "th", # Thai - # "zh-tw",# TC - "ur", # Urdu - "uk", # Ukrainian -] - - -def check_files(files: List[str]) -> List[str]: - files = [ - f for f in files if not f.startswith("http://") - ] # exclude online files, http - files = [ - f for f in files if not f.startswith("https://") - ] # exclude online files, https - missing_files = [file for file in files if not os.path.exists(file)] - return missing_files - - -def extract_text( - files: Iterable[str] = [], - pages: Optional[Container[int]] = None, - password: str = "", - debug: bool = False, - vfont: str = "", - vchar: str = "", - thread: int = 0, - lang_in: str = "", - lang_out: str = "", - service: str = "", - callback: object = None, - output: str = "", - **kwargs: Any, -): - if debug: - log.setLevel(logging.DEBUG) - - if not files: - raise PDFValueError("Must provide files to work upon!") - - for file in files: - if file is str and (file.startswith("http://") or file.startswith("https://")): - print("Online files detected, downloading...") - try: - r = requests.get(file, allow_redirects=True) - if r.status_code == 200: - if not os.path.exists("./pdf2zh_files"): - print("Making a temporary dir for downloading PDF files...") - os.mkdir(os.path.dirname("./pdf2zh_files")) - with open("./pdf2zh_files/tmp_download.pdf", "wb") as f: - print(f"Writing the file: {file}...") - f.write(r.content) - file = "./pdf2zh_files/tmp_download.pdf" - else: - r.raise_for_status() - except Exception as e: - raise PDFValueError( - f"Errors occur in downloading the PDF file. Please check the link(s).\nError:\n{e}" - ) - filename = os.path.splitext(os.path.basename(file))[0] - - font_list = [("tiro", None)] - noto = None - if lang_out.lower() in resfont_map: # CJK - resfont = resfont_map[lang_out.lower()] - font_list.append((resfont, None)) - elif lang_out.lower() in noto_list: # noto - resfont = "noto" - ttf_path = os.path.join(tempfile.gettempdir(), "GoNotoKurrent-Regular.ttf") - if not os.path.exists(ttf_path): - print("Downloading Noto font...") - urllib.request.urlretrieve( - "https://github.com/satbyy/go-noto-universal/releases/download/v7.0/GoNotoKurrent-Regular.ttf", - ttf_path, - ) - font_list.append(("noto", ttf_path)) - noto = pymupdf.Font("noto", ttf_path) - else: # fallback - resfont = "china-ss" - font_list.append(("china-ss", None)) - - doc_en = pymupdf.open(file) - page_count = doc_en.page_count - # font_list = [("china-ss", None), ("tiro", None)] - font_id = {} - for page in doc_en: - for font in font_list: - font_id[font[0]] = page.insert_font(font[0], font[1]) - xreflen = doc_en.xref_length() - for xref in range(1, xreflen): - for label in ["Resources/", ""]: # 可能是基于 xobj 的 res - try: # xref 读写可能出错 - font_res = doc_en.xref_get_key(xref, f"{label}Font") - if font_res[0] == "dict": - for font in font_list: - font_exist = doc_en.xref_get_key( - xref, f"{label}Font/{font[0]}" - ) - if font_exist[0] == "null": - doc_en.xref_set_key( - xref, - f"{label}Font/{font[0]}", - f"{font_id[font[0]]} 0 R", - ) - except Exception: - pass - doc_en.save(Path(output) / f"{filename}-en.pdf") - - with open(Path(output) / f"{filename}-en.pdf", "rb") as fp: - obj_patch: dict = extract_text_to_fp(fp, model=model, **locals()) - - for obj_id, ops_new in obj_patch.items(): - # ops_old=doc_en.xref_stream(obj_id) - # print(obj_id) - # print(ops_old) - # print(ops_new.encode()) - doc_en.update_stream(obj_id, ops_new.encode()) - - doc_zh = doc_en - doc_dual = pymupdf.open(Path(output) / f"{filename}-en.pdf") - doc_dual.insert_file(doc_zh) - for id in range(page_count): - doc_dual.move_page(page_count + id, id * 2 + 1) - doc_zh.save(Path(output) / f"{filename}-zh.pdf", deflate=1) - doc_dual.save(Path(output) / f"{filename}-dual.pdf", deflate=1) - doc_zh.close() - doc_dual.close() - os.remove(Path(output) / f"{filename}-en.pdf") - - return - def create_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__, add_help=True) @@ -308,19 +137,15 @@ def parse_args(args: Optional[List[str]]) -> argparse.Namespace: def main(args: Optional[List[str]] = None) -> int: parsed_args = parse_args(args) - missing_files = check_files(parsed_args.files) - if missing_files: - print("The following files do not exist:", file=sys.stderr) - for file in missing_files: - print(f" {file}", file=sys.stderr) - return -1 + if parsed_args.debug: + log.setLevel(logging.DEBUG) + if parsed_args.interactive: from pdf2zh.gui import setup_gui - setup_gui(parsed_args.share) return 0 - extract_text(**vars(parsed_args)) + translate(**vars(parsed_args)) return 0 From d6f96334b1fb16d5d20f475d24fd521974edd0ea Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Thu, 12 Dec 2024 13:21:58 +0800 Subject: [PATCH 030/114] fix: cache model --- pdf2zh/converter.py | 2 +- pdf2zh/pdf2zh.py | 1 + pdf2zh/translator.py | 53 ++++++++++++++++++++------------------------ 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 53d11d9b..db0bb933 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -144,7 +144,7 @@ def __init__( for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator, OpenAITranslator, ZhipuTranslator, SiliconTranslator, AzureTranslator, TencentTranslator]: if service_name == translator.name: - self.translator = translator(service, lang_out, lang_in, service_model) + self.translator = translator(lang_out, lang_in, service_model) if not self.translator: raise ValueError("Unsupported translation service") diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index d0a06733..04127b9b 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -142,6 +142,7 @@ def main(args: Optional[List[str]] = None) -> int: if parsed_args.interactive: from pdf2zh.gui import setup_gui + setup_gui(parsed_args.share) return 0 diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 2c80f9f8..b4e92456 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -25,10 +25,9 @@ class BaseTranslator: envs = {} lang_map = {} - def __init__(self, service, lang_out: str, lang_in: str, model): + def __init__(self, lang_out: str, lang_in: str, model): lang_out = self.lang_map.get(lang_out.lower(), lang_out) lang_in = self.lang_map.get(lang_in.lower(), lang_in) - self.service = service self.lang_out = lang_out self.lang_in = lang_in self.model = model @@ -49,15 +48,15 @@ def prompt(self, text): ] def __str__(self): - return f"{self.service} {self.lang_out} {self.lang_in}" + return f"{self.name} {self.lang_out} {self.lang_in} {self.model}" class GoogleTranslator(BaseTranslator): name = "google" lang_map = {"zh": "zh-CN"} - def __init__(self, service, lang_out, lang_in, model): - super().__init__(service, lang_out, lang_in, model) + def __init__(self, lang_out, lang_in, model): + super().__init__(lang_out, lang_in, model) self.session = requests.Session() self.endpoint = "http://translate.google.com/m" self.headers = { @@ -87,8 +86,8 @@ class BingTranslator(BaseTranslator): name = "bing" lang_map = {"zh": "zh-Hans"} - def __init__(self, service, lang_out, lang_in, model): - super().__init__(service, lang_out, lang_in, model) + def __init__(self, lang_out, lang_in, model): + super().__init__(lang_out, lang_in, model) self.session = requests.Session() self.endpoint = "https://www.bing.com/ttranslatev3" self.headers = { @@ -130,8 +129,8 @@ class DeepLTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, service, lang_out, lang_in, model): - super().__init__(service, lang_out, lang_in, model) + def __init__(self, lang_out, lang_in, model): + super().__init__(lang_out, lang_in, model) self.session = requests.Session() server_url = os.getenv("DEEPL_SERVER_URL", self.envs["DEEPL_SERVER_URL"]) auth_key = os.getenv("DEEPL_AUTH_KEY") @@ -152,8 +151,8 @@ class DeepLXTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, service, lang_out, lang_in, model): - super().__init__(service, lang_out, lang_in, model) + def __init__(self, lang_out, lang_in, model): + super().__init__(lang_out, lang_in, model) self.endpoint = os.getenv("DEEPLX_ENDPOINT", self.envs["DEEPLX_ENDPOINT"]) self.session = requests.Session() @@ -177,10 +176,10 @@ class OllamaTranslator(BaseTranslator): "OLLAMA_MODEL": "gemma2", } - def __init__(self, service, lang_out, lang_in, model): + def __init__(self, lang_out, lang_in, model): if not model: model = os.getenv("OLLAMA_MODEL", self.envs["OLLAMA_MODEL"]) - super().__init__(service, lang_out, lang_in, model) + super().__init__(lang_out, lang_in, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = ollama.Client() @@ -202,10 +201,10 @@ class OpenAITranslator(BaseTranslator): "OPENAI_MODEL": "gpt-4o-mini", } - def __init__(self, service, lang_out, lang_in, model, base_url=None, api_key=None): + def __init__(self, lang_out, lang_in, model, base_url=None, api_key=None): if not model: model = os.getenv("OPENAI_MODEL", self.envs["OPENAI_MODEL"]) - super().__init__(service, lang_out, lang_in, model) + super().__init__(lang_out, lang_in, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = openai.OpenAI(base_url=base_url, api_key=api_key) @@ -226,14 +225,14 @@ class AzureOpenAITranslator(BaseTranslator): "AZURE_OPENAI_MODEL": "gpt-4o-mini", } - def __init__(self, service, lang_out, lang_in, model, base_url=None, api_key=None): + def __init__(self, lang_out, lang_in, model, base_url=None, api_key=None): base_url = os.getenv( "AZURE_OPENAI_BASE_URL", self.envs["AZURE_OPENAI_BASE_URL"] ) api_version = os.getenv("AZURE_OPENAI_API_VERSION", "2024-06-01") if not model: model = os.getenv("AZURE_OPENAI_MODEL", self.envs["AZURE_OPENAI_MODEL"]) - super().__init__(service, lang_out, lang_in, model) + super().__init__(lang_out, lang_in, model) self.options = {"temperature": 0} self.client = openai.AzureOpenAI( azure_endpoint=base_url, @@ -259,14 +258,12 @@ class ZhipuTranslator(OpenAITranslator): "ZHIPU_MODEL": "glm-4-flash", } - def __init__(self, service, lang_out, lang_in, model): + def __init__(self, lang_out, lang_in, model): base_url = "https://open.bigmodel.cn/api/paas/v4" api_key = os.getenv("ZHIPU_API_KEY") if not model: model = os.getenv("ZHIPU_MODEL", self.envs["ZHIPU_MODEL"]) - super().__init__( - service, lang_out, lang_in, model, base_url=base_url, api_key=api_key - ) + super().__init__(lang_out, lang_in, model, base_url=base_url, api_key=api_key) class SiliconTranslator(OpenAITranslator): @@ -277,14 +274,12 @@ class SiliconTranslator(OpenAITranslator): "SILICON_MODEL": "Qwen/Qwen2.5-7B-Instruct", } - def __init__(self, service, lang_out, lang_in, model): + def __init__(self, lang_out, lang_in, model): base_url = "https://api.siliconflow.cn/v1" api_key = os.getenv("SILICON_API_KEY") if not model: model = os.getenv("SILICON_MODEL", self.envs["SILICON_MODEL"]) - super().__init__( - service, lang_out, lang_in, model, base_url=base_url, api_key=api_key - ) + super().__init__(lang_out, lang_in, model, base_url=base_url, api_key=api_key) class AzureTranslator(BaseTranslator): @@ -296,8 +291,8 @@ class AzureTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, service, lang_out, lang_in, model): - super().__init__(service, lang_out, lang_in, model) + def __init__(self, lang_out, lang_in, model): + super().__init__(lang_out, lang_in, model) endpoint = os.getenv("AZURE_ENDPOINT", self.envs["AZURE_ENDPOINT"]) api_key = os.getenv("AZURE_API_KEY") credential = AzureKeyCredential(api_key) @@ -326,8 +321,8 @@ class TencentTranslator(BaseTranslator): "TENCENTCLOUD_SECRET_KEY": None, } - def __init__(self, service, lang_out, lang_in, model): - super().__init__(service, lang_out, lang_in, model) + def __init__(self, lang_out, lang_in, model): + super().__init__(lang_out, lang_in, model) cred = credential.DefaultCredentialProvider().get_credential() self.client = TmtClient(cred, "ap-beijing") self.req = TextTranslateRequest() From be73033f1fa5fccde6808f2269132dc68d263f25 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 13 Dec 2024 00:07:28 +0800 Subject: [PATCH 031/114] chore: remove tmp file --- pdf2zh/high_level.py | 48 ++++++++++++++++++++++---------------------- pdf2zh/translator.py | 1 - 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 4c736fc1..c0e765f1 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -19,6 +19,7 @@ import requests import tempfile import os +import io model = DocLayoutModel.load_available() @@ -78,8 +79,7 @@ def translate_patch( vfont: str = "", vchar: str = "", thread: int = 0, - doc_en: Document = None, - model=None, + doc_zh: Document = None, lang_in: str = "", lang_out: str = "", service: str = "", @@ -112,7 +112,7 @@ def translate_patch( if callback: callback(progress) page.pageno = pageno - pix = doc_en[page.pageno].get_pixmap() + pix = doc_zh[page.pageno].get_pixmap() image = np.fromstring(pix.samples, np.uint8).reshape( pix.height, pix.width, 3 )[:, :, ::-1] @@ -143,10 +143,10 @@ def translate_patch( box[y0:y1, x0:x1] = 0 layout[page.pageno] = box # 新建一个 xref 存放新指令流 - page.page_xref = doc_en.get_new_xref() # hack 插入页面的新 xref - doc_en.update_object(page.page_xref, "<<>>") - doc_en.update_stream(page.page_xref, b"") - doc_en[page.pageno].set_contents(page.page_xref) + page.page_xref = doc_zh.get_new_xref() # hack 插入页面的新 xref + doc_zh.update_object(page.page_xref, "<<>>") + doc_zh.update_stream(page.page_xref, b"") + doc_zh[page.pageno].set_contents(page.page_xref) interpreter.process_page(page) device.close() @@ -220,51 +220,51 @@ def translate( font_list.append(("china-ss", None)) doc_en = Document(file) - page_count = doc_en.page_count + if doc_en.is_encrypted: + doc_en.authenticate(password) + doc_zh = Document(doc_en) + page_count = doc_zh.page_count # font_list = [("china-ss", None), ("tiro", None)] font_id = {} - for page in doc_en: + for page in doc_zh: for font in font_list: font_id[font[0]] = page.insert_font(font[0], font[1]) - xreflen = doc_en.xref_length() + xreflen = doc_zh.xref_length() for xref in range(1, xreflen): for label in ["Resources/", ""]: # 可能是基于 xobj 的 res try: # xref 读写可能出错 - font_res = doc_en.xref_get_key(xref, f"{label}Font") + font_res = doc_zh.xref_get_key(xref, f"{label}Font") if font_res[0] == "dict": for font in font_list: - font_exist = doc_en.xref_get_key( + font_exist = doc_zh.xref_get_key( xref, f"{label}Font/{font[0]}" ) if font_exist[0] == "null": - doc_en.xref_set_key( + doc_zh.xref_set_key( xref, f"{label}Font/{font[0]}", f"{font_id[font[0]]} 0 R", ) except Exception: pass - doc_en.save(Path(output) / f"{filename}-en.pdf") - with open(Path(output) / f"{filename}-en.pdf", "rb") as fp: - obj_patch: dict = translate_patch(fp, model=model, **locals()) + fp = io.BytesIO() + doc_zh.save(fp) + obj_patch: dict = translate_patch(fp, **locals()) for obj_id, ops_new in obj_patch.items(): # ops_old=doc_en.xref_stream(obj_id) # print(obj_id) # print(ops_old) # print(ops_new.encode()) - doc_en.update_stream(obj_id, ops_new.encode()) + doc_zh.update_stream(obj_id, ops_new.encode()) - doc_zh = doc_en - doc_dual = Document(Path(output) / f"{filename}-en.pdf") - doc_dual.insert_file(doc_zh) + doc_en.insert_file(doc_zh) for id in range(page_count): - doc_dual.move_page(page_count + id, id * 2 + 1) + doc_en.move_page(page_count + id, id * 2 + 1) doc_zh.save(Path(output) / f"{filename}-zh.pdf", deflate=1) - doc_dual.save(Path(output) / f"{filename}-dual.pdf", deflate=1) + doc_en.save(Path(output) / f"{filename}-dual.pdf", deflate=1) doc_zh.close() - doc_dual.close() - os.remove(Path(output) / f"{filename}-en.pdf") + doc_en.close() return diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index b4e92456..3e3b1627 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -82,7 +82,6 @@ def translate(self, text): class BingTranslator(BaseTranslator): # https://github.com/immersive-translate/old-immersive-translate/blob/6df13da22664bea2f51efe5db64c63aca59c4e79/src/background/translationService.js - # TODO: IID & IG name = "bing" lang_map = {"zh": "zh-Hans"} From f1c196592d3e7354bc4c7feccddb56078236cb2a Mon Sep 17 00:00:00 2001 From: Yadomin Jinta Date: Fri, 13 Dec 2024 00:03:30 +0800 Subject: [PATCH 032/114] add a simple backend --- pyproject.toml | 5 ++++ tools/backend.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++++ tools/tasks.py | 26 +++++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 tools/backend.py create mode 100644 tools/tasks.py diff --git a/pyproject.toml b/pyproject.toml index 594b33c0..d49a3eac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,11 @@ dev = [ "flake8", "pre-commit" ] +backend = [ + "flask", + "celery", + "redis" +] [project.urls] Homepage = "https://github.com/Byaidu/PDFMathTranslate" diff --git a/tools/backend.py b/tools/backend.py new file mode 100644 index 00000000..b7e24b11 --- /dev/null +++ b/tools/backend.py @@ -0,0 +1,76 @@ +import os +import tempfile + +from flask import Flask, request, send_file +from celery import Celery, Task +from celery.result import AsyncResult +from pathlib import Path +from tasks import translate_task + + +app = Flask("pdf2zh") +app.config.from_mapping( + CELERY=dict( + broker_url=os.environ.get("CELERY_BROKER", "redis://127.0.0.1:6379/0"), + result_backend=os.environ.get("CELERY_RESULT", "redis://127.0.0.1:6379/0"), + ignore_task_result=False, + ) +) + + +def celery_init_app(app: Flask) -> Celery: + class FlaskTask(Task): + def __call__(self, *args, **kwargs): + with app.app_context(): + return self.run(*args, **kwargs) + + celery_app = Celery(app.name) + celery_app.config_from_object(app.config["CELERY"]) + celery_app.Task = FlaskTask + celery_app.set_default() + celery_app.autodiscover_tasks() + app.extensions["celery"] = celery_app + return celery_app + + +celery_app = celery_init_app(app) + + +@app.route("/api/translate", methods=["POST"]) +def create_translate_tasks(): + f = request.files["source"] + output_dir = Path(tempfile.mkdtemp()) + file_basename = ".".join(f.filename.split(".")[:-1]) + if len(file_basename) == 0: + file_basename = "input" + origin_pdf = output_dir / f"{file_basename}.pdf" + f.save(origin_pdf) + lang_in = request.args.get("lang_in", "auto") + lang_out = request.args.get("lang_out", "zh") + service = request.args.get("service", "google") + task = translate_task.delay( + str(output_dir), file_basename, lang_in, lang_out, service + ) + return {"result_id": task.id} + + +@app.route("/api/results/", methods=["GET"]) +def check_translate_result(id: str): + result = AsyncResult(id) + return {"ready": result.ready(), "successful": result.successful()} + + +@app.route("/api/results//") +def get_translate_result(id: str, format: str): + result = celery_app.AsyncResult(id) + if not result.ready(): + return {"error": "task not finished"}, 400 + if not result.successful(): + return {"error": "task failed"}, 400 + translated_pdf, dual_pdf = result.get() + to_send = translated_pdf if format == "translated" else dual_pdf + return send_file(to_send, "application/pdf") + + +if __name__ == "__main__": + app.run() diff --git a/tools/tasks.py b/tools/tasks.py new file mode 100644 index 00000000..4a33dde9 --- /dev/null +++ b/tools/tasks.py @@ -0,0 +1,26 @@ +from celery import shared_task +from pathlib import Path +from pdf2zh.pdf2zh import translate + + +@shared_task(ignore_result=False) +def translate_task( + output_dir: str, + filename: str, + lang_in: str = "auto", + lang_out: str = "zh", + service: str = "google", +): + output_dir = Path(output_dir) + origin_pdf = output_dir / f"{filename}.pdf" + translated_pdf = output_dir / f"{filename}-zh.pdf" + dual_pdf = output_dir / f"{filename}-dual.pdf" + translate( + files=[str(origin_pdf)], + lang_in=lang_in, + lang_out=lang_out, + service=service, + thread=4, + output=str(output_dir) + ) + return str(translated_pdf), str(dual_pdf) From 8ebaaa99847a4101870f7d90d6c411cfe2dfb097 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 13 Dec 2024 02:42:35 +0800 Subject: [PATCH 033/114] refactor: translate stream --- README.md | 2 +- README_zh-CN.md | 2 +- pdf2zh/__init__.py | 2 + pdf2zh/gui.py | 12 ++-- pdf2zh/high_level.py | 166 ++++++++++++++++++++++++------------------- pdf2zh/pdf2zh.py | 4 +- tools/backend.py | 54 ++++++++------ tools/tasks.py | 26 ------- 8 files changed, 136 insertions(+), 132 deletions(-) delete mode 100644 tools/tasks.py diff --git a/README.md b/README.md index c566d3d5..f3af08ae 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ For docker deployment on cloud service:

Advanced Options

-Execute the translation command in the command line to generate the translated document `example-zh.pdf` and the bilingual document `example-dual.pdf` in the current working directory. Use Google as the default translation service. +Execute the translation command in the command line to generate the translated document `example-mono.pdf` and the bilingual document `example-dual.pdf` in the current working directory. Use Google as the default translation service. cmd diff --git a/README_zh-CN.md b/README_zh-CN.md index aff0e336..297f05d5 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -144,7 +144,7 @@

高级选项

-在命令行中执行翻译命令，在当前工作目录下生成译文文档 `example-zh.pdf` 和双语对照文档 `example-dual.pdf`，默认使用 Google 翻译服务 +在命令行中执行翻译命令，在当前工作目录下生成译文文档 `example-mono.pdf` 和双语对照文档 `example-dual.pdf`，默认使用 Google 翻译服务 cmd diff --git a/pdf2zh/__init__.py b/pdf2zh/__init__.py index cd0a125f..4fffb544 100644 --- a/pdf2zh/__init__.py +++ b/pdf2zh/__init__.py @@ -1,6 +1,8 @@ import logging +from pdf2zh.high_level import translate, translate_stream log = logging.getLogger(__name__) __version__ = "1.8.7" __author__ = "Byaidu" +__all__ = ["translate", "translate_stream"] diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 130c2579..5fb2b476 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -146,8 +146,8 @@ def translate_file( ) filename = os.path.splitext(os.path.basename(file_path))[0] - file_en = output / f"{filename}.pdf" - file_zh = output / f"{filename}-zh.pdf" + file_raw = output / f"{filename}.pdf" + file_mono = output / f"{filename}-mono.pdf" file_dual = output / f"{filename}-dual.pdf" translator = service_map[service] @@ -164,7 +164,7 @@ def progress_bar(t: tqdm.tqdm): progress(t.n / t.total, desc="Translating...") param = { - "files": [file_en], + "files": [file_raw], "pages": selected_page, "lang_in": lang_from, "lang_out": lang_to, @@ -177,18 +177,18 @@ def progress_bar(t: tqdm.tqdm): translate(**param) print(f"Files after translation: {os.listdir(output)}") - if not file_zh.exists() or not file_dual.exists(): + if not file_mono.exists() or not file_dual.exists(): raise gr.Error("No output") try: - translated_preview = pdf_preview(str(file_zh)) + translated_preview = pdf_preview(str(file_mono)) except Exception: raise gr.Error("No preview") progress(1.0, desc="Translation complete!") return ( - str(file_zh), + str(file_mono), translated_preview, str(file_dual), gr.update(visible=True), diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index c0e765f1..5db3b390 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -14,7 +14,7 @@ from pdf2zh.pdfinterp import PDFPageInterpreterEx from pdf2zh.doclayout import DocLayoutModel from pathlib import Path -from typing import Any, Container, Iterable, List, Optional +from typing import Any, Iterable, List import urllib.request import requests import tempfile @@ -75,7 +75,6 @@ def translate_patch( inf: BinaryIO, pages=None, password: str = "", - page_count: int = 0, vfont: str = "", vchar: str = "", thread: int = 0, @@ -86,7 +85,7 @@ def translate_patch( resfont: str = "", noto: Font = None, callback: object = None, - **kwarg, + **kwarg: Any, ) -> None: rsrcmgr = PDFResourceManager() layout = {} @@ -100,7 +99,7 @@ def translate_patch( if pages: total_pages = len(pages) else: - total_pages = page_count + total_pages = doc_zh.page_count parser = PDFParser(inf) doc = PDFDocument(parser, password=password) @@ -153,9 +152,89 @@ def translate_patch( return obj_patch +def translate_stream( + stream, + pages=None, + password: str = "", + vfont: str = "", + vchar: str = "", + thread: int = 0, + doc_zh: Document = None, + lang_in: str = "", + lang_out: str = "", + service: str = "", + callback: object = None, + **kwarg: Any, +): + font_list = [("tiro", None)] + noto = None + if lang_out.lower() in resfont_map: # CJK + resfont = resfont_map[lang_out.lower()] + font_list.append((resfont, None)) + elif lang_out.lower() in noto_list: # noto + resfont = "noto" + ttf_path = os.path.join(tempfile.gettempdir(), "GoNotoKurrent-Regular.ttf") + if not os.path.exists(ttf_path): + print("Downloading Noto font...") + urllib.request.urlretrieve( + "https://github.com/satbyy/go-noto-universal/releases/download/v7.0/GoNotoKurrent-Regular.ttf", + ttf_path, + ) + font_list.append(("noto", ttf_path)) + noto = Font("noto", ttf_path) + else: # fallback + resfont = "china-ss" + font_list.append(("china-ss", None)) + + doc_en = Document(stream=stream) + if doc_en.is_encrypted: + doc_en.authenticate(password) + doc_zh = Document(stream=stream) + page_count = doc_zh.page_count + # font_list = [("china-ss", None), ("tiro", None)] + font_id = {} + for page in doc_zh: + for font in font_list: + font_id[font[0]] = page.insert_font(font[0], font[1]) + xreflen = doc_zh.xref_length() + for xref in range(1, xreflen): + for label in ["Resources/", ""]: # 可能是基于 xobj 的 res + try: # xref 读写可能出错 + font_res = doc_zh.xref_get_key(xref, f"{label}Font") + if font_res[0] == "dict": + for font in font_list: + font_exist = doc_zh.xref_get_key(xref, f"{label}Font/{font[0]}") + if font_exist[0] == "null": + doc_zh.xref_set_key( + xref, + f"{label}Font/{font[0]}", + f"{font_id[font[0]]} 0 R", + ) + except Exception: + pass + + fp = io.BytesIO() + doc_zh.save(fp) + obj_patch: dict = translate_patch(fp, **locals()) + + for obj_id, ops_new in obj_patch.items(): + # ops_old=doc_en.xref_stream(obj_id) + # print(obj_id) + # print(ops_old) + # print(ops_new.encode()) + doc_zh.update_stream(obj_id, ops_new.encode()) + + doc_en.insert_file(doc_zh) + for id in range(page_count): + doc_en.move_page(page_count + id, id * 2 + 1) + + return doc_zh.write(deflate=1), doc_en.write(deflate=1) + + def translate( files: Iterable[str] = [], - pages: Optional[Container[int]] = None, + output: str = "", + pages=None, password: str = "", vfont: str = "", vchar: str = "", @@ -164,8 +243,7 @@ def translate( lang_out: str = "", service: str = "", callback: object = None, - output: str = "", - **kwargs: Any, + **kwarg: Any, ): if not files: raise PDFValueError("No files to process.") @@ -199,72 +277,12 @@ def translate( ) filename = os.path.splitext(os.path.basename(file))[0] - font_list = [("tiro", None)] - noto = None - if lang_out.lower() in resfont_map: # CJK - resfont = resfont_map[lang_out.lower()] - font_list.append((resfont, None)) - elif lang_out.lower() in noto_list: # noto - resfont = "noto" - ttf_path = os.path.join(tempfile.gettempdir(), "GoNotoKurrent-Regular.ttf") - if not os.path.exists(ttf_path): - print("Downloading Noto font...") - urllib.request.urlretrieve( - "https://github.com/satbyy/go-noto-universal/releases/download/v7.0/GoNotoKurrent-Regular.ttf", - ttf_path, - ) - font_list.append(("noto", ttf_path)) - noto = Font("noto", ttf_path) - else: # fallback - resfont = "china-ss" - font_list.append(("china-ss", None)) - - doc_en = Document(file) - if doc_en.is_encrypted: - doc_en.authenticate(password) - doc_zh = Document(doc_en) - page_count = doc_zh.page_count - # font_list = [("china-ss", None), ("tiro", None)] - font_id = {} - for page in doc_zh: - for font in font_list: - font_id[font[0]] = page.insert_font(font[0], font[1]) - xreflen = doc_zh.xref_length() - for xref in range(1, xreflen): - for label in ["Resources/", ""]: # 可能是基于 xobj 的 res - try: # xref 读写可能出错 - font_res = doc_zh.xref_get_key(xref, f"{label}Font") - if font_res[0] == "dict": - for font in font_list: - font_exist = doc_zh.xref_get_key( - xref, f"{label}Font/{font[0]}" - ) - if font_exist[0] == "null": - doc_zh.xref_set_key( - xref, - f"{label}Font/{font[0]}", - f"{font_id[font[0]]} 0 R", - ) - except Exception: - pass - - fp = io.BytesIO() - doc_zh.save(fp) - obj_patch: dict = translate_patch(fp, **locals()) - - for obj_id, ops_new in obj_patch.items(): - # ops_old=doc_en.xref_stream(obj_id) - # print(obj_id) - # print(ops_old) - # print(ops_new.encode()) - doc_zh.update_stream(obj_id, ops_new.encode()) - - doc_en.insert_file(doc_zh) - for id in range(page_count): - doc_en.move_page(page_count + id, id * 2 + 1) - doc_zh.save(Path(output) / f"{filename}-zh.pdf", deflate=1) - doc_en.save(Path(output) / f"{filename}-dual.pdf", deflate=1) - doc_zh.close() - doc_en.close() + doc_raw = open(file, "rb") + s_raw = doc_raw.read() + s_mono, s_dual = translate_stream(s_raw, **locals()) + doc_mono = open(Path(output) / f"{filename}-mono.pdf", "wb") + doc_dual = open(Path(output) / f"{filename}-dual.pdf", "wb") + doc_mono.write(s_mono) + doc_dual.write(s_dual) return diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index 04127b9b..905974af 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -12,8 +12,6 @@ from pdf2zh import __version__, log from pdf2zh.high_level import translate -logging.basicConfig() - def create_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__, add_help=True) @@ -135,6 +133,8 @@ def parse_args(args: Optional[List[str]]) -> argparse.Namespace: def main(args: Optional[List[str]] = None) -> int: + logging.basicConfig() + parsed_args = parse_args(args) if parsed_args.debug: diff --git a/tools/backend.py b/tools/backend.py index b7e24b11..d929a4e8 100644 --- a/tools/backend.py +++ b/tools/backend.py @@ -1,12 +1,8 @@ import os -import tempfile - from flask import Flask, request, send_file from celery import Celery, Task -from celery.result import AsyncResult -from pathlib import Path -from tasks import translate_task - +from pdf2zh import translate_stream +import tqdm app = Flask("pdf2zh") app.config.from_mapping( @@ -36,28 +32,42 @@ def __call__(self, *args, **kwargs): celery_app = celery_init_app(app) +@app.task(bind=True) +def translate_task( + stream: bytes, + lang_in: str = "", + lang_out: str = "", + service: str = "", +): + def progress_bar(t: tqdm.tqdm): + self.update_state(state="PROGRESS", meta={"n": t.n, "total": t.total}) # noqa + print(f"Translating {t.n} / {t.total} pages") + + doc_mono, doc_dual = translate_stream( + stream, + lang_in=lang_in, + lang_out=lang_out, + service=service, + thread=4, + callback=progress_bar, + ) + return doc_mono, doc_dual + + @app.route("/api/translate", methods=["POST"]) def create_translate_tasks(): - f = request.files["source"] - output_dir = Path(tempfile.mkdtemp()) - file_basename = ".".join(f.filename.split(".")[:-1]) - if len(file_basename) == 0: - file_basename = "input" - origin_pdf = output_dir / f"{file_basename}.pdf" - f.save(origin_pdf) - lang_in = request.args.get("lang_in", "auto") + stream = request.files["file"] + lang_in = request.args.get("lang_in", "en") lang_out = request.args.get("lang_out", "zh") service = request.args.get("service", "google") - task = translate_task.delay( - str(output_dir), file_basename, lang_in, lang_out, service - ) - return {"result_id": task.id} + task = translate_task.delay(stream, lang_in, lang_out, service) + return {"id": task.id} @app.route("/api/results/", methods=["GET"]) def check_translate_result(id: str): - result = AsyncResult(id) - return {"ready": result.ready(), "successful": result.successful()} + result = celery_app.AsyncResult(id) + return {"state": result.state, "info": result.info} @app.route("/api/results//") @@ -67,8 +77,8 @@ def get_translate_result(id: str, format: str): return {"error": "task not finished"}, 400 if not result.successful(): return {"error": "task failed"}, 400 - translated_pdf, dual_pdf = result.get() - to_send = translated_pdf if format == "translated" else dual_pdf + doc_mono, doc_dual = result.get() + to_send = doc_mono if format == "mono" else doc_dual return send_file(to_send, "application/pdf") diff --git a/tools/tasks.py b/tools/tasks.py deleted file mode 100644 index 4a33dde9..00000000 --- a/tools/tasks.py +++ /dev/null @@ -1,26 +0,0 @@ -from celery import shared_task -from pathlib import Path -from pdf2zh.pdf2zh import translate - - -@shared_task(ignore_result=False) -def translate_task( - output_dir: str, - filename: str, - lang_in: str = "auto", - lang_out: str = "zh", - service: str = "google", -): - output_dir = Path(output_dir) - origin_pdf = output_dir / f"{filename}.pdf" - translated_pdf = output_dir / f"{filename}-zh.pdf" - dual_pdf = output_dir / f"{filename}-dual.pdf" - translate( - files=[str(origin_pdf)], - lang_in=lang_in, - lang_out=lang_out, - service=service, - thread=4, - output=str(output_dir) - ) - return str(translated_pdf), str(dual_pdf) From 6f47b4fd608da31e8d9e3e6edab49eea57137e8c Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 13 Dec 2024 14:36:48 +0800 Subject: [PATCH 034/114] feat: backend option --- {tools => pdf2zh}/backend.py | 50 +++++++++++++++++++----------------- pdf2zh/pdf2zh.py | 22 ++++++++++++++++ 2 files changed, 49 insertions(+), 23 deletions(-) rename {tools => pdf2zh}/backend.py (64%) diff --git a/tools/backend.py b/pdf2zh/backend.py similarity index 64% rename from tools/backend.py rename to pdf2zh/backend.py index d929a4e8..98d47338 100644 --- a/tools/backend.py +++ b/pdf2zh/backend.py @@ -1,15 +1,16 @@ import os from flask import Flask, request, send_file from celery import Celery, Task +from celery.result import AsyncResult from pdf2zh import translate_stream import tqdm +import json -app = Flask("pdf2zh") -app.config.from_mapping( +flask_app = Flask("pdf2zh") +flask_app.config.from_mapping( CELERY=dict( broker_url=os.environ.get("CELERY_BROKER", "redis://127.0.0.1:6379/0"), result_backend=os.environ.get("CELERY_RESULT", "redis://127.0.0.1:6379/0"), - ignore_task_result=False, ) ) @@ -29,15 +30,14 @@ def __call__(self, *args, **kwargs): return celery_app -celery_app = celery_init_app(app) +celery_app = celery_init_app(flask_app) -@app.task(bind=True) +@celery_app.task(bind=True) def translate_task( + self: Task, stream: bytes, - lang_in: str = "", - lang_out: str = "", - service: str = "", + args: dict, ): def progress_bar(t: tqdm.tqdm): self.update_state(state="PROGRESS", meta={"n": t.n, "total": t.total}) # noqa @@ -45,32 +45,36 @@ def progress_bar(t: tqdm.tqdm): doc_mono, doc_dual = translate_stream( stream, - lang_in=lang_in, - lang_out=lang_out, - service=service, - thread=4, callback=progress_bar, + **args, ) return doc_mono, doc_dual -@app.route("/api/translate", methods=["POST"]) +@flask_app.route("/v1/translate", methods=["POST"]) def create_translate_tasks(): - stream = request.files["file"] - lang_in = request.args.get("lang_in", "en") - lang_out = request.args.get("lang_out", "zh") - service = request.args.get("service", "google") - task = translate_task.delay(stream, lang_in, lang_out, service) + file = request.files["file"] + stream = file.stream.read() + print(request.form.get("data")) + args = json.loads(request.form.get("data")) + task = translate_task.delay(stream, args) return {"id": task.id} -@app.route("/api/results/", methods=["GET"]) -def check_translate_result(id: str): - result = celery_app.AsyncResult(id) +@flask_app.route("/v1/tasks/", methods=["GET"]) +def get_translate_task(id: str): + result: AsyncResult = celery_app.AsyncResult(id) + return {"state": result.state, "info": result.info} + + +@flask_app.route("/v1/tasks/", methods=["DELETE"]) +def delete_translate_task(id: str): + result: AsyncResult = celery_app.AsyncResult(id) + result.revoke(terminate=True) return {"state": result.state, "info": result.info} -@app.route("/api/results//") +@flask_app.route("/v1/tasks//") def get_translate_result(id: str, format: str): result = celery_app.AsyncResult(id) if not result.ready(): @@ -83,4 +87,4 @@ def get_translate_result(id: str, format: str): if __name__ == "__main__": - app.run() + flask_app.run() diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index 905974af..f05170cf 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -112,6 +112,16 @@ def create_parser() -> argparse.ArgumentParser: action="store_true", help="Enable Gradio Share", ) + parse_params.add_argument( + "--flask", + action="store_true", + help="flask", + ) + parse_params.add_argument( + "--celery", + action="store_true", + help="celery", + ) return parser @@ -146,6 +156,18 @@ def main(args: Optional[List[str]] = None) -> int: setup_gui(parsed_args.share) return 0 + if parsed_args.flask: + from pdf2zh.backend import flask_app + + flask_app.run() + return 0 + + if parsed_args.celery: + from pdf2zh.backend import celery_app + + celery_app.start(argv=["worker", "--pool=prefork"]) + return 0 + translate(**vars(parsed_args)) return 0 From 9cb812e29cf10aff36d9820b6d52ea9260437470 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 13 Dec 2024 14:40:47 +0800 Subject: [PATCH 035/114] fix: files path --- pdf2zh/gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 5fb2b476..30a6c297 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -164,7 +164,7 @@ def progress_bar(t: tqdm.tqdm): progress(t.n / t.total, desc="Translating...") param = { - "files": [file_raw], + "files": [str(file_raw)], "pages": selected_page, "lang_in": lang_from, "lang_out": lang_to, From da678863a498ef53716c5ec3b0b48bd03d48bb32 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Fri, 13 Dec 2024 15:09:24 +0800 Subject: [PATCH 036/114] =?UTF-8?q?=E5=88=87=E6=8D=A2PDF=E9=A2=84=E8=A7=88?= =?UTF-8?q?=E4=B8=BAgradio-pdf=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pdf2zh/gui.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 30a6c297..bc542476 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -19,6 +19,7 @@ ) import gradio as gr +from gradio_pdf import PDF import numpy as np import pymupdf import tqdm @@ -88,7 +89,7 @@ def pdf_preview(file): def upload_file(file, service, progress=gr.Progress()): preview_image = pdf_preview(file) - return file, preview_image + return file, file def download_with_limit(url, save_path, size_limit): @@ -189,7 +190,7 @@ def progress_bar(t: tqdm.tqdm): return ( str(file_mono), - translated_preview, + str(file_mono), str(file_dual), gr.update(visible=True), gr.update(visible=True), @@ -380,7 +381,7 @@ def on_select_filetype(file_type): with gr.Column(scale=2): gr.Markdown("## Preview") - preview = gr.Image(label="Document Preview", visible=True) + preview = PDF(label="Document Preview", visible=True) # Event handlers file_input.upload( From afe7c53eaf75ac9f096cf9408142d7e754a7594e Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 13 Dec 2024 15:31:09 +0800 Subject: [PATCH 037/114] fix: dep --- pdf2zh/gui.py | 34 ++++++++-------------------------- pyproject.toml | 1 + 2 files changed, 9 insertions(+), 26 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index bc542476..118c50a6 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -20,8 +20,6 @@ import gradio as gr from gradio_pdf import PDF -import numpy as np -import pymupdf import tqdm import requests import cgi @@ -79,19 +77,6 @@ def verify_recaptcha(response): return result.get("success") -def pdf_preview(file): - doc = pymupdf.open(file) - page = doc[0] - pix = page.get_pixmap() - image = np.frombuffer(pix.samples, np.uint8).reshape(pix.height, pix.width, 3) - return image - - -def upload_file(file, service, progress=gr.Progress()): - preview_image = pdf_preview(file) - return file, file - - def download_with_limit(url, save_path, size_limit): chunk_size = 1024 total_size = 0 @@ -181,11 +166,6 @@ def progress_bar(t: tqdm.tqdm): if not file_mono.exists() or not file_dual.exists(): raise gr.Error("No output") - try: - translated_preview = pdf_preview(str(file_mono)) - except Exception: - raise gr.Error("No preview") - progress(1.0, desc="Translation complete!") return ( @@ -335,7 +315,9 @@ def on_select_filetype(file_type): ) output_title = gr.Markdown("## Translated", visible=False) - output_file = gr.File(label="Download Translation", visible=False) + output_file_mono = gr.File( + label="Download Translation (Mono)", visible=False + ) output_file_dual = gr.File( label="Download Translation (Dual)", visible=False ) @@ -385,9 +367,9 @@ def on_select_filetype(file_type): # Event handlers file_input.upload( - upload_file, - inputs=[file_input, service], - outputs=[file_input, preview], + lambda x: x, + inputs=file_input, + outputs=preview, js=( f""" (a,b)=>{{ @@ -419,10 +401,10 @@ def on_select_filetype(file_type): *envs, ], outputs=[ - output_file, + output_file_mono, preview, output_file_dual, - output_file, + output_file_mono, output_file_dual, output_title, ], diff --git a/pyproject.toml b/pyproject.toml index d49a3eac..516b3232 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "opencv-python-headless", "tencentcloud-sdk-python", "pdfminer.six>=20240706", + "gradio_pdf", ] [project.optional-dependencies] From fc6e34ca877aee10b8e6dbb669c9bfc5d97a9e84 Mon Sep 17 00:00:00 2001 From: Matt Wang Date: Fri, 13 Dec 2024 08:57:09 +0100 Subject: [PATCH 038/114] Do not send blank strings to translation services. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sending blank strings to an LLM will get back "请提供需要翻译的文本", which is highly undesirable. --- pdf2zh/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index db0bb933..58db7783 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -323,7 +323,7 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 @retry(wait=wait_fixed(1)) def worker(s: str): # 多线程翻译 - if re.match(r"^\$v\d+\$$", s): # 公式不翻译 + if not s.strip() or re.match(r"^\$v\d+\$$", s): # 空白和公式不翻译 return s try: hash_key_paragraph = cache.deterministic_hash( From 345ae4bcec2490f8071904c4bbf362ae9f5916ef Mon Sep 17 00:00:00 2001 From: hellofinch Date: Fri, 13 Dec 2024 17:03:47 +0800 Subject: [PATCH 039/114] =?UTF-8?q?=E6=9B=B4=E6=96=B0readme=EF=BC=8C?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0--share=E8=AF=B4=E6=98=8E=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + README_zh-CN.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index f3af08ae..63248b57 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,7 @@ In the following table, we list all advanced options for reference: | `-t` | [Multi-threads](#threads) | `pdf2zh example.pdf -t 1` | | `-o` | Output dir | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [Exceptions](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | +| `--share` | get gradio public link | `pdf2zh -i --share` |

Full / partial document translation

diff --git a/README_zh-CN.md b/README_zh-CN.md index 297f05d5..64faf36d 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -162,6 +162,7 @@ | `-t` | [多线程](#threads) | `pdf2zh example.pdf -t 1` | | `-o` | 输出目录 | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [例外规则](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | +| `--share` | 获取gradio公开链接 | `pdf2zh -i --share` |

全文或部分文档翻译

From 1e1335b9001d041fd3fbf6ab5a77cb6d5bfa6aa9 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 13 Dec 2024 20:43:26 +0800 Subject: [PATCH 040/114] chore: rm password --- pdf2zh/high_level.py | 40 ++++++++++++++++++---------------------- pdf2zh/pdf2zh.py | 7 ------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 5db3b390..a0730efc 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -14,7 +14,7 @@ from pdf2zh.pdfinterp import PDFPageInterpreterEx from pdf2zh.doclayout import DocLayoutModel from pathlib import Path -from typing import Any, Iterable, List +from typing import Any, List, Optional import urllib.request import requests import tempfile @@ -73,8 +73,7 @@ def check_files(files: List[str]) -> List[str]: def translate_patch( inf: BinaryIO, - pages=None, - password: str = "", + pages: Optional[list[int]] = None, vfont: str = "", vchar: str = "", thread: int = 0, @@ -102,7 +101,7 @@ def translate_patch( total_pages = doc_zh.page_count parser = PDFParser(inf) - doc = PDFDocument(parser, password=password) + doc = PDFDocument(parser) with tqdm.tqdm(total=total_pages) as progress: for pageno, page in enumerate(PDFPage.create_pages(doc)): if pages and (pageno not in pages): @@ -153,16 +152,14 @@ def translate_patch( def translate_stream( - stream, - pages=None, - password: str = "", - vfont: str = "", - vchar: str = "", - thread: int = 0, - doc_zh: Document = None, + stream: bytes, + pages: Optional[list[int]] = None, lang_in: str = "", lang_out: str = "", service: str = "", + thread: int = 0, + vfont: str = "", + vchar: str = "", callback: object = None, **kwarg: Any, ): @@ -187,8 +184,6 @@ def translate_stream( font_list.append(("china-ss", None)) doc_en = Document(stream=stream) - if doc_en.is_encrypted: - doc_en.authenticate(password) doc_zh = Document(stream=stream) page_count = doc_zh.page_count # font_list = [("china-ss", None), ("tiro", None)] @@ -232,16 +227,15 @@ def translate_stream( def translate( - files: Iterable[str] = [], + files: list[str], output: str = "", - pages=None, - password: str = "", - vfont: str = "", - vchar: str = "", - thread: int = 0, + pages: Optional[list[int]] = None, lang_in: str = "", lang_out: str = "", service: str = "", + thread: int = 0, + vfont: str = "", + vchar: str = "", callback: object = None, **kwarg: Any, ): @@ -280,9 +274,11 @@ def translate( doc_raw = open(file, "rb") s_raw = doc_raw.read() s_mono, s_dual = translate_stream(s_raw, **locals()) - doc_mono = open(Path(output) / f"{filename}-mono.pdf", "wb") - doc_dual = open(Path(output) / f"{filename}-dual.pdf", "wb") + file_mono = Path(output) / f"{filename}-mono.pdf" + file_dual = Path(output) / f"{filename}-dual.pdf" + doc_mono = open(file_mono, "wb") + doc_dual = open(file_dual, "wb") doc_mono.write(s_mono) doc_dual.write(s_dual) - return + return str(file_mono), str(file_dual) diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index f05170cf..89e101c5 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -45,13 +45,6 @@ def create_parser() -> argparse.ArgumentParser: type=str, help="The list of page numbers to parse.", ) - parse_params.add_argument( - "--password", - "-P", - type=str, - default="", - help="The password to use for decrypting PDF file.", - ) parse_params.add_argument( "--vfont", "-f", From a79b39ef27c1fd662aff15ec2550b0a40523ee04 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 13 Dec 2024 22:04:17 +0800 Subject: [PATCH 041/114] doc: backend --- README.md | 41 +++++++++++++++++++++++++++++++++++++++-- README_zh-CN.md | 39 +++++++++++++++++++++++++++++++++++++++ pdf2zh/backend.py | 16 ++++++++++------ pdf2zh/pdf2zh.py | 4 ++-- 4 files changed, 90 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 63248b57..68827446 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,45 @@ Use `-t` to specify how many threads to use in translation: pdf2zh example.pdf -t 1 ``` +

API

+ +### Python + +```python +from pdf2zh import translate, translate_stream + +params = {"lang_in": "en", "lang_out": "zh", "service": "google", "thread": 4} +doc_mono, doc_dual = translate(files=["example.pdf"], **params) +with open("example.pdf", "rb") as f: + stream_mono, stream_dual = translate_stream(stream=f.read(), **params) +``` + +### HTTP + +```bash +pip install pdf2zh[backend] +pdf2zh --flask +pdf2zh --celery worker +``` + +```bash +curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"l +ang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" +{"id":"d9894125-2f4e-45ea-9d93-1a9068d2045a"} + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a +{"info":{"n":13,"total":506},"state":"PROGRESS"} + +curl http://localhost:11008/v1/tasks/d9894125-2f4e-45ea-9d93-1a9068d2045a +{"state":"SUCCESS"} + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/mono --output example-mono.pdf + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/dual --output example-dual.pdf + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a -X DELETE +``` +

TODO

- [ ] Parse layout with DocLayNet based models, [PaddleX](https://github.com/PaddlePaddle/PaddleX/blob/17cc27ac3842e7880ca4aad92358d3ef8555429a/paddlex/repo_apis/PaddleDetection_api/object_det/official_categories.py#L81), [PaperMage](https://github.com/allenai/papermage/blob/9cd4bb48cbedab45d0f7a455711438f1632abebe/README.md?plain=1#L102), [SAM2](https://github.com/facebookresearch/sam2) @@ -247,8 +286,6 @@ pdf2zh example.pdf -t 1 - [ ] Support non-PDF/A files -- [ ] Provide API interface - - [ ] Plugins of [Zotero](https://github.com/zotero/zotero) and [Obsidian](https://github.com/obsidianmd/obsidian-releases)

Acknowledgements

diff --git a/README_zh-CN.md b/README_zh-CN.md index 64faf36d..1dd28f25 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -233,6 +233,45 @@ pdf2zh example.pdf -f "(CM[^RT].*|MS.*|.*Ital)" -c "(\(|\||\)|\+|=|\d|[\u0080-\u pdf2zh example.pdf -t 1 ``` +

API

+ +### Python + +```python +from pdf2zh import translate, translate_stream + +params = {"lang_in": "en", "lang_out": "zh", "service": "google", "thread": 4} +doc_mono, doc_dual = translate(files=["example.pdf"], **params) +with open("example.pdf", "rb") as f: + stream_mono, stream_dual = translate_stream(stream=f.read(), **params) +``` + +### HTTP + +```bash +pip install pdf2zh[backend] +pdf2zh --flask +pdf2zh --celery worker +``` + +```bash +curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"l +ang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" +{"id":"d9894125-2f4e-45ea-9d93-1a9068d2045a"} + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a +{"info":{"n":13,"total":506},"state":"PROGRESS"} + +curl http://localhost:11008/v1/tasks/d9894125-2f4e-45ea-9d93-1a9068d2045a +{"state":"SUCCESS"} + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/mono --output example-mono.pdf + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/dual --output example-dual.pdf + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a -X DELETE +``` +

致谢

- 文档合并：[PyMuPDF](https://github.com/pymupdf/PyMuPDF) diff --git a/pdf2zh/backend.py b/pdf2zh/backend.py index 98d47338..f33f510c 100644 --- a/pdf2zh/backend.py +++ b/pdf2zh/backend.py @@ -5,6 +5,7 @@ from pdf2zh import translate_stream import tqdm import json +import io flask_app = Flask("pdf2zh") flask_app.config.from_mapping( @@ -61,20 +62,23 @@ def create_translate_tasks(): return {"id": task.id} -@flask_app.route("/v1/tasks/", methods=["GET"]) +@flask_app.route("/v1/translate/", methods=["GET"]) def get_translate_task(id: str): result: AsyncResult = celery_app.AsyncResult(id) - return {"state": result.state, "info": result.info} + if str(result.state) == "PROGRESS": + return {"state": str(result.state), "info": result.info} + else: + return {"state": str(result.state)} -@flask_app.route("/v1/tasks/", methods=["DELETE"]) +@flask_app.route("/v1/translate/", methods=["DELETE"]) def delete_translate_task(id: str): result: AsyncResult = celery_app.AsyncResult(id) result.revoke(terminate=True) - return {"state": result.state, "info": result.info} + return {"state": str(result.state)} -@flask_app.route("/v1/tasks//") +@flask_app.route("/v1/translate//") def get_translate_result(id: str, format: str): result = celery_app.AsyncResult(id) if not result.ready(): @@ -83,7 +87,7 @@ def get_translate_result(id: str, format: str): return {"error": "task failed"}, 400 doc_mono, doc_dual = result.get() to_send = doc_mono if format == "mono" else doc_dual - return send_file(to_send, "application/pdf") + return send_file(io.BytesIO(to_send), "application/pdf") if __name__ == "__main__": diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index 89e101c5..351cf0ba 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -152,13 +152,13 @@ def main(args: Optional[List[str]] = None) -> int: if parsed_args.flask: from pdf2zh.backend import flask_app - flask_app.run() + flask_app.run(port=11008) return 0 if parsed_args.celery: from pdf2zh.backend import celery_app - celery_app.start(argv=["worker", "--pool=prefork"]) + celery_app.start(argv=sys.argv[2:]) return 0 translate(**vars(parsed_args)) From 833a8570b532f967909bdc1124cb424b72ae63b8 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Fri, 13 Dec 2024 22:07:44 +0800 Subject: [PATCH 042/114] doc: typo --- README.md | 2 +- README_zh-CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 68827446..10f41785 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ ang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a {"info":{"n":13,"total":506},"state":"PROGRESS"} -curl http://localhost:11008/v1/tasks/d9894125-2f4e-45ea-9d93-1a9068d2045a +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a {"state":"SUCCESS"} curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/mono --output example-mono.pdf diff --git a/README_zh-CN.md b/README_zh-CN.md index 1dd28f25..e955b8a3 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -262,7 +262,7 @@ ang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a {"info":{"n":13,"total":506},"state":"PROGRESS"} -curl http://localhost:11008/v1/tasks/d9894125-2f4e-45ea-9d93-1a9068d2045a +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a {"state":"SUCCESS"} curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/mono --output example-mono.pdf From b63ff573fcec4be3c9f0fc678942b2868eebec63 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 14 Dec 2024 00:01:32 +0800 Subject: [PATCH 043/114] fix: vmark --- pdf2zh/converter.py | 16 ++++++++-------- pdf2zh/translator.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 58db7783..7356e7ce 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -235,7 +235,7 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 or cls != xt_cls # 2. 当前字符与前一个字符不属于同一段落 # or (abs(child.x0 - xt.x0) > vmax and cls != 0) # 3. 段落内换行，可能是一长串斜体的段落，也可能是段内分式换行，这里设个阈值进行区分 # 禁止纯公式（代码）段落换行，直到文字开始再重开文字段落，保证只存在两种情况 - # A. 纯公式（代码）段落（锚定绝对位置）sstk[-1]=="" -> sstk[-1]=="$v*$" + # A. 纯公式（代码）段落（锚定绝对位置）sstk[-1]=="" -> sstk[-1]=="{v*}" # B. 文字开头段落（排版相对位置）sstk[-1]!="" or (sstk[-1] != "" and abs(child.x0 - xt.x0) > vmax) # 因为 cls==xt_cls==0 一定有 sstk[-1]==""，所以这里不需要再判定 cls!=0 ): @@ -247,8 +247,8 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 ): vfix = vstk[0].y0 - child.y0 if sstk[-1] == "": - xt_cls = -1 # 禁止纯公式段落（sstk[-1]=="$v*$"）的后续连接，但是要考虑新字符和后续字符的连接，所以这里修改的是上个字符的类别 - sstk[-1] += f"$v{len(var)}$" + xt_cls = -1 # 禁止纯公式段落（sstk[-1]=="{v*}"）的后续连接，但是要考虑新字符和后续字符的连接，所以这里修改的是上个字符的类别 + sstk[-1] += f"{{v{len(var)}}}" var.append(vstk) varl.append(vlstk) varf.append(vfix) @@ -305,14 +305,14 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 pass # 处理结尾 if vstk: # 公式出栈 - sstk[-1] += f"$v{len(var)}$" + sstk[-1] += f"{{v{len(var)}}}" var.append(vstk) varl.append(vlstk) varf.append(vfix) log.debug("\n==========[VSTACK]==========\n") for id, v in enumerate(var): # 计算公式宽度 l = max([vch.x1 for vch in v]) - v[0].x0 - log.debug(f'< {l:.1f} {v[0].x0:.1f} {v[0].y0:.1f} {v[0].cid} {v[0].fontname} {len(varl[id])} > $v{id}$ = {"".join([ch.get_text() for ch in v])}') + log.debug(f'< {l:.1f} {v[0].x0:.1f} {v[0].y0:.1f} {v[0].cid} {v[0].fontname} {len(varl[id])} > v{id} = {"".join([ch.get_text() for ch in v])}') vlen.append(l) ############################################################ @@ -323,7 +323,7 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 @retry(wait=wait_fixed(1)) def worker(s: str): # 多线程翻译 - if not s.strip() or re.match(r"^\$v\d+\$$", s): # 空白和公式不翻译 + if not s.strip() or re.match(r"^\{v\d+\}$", s): # 空白和公式不翻译 return s try: hash_key_paragraph = cache.deterministic_hash( @@ -371,8 +371,8 @@ def raw_string(fcur: str, cstk: str): # 编码字符串 log.debug(f"< {y} {x} {x0} {x1} {size} {brk} > {sstk[id]} | {new}") while ptr < len(new): vy_regex = re.match( - r"\$?\s*v([\d\s]+)\$", new[ptr:], re.IGNORECASE - ) # 匹配 $vn$ 公式标记，前面的 $ 有的时候会被丢掉 + r"\{\s*v([\d\s]+)\}", new[ptr:], re.IGNORECASE + ) # 匹配 {vn} 公式标记 mod = 0 # 文字修饰符 if vy_regex: # 加载公式 ptr += len(vy_regex.group(0)) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 3e3b1627..0b5b52b4 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -43,7 +43,7 @@ def prompt(self, text): }, { "role": "user", - "content": f"Translate the following markdown source text to {self.lang_out}. Keep the formula notation $v*$ unchanged. Output translation directly without any additional text.\nSource Text: {text}\nTranslated Text:", # noqa: E501 + "content": f"Translate the following markdown source text to {self.lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: {text}\nTranslated Text:", # noqa: E501 }, ] From c6c0b4fa8ff780c4d197f5feb9fc7e71e0a58218 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 14 Dec 2024 00:17:33 +0800 Subject: [PATCH 044/114] fix: translate list --- README.md | 2 +- README_zh-CN.md | 2 +- pdf2zh/high_level.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 10f41785..093beca4 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,7 @@ pdf2zh example.pdf -t 1 from pdf2zh import translate, translate_stream params = {"lang_in": "en", "lang_out": "zh", "service": "google", "thread": 4} -doc_mono, doc_dual = translate(files=["example.pdf"], **params) +file_mono, file_dual = translate(files=["example.pdf"], **params)[0] with open("example.pdf", "rb") as f: stream_mono, stream_dual = translate_stream(stream=f.read(), **params) ``` diff --git a/README_zh-CN.md b/README_zh-CN.md index e955b8a3..ed70794d 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -241,7 +241,7 @@ pdf2zh example.pdf -t 1 from pdf2zh import translate, translate_stream params = {"lang_in": "en", "lang_out": "zh", "service": "google", "thread": 4} -doc_mono, doc_dual = translate(files=["example.pdf"], **params) +file_mono, file_dual = translate(files=["example.pdf"], **params)[0] with open("example.pdf", "rb") as f: stream_mono, stream_dual = translate_stream(stream=f.read(), **params) ``` diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index a0730efc..1d108a75 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -250,6 +250,8 @@ def translate( print(f" {file}", file=sys.stderr) raise PDFValueError("Some files do not exist.") + result_files = [] + for file in files: if file is str and (file.startswith("http://") or file.startswith("https://")): print("Online files detected, downloading...") @@ -280,5 +282,6 @@ def translate( doc_dual = open(file_dual, "wb") doc_mono.write(s_mono) doc_dual.write(s_dual) + result_files.append((str(file_mono), str(file_dual))) - return str(file_mono), str(file_dual) + return result_files From 036f55acd98092b873c09412a3cd02888d37e246 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 14 Dec 2024 17:01:30 +0800 Subject: [PATCH 045/114] doc: vfont --- README.md | 8 +++++++- README_zh-CN.md | 8 +++++++- pdf2zh/gui.py | 10 ---------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 093beca4..f2fd9b9f 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Note that the computing resources of the demo are limited, so please avoid abusi

Installation and Usage

-We provide three methods for using this project: [Commandline](#cmd), [Portable](#portable), [GUI](#gui), and [Docker](#docker). +We provide four methods for using this project: [Commandline](#cmd), [Portable](#portable), [GUI](#gui), and [Docker](#docker).

Method I. Commandline

@@ -225,6 +225,12 @@ Use regex to specify formula fonts and characters that need to be preserved: pdf2zh example.pdf -f "(CM[^RT].*|MS.*|.*Ital)" -c "(\(|\||\)|\+|=|\d|[\u0080-\ufaff])" ``` +Preserve `Latex`, `Mono`, `Code`, `Italic`, `Symbol` and `Math` fonts by default: + +```bash +pdf2zh example.pdf -f "(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-|rsfs|txsy|wasy|stmary|.*Mono|.*Code|.*Ital|.*Sym|.*Math)" +``` +

Specify threads

Use `-t` to specify how many threads to use in translation: diff --git a/README_zh-CN.md b/README_zh-CN.md index ed70794d..7a870612 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -66,7 +66,7 @@

安装和使用

-我们提供了三种使用该项目的方法：[命令行工具](#cmd)、[便携式安装](#portable)、[图形交互界面](#gui) 和 [容器化部署](#docker). +我们提供了四种使用该项目的方法：[命令行工具](#cmd)、[便携式安装](#portable)、[图形交互界面](#gui) 和 [容器化部署](#docker).

方法一、命令行工具

@@ -225,6 +225,12 @@ pdf2zh example.pdf -s openai pdf2zh example.pdf -f "(CM[^RT].*|MS.*|.*Ital)" -c "(\(|\||\)|\+|=|\d|[\u0080-\ufaff])" ``` +默认保留 `Latex`, `Mono`, `Code`, `Italic`, `Symbol` 以及 `Math` 字体： + +```bash +pdf2zh example.pdf -f "(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-|rsfs|txsy|wasy|stmary|.*Mono|.*Code|.*Ital|.*Sym|.*Math)" +``` +

指定线程数量

使用 `-t` 指定翻译时使用的线程数量： diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 118c50a6..1ffd2551 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -208,16 +208,6 @@ def progress_bar(t: tqdm.tqdm): .input-file { border: 1.2px dashed #165DFF !important; border-radius: 6px !important; - # background-color: #ffffff !important; - transition: background-color 0.4s ease-out; - } - - .input-file:hover { - border: 1.2px dashed #165DFF !important; - border-radius: 6px !important; - color: #165DFF !important; - background-color: #E8F3FF !important; - transition: background-color 0.2s ease-in; } .progress-bar-wrap { From 261dd6c56c7ebe390e5361bbd83910dcb706c886 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Sat, 14 Dec 2024 18:38:27 +0900 Subject: [PATCH 046/114] docs: add Japanese README I created Japanese translated README. --- README.md | 2 +- README_ja-JP.md | 313 ++++++++++++++++++++++++++++++++++++++++++++++++ README_zh-CN.md | 2 +- 3 files changed, 315 insertions(+), 2 deletions(-) create mode 100644 README_ja-JP.md diff --git a/README.md b/README.md index f2fd9b9f..f7f4a5a6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
-English | [简体中文](README_zh-CN.md) +English | [简体中文](README_zh-CN.md) | [日本語](README_ja-JP.md) PDF2ZH diff --git a/README_ja-JP.md b/README_ja-JP.md new file mode 100644 index 00000000..d342a4f4 --- /dev/null +++ b/README_ja-JP.md @@ -0,0 +1,313 @@ +
+ +[English](README.md) | [简体中文](README_zh-CN.md) | 日本語 + +PDF2ZH + +

PDFMathTranslate

+ +

+ + + + + + + + + + + + + + + + +

+ +
+ +科学 PDF 文書の翻訳およびバイリンガル比較ツール + +- 📊 数式、チャート、目次、注釈を保持 *([プレビュー](#preview))* +- 🌐 [複数の言語](#language) と [多様な翻訳サービス](#services) をサポート +- 🤖 [コマンドラインツール](#usage)、[インタラクティブユーザーインターフェース](#gui)、および [Docker](#docker) を提供 + +フィードバックは [GitHub Issues](https://github.com/Byaidu/PDFMathTranslate/issues)、[Telegram グループ](https://t.me/+Z9_SgnxmsmA5NzBl) または [QQ グループ](https://qm.qq.com/q/DixZCxQej0) でお気軽にどうぞ + +

最近の更新

+ +- [2024年11月26日] CLIがオンラインファイルをサポートするようになりました *(by [@reycn](https://github.com/reycn))* +- [2024年11月24日] 依存関係のサイズを削減するために [ONNX](https://github.com/onnx/onnx) サポートを追加しました *(by [@Wybxc](https://github.com/Wybxc))* +- [2024年11月23日] 🌟 [公共サービス](#demo) がオンラインになりました! *(by [@Byaidu](https://github.com/Byaidu))* +- [2024年11月23日] ウェブボットを防ぐためのファイアウォールを追加しました *(by [@Byaidu](https://github.com/Byaidu))* +- [2024年11月22日] GUIがイタリア語をサポートし、改善されました *(by [@Byaidu](https://github.com/Byaidu), [@reycn](https://github.com/reycn))* +- [2024年11月22日] デプロイされたサービスを他の人と共有できるようになりました *(by [@Zxis233](https://github.com/Zxis233))* +- [2024年11月22日] Tencent翻訳をサポートしました *(by [@hellofinch](https://github.com/hellofinch))* +- [2024年11月21日] GUIがバイリンガルドキュメントのダウンロードをサポートするようになりました *(by [@reycn](https://github.com/reycn))* +- [2024年11月20日] 🌟 [デモ](#demo) がオンラインになりました! *(by [@reycn](https://github.com/reycn))* + +

プレビュー

+ +
+ +
+ +

公共サービス 🌟

+ +### 無料サービス () + +インストールなしで [公共サービス](https://pdf2zh.com/) をオンラインで試すことができます。 + +### Hugging Face デモ + +インストールなしで [HuggingFace上のデモ](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker) を試すことができます。 +デモの計算リソースは限られているため、乱用しないようにしてください。 + +

インストールと使用方法

+ +このプロジェクトを使用するための4つの方法を提供しています：[コマンドライン](#cmd)、[ポータブル](#portable)、[GUI](#gui)、および [Docker](#docker)。 + +

方法1. コマンドライン

+ + 1. Pythonがインストールされていること (バージョン3.8 <= バージョン <= 3.12) + 2. パッケージをインストールします： + + ```bash + pip install pdf2zh + ``` + + 3. 翻訳を実行し、[現在の作業ディレクトリ](https://chatgpt.com/share/6745ed36-9acc-800e-8a90-59204bd13444) にファイルを生成します： + + ```bash + pdf2zh document.pdf + ``` + +

方法2. ポータブル

+ +Python環境を事前にインストールする必要はありません + +[setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/setup.bat) をダウンロードしてダブルクリックして実行します + +

方法3. GUI

+ +1. Pythonがインストールされていること (バージョン3.8 <= バージョン <= 3.12) +2. パッケージをインストールします： + + ```bash + pip install pdf2zh + ``` + +3. ブラウザで使用を開始します： + + ```bash + pdf2zh -i + ``` + +4. ブラウザが自動的に起動しない場合は、次のURLを開きます： + + ```bash + http://localhost:7860/ + ``` + + + +詳細については、[GUIのドキュメント](./docs/README_GUI.md) を参照してください。 + +

方法4. Docker

+ +1. プルして実行します： + + ```bash + docker pull byaidu/pdf2zh + docker run -d -p 7860:7860 byaidu/pdf2zh + ``` + +2. ブラウザで開きます： + + ``` + http://localhost:7860/ + ``` + +クラウドサービスでのDockerデプロイメント用： + + + +

高度なオプション

+ +コマンドラインで翻訳コマンドを実行し、現在の作業ディレクトリに翻訳されたドキュメント `example-mono.pdf` とバイリンガルドキュメント `example-dual.pdf` を生成します。デフォルトではGoogle翻訳サービスを使用します。 + +cmd + +以下の表に、参考のためにすべての高度なオプションをリストしました： + +| オプション | 機能 | 例 | +| -------- | ------- |------- | +| files | ローカルファイル | `pdf2zh ~/local.pdf` | +| links | オンラインファイル | `pdf2zh http://arxiv.org/paper.pdf` | +| `-i` | [GUIに入る](#gui) | `pdf2zh -i` | +| `-p` | [部分的なドキュメント翻訳](#partial) | `pdf2zh example.pdf -p 1` | +| `-li` | [ソース言語](#languages) | `pdf2zh example.pdf -li en` | +| `-lo` | [ターゲット言語](#languages) | `pdf2zh example.pdf -lo zh` | +| `-s` | [翻訳サービス](#services) | `pdf2zh example.pdf -s deepl` | +| `-t` | [マルチスレッド](#threads) | `pdf2zh example.pdf -t 1` | +| `-o` | 出力ディレクトリ | `pdf2zh example.pdf -o output` | +| `-f`, `-c` | [例外](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | +| `--share` | gradio公開リンクを取得 | `pdf2zh -i --share` | + +

全文または部分的なドキュメント翻訳

+ +- **全文翻訳** + +```bash +pdf2zh example.pdf +``` + +- **部分翻訳** + +```bash +pdf2zh example.pdf -p 1-3,5 +``` + +

ソース言語とターゲット言語を指定

+ +[Google Languages Codes](https://developers.google.com/admin-sdk/directory/v1/languages)、[DeepL Languages Codes](https://developers.deepl.com/docs/resources/supported-languages) を参照してください + +```bash +pdf2zh example.pdf -li en -lo ja +``` + +

異なるサービスで翻訳

+ +以下の表は、各翻訳サービスに必要な [環境変数](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4) を示しています。各サービスを使用する前に、これらの変数を設定してください。 + +|**翻訳者**|**サービス**|**環境変数**|**デフォルト値**|**備考**| +|-|-|-|-|-| +|**Google (デフォルト)**|`google`|なし|N/A|なし| +|**Bing**|`bing`|なし|N/A|なし| +|**DeepL**|`deepl`|`DEEPL_SERVER_URL`,`DEEPL_AUTH_KEY`|`https://api.deepl.com`, `[Your Key]`|[DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API) を参照| +|**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|[DeepLX](https://github.com/OwO-Network/DeepLX) を参照| +|**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|[Ollama](https://github.com/ollama/ollama) を参照| +|**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|[OpenAI](https://platform.openai.com/docs/overview) を参照| +|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|[Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python) を参照| +|**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|[Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk) を参照| +|**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|[SiliconCloud](https://docs.siliconflow.cn/quickstart) を参照| +|**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|[Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview) を参照| +|**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|[Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104) を参照| + +`-s service` または `-s service:model` を使用してサービスを指定します： + +```bash +pdf2zh example.pdf -s openai:gpt-4o-mini +``` + +または環境変数でモデルを指定します： + +```bash +set OPENAI_MODEL=gpt-4o-mini +pdf2zh example.pdf -s openai +``` + +

例外を指定して翻訳

+ +正規表現を使用して保持する必要がある数式フォントと文字を指定します： + +```bash +pdf2zh example.pdf -f "(CM[^RT].*|MS.*|.*Ital)" -c "(\(|\||\)|\+|=|\d|[\u0080-\ufaff])" +``` + +デフォルトで `Latex`、`Mono`、`Code`、`Italic`、`Symbol` および `Math` フォントを保持します： + +```bash +pdf2zh example.pdf -f "(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-|rsfs|txsy|wasy|stmary|.*Mono|.*Code|.*Ital|.*Sym|.*Math)" +``` + +

スレッド数を指定

+ +`-t` を使用して翻訳に使用するスレッド数を指定します： + +```bash +pdf2zh example.pdf -t 1 +``` + +

API

+ +### Python + +```python +from pdf2zh import translate, translate_stream + +params = {"lang_in": "en", "lang_out": "zh", "service": "google", "thread": 4} +file_mono, file_dual = translate(files=["example.pdf"], **params)[0] +with open("example.pdf", "rb") as f: + stream_mono, stream_dual = translate_stream(stream=f.read(), **params) +``` + +### HTTP + +```bash +pip install pdf2zh[backend] +pdf2zh --flask +pdf2zh --celery worker +``` + +```bash +curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"l +ang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" +{"id":"d9894125-2f4e-45ea-9d93-1a9068d2045a"} + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a +{"info":{"n":13,"total":506},"state":"PROGRESS"} + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a +{"state":"SUCCESS"} + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/mono --output example-mono.pdf + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/dual --output example-dual.pdf + +curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a -X DELETE +``` + +

謝辞

+ +- ドキュメントのマージ：[PyMuPDF](https://github.com/pymupdf/PyMuPDF) + +- ドキュメントの解析：[Pdfminer.six](https://github.com/pdfminer/pdfminer.six) + +- ドキュメントの抽出：[MinerU](https://github.com/opendatalab/MinerU) + +- マルチスレッド翻訳：[MathTranslate](https://github.com/SUSYUSTC/MathTranslate) + +- レイアウト解析：[DocLayout-YOLO](https://github.com/opendatalab/DocLayout-YOLO) + +- ドキュメント標準：[PDF Explained](https://zxyle.github.io/PDF-Explained/)、[PDF Cheat Sheets](https://pdfa.org/resource/pdf-cheat-sheets/) + +- 多言語フォント：[Go Noto Universal](https://github.com/satbyy/go-noto-universal) + +

貢献者

+ + + + + +![Alt](https://repobeats.axiom.co/api/embed/dfa7583da5332a11468d686fbd29b92320a6a869.svg "Repobeats analytics image") + +

スター履歴

+ + + + + + Star History Chart + + diff --git a/README_zh-CN.md b/README_zh-CN.md index 7a870612..34a48fe1 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -1,6 +1,6 @@
-[English](README.md) | 简体中文 +[English](README.md) | 简体中文 | [日本語](README_ja-JP.md) PDF2ZH From b8817ffe21bd5a07c35b7f8655c8bf24b1eaba3d Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sat, 14 Dec 2024 23:30:53 +0800 Subject: [PATCH 047/114] chore: check http status --- README.md | 2 +- README_zh-CN.md | 2 +- pdf2zh/converter.py | 2 +- pdf2zh/translator.py | 73 +++++++++++++++++++++++--------------------- 4 files changed, 41 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index f2fd9b9f..403a2be5 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ In the following table, we list all advanced options for reference: | `-t` | [Multi-threads](#threads) | `pdf2zh example.pdf -t 1` | | `-o` | Output dir | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [Exceptions](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | -| `--share` | get gradio public link | `pdf2zh -i --share` | +| `--share` | Get gradio public link | `pdf2zh -i --share` |

Full / partial document translation

diff --git a/README_zh-CN.md b/README_zh-CN.md index 7a870612..c0aa949c 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -162,7 +162,7 @@ | `-t` | [多线程](#threads) | `pdf2zh example.pdf -t 1` | | `-o` | 输出目录 | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [例外规则](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | -| `--share` | 获取gradio公开链接 | `pdf2zh -i --share` | +| `--share` | 获取 gradio 公开链接 | `pdf2zh -i --share` |

全文或部分文档翻译

diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 7356e7ce..51c011af 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -144,7 +144,7 @@ def __init__( for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator, OpenAITranslator, ZhipuTranslator, SiliconTranslator, AzureTranslator, TencentTranslator]: if service_name == translator.name: - self.translator = translator(lang_out, lang_in, service_model) + self.translator = translator(lang_in, lang_out, service_model) if not self.translator: raise ValueError("Unsupported translation service") diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 0b5b52b4..bb4f9da4 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -25,11 +25,11 @@ class BaseTranslator: envs = {} lang_map = {} - def __init__(self, lang_out: str, lang_in: str, model): - lang_out = self.lang_map.get(lang_out.lower(), lang_out) + def __init__(self, lang_in, lang_out, model): lang_in = self.lang_map.get(lang_in.lower(), lang_in) - self.lang_out = lang_out + lang_out = self.lang_map.get(lang_out.lower(), lang_out) self.lang_in = lang_in + self.lang_out = lang_out self.model = model def translate(self, text): @@ -48,15 +48,15 @@ def prompt(self, text): ] def __str__(self): - return f"{self.name} {self.lang_out} {self.lang_in} {self.model}" + return f"{self.name} {self.lang_in} {self.lang_out} {self.model}" class GoogleTranslator(BaseTranslator): name = "google" lang_map = {"zh": "zh-CN"} - def __init__(self, lang_out, lang_in, model): - super().__init__(lang_out, lang_in, model) + def __init__(self, lang_in, lang_out, model): + super().__init__(lang_in, lang_out, model) self.session = requests.Session() self.endpoint = "http://translate.google.com/m" self.headers = { @@ -76,6 +76,7 @@ def translate(self, text): if response.status_code == 400: result = "IRREPARABLE TRANSLATION ERROR" else: + response.raise_for_status() result = html.unescape(re_result[0]) return remove_control_characters(result) @@ -85,8 +86,8 @@ class BingTranslator(BaseTranslator): name = "bing" lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_out, lang_in, model): - super().__init__(lang_out, lang_in, model) + def __init__(self, lang_in, lang_out, model): + super().__init__(lang_in, lang_out, model) self.session = requests.Session() self.endpoint = "https://www.bing.com/ttranslatev3" self.headers = { @@ -94,18 +95,19 @@ def __init__(self, lang_out, lang_in, model): } def fineSID(self): - resp = self.session.get("https://www.bing.com/translator") - ig = re.findall(r"\"ig\":\"(.*?)\"", resp.text)[0] - iid = re.findall(r"data-iid=\"(.*?)\"", resp.text)[-1] + response = self.session.get("https://www.bing.com/translator") + response.raise_for_status() + ig = re.findall(r"\"ig\":\"(.*?)\"", response.text)[0] + iid = re.findall(r"data-iid=\"(.*?)\"", response.text)[-1] key, token = re.findall( - r"params_AbusePreventionHelper\s=\s\[(.*?),\"(.*?)\",", resp.text + r"params_AbusePreventionHelper\s=\s\[(.*?),\"(.*?)\",", response.text )[0] return ig, iid, key, token def translate(self, text): text = text[:1000] # bing translate max length ig, iid, key, token = self.fineSID() - resp = self.session.post( + response = self.session.post( f"{self.endpoint}?IG={ig}&IID={iid}", data={ "fromLang": self.lang_in, @@ -116,7 +118,8 @@ def translate(self, text): }, headers=self.headers, ) - return resp.json()[0]["translations"][0]["text"] + response.raise_for_status() + return response.json()[0]["translations"][0]["text"] class DeepLTranslator(BaseTranslator): @@ -128,9 +131,8 @@ class DeepLTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_out, lang_in, model): - super().__init__(lang_out, lang_in, model) - self.session = requests.Session() + def __init__(self, lang_in, lang_out, model): + super().__init__(lang_in, lang_out, model) server_url = os.getenv("DEEPL_SERVER_URL", self.envs["DEEPL_SERVER_URL"]) auth_key = os.getenv("DEEPL_AUTH_KEY") self.client = deepl.Translator(auth_key, server_url=server_url) @@ -150,13 +152,13 @@ class DeepLXTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_out, lang_in, model): - super().__init__(lang_out, lang_in, model) + def __init__(self, lang_in, lang_out, model): + super().__init__(lang_in, lang_out, model) self.endpoint = os.getenv("DEEPLX_ENDPOINT", self.envs["DEEPLX_ENDPOINT"]) self.session = requests.Session() def translate(self, text): - resp = self.session.post( + response = self.session.post( self.endpoint, json={ "source_lang": self.lang_in, @@ -164,7 +166,8 @@ def translate(self, text): "text": text, }, ) - return resp.json()["data"] + response.raise_for_status() + return response.json()["data"] class OllamaTranslator(BaseTranslator): @@ -175,10 +178,10 @@ class OllamaTranslator(BaseTranslator): "OLLAMA_MODEL": "gemma2", } - def __init__(self, lang_out, lang_in, model): + def __init__(self, lang_in, lang_out, model): if not model: model = os.getenv("OLLAMA_MODEL", self.envs["OLLAMA_MODEL"]) - super().__init__(lang_out, lang_in, model) + super().__init__(lang_in, lang_out, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = ollama.Client() @@ -200,10 +203,10 @@ class OpenAITranslator(BaseTranslator): "OPENAI_MODEL": "gpt-4o-mini", } - def __init__(self, lang_out, lang_in, model, base_url=None, api_key=None): + def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): if not model: model = os.getenv("OPENAI_MODEL", self.envs["OPENAI_MODEL"]) - super().__init__(lang_out, lang_in, model) + super().__init__(lang_in, lang_out, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = openai.OpenAI(base_url=base_url, api_key=api_key) @@ -224,14 +227,14 @@ class AzureOpenAITranslator(BaseTranslator): "AZURE_OPENAI_MODEL": "gpt-4o-mini", } - def __init__(self, lang_out, lang_in, model, base_url=None, api_key=None): + def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): base_url = os.getenv( "AZURE_OPENAI_BASE_URL", self.envs["AZURE_OPENAI_BASE_URL"] ) api_version = os.getenv("AZURE_OPENAI_API_VERSION", "2024-06-01") if not model: model = os.getenv("AZURE_OPENAI_MODEL", self.envs["AZURE_OPENAI_MODEL"]) - super().__init__(lang_out, lang_in, model) + super().__init__(lang_in, lang_out, model) self.options = {"temperature": 0} self.client = openai.AzureOpenAI( azure_endpoint=base_url, @@ -257,12 +260,12 @@ class ZhipuTranslator(OpenAITranslator): "ZHIPU_MODEL": "glm-4-flash", } - def __init__(self, lang_out, lang_in, model): + def __init__(self, lang_in, lang_out, model): base_url = "https://open.bigmodel.cn/api/paas/v4" api_key = os.getenv("ZHIPU_API_KEY") if not model: model = os.getenv("ZHIPU_MODEL", self.envs["ZHIPU_MODEL"]) - super().__init__(lang_out, lang_in, model, base_url=base_url, api_key=api_key) + super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) class SiliconTranslator(OpenAITranslator): @@ -273,12 +276,12 @@ class SiliconTranslator(OpenAITranslator): "SILICON_MODEL": "Qwen/Qwen2.5-7B-Instruct", } - def __init__(self, lang_out, lang_in, model): + def __init__(self, lang_in, lang_out, model): base_url = "https://api.siliconflow.cn/v1" api_key = os.getenv("SILICON_API_KEY") if not model: model = os.getenv("SILICON_MODEL", self.envs["SILICON_MODEL"]) - super().__init__(lang_out, lang_in, model, base_url=base_url, api_key=api_key) + super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) class AzureTranslator(BaseTranslator): @@ -290,8 +293,8 @@ class AzureTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_out, lang_in, model): - super().__init__(lang_out, lang_in, model) + def __init__(self, lang_in, lang_out, model): + super().__init__(lang_in, lang_out, model) endpoint = os.getenv("AZURE_ENDPOINT", self.envs["AZURE_ENDPOINT"]) api_key = os.getenv("AZURE_API_KEY") credential = AzureKeyCredential(api_key) @@ -320,8 +323,8 @@ class TencentTranslator(BaseTranslator): "TENCENTCLOUD_SECRET_KEY": None, } - def __init__(self, lang_out, lang_in, model): - super().__init__(lang_out, lang_in, model) + def __init__(self, lang_in, lang_out, model): + super().__init__(lang_in, lang_out, model) cred = credential.DefaultCredentialProvider().get_credential() self.client = TmtClient(cred, "ap-beijing") self.req = TextTranslateRequest() From 3a23b4fbef403ba935aeddbe1519004f8e89100e Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 15 Dec 2024 00:44:12 +0800 Subject: [PATCH 048/114] feat: add gemini --- README.md | 1 + README_ja-JP.md | 25 +++++++++++++------------ README_zh-CN.md | 1 + pdf2zh/converter.py | 5 ++++- pdf2zh/gui.py | 2 ++ pdf2zh/translator.py | 16 ++++++++++++++++ 6 files changed, 37 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 577e5d6c..ddac1586 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,7 @@ The table below outlines the required [environment variables](https://chatgpt.co |**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| |**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| |**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| +|**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| |**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| diff --git a/README_ja-JP.md b/README_ja-JP.md index d342a4f4..1a7c15a6 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -190,19 +190,20 @@ pdf2zh example.pdf -li en -lo ja 以下の表は、各翻訳サービスに必要な [環境変数](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4) を示しています。各サービスを使用する前に、これらの変数を設定してください。 -|**翻訳者**|**サービス**|**環境変数**|**デフォルト値**|**備考**| +|**Translator**|**Service**|**Environment Variables**|**Default Values**|**Notes**| |-|-|-|-|-| -|**Google (デフォルト)**|`google`|なし|N/A|なし| -|**Bing**|`bing`|なし|N/A|なし| -|**DeepL**|`deepl`|`DEEPL_SERVER_URL`,`DEEPL_AUTH_KEY`|`https://api.deepl.com`, `[Your Key]`|[DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API) を参照| -|**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|[DeepLX](https://github.com/OwO-Network/DeepLX) を参照| -|**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|[Ollama](https://github.com/ollama/ollama) を参照| -|**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|[OpenAI](https://platform.openai.com/docs/overview) を参照| -|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|[Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python) を参照| -|**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|[Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk) を参照| -|**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|[SiliconCloud](https://docs.siliconflow.cn/quickstart) を参照| -|**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|[Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview) を参照| -|**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|[Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104) を参照| +|**Google (Default)**|`google`|None|N/A|None| +|**Bing**|`bing`|None|N/A|None| +|**DeepL**|`deepl`|`DEEPL_SERVER_URL`,`DEEPL_AUTH_KEY`|`https://api.deepl.com`, `[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| +|**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| +|**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| +|**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| +|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| +|**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| +|**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| +|**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| +|**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| +|**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| `-s service` または `-s service:model` を使用してサービスを指定します： diff --git a/README_zh-CN.md b/README_zh-CN.md index 65754c9c..fbb7725f 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -201,6 +201,7 @@ pdf2zh example.pdf -li en -lo ja |**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| |**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| |**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| +|**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| |**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 51c011af..294bd6a9 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -27,6 +27,7 @@ OpenAITranslator, ZhipuTranslator, SiliconTranslator, + GeminiTranslator, AzureTranslator, TencentTranslator, ) @@ -142,7 +143,7 @@ def __init__( service_name = param[0] service_model = param[1] if len(param) > 1 else None for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator, - OpenAITranslator, ZhipuTranslator, SiliconTranslator, AzureTranslator, TencentTranslator]: + OpenAITranslator, ZhipuTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator]: if service_name == translator.name: self.translator = translator(lang_in, lang_out, service_model) if not self.translator: @@ -170,6 +171,8 @@ def receive_layout(self, ltpage: LTPage): ops: str = "" # 渲染结果 def vflag(font: str, char: str): # 匹配公式（和角标）字体 + if isinstance(font, bytes): # hack 嵌入的 china-ss 会变成 b'Song' + font = font.decode() font = font.split("+")[-1] # 字体名截断 if re.match(r"\(cid:", char): return True diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 1ffd2551..aef853b4 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -14,6 +14,7 @@ OpenAITranslator, ZhipuTranslator, SiliconTranslator, + GeminiTranslator, AzureTranslator, TencentTranslator, ) @@ -34,6 +35,7 @@ "OpenAI": OpenAITranslator, "Zhipu": ZhipuTranslator, "Silicon": SiliconTranslator, + "Gemini": GeminiTranslator, "Azure": AzureTranslator, "Tencent": TencentTranslator, } diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index bb4f9da4..8ad21ca7 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -284,6 +284,22 @@ def __init__(self, lang_in, lang_out, model): super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) +class GeminiTranslator(OpenAITranslator): + # https://ai.google.dev/gemini-api/docs/openai + name = "gemini" + envs = { + "GEMINI_API_KEY": None, + "GEMINI_MODEL": "gemini-1.5-flash", + } + + def __init__(self, lang_in, lang_out, model): + base_url = "https://generativelanguage.googleapis.com/v1beta/openai/" + api_key = os.getenv("GEMINI_API_KEY") + if not model: + model = os.getenv("GEMINI_MODEL", self.envs["GEMINI_MODEL"]) + super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) + + class AzureTranslator(BaseTranslator): # https://github.com/Azure/azure-sdk-for-python name = "azure" From 1ec8db4b9c192296900a9319a8c54c82c3d3b2a8 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 15 Dec 2024 17:33:31 +0800 Subject: [PATCH 049/114] doc: add badge --- README.md | 4 ++++ README_ja-JP.md | 4 ++++ README_zh-CN.md | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/README.md b/README.md index ddac1586..6ca8de42 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@
+

English | [简体中文](README_zh-CN.md) | [日本語](README_ja-JP.md) +

PDF2ZH @@ -25,6 +27,8 @@ English | [简体中文](README_zh-CN.md) | [日本語](README_ja-JP.md)

+Byaidu%2FPDFMathTranslate | Trendshift +
PDF scientific paper translation and bilingual comparison. diff --git a/README_ja-JP.md b/README_ja-JP.md index 1a7c15a6..b2c33c48 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -1,6 +1,8 @@
+

[English](README.md) | [简体中文](README_zh-CN.md) | 日本語 +

PDF2ZH @@ -25,6 +27,8 @@

+Byaidu%2FPDFMathTranslate | Trendshift +
科学 PDF 文書の翻訳およびバイリンガル比較ツール diff --git a/README_zh-CN.md b/README_zh-CN.md index fbb7725f..ec2e6ef1 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -1,6 +1,8 @@
+

[English](README.md) | 简体中文 | [日本語](README_ja-JP.md) +

PDF2ZH @@ -25,6 +27,8 @@

+Byaidu%2FPDFMathTranslate | Trendshift +
科学 PDF 文档翻译及双语对照工具 From ae471cb6b492c83ec88d83e5c71a0d539132e860 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 15 Dec 2024 17:34:22 +0800 Subject: [PATCH 050/114] doc: fix link --- README.md | 2 -- README_ja-JP.md | 2 -- README_zh-CN.md | 2 -- 3 files changed, 6 deletions(-) diff --git a/README.md b/README.md index 6ca8de42..fd3aee37 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@
-

English | [简体中文](README_zh-CN.md) | [日本語](README_ja-JP.md) -

PDF2ZH diff --git a/README_ja-JP.md b/README_ja-JP.md index b2c33c48..02385bbe 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -1,8 +1,6 @@
-

[English](README.md) | [简体中文](README_zh-CN.md) | 日本語 -

PDF2ZH diff --git a/README_zh-CN.md b/README_zh-CN.md index ec2e6ef1..71ba0a81 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -1,8 +1,6 @@
-

[English](README.md) | 简体中文 | [日本語](README_ja-JP.md) -

PDF2ZH From d617c16c40297fe84a46eda00cc1f14bccca78c7 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 15 Dec 2024 21:34:11 +0800 Subject: [PATCH 051/114] fix: bing host --- pdf2zh/translator.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 8ad21ca7..f43f2433 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -89,26 +89,27 @@ class BingTranslator(BaseTranslator): def __init__(self, lang_in, lang_out, model): super().__init__(lang_in, lang_out, model) self.session = requests.Session() - self.endpoint = "https://www.bing.com/ttranslatev3" + self.endpoint = "https://www.bing.com/translator" self.headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0", # noqa: E501 } def fineSID(self): - response = self.session.get("https://www.bing.com/translator") + response = self.session.get(self.endpoint) response.raise_for_status() + url = response.url[:-10] ig = re.findall(r"\"ig\":\"(.*?)\"", response.text)[0] iid = re.findall(r"data-iid=\"(.*?)\"", response.text)[-1] key, token = re.findall( r"params_AbusePreventionHelper\s=\s\[(.*?),\"(.*?)\",", response.text )[0] - return ig, iid, key, token + return url, ig, iid, key, token def translate(self, text): text = text[:1000] # bing translate max length - ig, iid, key, token = self.fineSID() + url, ig, iid, key, token = self.fineSID() response = self.session.post( - f"{self.endpoint}?IG={ig}&IID={iid}", + f"{url}ttranslatev3?IG={ig}&IID={iid}", data={ "fromLang": self.lang_in, "to": self.lang_out, From beb85174ca082000f6a41f6d50ab960f3f755585 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Sun, 15 Dec 2024 22:28:20 +0800 Subject: [PATCH 052/114] release: 1.8.8 --- README.md | 3 +-- README_ja-JP.md | 3 +-- README_zh-CN.md | 3 +-- pdf2zh/__init__.py | 2 +- pdf2zh/translator.py | 7 +++---- pyproject.toml | 2 +- 6 files changed, 8 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index fd3aee37..c8b15537 100644 --- a/README.md +++ b/README.md @@ -264,8 +264,7 @@ pdf2zh --celery worker ``` ```bash -curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"l -ang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" +curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"lang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" {"id":"d9894125-2f4e-45ea-9d93-1a9068d2045a"} curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a diff --git a/README_ja-JP.md b/README_ja-JP.md index 02385bbe..d2c329a9 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -264,8 +264,7 @@ pdf2zh --celery worker ``` ```bash -curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"l -ang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" +curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"lang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" {"id":"d9894125-2f4e-45ea-9d93-1a9068d2045a"} curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a diff --git a/README_zh-CN.md b/README_zh-CN.md index 71ba0a81..bfe4cc8f 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -264,8 +264,7 @@ pdf2zh --celery worker ``` ```bash -curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"l -ang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" +curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"lang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" {"id":"d9894125-2f4e-45ea-9d93-1a9068d2045a"} curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a diff --git a/pdf2zh/__init__.py b/pdf2zh/__init__.py index 4fffb544..8483f481 100644 --- a/pdf2zh/__init__.py +++ b/pdf2zh/__init__.py @@ -3,6 +3,6 @@ log = logging.getLogger(__name__) -__version__ = "1.8.7" +__version__ = "1.8.8" __author__ = "Byaidu" __all__ = ["translate", "translate_stream"] diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index f43f2433..3667a920 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -94,7 +94,7 @@ def __init__(self, lang_in, lang_out, model): "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0", # noqa: E501 } - def fineSID(self): + def findSID(self): response = self.session.get(self.endpoint) response.raise_for_status() url = response.url[:-10] @@ -107,7 +107,7 @@ def fineSID(self): def translate(self, text): text = text[:1000] # bing translate max length - url, ig, iid, key, token = self.fineSID() + url, ig, iid, key, token = self.findSID() response = self.session.post( f"{url}ttranslatev3?IG={ig}&IID={iid}", data={ @@ -232,7 +232,6 @@ def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): base_url = os.getenv( "AZURE_OPENAI_BASE_URL", self.envs["AZURE_OPENAI_BASE_URL"] ) - api_version = os.getenv("AZURE_OPENAI_API_VERSION", "2024-06-01") if not model: model = os.getenv("AZURE_OPENAI_MODEL", self.envs["AZURE_OPENAI_MODEL"]) super().__init__(lang_in, lang_out, model) @@ -240,7 +239,7 @@ def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): self.client = openai.AzureOpenAI( azure_endpoint=base_url, azure_deployment=model, - api_version=api_version, + api_version="2024-06-01", api_key=api_key, ) diff --git a/pyproject.toml b/pyproject.toml index 516b3232..c8bdefa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pdf2zh" -version = "1.8.7" +version = "1.8.8" description = "Latex PDF Translator" authors = [{ name = "Byaidu", email = "byaidux@gmail.com" }] license = "AGPL-3.0" From 4ebc0b8e88aef1ce0b90134105797b71e6c20bc0 Mon Sep 17 00:00:00 2001 From: charles Date: Mon, 16 Dec 2024 01:30:38 +0000 Subject: [PATCH 053/114] Add cancel support for gui --- pdf2zh/gui.py | 31 ++++++++++++++++++++++++++++++- pdf2zh/high_level.py | 8 +++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 1ffd2551..e47ad707 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -1,5 +1,8 @@ import os import shutil +import uuid +import asyncio +from asyncio import CancelledError from pathlib import Path from pdf2zh import __version__ from pdf2zh.high_level import translate @@ -96,6 +99,12 @@ def download_with_limit(url, save_path, size_limit): file.write(chunk) return save_path / filename +def stop_translate_file(state): + session_id = state["session_id"] + if session_id is None: + return + if session_id in cancellation_event_map: + cancellation_event_map[session_id].set() def translate_file( file_type, @@ -106,9 +115,13 @@ def translate_file( lang_to, page_range, recaptcha_response, + state, progress=gr.Progress(), *envs, ): + session_id = uuid.uuid4() + state["session_id"] = session_id + cancellation_event_map[session_id] = asyncio.Event() """Translate PDF content using selected service.""" if flag_demo and not verify_recaptcha(recaptcha_response): raise gr.Error("reCAPTCHA fail") @@ -158,9 +171,14 @@ def progress_bar(t: tqdm.tqdm): "output": output, "thread": 4, "callback": progress_bar, + "cancellation_event": cancellation_event_map[session_id], } print(param) - translate(**param) + try: + translate(**param) + except CancelledError as e: + del cancellation_event_map[session_id] + raise gr.Error("Translation cancelled") print(f"Files after translation: {os.listdir(output)}") if not file_mono.exists() or not file_dual.exists(): @@ -193,6 +211,8 @@ def progress_bar(t: tqdm.tqdm): c950="#020B33", ) +cancellation_event_map = {} + with gr.Blocks( title="PDFMathTranslate - PDF Translation with preserved formats", theme=gr.themes.Default( @@ -316,6 +336,7 @@ def on_select_filetype(file_type): ) recaptcha_box = gr.HTML('
') translate_btn = gr.Button("Translate", variant="primary") + cancellation_btn = gr.Button("Cancel", variant="secondary") tech_details_tog = gr.Markdown( f""" Technical details @@ -377,6 +398,8 @@ def on_select_filetype(file_type): ), ) + state = gr.State({"session_id": None}) + translate_btn.click( translate_file, inputs=[ @@ -388,6 +411,7 @@ def on_select_filetype(file_type): lang_to, page_range, recaptcha_response, + state, *envs, ], outputs=[ @@ -400,6 +424,11 @@ def on_select_filetype(file_type): ], ).then(lambda: None, js="()=>{grecaptcha.reset()}" if flag_demo else "") + cancellation_btn.click( + stop_translate_file, + inputs=[state], + ) + def setup_gui(share=False): if flag_demo: diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 1d108a75..06e184bf 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -1,5 +1,6 @@ """Functions that can be used for the most common use-cases for pdf2zh.six""" - +import asyncio +from asyncio import CancelledError from typing import BinaryIO import numpy as np import tqdm @@ -84,6 +85,7 @@ def translate_patch( resfont: str = "", noto: Font = None, callback: object = None, + cancellation_event : asyncio.Event = None, **kwarg: Any, ) -> None: rsrcmgr = PDFResourceManager() @@ -104,6 +106,8 @@ def translate_patch( doc = PDFDocument(parser) with tqdm.tqdm(total=total_pages) as progress: for pageno, page in enumerate(PDFPage.create_pages(doc)): + if cancellation_event and cancellation_event.is_set(): + raise CancelledError("task cancelled") if pages and (pageno not in pages): continue progress.update() @@ -161,6 +165,7 @@ def translate_stream( vfont: str = "", vchar: str = "", callback: object = None, + cancellation_event: asyncio.Event = None, **kwarg: Any, ): font_list = [("tiro", None)] @@ -237,6 +242,7 @@ def translate( vfont: str = "", vchar: str = "", callback: object = None, + cancellation_event: asyncio.Event = None, **kwarg: Any, ): if not files: From 8096e971c2c1683374e4e293697b622e11095200 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Mon, 16 Dec 2024 11:29:48 +0800 Subject: [PATCH 054/114] =?UTF-8?q?openai=20api=E7=8A=B6=E6=80=81=E5=88=A4?= =?UTF-8?q?=E6=96=AD=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pdf2zh/translator.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 3667a920..c30a8a42 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -212,11 +212,21 @@ def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): self.client = openai.OpenAI(base_url=base_url, api_key=api_key) def translate(self, text) -> str: - response = self.client.chat.completions.create( - model=self.model, - **self.options, - messages=self.prompt(text), - ) + try: + response = self.client.chat.completions.create( + model=self.model, + **self.options, + messages=self.prompt(text), + ) + except openai.BadRequestError as e: + print("400 API BadRequestError") + return "" + except openai.APIStatusError as e: + print("API Status Error.") + return "" + except openai.APIConnectionError as e: + print("API Connection Error") + return "" return response.choices[0].message.content.strip() From 813c594a341285c199fda4b1729e943aab39eb45 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Mon, 16 Dec 2024 11:31:26 +0800 Subject: [PATCH 055/114] =?UTF-8?q?openai=20api=20=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pdf2zh/translator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index c30a8a42..1e5d522b 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -218,13 +218,13 @@ def translate(self, text) -> str: **self.options, messages=self.prompt(text), ) - except openai.BadRequestError as e: + except openai.BadRequestError: print("400 API BadRequestError") return "" - except openai.APIStatusError as e: + except openai.APIStatusError: print("API Status Error.") return "" - except openai.APIConnectionError as e: + except openai.APIConnectionError: print("API Connection Error") return "" return response.choices[0].message.content.strip() From 80cd323e53f655f2209334e4309ab2def459e280 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Mon, 16 Dec 2024 14:10:20 +0800 Subject: [PATCH 056/114] =?UTF-8?q?=E5=8F=AA=E6=A3=80=E6=B5=8B400=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pdf2zh/translator.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 1e5d522b..e7df4c40 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -221,12 +221,6 @@ def translate(self, text) -> str: except openai.BadRequestError: print("400 API BadRequestError") return "" - except openai.APIStatusError: - print("API Status Error.") - return "" - except openai.APIConnectionError: - print("API Connection Error") - return "" return response.choices[0].message.content.strip() From 2e11e33e5161f25bbb24643d7ac9dd71e5aaa39c Mon Sep 17 00:00:00 2001 From: CCcool Date: Sun, 15 Dec 2024 22:40:57 -0800 Subject: [PATCH 057/114] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86Dify?= =?UTF-8?q?=E5=92=8CAngthingLLM=E6=8E=A5=E5=8F=A3=EF=BC=8C=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E6=90=BA=E5=B8=A6=E7=9F=A5=E8=AF=86=E5=BA=93=E7=9A=84?= =?UTF-8?q?=E7=BF=BB=E8=AF=91=E8=BF=87=E7=A8=8B=EF=BC=8C=E6=8F=90=E9=AB=98?= =?UTF-8?q?=E7=BF=BB=E8=AF=91=E8=B4=A8=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pdf2zh/converter.py | 4 ++- pdf2zh/gui.py | 4 +++ pdf2zh/translator.py | 71 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 294bd6a9..bd60b519 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -30,6 +30,8 @@ GeminiTranslator, AzureTranslator, TencentTranslator, + DifyTranslator, + AngthingLLMTranslator, ) from pymupdf import Font @@ -143,7 +145,7 @@ def __init__( service_name = param[0] service_model = param[1] if len(param) > 1 else None for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator, - OpenAITranslator, ZhipuTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator]: + OpenAITranslator, ZhipuTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator, DifyTranslator, AngthingLLMTranslator]: if service_name == translator.name: self.translator = translator(lang_in, lang_out, service_model) if not self.translator: diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index aef853b4..6881e9f3 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -17,6 +17,8 @@ GeminiTranslator, AzureTranslator, TencentTranslator, + DifyTranslator, + AngthingLLMTranslator, ) import gradio as gr @@ -38,6 +40,8 @@ "Gemini": GeminiTranslator, "Azure": AzureTranslator, "Tencent": TencentTranslator, + "Dify": DifyTranslator, + "AnythingLLM": AngthingLLMTranslator, } lang_map = { "Chinese": "zh", diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 3667a920..de98efcf 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -352,3 +352,74 @@ def translate(self, text): self.req.SourceText = text resp: TextTranslateResponse = self.client.TextTranslate(self.req) return resp.TargetText +class AngthingLLMTranslator(BaseTranslator): + name = "angthingllm" + envs = { + "AngthingLLM_URL": None, + "AngthingLLM_APIKEY": "api_key", + } + + def __init__(self, lang_out, lang_in, model): + super().__init__(lang_out, lang_in, model) + self.api_url = os.getenv("AngthingLLM_URL", self.envs["AngthingLLM_URL"]) + self.api_key = os.getenv("AngthingLLM_APIKEY", self.envs["AngthingLLM_APIKEY"]) + self.headers = { + "accept": "application/json", + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + + def translate(self, text): + messages = self.prompt(text) + payload = { + "message": messages, + "mode": "chat", + "sessionId": "translation_expert", + } + + response = requests.post(self.api_url, headers=self.headers, data=json.dumps(payload)) + response.raise_for_status() + data = response.json() + + if "textResponse" in data: + return data["textResponse"].strip() + +class DifyTranslator(BaseTranslator): + name = "dify" + envs = { + "DIFY_API_URL": None, # 填写实际 Dify API 地址 + "DIFY_API_KEY": "api_key" # 替换为实际 API 密钥 + } + + def __init__(self, lang_out, lang_in, model): + super().__init__(lang_out, lang_in, model) + self.api_url = os.getenv("DIFY_API_URL", self.envs["DIFY_API_URL"]) + self.api_key = os.getenv("DIFY_API_KEY", self.envs["DIFY_API_KEY"]) + + def translate(self, text): + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json" + } + + payload = { + "inputs": { + "lang_out": self.lang_out, + "lang_in": self.lang_in, + "text": text + }, + "response_mode": "blocking", + "user": "translator-service" + } + + # 向 Dify 服务器发送请求 + response = requests.post( + self.api_url, + headers=headers, + data=json.dumps(payload) + ) + response.raise_for_status() + response_data = response.json() + + # 解析响应 + return response_data.get('data', {}).get('outputs', {}).get('text', []) From 14c2508bf17ad76431ce584cf6e89f74ef12bcd9 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Mon, 16 Dec 2024 15:03:09 +0800 Subject: [PATCH 058/114] =?UTF-8?q?=E5=8F=AA=E5=AF=B9=E6=99=BA=E8=B0=B1?= =?UTF-8?q?=E6=A3=80=E6=9F=A51301=E7=9A=84=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pdf2zh/translator.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index e7df4c40..0bf00f50 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -15,6 +15,8 @@ from tencentcloud.tmt.v20180321.models import TextTranslateRequest from tencentcloud.tmt.v20180321.models import TextTranslateResponse +import json + def remove_control_characters(s): return "".join(ch for ch in s if unicodedata.category(ch)[0] != "C") @@ -212,15 +214,11 @@ def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): self.client = openai.OpenAI(base_url=base_url, api_key=api_key) def translate(self, text) -> str: - try: - response = self.client.chat.completions.create( - model=self.model, - **self.options, - messages=self.prompt(text), - ) - except openai.BadRequestError: - print("400 API BadRequestError") - return "" + response = self.client.chat.completions.create( + model=self.model, + **self.options, + messages=self.prompt(text), + ) return response.choices[0].message.content.strip() @@ -271,6 +269,22 @@ def __init__(self, lang_in, lang_out, model): model = os.getenv("ZHIPU_MODEL", self.envs["ZHIPU_MODEL"]) super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) + def translate(self, text) -> str: + try: + response = self.client.chat.completions.create( + model=self.model, + **self.options, + messages=self.prompt(text), + ) + except openai.APIError: + if ( + json.loads(response.choices[0].message.content.strip())["error"]["code"] + == "1301" + ): + return "" + print("openai api error.") + return response.choices[0].message.content.strip() + class SiliconTranslator(OpenAITranslator): # https://docs.siliconflow.cn/quickstart From 25be98432d3bd4febdd5810cb16e832cdab7da27 Mon Sep 17 00:00:00 2001 From: CCcool Date: Sun, 15 Dec 2024 23:43:35 -0800 Subject: [PATCH 059/114] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86Dify?= =?UTF-8?q?=E5=92=8CAngthingLLM=E6=8E=A5=E5=8F=A3=EF=BC=8C=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E6=90=BA=E5=B8=A6=E7=9F=A5=E8=AF=86=E5=BA=93=E7=9A=84?= =?UTF-8?q?=E7=BF=BB=E8=AF=91=E8=BF=87=E7=A8=8B=EF=BC=8C=E6=8F=90=E9=AB=98?= =?UTF-8?q?=E7=BF=BB=E8=AF=91=E8=B4=A8=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pdf2zh/converter.py | 4 ++-- pdf2zh/gui.py | 4 ++-- pdf2zh/translator.py | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index bd60b519..f7d39ffa 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -31,7 +31,7 @@ AzureTranslator, TencentTranslator, DifyTranslator, - AngthingLLMTranslator, + AnythingLLMTranslator, ) from pymupdf import Font @@ -145,7 +145,7 @@ def __init__( service_name = param[0] service_model = param[1] if len(param) > 1 else None for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator, - OpenAITranslator, ZhipuTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator, DifyTranslator, AngthingLLMTranslator]: + OpenAITranslator, ZhipuTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator, DifyTranslator, AnythingLLMTranslator]: if service_name == translator.name: self.translator = translator(lang_in, lang_out, service_model) if not self.translator: diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 6881e9f3..cd080a6f 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -18,7 +18,7 @@ AzureTranslator, TencentTranslator, DifyTranslator, - AngthingLLMTranslator, + AnythingLLMTranslator, ) import gradio as gr @@ -41,7 +41,7 @@ "Azure": AzureTranslator, "Tencent": TencentTranslator, "Dify": DifyTranslator, - "AnythingLLM": AngthingLLMTranslator, + "AnythingLLM": AnythingLLMTranslator, } lang_map = { "Chinese": "zh", diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index de98efcf..9c5bc03e 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -352,17 +352,17 @@ def translate(self, text): self.req.SourceText = text resp: TextTranslateResponse = self.client.TextTranslate(self.req) return resp.TargetText -class AngthingLLMTranslator(BaseTranslator): - name = "angthingllm" +class AnythingLLMTranslator(BaseTranslator): + name = "anythingllm" envs = { - "AngthingLLM_URL": None, - "AngthingLLM_APIKEY": "api_key", + "AnythingLLM_URL": None, + "AnythingLLM_APIKEY": "api_key", } def __init__(self, lang_out, lang_in, model): super().__init__(lang_out, lang_in, model) - self.api_url = os.getenv("AngthingLLM_URL", self.envs["AngthingLLM_URL"]) - self.api_key = os.getenv("AngthingLLM_APIKEY", self.envs["AngthingLLM_APIKEY"]) + self.api_url = os.getenv("AnythingLLM_URL", self.envs["AnythingLLM_URL"]) + self.api_key = os.getenv("AnythingLLM_APIKEY", self.envs["AnythingLLM_APIKEY"]) self.headers = { "accept": "application/json", "Authorization": f"Bearer {self.api_key}", From 0ae1692abcf775d0fe9f7b0ae5f931e35cbbdd56 Mon Sep 17 00:00:00 2001 From: CCcool Date: Mon, 16 Dec 2024 00:20:40 -0800 Subject: [PATCH 060/114] Update README files: English, Japanese, and Chinese versions --- README.md | 2 ++ README_ja-JP.md | 2 ++ README_zh-CN.md | 2 ++ 3 files changed, 6 insertions(+) diff --git a/README.md b/README.md index c8b15537..13391a12 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,8 @@ The table below outlines the required [environment variables](https://chatgpt.co |**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| |**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| +|**Dify**|`dify`|`DIFY_API_URL`, `DIFY_API_KEY`|`[Your DIFY URL]`, `[Your Key]`|See [Dify](https://github.com/langgenius/dify),Three variables, lang_out, lang_in, and text, need to be defined in Dify's workflow input.| +|**AnythingLLM**|`anythingllm`|`AnythingLLM_URL`, `AnythingLLM_APIKEY`|`[Your AnythingLLM URL]`, `[Your Key]`|See [anything-llm](https://github.com/Mintplex-Labs/anything-llm)| Use `-s service` or `-s service:model` to specify service: diff --git a/README_ja-JP.md b/README_ja-JP.md index d2c329a9..3b5284a3 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -206,6 +206,8 @@ pdf2zh example.pdf -li en -lo ja |**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| |**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| +|**Dify**|`dify`|`DIFY_API_URL`, `DIFY_API_KEY`|`[Your DIFY URL]`, `[Your Key]`|See [Dify](https://github.com/langgenius/dify),Three variables, lang_out, lang_in, and text, need to be defined in Dify's workflow input.| +|**AnythingLLM**|`anythingllm`|`AnythingLLM_URL`, `AnythingLLM_APIKEY`|`[Your AnythingLLM URL]`, `[Your Key]`|See [anything-llm](https://github.com/Mintplex-Labs/anything-llm)| `-s service` または `-s service:model` を使用してサービスを指定します： diff --git a/README_zh-CN.md b/README_zh-CN.md index bfe4cc8f..8252d197 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -206,6 +206,8 @@ pdf2zh example.pdf -li en -lo ja |**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| |**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| +|**Dify**|`dify`|`DIFY_API_URL`, `DIFY_API_KEY`|`[Your DIFY URL]`, `[Your Key]`|See [Dify](https://github.com/langgenius/dify),Three variables, lang_out, lang_in, and text, need to be defined in Dify's workflow input.| +|**AnythingLLM**|`anythingllm`|`AnythingLLM_URL`, `AnythingLLM_APIKEY`|`[Your AnythingLLM URL]`, `[Your Key]`|See [anything-llm](https://github.com/Mintplex-Labs/anything-llm)| 使用 `-s service` 或 `-s service:model` 指定翻译服务: From 2e84cda6ca827427976c715698bbc03314531be6 Mon Sep 17 00:00:00 2001 From: borcation Date: Mon, 16 Dec 2024 16:40:54 +0800 Subject: [PATCH 061/114] fix the translation api request for DeepL --- README.md | 2 +- README_ja-JP.md | 2 +- README_zh-CN.md | 2 +- pdf2zh/translator.py | 4 +--- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 13391a12..09117308 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ The table below outlines the required [environment variables](https://chatgpt.co |-|-|-|-|-| |**Google (Default)**|`google`|None|N/A|None| |**Bing**|`bing`|None|N/A|None| -|**DeepL**|`deepl`|`DEEPL_SERVER_URL`,`DEEPL_AUTH_KEY`|`https://api.deepl.com`, `[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| +|**DeepL**|`deepl`|`DEEPL_AUTH_KEY`|`[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| |**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| |**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| |**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| diff --git a/README_ja-JP.md b/README_ja-JP.md index 3b5284a3..b232fae0 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -196,7 +196,7 @@ pdf2zh example.pdf -li en -lo ja |-|-|-|-|-| |**Google (Default)**|`google`|None|N/A|None| |**Bing**|`bing`|None|N/A|None| -|**DeepL**|`deepl`|`DEEPL_SERVER_URL`,`DEEPL_AUTH_KEY`|`https://api.deepl.com`, `[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| +|**DeepL**|`deepl`|`DEEPL_AUTH_KEY`|`[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| |**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| |**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| |**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| diff --git a/README_zh-CN.md b/README_zh-CN.md index 8252d197..89d56550 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -196,7 +196,7 @@ pdf2zh example.pdf -li en -lo ja |-|-|-|-|-| |**Google (Default)**|`google`|None|N/A|None| |**Bing**|`bing`|None|N/A|None| -|**DeepL**|`deepl`|`DEEPL_SERVER_URL`,`DEEPL_AUTH_KEY`|`https://api.deepl.com`, `[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| +|**DeepL**|`deepl`|`DEEPL_AUTH_KEY`|`[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| |**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| |**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| |**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 9c5bc03e..d776752c 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -127,16 +127,14 @@ class DeepLTranslator(BaseTranslator): # https://github.com/DeepLcom/deepl-python name = "deepl" envs = { - "DEEPL_SERVER_URL": "https://api.deepl.com", "DEEPL_AUTH_KEY": None, } lang_map = {"zh": "zh-Hans"} def __init__(self, lang_in, lang_out, model): super().__init__(lang_in, lang_out, model) - server_url = os.getenv("DEEPL_SERVER_URL", self.envs["DEEPL_SERVER_URL"]) auth_key = os.getenv("DEEPL_AUTH_KEY") - self.client = deepl.Translator(auth_key, server_url=server_url) + self.client = deepl.Translator(auth_key) def translate(self, text): response = self.client.translate_text( From 6952684e8bf4219c6e2d0d87215312042be98b9e Mon Sep 17 00:00:00 2001 From: hellofinch Date: Mon, 16 Dec 2024 17:49:24 +0800 Subject: [PATCH 062/114] raise error. --- pdf2zh/translator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 0bf00f50..05681567 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -282,7 +282,7 @@ def translate(self, text) -> str: == "1301" ): return "" - print("openai api error.") + raise ValueError("openai api error.") return response.choices[0].message.content.strip() From 0b7a789f5f741372bf42ed6b9c82ae6e36eb224e Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Mon, 16 Dec 2024 18:00:35 +0800 Subject: [PATCH 063/114] fix: reraise --- pdf2zh/gui.py | 4 +++- pdf2zh/high_level.py | 3 ++- pdf2zh/translator.py | 27 +++++++++++++++------------ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 878f15bb..2ad3a0cc 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -105,6 +105,7 @@ def download_with_limit(url, save_path, size_limit): file.write(chunk) return save_path / filename + def stop_translate_file(state): session_id = state["session_id"] if session_id is None: @@ -112,6 +113,7 @@ def stop_translate_file(state): if session_id in cancellation_event_map: cancellation_event_map[session_id].set() + def translate_file( file_type, file_input, @@ -182,7 +184,7 @@ def progress_bar(t: tqdm.tqdm): print(param) try: translate(**param) - except CancelledError as e: + except CancelledError: del cancellation_event_map[session_id] raise gr.Error("Translation cancelled") print(f"Files after translation: {os.listdir(output)}") diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 06e184bf..e536418c 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -1,4 +1,5 @@ """Functions that can be used for the most common use-cases for pdf2zh.six""" + import asyncio from asyncio import CancelledError from typing import BinaryIO @@ -85,7 +86,7 @@ def translate_patch( resfont: str = "", noto: Font = None, callback: object = None, - cancellation_event : asyncio.Event = None, + cancellation_event: asyncio.Event = None, **kwarg: Any, ) -> None: rsrcmgr = PDFResourceManager() diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 9cfb381c..411f6e11 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -274,13 +274,13 @@ def translate(self, text) -> str: **self.options, messages=self.prompt(text), ) - except openai.APIError: + except openai.BadRequestError as e: if ( json.loads(response.choices[0].message.content.strip())["error"]["code"] == "1301" ): - return "" - raise ValueError("openai api error.") + return "IRREPARABLE TRANSLATION ERROR" + raise e return response.choices[0].message.content.strip() @@ -368,6 +368,8 @@ def translate(self, text): self.req.SourceText = text resp: TextTranslateResponse = self.client.TextTranslate(self.req) return resp.TargetText + + class AnythingLLMTranslator(BaseTranslator): name = "anythingllm" envs = { @@ -393,18 +395,21 @@ def translate(self, text): "sessionId": "translation_expert", } - response = requests.post(self.api_url, headers=self.headers, data=json.dumps(payload)) + response = requests.post( + self.api_url, headers=self.headers, data=json.dumps(payload) + ) response.raise_for_status() data = response.json() if "textResponse" in data: return data["textResponse"].strip() + class DifyTranslator(BaseTranslator): name = "dify" envs = { "DIFY_API_URL": None, # 填写实际 Dify API 地址 - "DIFY_API_KEY": "api_key" # 替换为实际 API 密钥 + "DIFY_API_KEY": "api_key", # 替换为实际 API 密钥 } def __init__(self, lang_out, lang_in, model): @@ -415,27 +420,25 @@ def __init__(self, lang_out, lang_in, model): def translate(self, text): headers = { "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" + "Content-Type": "application/json", } payload = { "inputs": { "lang_out": self.lang_out, "lang_in": self.lang_in, - "text": text + "text": text, }, "response_mode": "blocking", - "user": "translator-service" + "user": "translator-service", } # 向 Dify 服务器发送请求 response = requests.post( - self.api_url, - headers=headers, - data=json.dumps(payload) + self.api_url, headers=headers, data=json.dumps(payload) ) response.raise_for_status() response_data = response.json() # 解析响应 - return response_data.get('data', {}).get('outputs', {}).get('text', []) + return response_data.get("data", {}).get("outputs", {}).get("text", []) From bf252754a215f4c7971529661b08876ce7324ad3 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Mon, 16 Dec 2024 20:15:01 +0800 Subject: [PATCH 064/114] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=BD=91=E9=A1=B5?= =?UTF-8?q?=E8=AE=A4=E8=AF=81=E9=80=89=E9=A1=B9=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +- README_zh-CN.md | 3 +- pdf2zh/gui.py | 77 +++++++++++++++++++++++++++++++++++++++++------- pdf2zh/pdf2zh.py | 10 ++++++- 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 09117308..f6c215b3 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,8 @@ In the following table, we list all advanced options for reference: | `-t` | [Multi-threads](#threads) | `pdf2zh example.pdf -t 1` | | `-o` | Output dir | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [Exceptions](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | -| `--share` | Get gradio public link | `pdf2zh -i --share` | +| `--share` | [Get gradio public link] | `pdf2zh -i --share` | +| `-a` | [add authorization and custom login page] | `pdf2zh -i -a users.txt [auth.html]` |

Full / partial document translation

diff --git a/README_zh-CN.md b/README_zh-CN.md index 89d56550..923eb42d 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -164,7 +164,8 @@ | `-t` | [多线程](#threads) | `pdf2zh example.pdf -t 1` | | `-o` | 输出目录 | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [例外规则](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | -| `--share` | 获取 gradio 公开链接 | `pdf2zh -i --share` | +| `--share` | [获取 gradio 公开链接] | `pdf2zh -i --share` | +| `-a` | [添加网页认证和自定义认证页] | `pdf2zh -i -a users.txt [auth.html]` |

全文或部分文档翻译

diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 2ad3a0cc..2b010447 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -438,25 +438,82 @@ def on_select_filetype(file_type): ) -def setup_gui(share=False): +def readuserandpasswd(file_path): + tuple_list = [] + content = "" + if len(file_path) == 2: + try: + with open(file_path[1], "r", encoding="utf-8") as file: + content = file.read() + except FileNotFoundError: + print(f"Error: File '{file_path[1]}' not found.") + try: + with open(file_path[0], "r", encoding="utf-8") as file: + tuple_list = [ + tuple(line.strip().split(",")) for line in file if line.strip() + ] + except FileNotFoundError: + print(f"Error: File '{file_path[0]}' not found.") + return tuple_list, content + + +def setup_gui(share=False, authfile=["", ""]): + userlist, html = readuserandpasswd(authfile) if flag_demo: demo.launch(server_name="0.0.0.0", max_file_size="5mb", inbrowser=True) else: - try: - demo.launch(server_name="0.0.0.0", debug=True, inbrowser=True, share=share) - except Exception: - print( - "Error launching GUI using 0.0.0.0.\nThis may be caused by global mode of proxy software." - ) + if len(userlist) == 0: try: demo.launch( - server_name="127.0.0.1", debug=True, inbrowser=True, share=share + server_name="0.0.0.0", debug=True, inbrowser=True, share=share ) except Exception: print( - "Error launching GUI using 127.0.0.1.\nThis may be caused by global mode of proxy software." + "Error launching GUI using 0.0.0.0.\nThis may be caused by global mode of proxy software." ) - demo.launch(debug=True, inbrowser=True, share=True) + try: + demo.launch( + server_name="127.0.0.1", debug=True, inbrowser=True, share=share + ) + except Exception: + print( + "Error launching GUI using 127.0.0.1.\nThis may be caused by global mode of proxy software." + ) + demo.launch(debug=True, inbrowser=True, share=True) + else: + try: + demo.launch( + server_name="0.0.0.0", + debug=True, + inbrowser=True, + share=share, + auth=userlist, + auth_message=html, + ) + except Exception: + print( + "Error launching GUI using 0.0.0.0.\nThis may be caused by global mode of proxy software." + ) + try: + demo.launch( + server_name="127.0.0.1", + debug=True, + inbrowser=True, + share=share, + auth=userlist, + auth_message=html, + ) + except Exception: + print( + "Error launching GUI using 127.0.0.1.\nThis may be caused by global mode of proxy software." + ) + demo.launch( + debug=True, + inbrowser=True, + share=True, + auth=userlist, + auth_message=html, + ) # For auto-reloading while developing diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index 351cf0ba..40f810a9 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -115,6 +115,14 @@ def create_parser() -> argparse.ArgumentParser: action="store_true", help="celery", ) + parse_params.add_argument( + "--authorized", + "-a", + type=str, + nargs="+", + default=["./users.txt", "./auth.html"], + help="user name and password.", + ) return parser @@ -146,7 +154,7 @@ def main(args: Optional[List[str]] = None) -> int: if parsed_args.interactive: from pdf2zh.gui import setup_gui - setup_gui(parsed_args.share) + setup_gui(parsed_args.share, parsed_args.authorized) return 0 if parsed_args.flask: From d2d36bda0cf567511df375c0dff716b36a223a8c Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Mon, 16 Dec 2024 21:06:58 +0800 Subject: [PATCH 065/114] fix: decode --- pdf2zh/converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index f7d39ffa..00544ea7 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -173,8 +173,8 @@ def receive_layout(self, ltpage: LTPage): ops: str = "" # 渲染结果 def vflag(font: str, char: str): # 匹配公式（和角标）字体 - if isinstance(font, bytes): # hack 嵌入的 china-ss 会变成 b'Song' - font = font.decode() + if isinstance(font, bytes): # 不一定能 decode，直接转 str + font = str(font) font = font.split("+")[-1] # 字体名截断 if re.match(r"\(cid:", char): return True From 2dc86991a3658ebce2b7ae74caf648ea286b2aff Mon Sep 17 00:00:00 2001 From: hellofinch Date: Tue, 17 Dec 2024 10:18:38 +0800 Subject: [PATCH 066/114] =?UTF-8?q?readme=20=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README_ja-JP.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README_ja-JP.md b/README_ja-JP.md index b232fae0..fa9c96de 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -164,7 +164,8 @@ Python環境を事前にインストールする必要はありません | `-t` | [マルチスレッド](#threads) | `pdf2zh example.pdf -t 1` | | `-o` | 出力ディレクトリ | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [例外](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | -| `--share` | gradio公開リンクを取得 | `pdf2zh -i --share` | +| `--share` | [gradio公開リンクを取得] | `pdf2zh -i --share` | +| `-a` | [ウェブ認証とカスタム認証ページの追加] | `pdf2zh -i -a users.txt [auth.html]` |

全文または部分的なドキュメント翻訳

From 9ae6f906d70228856161585112227b248bf960f2 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 11:11:24 +0800 Subject: [PATCH 067/114] support ms --- pdf2zh/converter.py | 3 ++- pdf2zh/gui.py | 2 ++ pdf2zh/translator.py | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 00544ea7..c466f97f 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -26,6 +26,7 @@ OllamaTranslator, OpenAITranslator, ZhipuTranslator, + ModelScopeTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, @@ -145,7 +146,7 @@ def __init__( service_name = param[0] service_model = param[1] if len(param) > 1 else None for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator, - OpenAITranslator, ZhipuTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator, DifyTranslator, AnythingLLMTranslator]: + OpenAITranslator, ZhipuTranslator, ModelScopeTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator, DifyTranslator, AnythingLLMTranslator]: if service_name == translator.name: self.translator = translator(lang_in, lang_out, service_model) if not self.translator: diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 2ad3a0cc..a901ecc5 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -16,6 +16,7 @@ AzureOpenAITranslator, OpenAITranslator, ZhipuTranslator, + ModelScopeTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, @@ -39,6 +40,7 @@ "AzureOpenAI": AzureOpenAITranslator, "OpenAI": OpenAITranslator, "Zhipu": ZhipuTranslator, + "ModelScope": ModelScopeTranslator, "Silicon": SiliconTranslator, "Gemini": GeminiTranslator, "Azure": AzureTranslator, diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 411f6e11..c0d6249e 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -252,6 +252,22 @@ def translate(self, text) -> str: return response.choices[0].message.content.strip() +class ModelScopeTranslator(OpenAITranslator): + name = "modelscope" + envs = { + "MODELSCOPE_BASE_URL": "https://api-inference.modelscope.cn/v1", + "MODELSCOPE_API_KEY": None, + "MODELSCOPE_MODEL": "Qwen/Qwen2.5-Coder-32B-Instruct", + } + + def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): + base_url = "https://api-inference.modelscope.cn/v1" + api_key = os.getenv("MODELSCOPE_API_KEY") + if not model: + model = os.getenv("MODELSCOPE_MODEL", self.envs["MODELSCOPE_MODEL"]) + super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) + + class ZhipuTranslator(OpenAITranslator): # https://bigmodel.cn/dev/api/thirdparty-frame/openai-sdk name = "zhipu" From 291e23d8a5246594a88cb583fc549640d05c0968 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 11:32:20 +0800 Subject: [PATCH 068/114] update readme --- README.md | 1 + README_ja-JP.md | 1 + README_zh-CN.md | 33 +++++++++++++++++---------------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 09117308..028d6526 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,7 @@ The table below outlines the required [environment variables](https://chatgpt.co |**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| |**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| |**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| +| **ModelScope** | `ModelScope` |`MODELSCOPE_API_KEY`, `MODELSCOPE_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-Coder-32B-Instruct`| See [ModelScope](https://www.modelscope.cn/docs/model-service/API-Inference/intro)| |**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| |**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| diff --git a/README_ja-JP.md b/README_ja-JP.md index b232fae0..77cacaa6 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -202,6 +202,7 @@ pdf2zh example.pdf -li en -lo ja |**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| |**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| |**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| +| **ModelScope** | `ModelScope` |`MODELSCOPE_API_KEY`, `MODELSCOPE_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-Coder-32B-Instruct`| See [ModelScope](https://www.modelscope.cn/docs/model-service/API-Inference/intro)| |**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| |**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| |**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| diff --git a/README_zh-CN.md b/README_zh-CN.md index 89d56550..8467f1ed 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -192,22 +192,23 @@ pdf2zh example.pdf -li en -lo ja 下表列出了每个翻译服务所需的 [环境变量](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4)，在使用相应服务之前，请确保已设置这些变量 -|**Translator**|**Service**|**Environment Variables**|**Default Values**|**Notes**| -|-|-|-|-|-| -|**Google (Default)**|`google`|None|N/A|None| -|**Bing**|`bing`|None|N/A|None| -|**DeepL**|`deepl`|`DEEPL_AUTH_KEY`|`[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| -|**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| -|**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| -|**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| -|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| -|**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| -|**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| -|**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| -|**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| -|**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| -|**Dify**|`dify`|`DIFY_API_URL`, `DIFY_API_KEY`|`[Your DIFY URL]`, `[Your Key]`|See [Dify](https://github.com/langgenius/dify),Three variables, lang_out, lang_in, and text, need to be defined in Dify's workflow input.| -|**AnythingLLM**|`anythingllm`|`AnythingLLM_URL`, `AnythingLLM_APIKEY`|`[Your AnythingLLM URL]`, `[Your Key]`|See [anything-llm](https://github.com/Mintplex-Labs/anything-llm)| +| **Translator** | **Service** |**Environment Variables**|**Default Values**| **Notes** | +|----------------------|----------------|-|-|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Google (Default)** | `google` |None|N/A| None | +| **Bing** | `bing` |None|N/A| None | +| **DeepL** | `deepl` |`DEEPL_AUTH_KEY`|`[Your Key]`| See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API) | +| **DeepLX** | `deeplx` |`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`| See [DeepLX](https://github.com/OwO-Network/DeepLX) | +| **Ollama** | `ollama` |`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`| See [Ollama](https://github.com/ollama/ollama) | +| **OpenAI** | `openai` |`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`| See [OpenAI](https://platform.openai.com/docs/overview) | +| **AzureOpenAI** | `azure-openai` |`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`| See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python) | +| **Zhipu** | `zhipu` |`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`| See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk) | +| **ModelScope** | `ModelScope` |`MODELSCOPE_API_KEY`, `MODELSCOPE_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-Coder-32B-Instruct`| See [ModelScope](https://www.modelscope.cn/docs/model-service/API-Inference/intro) | +| **Silicon** | `silicon` |`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`| See [SiliconCloud](https://docs.siliconflow.cn/quickstart) | +| **Gemini** | `gemini` |`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`| See [Gemini](https://ai.google.dev/gemini-api/docs/openai) | +| **Azure** | `azure` |`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`| See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview) | +| **Tencent** | `tencent` |`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`| See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104) | +| **Dify** | `dify` |`DIFY_API_URL`, `DIFY_API_KEY`|`[Your DIFY URL]`, `[Your Key]`| See [Dify](https://github.com/langgenius/dify),Three variables, lang_out, lang_in, and text, need to be defined in Dify's workflow input. | +| **AnythingLLM** | `anythingllm` |`AnythingLLM_URL`, `AnythingLLM_APIKEY`|`[Your AnythingLLM URL]`, `[Your Key]`| See [anything-llm](https://github.com/Mintplex-Labs/anything-llm) | 使用 `-s service` 或 `-s service:model` 指定翻译服务: From 198e4fd1054385b012fed5ad7ed175311d2d9ad8 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 11:33:19 +0800 Subject: [PATCH 069/114] fix lint --- README_zh-CN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README_zh-CN.md b/README_zh-CN.md index 8467f1ed..2f3be2ff 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -193,7 +193,7 @@ pdf2zh example.pdf -li en -lo ja 下表列出了每个翻译服务所需的 [环境变量](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4)，在使用相应服务之前，请确保已设置这些变量 | **Translator** | **Service** |**Environment Variables**|**Default Values**| **Notes** | -|----------------------|----------------|-|-|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +|-|-|-|-|-| | **Google (Default)** | `google` |None|N/A| None | | **Bing** | `bing` |None|N/A| None | | **DeepL** | `deepl` |`DEEPL_AUTH_KEY`|`[Your Key]`| See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API) | @@ -202,7 +202,7 @@ pdf2zh example.pdf -li en -lo ja | **OpenAI** | `openai` |`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`| See [OpenAI](https://platform.openai.com/docs/overview) | | **AzureOpenAI** | `azure-openai` |`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`| See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python) | | **Zhipu** | `zhipu` |`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`| See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk) | -| **ModelScope** | `ModelScope` |`MODELSCOPE_API_KEY`, `MODELSCOPE_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-Coder-32B-Instruct`| See [ModelScope](https://www.modelscope.cn/docs/model-service/API-Inference/intro) | +| **ModelScope** | `ModelScope` |`MODELSCOPE_API_KEY`, `MODELSCOPE_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-Coder-32B-Instruct`| See [ModelScope](https://www.modelscope.cn/docs/model-service/API-Inference/intro)| | **Silicon** | `silicon` |`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`| See [SiliconCloud](https://docs.siliconflow.cn/quickstart) | | **Gemini** | `gemini` |`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`| See [Gemini](https://ai.google.dev/gemini-api/docs/openai) | | **Azure** | `azure` |`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`| See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview) | From d92aa77900305a23527232f3f432969c14de6205 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 11:35:02 +0800 Subject: [PATCH 070/114] fix lint --- README_zh-CN.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README_zh-CN.md b/README_zh-CN.md index 2f3be2ff..6d02df4c 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -192,23 +192,23 @@ pdf2zh example.pdf -li en -lo ja 下表列出了每个翻译服务所需的 [环境变量](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4)，在使用相应服务之前，请确保已设置这些变量 -| **Translator** | **Service** |**Environment Variables**|**Default Values**| **Notes** | +|**Translator**|**Service**|**Environment Variables**|**Default Values**|**Notes**| |-|-|-|-|-| -| **Google (Default)** | `google` |None|N/A| None | -| **Bing** | `bing` |None|N/A| None | -| **DeepL** | `deepl` |`DEEPL_AUTH_KEY`|`[Your Key]`| See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API) | -| **DeepLX** | `deeplx` |`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`| See [DeepLX](https://github.com/OwO-Network/DeepLX) | -| **Ollama** | `ollama` |`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`| See [Ollama](https://github.com/ollama/ollama) | -| **OpenAI** | `openai` |`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`| See [OpenAI](https://platform.openai.com/docs/overview) | -| **AzureOpenAI** | `azure-openai` |`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`| See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python) | -| **Zhipu** | `zhipu` |`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`| See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk) | +|**Google (Default)**|`google`|None|N/A|None| +|**Bing**|`bing`|None|N/A|None| +|**DeepL**|`deepl`|`DEEPL_AUTH_KEY`|`[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| +|**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| +|**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| +|**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| +|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| +|**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| | **ModelScope** | `ModelScope` |`MODELSCOPE_API_KEY`, `MODELSCOPE_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-Coder-32B-Instruct`| See [ModelScope](https://www.modelscope.cn/docs/model-service/API-Inference/intro)| -| **Silicon** | `silicon` |`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`| See [SiliconCloud](https://docs.siliconflow.cn/quickstart) | -| **Gemini** | `gemini` |`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`| See [Gemini](https://ai.google.dev/gemini-api/docs/openai) | -| **Azure** | `azure` |`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`| See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview) | -| **Tencent** | `tencent` |`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`| See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104) | -| **Dify** | `dify` |`DIFY_API_URL`, `DIFY_API_KEY`|`[Your DIFY URL]`, `[Your Key]`| See [Dify](https://github.com/langgenius/dify),Three variables, lang_out, lang_in, and text, need to be defined in Dify's workflow input. | -| **AnythingLLM** | `anythingllm` |`AnythingLLM_URL`, `AnythingLLM_APIKEY`|`[Your AnythingLLM URL]`, `[Your Key]`| See [anything-llm](https://github.com/Mintplex-Labs/anything-llm) | +|**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| +|**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| +|**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| +|**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| +|**Dify**|`dify`|`DIFY_API_URL`, `DIFY_API_KEY`|`[Your DIFY URL]`, `[Your Key]`|See [Dify](https://github.com/langgenius/dify),Three variables, lang_out, lang_in, and text, need to be defined in Dify's workflow input.| +|**AnythingLLM**|`anythingllm`|`AnythingLLM_URL`, `AnythingLLM_APIKEY`|`[Your AnythingLLM URL]`, `[Your Key]`|See [anything-llm](https://github.com/Mintplex-Labs/anything-llm)| 使用 `-s service` 或 `-s service:model` 指定翻译服务: From 872e2dafec69ecd24ebbe3ce9462482b1a555376 Mon Sep 17 00:00:00 2001 From: charles7668 Date: Tue, 17 Dec 2024 03:47:51 +0000 Subject: [PATCH 071/114] Fix error when using the `mat` method with NumPy version >= 2.0.0. --- pdf2zh/pdfinterp.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pdf2zh/pdfinterp.py b/pdf2zh/pdfinterp.py index fe4f7568..62ff4ed3 100644 --- a/pdf2zh/pdfinterp.py +++ b/pdf2zh/pdfinterp.py @@ -229,7 +229,11 @@ def do_Do(self, xobjid_arg: PDFStackT) -> None: self.device.fontmap = interpreter.fontmap ops_new = self.device.end_figure(xobjid) ctm_inv = np.linalg.inv(np.array(ctm[:4]).reshape(2, 2)) - pos_inv = -np.mat(ctm[4:]) * ctm_inv + np_version = np.__version__ + if np_version.split(".")[0] >= "2": + pos_inv = -np.asmatrix(ctm[4:]) * ctm_inv + else: + pos_inv = -np.mat(ctm[4:]) * ctm_inv a, b, c, d = ctm_inv.reshape(4).tolist() e, f = pos_inv.tolist()[0] self.obj_patch[self.xobjmap[xobjid].objid] = ( From ca39f01f03cbb31ac7ef5aac946ae5cfaa2d6872 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 14:05:32 +0800 Subject: [PATCH 072/114] change default value --- pdf2zh/gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index a901ecc5..50f4eff3 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -290,7 +290,7 @@ def progress_bar(t: tqdm.tqdm): service = gr.Dropdown( label="Service", choices=service_map.keys(), - value="Google", + value="ModelScope", ) envs = [] for i in range(3): From a3baa119465e36563c246697247d7b0ac953f075 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 14:09:31 +0800 Subject: [PATCH 073/114] default shows envs --- pdf2zh/gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 50f4eff3..82deb980 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -296,7 +296,7 @@ def progress_bar(t: tqdm.tqdm): for i in range(3): envs.append( gr.Textbox( - visible=False, + visible=True, interactive=True, ) ) From 0c69753898bf826907efa7baec640bbf5e724024 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 14:13:20 +0800 Subject: [PATCH 074/114] fix --- pdf2zh/gui.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 82deb980..f52bbaf1 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -296,7 +296,7 @@ def progress_bar(t: tqdm.tqdm): for i in range(3): envs.append( gr.Textbox( - visible=True, + visible=False, interactive=True, ) ) @@ -438,6 +438,7 @@ def on_select_filetype(file_type): stop_translate_file, inputs=[state], ) + demo.load(on_select_service, service, envs) def setup_gui(share=False): From 4559181f84fa537f499a7aa31896689ee738b0b3 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 15:02:41 +0800 Subject: [PATCH 075/114] fix --- pdf2zh/doclayout.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pdf2zh/doclayout.py b/pdf2zh/doclayout.py index c00f5e0e..eb6fff46 100644 --- a/pdf2zh/doclayout.py +++ b/pdf2zh/doclayout.py @@ -1,4 +1,6 @@ import abc +import os.path + import cv2 import numpy as np import ast @@ -11,7 +13,7 @@ class DocLayoutModel(abc.ABC): @staticmethod def load_onnx(): model = OnnxModel.from_pretrained( - repo_id="wybxc/DocLayout-YOLO-DocStructBench-onnx", + repo_id='AI-ModelScope/DocLayout-YOLO-DocStructBench-onnx', filename="doclayout_yolo_docstructbench_imgsz1024.onnx", ) return model @@ -70,7 +72,9 @@ def __init__(self, model_path: str): @staticmethod def from_pretrained(repo_id: str, filename: str): - pth = hf_hub_download(repo_id=repo_id, filename=filename, etag_timeout=1) + from modelscope import snapshot_download + model_dir = snapshot_download(repo_id) + pth = os.path.join(model_dir, filename) return OnnxModel(pth) @property From cae245c7cc685e300c26b0993bf41a45112611a6 Mon Sep 17 00:00:00 2001 From: chiu0602 Date: Tue, 17 Dec 2024 15:13:04 +0800 Subject: [PATCH 076/114] Support Traditional Chinese --- pdf2zh/gui.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index cff3d2dc..bf4c557c 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -49,7 +49,8 @@ "AnythingLLM": AnythingLLMTranslator, } lang_map = { - "Chinese": "zh", + "Simplified Chinese": "zh", + "Traditional Chinese": "zh-TW", "English": "en", "French": "fr", "German": "de", @@ -309,7 +310,7 @@ def progress_bar(t: tqdm.tqdm): lang_to = gr.Dropdown( label="Translate to", choices=lang_map.keys(), - value="Chinese", + value="Simplified Chinese", ) page_range = gr.Radio( choices=page_map.keys(), From fb3b70a15f411ede05ce3c4c9696f9d33b7a46ee Mon Sep 17 00:00:00 2001 From: charles7668 Date: Tue, 17 Dec 2024 07:27:31 +0000 Subject: [PATCH 077/114] Fix bug where the font size could become too large. --- pdf2zh/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index c466f97f..577bb1ca 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -276,7 +276,7 @@ def vflag(font: str, char: str): # 匹配公式（和角标）字体 if ( # 根据当前字符修正段落属性 child.size > pstk[-1].size / 0.79 # 1. 当前字符显著比段落字体大 or len(sstk[-1].strip()) == 1 # 2. 当前字符为段落第二个文字（考虑首字母放大的情况） - ): + ) and child.get_text() != " ": # 3. 当前字符不是空格 pstk[-1].y -= child.size - pstk[-1].size # 修正段落初始纵坐标，假设两个不同大小字符的上边界对齐 pstk[-1].size = child.size sstk[-1] += child.get_text() From cbccf1686c17c69016c0c6f22543077744ad979b Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 16:07:24 +0800 Subject: [PATCH 078/114] fix default model --- pdf2zh/gui.py | 3 --- pdf2zh/translator.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index f52bbaf1..962462c7 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -164,9 +164,6 @@ def translate_file( lang_from = lang_map[lang_from] lang_to = lang_map[lang_to] - for i, env in enumerate(translator.envs.items()): - os.environ[env[0]] = envs[i] - print(f"Files before translation: {os.listdir(output)}") def progress_bar(t: tqdm.tqdm): diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index c0d6249e..ab5edec4 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -257,7 +257,7 @@ class ModelScopeTranslator(OpenAITranslator): envs = { "MODELSCOPE_BASE_URL": "https://api-inference.modelscope.cn/v1", "MODELSCOPE_API_KEY": None, - "MODELSCOPE_MODEL": "Qwen/Qwen2.5-Coder-32B-Instruct", + "MODELSCOPE_MODEL": "Qwen/Qwen2.5-32B-Instruct", } def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): From 16c2426012a80c81d14705500f49612dc13444bf Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 17:10:31 +0800 Subject: [PATCH 079/114] refactor code --- pdf2zh/__init__.py | 3 - pdf2zh/converter.py | 5 +- pdf2zh/{pdf2zh.py => entrance.py} | 3 +- pdf2zh/gui.py | 6 +- pdf2zh/high_level.py | 2 +- pdf2zh/translator.py | 91 +++++++++++++++++++------------ pyproject.toml | 2 +- 7 files changed, 68 insertions(+), 44 deletions(-) rename pdf2zh/{pdf2zh.py => entrance.py} (98%) diff --git a/pdf2zh/__init__.py b/pdf2zh/__init__.py index 8483f481..0e1d9701 100644 --- a/pdf2zh/__init__.py +++ b/pdf2zh/__init__.py @@ -1,8 +1,5 @@ import logging -from pdf2zh.high_level import translate, translate_stream - log = logging.getLogger(__name__) __version__ = "1.8.8" __author__ = "Byaidu" -__all__ = ["translate", "translate_stream"] diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index c466f97f..106833c6 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -1,3 +1,5 @@ +from typing import List, Dict + from pdfminer.pdfinterp import PDFGraphicState, PDFResourceManager from pdfminer.pdffont import PDFCIDFont from pdfminer.converter import PDFConverter @@ -133,6 +135,7 @@ def __init__( service: str = "", resfont: str = "", noto: Font = None, + envs: Dict = None, ) -> None: super().__init__(rsrcmgr) self.vfont = vfont @@ -148,7 +151,7 @@ def __init__( for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator, OpenAITranslator, ZhipuTranslator, ModelScopeTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator, DifyTranslator, AnythingLLMTranslator]: if service_name == translator.name: - self.translator = translator(lang_in, lang_out, service_model) + self.translator = translator(lang_in, lang_out, service_model, envs=envs) if not self.translator: raise ValueError("Unsupported translation service") diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/entrance.py similarity index 98% rename from pdf2zh/pdf2zh.py rename to pdf2zh/entrance.py index 351cf0ba..a107c8a1 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/entrance.py @@ -9,11 +9,11 @@ import sys import logging from typing import List, Optional -from pdf2zh import __version__, log from pdf2zh.high_level import translate def create_parser() -> argparse.ArgumentParser: + from pdf2zh import __version__ parser = argparse.ArgumentParser(description=__doc__, add_help=True) parser.add_argument( "files", @@ -136,6 +136,7 @@ def parse_args(args: Optional[List[str]]) -> argparse.Namespace: def main(args: Optional[List[str]] = None) -> int: + from pdf2zh import log logging.basicConfig() parsed_args = parse_args(args) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 962462c7..3d3c454d 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -164,6 +164,10 @@ def translate_file( lang_from = lang_map[lang_from] lang_to = lang_map[lang_to] + _envs = {} + for i, env in enumerate(translator.envs.items()): + _envs[env[0]] = envs[i] + print(f"Files before translation: {os.listdir(output)}") def progress_bar(t: tqdm.tqdm): @@ -179,8 +183,8 @@ def progress_bar(t: tqdm.tqdm): "thread": 4, "callback": progress_bar, "cancellation_event": cancellation_event_map[session_id], + "envs": _envs, } - print(param) try: translate(**param) except CancelledError: diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index e536418c..5d77f950 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -92,7 +92,7 @@ def translate_patch( rsrcmgr = PDFResourceManager() layout = {} device = TranslateConverter( - rsrcmgr, vfont, vchar, thread, layout, lang_in, lang_out, service, resfont, noto + rsrcmgr, vfont, vchar, thread, layout, lang_in, lang_out, service, resfont, noto, kwarg.get('envs', {}) ) assert device is not None diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index ab5edec4..315644ee 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -34,6 +34,14 @@ def __init__(self, lang_in, lang_out, model): self.lang_out = lang_out self.model = model + def set_envs(self, envs): + for key in self.envs: + if key in os.environ: + self.envs[key] = os.environ[key] + if envs is not None: + for key in envs: + self.envs[key] = envs[key] + def translate(self, text): pass @@ -57,7 +65,7 @@ class GoogleTranslator(BaseTranslator): name = "google" lang_map = {"zh": "zh-CN"} - def __init__(self, lang_in, lang_out, model): + def __init__(self, lang_in, lang_out, model, **kwargs): super().__init__(lang_in, lang_out, model) self.session = requests.Session() self.endpoint = "http://translate.google.com/m" @@ -88,7 +96,7 @@ class BingTranslator(BaseTranslator): name = "bing" lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_in, lang_out, model): + def __init__(self, lang_in, lang_out, model, **kwargs): super().__init__(lang_in, lang_out, model) self.session = requests.Session() self.endpoint = "https://www.bing.com/translator" @@ -133,9 +141,10 @@ class DeepLTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_in, lang_out, model): + def __init__(self, lang_in, lang_out, model, envs=None): + self.set_envs(envs) super().__init__(lang_in, lang_out, model) - auth_key = os.getenv("DEEPL_AUTH_KEY") + auth_key = self.envs["DEEPL_AUTH_KEY"] self.client = deepl.Translator(auth_key) def translate(self, text): @@ -153,9 +162,10 @@ class DeepLXTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_in, lang_out, model): + def __init__(self, lang_in, lang_out, model, envs=None): + self.set_envs(envs) super().__init__(lang_in, lang_out, model) - self.endpoint = os.getenv("DEEPLX_ENDPOINT", self.envs["DEEPLX_ENDPOINT"]) + self.endpoint = self.envs["DEEPLX_ENDPOINT"] self.session = requests.Session() def translate(self, text): @@ -179,9 +189,10 @@ class OllamaTranslator(BaseTranslator): "OLLAMA_MODEL": "gemma2", } - def __init__(self, lang_in, lang_out, model): + def __init__(self, lang_in, lang_out, model, envs=None): + self.set_envs(envs) if not model: - model = os.getenv("OLLAMA_MODEL", self.envs["OLLAMA_MODEL"]) + model = self.envs["OLLAMA_MODEL"] super().__init__(lang_in, lang_out, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = ollama.Client() @@ -204,9 +215,10 @@ class OpenAITranslator(BaseTranslator): "OPENAI_MODEL": "gpt-4o-mini", } - def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): + def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None): + self.set_envs(envs) if not model: - model = os.getenv("OPENAI_MODEL", self.envs["OPENAI_MODEL"]) + model = self.envs["OPENAI_MODEL"] super().__init__(lang_in, lang_out, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = openai.OpenAI(base_url=base_url, api_key=api_key) @@ -228,12 +240,11 @@ class AzureOpenAITranslator(BaseTranslator): "AZURE_OPENAI_MODEL": "gpt-4o-mini", } - def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): - base_url = os.getenv( - "AZURE_OPENAI_BASE_URL", self.envs["AZURE_OPENAI_BASE_URL"] - ) + def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None): + self.set_envs(envs) + base_url = self.envs["AZURE_OPENAI_BASE_URL"] if not model: - model = os.getenv("AZURE_OPENAI_MODEL", self.envs["AZURE_OPENAI_MODEL"]) + model = self.envs["AZURE_OPENAI_MODEL"] super().__init__(lang_in, lang_out, model) self.options = {"temperature": 0} self.client = openai.AzureOpenAI( @@ -260,11 +271,12 @@ class ModelScopeTranslator(OpenAITranslator): "MODELSCOPE_MODEL": "Qwen/Qwen2.5-32B-Instruct", } - def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): + def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None): + self.set_envs(envs) base_url = "https://api-inference.modelscope.cn/v1" - api_key = os.getenv("MODELSCOPE_API_KEY") + api_key = self.envs["MODELSCOPE_API_KEY"] if not model: - model = os.getenv("MODELSCOPE_MODEL", self.envs["MODELSCOPE_MODEL"]) + model = self.envs["MODELSCOPE_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) @@ -276,11 +288,12 @@ class ZhipuTranslator(OpenAITranslator): "ZHIPU_MODEL": "glm-4-flash", } - def __init__(self, lang_in, lang_out, model): + def __init__(self, lang_in, lang_out, model, envs=None): + self.set_envs(envs) base_url = "https://open.bigmodel.cn/api/paas/v4" - api_key = os.getenv("ZHIPU_API_KEY") + api_key = self.envs["ZHIPU_API_KEY"] if not model: - model = os.getenv("ZHIPU_MODEL", self.envs["ZHIPU_MODEL"]) + model = self.envs["ZHIPU_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) def translate(self, text) -> str: @@ -308,11 +321,12 @@ class SiliconTranslator(OpenAITranslator): "SILICON_MODEL": "Qwen/Qwen2.5-7B-Instruct", } - def __init__(self, lang_in, lang_out, model): + def __init__(self, lang_in, lang_out, model, envs=None): + self.set_envs(envs) base_url = "https://api.siliconflow.cn/v1" - api_key = os.getenv("SILICON_API_KEY") + api_key = self.envs["SILICON_API_KEY"] if not model: - model = os.getenv("SILICON_MODEL", self.envs["SILICON_MODEL"]) + model = self.envs["SILICON_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) @@ -324,11 +338,12 @@ class GeminiTranslator(OpenAITranslator): "GEMINI_MODEL": "gemini-1.5-flash", } - def __init__(self, lang_in, lang_out, model): + def __init__(self, lang_in, lang_out, model, envs=None): + self.set_envs(envs) base_url = "https://generativelanguage.googleapis.com/v1beta/openai/" - api_key = os.getenv("GEMINI_API_KEY") + api_key = self.envs["GEMINI_API_KEY"] if not model: - model = os.getenv("GEMINI_MODEL", self.envs["GEMINI_MODEL"]) + model = self.envs["GEMINI_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) @@ -341,9 +356,10 @@ class AzureTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_in, lang_out, model): + def __init__(self, lang_in, lang_out, model, envs=None): + self.set_envs(envs) super().__init__(lang_in, lang_out, model) - endpoint = os.getenv("AZURE_ENDPOINT", self.envs["AZURE_ENDPOINT"]) + endpoint = self.envs["AZURE_ENDPOINT"] api_key = os.getenv("AZURE_API_KEY") credential = AzureKeyCredential(api_key) self.client = TextTranslationClient( @@ -371,7 +387,8 @@ class TencentTranslator(BaseTranslator): "TENCENTCLOUD_SECRET_KEY": None, } - def __init__(self, lang_in, lang_out, model): + def __init__(self, lang_in, lang_out, model, envs=None): + self.set_envs(envs) super().__init__(lang_in, lang_out, model) cred = credential.DefaultCredentialProvider().get_credential() self.client = TmtClient(cred, "ap-beijing") @@ -393,10 +410,11 @@ class AnythingLLMTranslator(BaseTranslator): "AnythingLLM_APIKEY": "api_key", } - def __init__(self, lang_out, lang_in, model): + def __init__(self, lang_out, lang_in, model, envs=None): + self.set_envs(envs) super().__init__(lang_out, lang_in, model) - self.api_url = os.getenv("AnythingLLM_URL", self.envs["AnythingLLM_URL"]) - self.api_key = os.getenv("AnythingLLM_APIKEY", self.envs["AnythingLLM_APIKEY"]) + self.api_url = self.envs["AnythingLLM_URL"] + self.api_key = self.envs["AnythingLLM_APIKEY"] self.headers = { "accept": "application/json", "Authorization": f"Bearer {self.api_key}", @@ -428,10 +446,11 @@ class DifyTranslator(BaseTranslator): "DIFY_API_KEY": "api_key", # 替换为实际 API 密钥 } - def __init__(self, lang_out, lang_in, model): + def __init__(self, lang_out, lang_in, model, envs=None): + self.set_envs(envs) super().__init__(lang_out, lang_in, model) - self.api_url = os.getenv("DIFY_API_URL", self.envs["DIFY_API_URL"]) - self.api_key = os.getenv("DIFY_API_KEY", self.envs["DIFY_API_KEY"]) + self.api_url = self.envs["DIFY_API_URL"] + self.api_key = self.envs["DIFY_API_KEY"] def translate(self, text): headers = { diff --git a/pyproject.toml b/pyproject.toml index c8bdefa5..2eac73a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,4 +50,4 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project.scripts] -pdf2zh = "pdf2zh.pdf2zh:main" +pdf2zh = "pdf2zh.entrance:main" From 7c7e0d17196a25e5960227f5a59067d6cd0df54e Mon Sep 17 00:00:00 2001 From: tastelikefeet Date: Tue, 17 Dec 2024 17:52:01 +0800 Subject: [PATCH 080/114] fix --- pdf2zh/gui.py | 2 +- pdf2zh/high_level.py | 4 ++-- pdf2zh/translator.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 3d3c454d..6299fd64 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -167,7 +167,7 @@ def translate_file( _envs = {} for i, env in enumerate(translator.envs.items()): _envs[env[0]] = envs[i] - + print(f"Files before translation: {os.listdir(output)}") def progress_bar(t: tqdm.tqdm): diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 5d77f950..1e3be409 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -216,7 +216,7 @@ def translate_stream( fp = io.BytesIO() doc_zh.save(fp) - obj_patch: dict = translate_patch(fp, **locals()) + obj_patch: dict = translate_patch(fp, envs=kwarg['envs'], **locals()) for obj_id, ops_new in obj_patch.items(): # ops_old=doc_en.xref_stream(obj_id) @@ -282,7 +282,7 @@ def translate( doc_raw = open(file, "rb") s_raw = doc_raw.read() - s_mono, s_dual = translate_stream(s_raw, **locals()) + s_mono, s_dual = translate_stream(s_raw, envs=kwarg['envs'], **locals()) file_mono = Path(output) / f"{filename}-mono.pdf" file_dual = Path(output) / f"{filename}-dual.pdf" doc_mono = open(file_mono, "wb") diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 315644ee..5f1450eb 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -3,7 +3,7 @@ import os import re import unicodedata - +from copy import copy import deepl import ollama import openai @@ -35,6 +35,7 @@ def __init__(self, lang_in, lang_out, model): self.model = model def set_envs(self, envs): + self.envs = copy(self.__class__.envs) for key in self.envs: if key in os.environ: self.envs[key] = os.environ[key] @@ -215,8 +216,7 @@ class OpenAITranslator(BaseTranslator): "OPENAI_MODEL": "gpt-4o-mini", } - def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None): - self.set_envs(envs) + def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): if not model: model = self.envs["OPENAI_MODEL"] super().__init__(lang_in, lang_out, model) From cc27355ad7538e0b1e6322389d47e04775fa031b Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 19:00:02 +0800 Subject: [PATCH 081/114] fix --- pdf2zh/__init__.py | 3 +++ pdf2zh/converter.py | 2 +- pdf2zh/doclayout.py | 15 +++++++++++---- pdf2zh/gui.py | 3 +-- pdf2zh/{entrance.py => pdf2zh.py} | 3 +-- pyproject.toml | 2 +- 6 files changed, 18 insertions(+), 10 deletions(-) rename pdf2zh/{entrance.py => pdf2zh.py} (98%) diff --git a/pdf2zh/__init__.py b/pdf2zh/__init__.py index 0e1d9701..8483f481 100644 --- a/pdf2zh/__init__.py +++ b/pdf2zh/__init__.py @@ -1,5 +1,8 @@ import logging +from pdf2zh.high_level import translate, translate_stream + log = logging.getLogger(__name__) __version__ = "1.8.8" __author__ = "Byaidu" +__all__ = ["translate", "translate_stream"] diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 733d7827..3fdef500 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -1,4 +1,4 @@ -from typing import List, Dict +from typing import Dict from pdfminer.pdfinterp import PDFGraphicState, PDFResourceManager from pdfminer.pdffont import PDFCIDFont diff --git a/pdf2zh/doclayout.py b/pdf2zh/doclayout.py index eb6fff46..ee050e81 100644 --- a/pdf2zh/doclayout.py +++ b/pdf2zh/doclayout.py @@ -13,7 +13,7 @@ class DocLayoutModel(abc.ABC): @staticmethod def load_onnx(): model = OnnxModel.from_pretrained( - repo_id='AI-ModelScope/DocLayout-YOLO-DocStructBench-onnx', + repo_id="wybxc/DocLayout-YOLO-DocStructBench-onnx", filename="doclayout_yolo_docstructbench_imgsz1024.onnx", ) return model @@ -72,9 +72,16 @@ def __init__(self, model_path: str): @staticmethod def from_pretrained(repo_id: str, filename: str): - from modelscope import snapshot_download - model_dir = snapshot_download(repo_id) - pth = os.path.join(model_dir, filename) + if os.environ.get("USE_MODELSCOPE", "0") == "1": + repo_mapping = { + # Edit here to add more models + "wybxc/DocLayout-YOLO-DocStructBench-onnx": "AI-ModelScope/DocLayout-YOLO-DocStructBench-onnx" + } + from modelscope import snapshot_download + model_dir = snapshot_download(repo_mapping[repo_id]) + pth = os.path.join(model_dir, filename) + else: + pth = hf_hub_download(repo_id=repo_id, filename=filename, etag_timeout=1) return OnnxModel(pth) @property diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index c51f4856..cc582148 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -292,7 +292,7 @@ def progress_bar(t: tqdm.tqdm): service = gr.Dropdown( label="Service", choices=service_map.keys(), - value="ModelScope", + value="Google", ) envs = [] for i in range(3): @@ -440,7 +440,6 @@ def on_select_filetype(file_type): stop_translate_file, inputs=[state], ) - demo.load(on_select_service, service, envs) def readuserandpasswd(file_path): diff --git a/pdf2zh/entrance.py b/pdf2zh/pdf2zh.py similarity index 98% rename from pdf2zh/entrance.py rename to pdf2zh/pdf2zh.py index 81166e5b..40f810a9 100644 --- a/pdf2zh/entrance.py +++ b/pdf2zh/pdf2zh.py @@ -9,11 +9,11 @@ import sys import logging from typing import List, Optional +from pdf2zh import __version__, log from pdf2zh.high_level import translate def create_parser() -> argparse.ArgumentParser: - from pdf2zh import __version__ parser = argparse.ArgumentParser(description=__doc__, add_help=True) parser.add_argument( "files", @@ -144,7 +144,6 @@ def parse_args(args: Optional[List[str]]) -> argparse.Namespace: def main(args: Optional[List[str]] = None) -> int: - from pdf2zh import log logging.basicConfig() parsed_args = parse_args(args) diff --git a/pyproject.toml b/pyproject.toml index 2eac73a9..c8bdefa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,4 +50,4 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project.scripts] -pdf2zh = "pdf2zh.entrance:main" +pdf2zh = "pdf2zh.pdf2zh:main" From b86c9a8b62e0649d6b2324a9ae0fdc2fba059bdf Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 19:13:12 +0800 Subject: [PATCH 082/114] update readme --- README.md | 6 ++++-- README_ja-JP.md | 4 +++- README_zh-CN.md | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7e4490c0..b61f8931 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ English | [简体中文](README_zh-CN.md) | [日本語](README_ja-JP.md) + + @@ -61,9 +63,9 @@ Feel free to provide feedback in [GitHub Issues](https://github.com/Byaidu/PDFMa You can try our [public service](https://pdf2zh.com/) online without installation. -### Hugging Face Demo +### Demos -You can try [our demo on HuggingFace](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker) without installation. +You can try [our demo on HuggingFace](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker) or [our demo on ModelScope](https://www.modelscope.cn/studios/AI-ModelScope/PDFMathTranslate) without installation. Note that the computing resources of the demo are limited, so please avoid abusing them.

Installation and Usage

diff --git a/README_ja-JP.md b/README_ja-JP.md index d0124ebb..2f2d6f5a 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -19,6 +19,8 @@ + + @@ -63,7 +65,7 @@ ### Hugging Face デモ -インストールなしで [HuggingFace上のデモ](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker) を試すことができます。 +インストールなしで [HuggingFace上のデモ](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker), [ModelScope上のデモ](https://www.modelscope.cn/studios/AI-ModelScope/PDFMathTranslate) を試すことができます。 デモの計算リソースは限られているため、乱用しないようにしてください。

インストールと使用方法

diff --git a/README_zh-CN.md b/README_zh-CN.md index 6db25b38..6e25bc18 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -19,6 +19,8 @@ + + @@ -63,7 +65,7 @@ ### Hugging Face 在线演示 -你可以立即尝试 [在 HuggingFace 上的在线演示](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker) 而无需安装 +你可以立即尝试 [在 HuggingFace 上的在线演示](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker)和[魔搭的在线演示](https://www.modelscope.cn/studios/AI-ModelScope/PDFMathTranslate)而无需安装 请注意，演示的计算资源有限，因此请避免滥用

安装和使用

From a89846563ec4383183ed534515c200149ab2a3d8 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 19:24:44 +0800 Subject: [PATCH 083/114] update readme --- README.md | 5 +++++ README_ja-JP.md | 6 ++++++ README_zh-CN.md | 5 +++++ 3 files changed, 16 insertions(+) diff --git a/README.md b/README.md index b61f8931..7f7cb3eb 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,11 @@ Note that the computing resources of the demo are limited, so please avoid abusi We provide four methods for using this project: [Commandline](#cmd), [Portable](#portable), [GUI](#gui), and [Docker](#docker). +pdf2zh needs an extra model(`wybxc/DocLayout-YOLO-DocStructBench-onnx`), which can be found in modelscope. if you have a problem with downloading this model, try this environment variable: +```shell +USE_MODELSCOPE=1 pdf2zh +``` +

Method I. Commandline

1. Python installed (3.8 <= version <= 3.12) diff --git a/README_ja-JP.md b/README_ja-JP.md index 2f2d6f5a..1fc6c428 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -72,6 +72,12 @@ このプロジェクトを使用するための4つの方法を提供しています：[コマンドライン](#cmd)、[ポータブル](#portable)、[GUI](#gui)、および [Docker](#docker)。 +pdf2zhの実行には追加モデル（`wybxc/DocLayout-YOLO-DocStructBench-onnx`）が必要です。このモデルはModelScopeでも見つけることができます。起動時にこのモデルのダウンロードに問題がある場合は、以下の環境変数を使用してください： + +```shell +USE_MODELSCOPE=1 pdf2zh +``` +

方法1. コマンドライン

1. Pythonがインストールされていること (バージョン3.8 <= バージョン <= 3.12) diff --git a/README_zh-CN.md b/README_zh-CN.md index 6e25bc18..78ba39a2 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -72,6 +72,11 @@ 我们提供了四种使用该项目的方法：[命令行工具](#cmd)、[便携式安装](#portable)、[图形交互界面](#gui) 和 [容器化部署](#docker). +pdf2zh的运行依赖于额外模型(`wybxc/DocLayout-YOLO-DocStructBench-onnx`)，该模型在魔搭上也可以找到。如果你在启动时下载该模型遇到问题，请使用如下环境变量： +```shell +USE_MODELSCOPE=1 pdf2zh +``` +

方法一、命令行工具

1. 确保安装了版本大于 3.8 且小于 3.12 的 Python From da93cfc1d0a647b5a20860e9ef44fba18e0193cb Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 19:40:33 +0800 Subject: [PATCH 084/114] lint --- pdf2zh/doclayout.py | 1 + pdf2zh/gui.py | 2 +- pdf2zh/high_level.py | 12 +++++++++++- pdf2zh/translator.py | 8 ++++++-- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/pdf2zh/doclayout.py b/pdf2zh/doclayout.py index ee050e81..55625615 100644 --- a/pdf2zh/doclayout.py +++ b/pdf2zh/doclayout.py @@ -78,6 +78,7 @@ def from_pretrained(repo_id: str, filename: str): "wybxc/DocLayout-YOLO-DocStructBench-onnx": "AI-ModelScope/DocLayout-YOLO-DocStructBench-onnx" } from modelscope import snapshot_download + model_dir = snapshot_download(repo_mapping[repo_id]) pth = os.path.join(model_dir, filename) else: diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 8330d352..cc582148 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -168,7 +168,7 @@ def translate_file( _envs = {} for i, env in enumerate(translator.envs.items()): _envs[env[0]] = envs[i] - + print(f"Files before translation: {os.listdir(output)}") def progress_bar(t: tqdm.tqdm): diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 1e3be409..380fdb77 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -92,7 +92,17 @@ def translate_patch( rsrcmgr = PDFResourceManager() layout = {} device = TranslateConverter( - rsrcmgr, vfont, vchar, thread, layout, lang_in, lang_out, service, resfont, noto, kwarg.get('envs', {}) + rsrcmgr, + vfont, + vchar, + thread, + layout, + lang_in, + lang_out, + service, + resfont, + noto, + kwarg.get('envs', {}), ) assert device is not None diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 5f1450eb..4233065d 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -240,7 +240,9 @@ class AzureOpenAITranslator(BaseTranslator): "AZURE_OPENAI_MODEL": "gpt-4o-mini", } - def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None): + def __init__( + self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None + ): self.set_envs(envs) base_url = self.envs["AZURE_OPENAI_BASE_URL"] if not model: @@ -271,7 +273,9 @@ class ModelScopeTranslator(OpenAITranslator): "MODELSCOPE_MODEL": "Qwen/Qwen2.5-32B-Instruct", } - def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None): + def __init__( + self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None + ): self.set_envs(envs) base_url = "https://api-inference.modelscope.cn/v1" api_key = self.envs["MODELSCOPE_API_KEY"] From 0bfd0fe7afc79dad4f3c1b8a1f580ce4755dbb5d Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Tue, 17 Dec 2024 19:41:32 +0800 Subject: [PATCH 085/114] lint code --- pdf2zh/high_level.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 380fdb77..fa6611f4 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -102,7 +102,7 @@ def translate_patch( service, resfont, noto, - kwarg.get('envs', {}), + kwarg.get("envs", {}), ) assert device is not None @@ -226,7 +226,7 @@ def translate_stream( fp = io.BytesIO() doc_zh.save(fp) - obj_patch: dict = translate_patch(fp, envs=kwarg['envs'], **locals()) + obj_patch: dict = translate_patch(fp, envs=kwarg["envs"], **locals()) for obj_id, ops_new in obj_patch.items(): # ops_old=doc_en.xref_stream(obj_id) @@ -292,7 +292,7 @@ def translate( doc_raw = open(file, "rb") s_raw = doc_raw.read() - s_mono, s_dual = translate_stream(s_raw, envs=kwarg['envs'], **locals()) + s_mono, s_dual = translate_stream(s_raw, envs=kwarg["envs"], **locals()) file_mono = Path(output) / f"{filename}-mono.pdf" file_dual = Path(output) / f"{filename}-dual.pdf" doc_mono = open(file_mono, "wb") From 39fe9402f44abfae0f891abf007c58a572c42a34 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Wed, 18 Dec 2024 01:02:31 +0800 Subject: [PATCH 086/114] fix comment --- pdf2zh/translator.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 4233065d..5741c880 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -35,7 +35,10 @@ def __init__(self, lang_in, lang_out, model): self.model = model def set_envs(self, envs): - self.envs = copy(self.__class__.envs) + # Detach from self.__class__.envs + # Cannot use self.envs = copy(self.__class__.envs) + # because if set_envs called twice, the second call will override the first call + self.envs = copy(self.envs) for key in self.envs: if key in os.environ: self.envs[key] = os.environ[key] @@ -216,7 +219,10 @@ class OpenAITranslator(BaseTranslator): "OPENAI_MODEL": "gpt-4o-mini", } - def __init__(self, lang_in, lang_out, model, base_url=None, api_key=None): + def __init__( + self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None + ): + self.set_envs(envs) if not model: model = self.envs["OPENAI_MODEL"] super().__init__(lang_in, lang_out, model) From e1f1c7167e725307fdc0fec1db53ee5887a56cdc Mon Sep 17 00:00:00 2001 From: hellofinch Date: Wed, 18 Dec 2024 09:51:18 +0800 Subject: [PATCH 087/114] =?UTF-8?q?=E6=9B=B4=E6=96=B0service=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=E9=BB=98=E8=AE=A4=E5=80=BC=EF=BC=8C=E6=9B=B4=E6=96=B0?= =?UTF-8?q?readme=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README_ja-JP.md | 2 +- README_zh-CN.md | 2 +- pdf2zh/translator.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README_ja-JP.md b/README_ja-JP.md index 1fc6c428..101a02c2 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -63,7 +63,7 @@ インストールなしで [公共サービス](https://pdf2zh.com/) をオンラインで試すことができます。 -### Hugging Face デモ +### デモ インストールなしで [HuggingFace上のデモ](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker), [ModelScope上のデモ](https://www.modelscope.cn/studios/AI-ModelScope/PDFMathTranslate) を試すことができます。 デモの計算リソースは限られているため、乱用しないようにしてください。 diff --git a/README_zh-CN.md b/README_zh-CN.md index 78ba39a2..838f1718 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -63,7 +63,7 @@ 你可以立即尝试 [免费公共服务](https://pdf2zh.com/) 而无需安装 -### Hugging Face 在线演示 +### 在线演示 你可以立即尝试 [在 HuggingFace 上的在线演示](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker)和[魔搭的在线演示](https://www.modelscope.cn/studios/AI-ModelScope/PDFMathTranslate)而无需安装 请注意，演示的计算资源有限，因此请避免滥用 diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 5741c880..8aaa0090 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -417,7 +417,7 @@ class AnythingLLMTranslator(BaseTranslator): name = "anythingllm" envs = { "AnythingLLM_URL": None, - "AnythingLLM_APIKEY": "api_key", + "AnythingLLM_APIKEY": None, } def __init__(self, lang_out, lang_in, model, envs=None): @@ -453,7 +453,7 @@ class DifyTranslator(BaseTranslator): name = "dify" envs = { "DIFY_API_URL": None, # 填写实际 Dify API 地址 - "DIFY_API_KEY": "api_key", # 替换为实际 API 密钥 + "DIFY_API_KEY": None, # 替换为实际 API 密钥 } def __init__(self, lang_out, lang_in, model, envs=None): From 752e9b37aed8a2fcb56350adec7acc2b606a5bff Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Wed, 18 Dec 2024 11:47:42 +0800 Subject: [PATCH 088/114] fix cli --- pdf2zh/high_level.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index fa6611f4..06084161 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -292,7 +292,7 @@ def translate( doc_raw = open(file, "rb") s_raw = doc_raw.read() - s_mono, s_dual = translate_stream(s_raw, envs=kwarg["envs"], **locals()) + s_mono, s_dual = translate_stream(s_raw, envs=kwarg.get('envs'), **locals()) file_mono = Path(output) / f"{filename}-mono.pdf" file_dual = Path(output) / f"{filename}-dual.pdf" doc_mono = open(file_mono, "wb") From bc8dd2455046bacf3d51eba615c2e82b9504ee54 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Wed, 18 Dec 2024 14:36:55 +0800 Subject: [PATCH 089/114] =?UTF-8?q?=E6=B7=BB=E5=8A=A0cli=E7=9A=84=E5=AD=97?= =?UTF-8?q?=E5=AF=B9=E5=BA=94prompt=E6=94=AF=E6=8C=81=E3=80=82=E6=9B=B4?= =?UTF-8?q?=E6=96=B0readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 28 ++++++++++++++++ README_ja-JP.md | 30 +++++++++++++++++ README_zh-CN.md | 29 ++++++++++++++++ pdf2zh/converter.py | 5 +-- pdf2zh/high_level.py | 5 +-- pdf2zh/pdf2zh.py | 16 ++++++++- pdf2zh/translator.py | 79 +++++++++++++++++++++++++++----------------- 7 files changed, 156 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 7f7cb3eb..d29d62b2 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,7 @@ In the following table, we list all advanced options for reference: | `-f`, `-c` | [Exceptions](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | | `--share` | [Get gradio public link] | `pdf2zh -i --share` | | `-a` | [add authorization and custom login page] | `pdf2zh -i -a users.txt [auth.html]` | +| `-pr` | [custom llm prompt] | `pdf2zh -pr [prompt.txt]` |

Full / partial document translation

@@ -252,7 +253,34 @@ Use `-t` to specify how many threads to use in translation: ```bash pdf2zh example.pdf -t 1 ``` +

custom prompt

+Use `-pr` or `--prompt` to specify which prompt to use in llm: +```bash +pdf2zh example.pdf -pr prompt.txt +``` + + +example prompt.txt +``` +[ + { + "role": "system", + "content": "You are a professional,authentic machine translation engine.", + }, + { + "role": "user", + "content": "Translate the following markdown source text to ${lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: ${text}\nTranslated Text:", + }, +] +``` + +In custom prompt file, there are three variables can be used. +|**variables**|**comment**| +|-|-| +|`lang_in`|input language| +|`lang_out`|output language| +|`text`|text need to be translated|

API

### Python diff --git a/README_ja-JP.md b/README_ja-JP.md index 101a02c2..fd3efd07 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -174,6 +174,7 @@ Python環境を事前にインストールする必要はありません | `-f`, `-c` | [例外](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | | `--share` | [gradio公開リンクを取得] | `pdf2zh -i --share` | | `-a` | [ウェブ認証とカスタム認証ページの追加] | `pdf2zh -i -a users.txt [auth.html]` | +| `-pr` | [カスタムビッグモデルのプロンプトを使用する] | `pdf2zh -pr [prompt.txt]` |

全文または部分的なドキュメント翻訳

@@ -254,6 +255,35 @@ pdf2zh example.pdf -f "(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-| pdf2zh example.pdf -t 1 ``` +

custom prompt

+(need Japenese translation) +Use `-pr` or `--prompt` to specify which prompt to use in llm: +```bash +pdf2zh example.pdf -pr prompt.txt +``` + + +example prompt.txt +``` +[ + { + "role": "system", + "content": "You are a professional,authentic machine translation engine.", + }, + { + "role": "user", + "content": "Translate the following markdown source text to ${lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: ${text}\nTranslated Text:", + }, +] +``` + + +In custom prompt file, there are three variables can be used. +|**variables**|**comment**| +|-|-| +|`lang_in`|input language| +|`lang_out`|output language| +|`text`|text need to be translated|

API

### Python diff --git a/README_zh-CN.md b/README_zh-CN.md index 838f1718..2a9db04e 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -173,6 +173,7 @@ USE_MODELSCOPE=1 pdf2zh | `-f`, `-c` | [例外规则](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | | `--share` | [获取 gradio 公开链接] | `pdf2zh -i --share` | | `-a` | [添加网页认证和自定义认证页] | `pdf2zh -i -a users.txt [auth.html]` | +| `-pr` | [使用自定义的大模型prompt] | `pdf2zh -pr [prompt.txt]` |

全文或部分文档翻译

@@ -252,6 +253,34 @@ pdf2zh example.pdf -f "(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-| ```bash pdf2zh example.pdf -t 1 ``` +

自定义大模型prompt

+使用 `-pr` 或 `--prompt` 指定使用大模型翻译时使用的prompt文件。 +```bash +pdf2zh example.pdf -pr prompt.txt +``` + + +示例prompt.txt文件 +``` +[ + { + "role": "system", + "content": "You are a professional,authentic machine translation engine.", + }, + { + "role": "user", + "content": "Translate the following markdown source text to ${lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: ${text}\nTranslated Text:", + }, +] +``` + + +自定义prompt文件中，可以使用三个内置变量用来传递参数。 +|**变量名**|**说明**| +|-|-| +|`lang_in`|输入的语言| +|`lang_out`|输出的语言| +|`text`|需要翻译的文本|

API

diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 3fdef500..ab5789ea 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict,List from pdfminer.pdfinterp import PDFGraphicState, PDFResourceManager from pdfminer.pdffont import PDFCIDFont @@ -136,6 +136,7 @@ def __init__( resfont: str = "", noto: Font = None, envs: Dict = None, + prompt: List = None, ) -> None: super().__init__(rsrcmgr) self.vfont = vfont @@ -151,7 +152,7 @@ def __init__( for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator, OpenAITranslator, ZhipuTranslator, ModelScopeTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator, DifyTranslator, AnythingLLMTranslator]: if service_name == translator.name: - self.translator = translator(lang_in, lang_out, service_model, envs=envs) + self.translator = translator(lang_in, lang_out, service_model, envs=envs,prompt=prompt) if not self.translator: raise ValueError("Unsupported translation service") diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 06084161..d10aa04f 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -103,6 +103,7 @@ def translate_patch( resfont, noto, kwarg.get("envs", {}), + kwarg.get("prompt", []), ) assert device is not None @@ -226,7 +227,7 @@ def translate_stream( fp = io.BytesIO() doc_zh.save(fp) - obj_patch: dict = translate_patch(fp, envs=kwarg["envs"], **locals()) + obj_patch: dict = translate_patch(fp, prompt=kwarg["prompt"], **locals()) for obj_id, ops_new in obj_patch.items(): # ops_old=doc_en.xref_stream(obj_id) @@ -292,7 +293,7 @@ def translate( doc_raw = open(file, "rb") s_raw = doc_raw.read() - s_mono, s_dual = translate_stream(s_raw, envs=kwarg.get('envs'), **locals()) + s_mono, s_dual = translate_stream(s_raw, envs=kwarg.get('envs'), prompt=kwarg["prompt"], **locals()) file_mono = Path(output) / f"{filename}-mono.pdf" file_dual = Path(output) / f"{filename}-dual.pdf" doc_mono = open(file_mono, "wb") diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index 40f810a9..fd1a185a 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -11,6 +11,7 @@ from typing import List, Optional from pdf2zh import __version__, log from pdf2zh.high_level import translate +from string import Template def create_parser() -> argparse.ArgumentParser: @@ -120,9 +121,14 @@ def create_parser() -> argparse.ArgumentParser: "-a", type=str, nargs="+", - default=["./users.txt", "./auth.html"], help="user name and password.", ) + parse_params.add_argument( + "--prompt", + "-pr", + type=str, + help="user custom prompt.", + ) return parser @@ -169,6 +175,14 @@ def main(args: Optional[List[str]] = None) -> int: celery_app.start(argv=sys.argv[2:]) return 0 + if parsed_args.prompt: + try: + with open(parsed_args.prompt,'r',encoding='utf-8') as file: + content=file.read() + parsed_args.prompt=Template(content) + except Exception as e: + raise ValueError("prompt error.") + translate(**vars(parsed_args)) return 0 diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 8aaa0090..75cec757 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -17,7 +17,6 @@ import json - def remove_control_characters(s): return "".join(ch for ch in s if unicodedata.category(ch)[0] != "C") @@ -49,17 +48,25 @@ def set_envs(self, envs): def translate(self, text): pass - def prompt(self, text): - return [ - { - "role": "system", - "content": "You are a professional,authentic machine translation engine.", - }, - { - "role": "user", - "content": f"Translate the following markdown source text to {self.lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: {text}\nTranslated Text:", # noqa: E501 - }, - ] + def prompt(self, text, prompt): + if prompt: + context={ + "lang_in":self.lang_in, + "lang_out":self.lang_out, + "text":text, + } + return eval(prompt.safe_substitute(context)) + else: + return [ + { + "role": "system", + "content": "You are a professional,authentic machine translation engine.", + }, + { + "role": "user", + "content": f"Translate the following markdown source text to {self.lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: {text}\nTranslated Text:", # noqa: E501 + }, + ] def __str__(self): return f"{self.name} {self.lang_in} {self.lang_out} {self.model}" @@ -145,7 +152,7 @@ class DeepLTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_in, lang_out, model, envs=None): + def __init__(self, lang_in, lang_out, model, envs=None, **kwargs): self.set_envs(envs) super().__init__(lang_in, lang_out, model) auth_key = self.envs["DEEPL_AUTH_KEY"] @@ -166,7 +173,7 @@ class DeepLXTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_in, lang_out, model, envs=None): + def __init__(self, lang_in, lang_out, model, envs=None, **kwargs): self.set_envs(envs) super().__init__(lang_in, lang_out, model) self.endpoint = self.envs["DEEPLX_ENDPOINT"] @@ -193,19 +200,23 @@ class OllamaTranslator(BaseTranslator): "OLLAMA_MODEL": "gemma2", } - def __init__(self, lang_in, lang_out, model, envs=None): + def __init__(self, lang_in, lang_out, model, envs=None,prompt=None): self.set_envs(envs) if not model: model = self.envs["OLLAMA_MODEL"] super().__init__(lang_in, lang_out, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = ollama.Client() + self.prompttext=prompt def translate(self, text): + print(len(self.prompt(text,self.prompttext))) + print(self.prompt(text,self.prompttext)[0]) + print(self.prompt(text,self.prompttext)[1]) response = self.client.chat( model=self.model, options=self.options, - messages=self.prompt(text), + messages=self.prompt(text,self.prompttext), ) return response["message"]["content"].strip() @@ -220,7 +231,7 @@ class OpenAITranslator(BaseTranslator): } def __init__( - self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None + self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None,prompt=None ): self.set_envs(envs) if not model: @@ -228,12 +239,13 @@ def __init__( super().__init__(lang_in, lang_out, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = openai.OpenAI(base_url=base_url, api_key=api_key) + self.prompttext=prompt def translate(self, text) -> str: response = self.client.chat.completions.create( model=self.model, **self.options, - messages=self.prompt(text), + messages=self.prompt(text,self.prompttext), ) return response.choices[0].message.content.strip() @@ -247,7 +259,7 @@ class AzureOpenAITranslator(BaseTranslator): } def __init__( - self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None + self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None,prompt=None ): self.set_envs(envs) base_url = self.envs["AZURE_OPENAI_BASE_URL"] @@ -261,12 +273,13 @@ def __init__( api_version="2024-06-01", api_key=api_key, ) + self.prompttext=prompt def translate(self, text) -> str: response = self.client.chat.completions.create( model=self.model, **self.options, - messages=self.prompt(text), + messages=self.prompt(text,self.prompttext), ) return response.choices[0].message.content.strip() @@ -280,7 +293,7 @@ class ModelScopeTranslator(OpenAITranslator): } def __init__( - self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None + self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None,prompt=None ): self.set_envs(envs) base_url = "https://api-inference.modelscope.cn/v1" @@ -288,6 +301,7 @@ def __init__( if not model: model = self.envs["MODELSCOPE_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) + self.prompttext=prompt class ZhipuTranslator(OpenAITranslator): @@ -298,20 +312,21 @@ class ZhipuTranslator(OpenAITranslator): "ZHIPU_MODEL": "glm-4-flash", } - def __init__(self, lang_in, lang_out, model, envs=None): + def __init__(self, lang_in, lang_out, model, envs=None,prompt=None): self.set_envs(envs) base_url = "https://open.bigmodel.cn/api/paas/v4" api_key = self.envs["ZHIPU_API_KEY"] if not model: model = self.envs["ZHIPU_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) + self.prompttext=prompt def translate(self, text) -> str: try: response = self.client.chat.completions.create( model=self.model, **self.options, - messages=self.prompt(text), + messages=self.prompt(text,self.prompttext), ) except openai.BadRequestError as e: if ( @@ -331,13 +346,14 @@ class SiliconTranslator(OpenAITranslator): "SILICON_MODEL": "Qwen/Qwen2.5-7B-Instruct", } - def __init__(self, lang_in, lang_out, model, envs=None): + def __init__(self, lang_in, lang_out, model, envs=None,prompt=None): self.set_envs(envs) base_url = "https://api.siliconflow.cn/v1" api_key = self.envs["SILICON_API_KEY"] if not model: model = self.envs["SILICON_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) + self.prompttext=prompt class GeminiTranslator(OpenAITranslator): @@ -348,14 +364,14 @@ class GeminiTranslator(OpenAITranslator): "GEMINI_MODEL": "gemini-1.5-flash", } - def __init__(self, lang_in, lang_out, model, envs=None): + def __init__(self, lang_in, lang_out, model, envs=None,prompt=None): self.set_envs(envs) base_url = "https://generativelanguage.googleapis.com/v1beta/openai/" api_key = self.envs["GEMINI_API_KEY"] if not model: model = self.envs["GEMINI_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) - + self.prompttext=prompt class AzureTranslator(BaseTranslator): # https://github.com/Azure/azure-sdk-for-python @@ -366,7 +382,7 @@ class AzureTranslator(BaseTranslator): } lang_map = {"zh": "zh-Hans"} - def __init__(self, lang_in, lang_out, model, envs=None): + def __init__(self, lang_in, lang_out, model, envs=None, **kwargs): self.set_envs(envs) super().__init__(lang_in, lang_out, model) endpoint = self.envs["AZURE_ENDPOINT"] @@ -397,7 +413,7 @@ class TencentTranslator(BaseTranslator): "TENCENTCLOUD_SECRET_KEY": None, } - def __init__(self, lang_in, lang_out, model, envs=None): + def __init__(self, lang_in, lang_out, model, envs=None, **kwargs): self.set_envs(envs) super().__init__(lang_in, lang_out, model) cred = credential.DefaultCredentialProvider().get_credential() @@ -420,7 +436,7 @@ class AnythingLLMTranslator(BaseTranslator): "AnythingLLM_APIKEY": None, } - def __init__(self, lang_out, lang_in, model, envs=None): + def __init__(self, lang_out, lang_in, model, envs=None,prompt=None): self.set_envs(envs) super().__init__(lang_out, lang_in, model) self.api_url = self.envs["AnythingLLM_URL"] @@ -430,9 +446,10 @@ def __init__(self, lang_out, lang_in, model, envs=None): "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", } + self.prompttext=prompt def translate(self, text): - messages = self.prompt(text) + messages = self.prompt(text,self.prompttext) payload = { "message": messages, "mode": "chat", @@ -456,7 +473,7 @@ class DifyTranslator(BaseTranslator): "DIFY_API_KEY": None, # 替换为实际 API 密钥 } - def __init__(self, lang_out, lang_in, model, envs=None): + def __init__(self, lang_out, lang_in, model, envs=None, **kwargs): self.set_envs(envs) super().__init__(lang_out, lang_in, model) self.api_url = self.envs["DIFY_API_URL"] From 21a68506b126ca7b72e9237fe3743bf5be799502 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Wed, 18 Dec 2024 14:43:58 +0800 Subject: [PATCH 090/114] format --- pdf2zh/converter.py | 2 +- pdf2zh/high_level.py | 4 ++- pdf2zh/pdf2zh.py | 6 ++-- pdf2zh/pdfinterp.py | 12 +++---- pdf2zh/translator.py | 79 ++++++++++++++++++++++++++++---------------- 5 files changed, 64 insertions(+), 39 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index ab5789ea..551db0ce 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -1,4 +1,4 @@ -from typing import Dict,List +from typing import Dict, List from pdfminer.pdfinterp import PDFGraphicState, PDFResourceManager from pdfminer.pdffont import PDFCIDFont diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index d10aa04f..d6ebd79c 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -293,7 +293,9 @@ def translate( doc_raw = open(file, "rb") s_raw = doc_raw.read() - s_mono, s_dual = translate_stream(s_raw, envs=kwarg.get('envs'), prompt=kwarg["prompt"], **locals()) + s_mono, s_dual = translate_stream( + s_raw, envs=kwarg.get("envs"), prompt=kwarg["prompt"], **locals() + ) file_mono = Path(output) / f"{filename}-mono.pdf" file_dual = Path(output) / f"{filename}-dual.pdf" doc_mono = open(file_mono, "wb") diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index fd1a185a..ef86bb97 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -177,9 +177,9 @@ def main(args: Optional[List[str]] = None) -> int: if parsed_args.prompt: try: - with open(parsed_args.prompt,'r',encoding='utf-8') as file: - content=file.read() - parsed_args.prompt=Template(content) + with open(parsed_args.prompt, "r", encoding="utf-8") as file: + content = file.read() + parsed_args.prompt = Template(content) except Exception as e: raise ValueError("prompt error.") diff --git a/pdf2zh/pdfinterp.py b/pdf2zh/pdfinterp.py index 62ff4ed3..adbef1ee 100644 --- a/pdf2zh/pdfinterp.py +++ b/pdf2zh/pdfinterp.py @@ -236,9 +236,9 @@ def do_Do(self, xobjid_arg: PDFStackT) -> None: pos_inv = -np.mat(ctm[4:]) * ctm_inv a, b, c, d = ctm_inv.reshape(4).tolist() e, f = pos_inv.tolist()[0] - self.obj_patch[self.xobjmap[xobjid].objid] = ( - f"q {ops_base}Q {a} {b} {c} {d} {e} {f} cm {ops_new}" - ) + self.obj_patch[ + self.xobjmap[xobjid].objid + ] = f"q {ops_base}Q {a} {b} {c} {d} {e} {f} cm {ops_new}" except Exception: pass elif subtype is LITERAL_IMAGE and "Width" in xobj and "Height" in xobj: @@ -269,9 +269,9 @@ def process_page(self, page: PDFPage) -> None: self.device.fontmap = self.fontmap ops_new = self.device.end_page(page) # 上面渲染的时候会根据 cropbox 减掉页面偏移得到真实坐标，这里输出的时候需要用 cm 把页面偏移加回来 - self.obj_patch[page.page_xref] = ( - f"q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}" # ops_base 里可能有图，需要让 ops_new 里的文字覆盖在上面，使用 q/Q 重置位置矩阵 - ) + self.obj_patch[ + page.page_xref + ] = f"q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}" # ops_base 里可能有图，需要让 ops_new 里的文字覆盖在上面，使用 q/Q 重置位置矩阵 for obj in page.contents: self.obj_patch[obj.objid] = "" diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index 75cec757..fa2cd44f 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -17,6 +17,7 @@ import json + def remove_control_characters(s): return "".join(ch for ch in s if unicodedata.category(ch)[0] != "C") @@ -50,10 +51,10 @@ def translate(self, text): def prompt(self, text, prompt): if prompt: - context={ - "lang_in":self.lang_in, - "lang_out":self.lang_out, - "text":text, + context = { + "lang_in": self.lang_in, + "lang_out": self.lang_out, + "text": text, } return eval(prompt.safe_substitute(context)) else: @@ -200,23 +201,23 @@ class OllamaTranslator(BaseTranslator): "OLLAMA_MODEL": "gemma2", } - def __init__(self, lang_in, lang_out, model, envs=None,prompt=None): + def __init__(self, lang_in, lang_out, model, envs=None, prompt=None): self.set_envs(envs) if not model: model = self.envs["OLLAMA_MODEL"] super().__init__(lang_in, lang_out, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = ollama.Client() - self.prompttext=prompt + self.prompttext = prompt def translate(self, text): - print(len(self.prompt(text,self.prompttext))) - print(self.prompt(text,self.prompttext)[0]) - print(self.prompt(text,self.prompttext)[1]) + print(len(self.prompt(text, self.prompttext))) + print(self.prompt(text, self.prompttext)[0]) + print(self.prompt(text, self.prompttext)[1]) response = self.client.chat( model=self.model, options=self.options, - messages=self.prompt(text,self.prompttext), + messages=self.prompt(text, self.prompttext), ) return response["message"]["content"].strip() @@ -231,7 +232,14 @@ class OpenAITranslator(BaseTranslator): } def __init__( - self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None,prompt=None + self, + lang_in, + lang_out, + model, + base_url=None, + api_key=None, + envs=None, + prompt=None, ): self.set_envs(envs) if not model: @@ -239,13 +247,13 @@ def __init__( super().__init__(lang_in, lang_out, model) self.options = {"temperature": 0} # 随机采样可能会打断公式标记 self.client = openai.OpenAI(base_url=base_url, api_key=api_key) - self.prompttext=prompt + self.prompttext = prompt def translate(self, text) -> str: response = self.client.chat.completions.create( model=self.model, **self.options, - messages=self.prompt(text,self.prompttext), + messages=self.prompt(text, self.prompttext), ) return response.choices[0].message.content.strip() @@ -259,7 +267,14 @@ class AzureOpenAITranslator(BaseTranslator): } def __init__( - self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None,prompt=None + self, + lang_in, + lang_out, + model, + base_url=None, + api_key=None, + envs=None, + prompt=None, ): self.set_envs(envs) base_url = self.envs["AZURE_OPENAI_BASE_URL"] @@ -273,13 +288,13 @@ def __init__( api_version="2024-06-01", api_key=api_key, ) - self.prompttext=prompt + self.prompttext = prompt def translate(self, text) -> str: response = self.client.chat.completions.create( model=self.model, **self.options, - messages=self.prompt(text,self.prompttext), + messages=self.prompt(text, self.prompttext), ) return response.choices[0].message.content.strip() @@ -293,7 +308,14 @@ class ModelScopeTranslator(OpenAITranslator): } def __init__( - self, lang_in, lang_out, model, base_url=None, api_key=None, envs=None,prompt=None + self, + lang_in, + lang_out, + model, + base_url=None, + api_key=None, + envs=None, + prompt=None, ): self.set_envs(envs) base_url = "https://api-inference.modelscope.cn/v1" @@ -301,7 +323,7 @@ def __init__( if not model: model = self.envs["MODELSCOPE_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) - self.prompttext=prompt + self.prompttext = prompt class ZhipuTranslator(OpenAITranslator): @@ -312,21 +334,21 @@ class ZhipuTranslator(OpenAITranslator): "ZHIPU_MODEL": "glm-4-flash", } - def __init__(self, lang_in, lang_out, model, envs=None,prompt=None): + def __init__(self, lang_in, lang_out, model, envs=None, prompt=None): self.set_envs(envs) base_url = "https://open.bigmodel.cn/api/paas/v4" api_key = self.envs["ZHIPU_API_KEY"] if not model: model = self.envs["ZHIPU_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) - self.prompttext=prompt + self.prompttext = prompt def translate(self, text) -> str: try: response = self.client.chat.completions.create( model=self.model, **self.options, - messages=self.prompt(text,self.prompttext), + messages=self.prompt(text, self.prompttext), ) except openai.BadRequestError as e: if ( @@ -346,14 +368,14 @@ class SiliconTranslator(OpenAITranslator): "SILICON_MODEL": "Qwen/Qwen2.5-7B-Instruct", } - def __init__(self, lang_in, lang_out, model, envs=None,prompt=None): + def __init__(self, lang_in, lang_out, model, envs=None, prompt=None): self.set_envs(envs) base_url = "https://api.siliconflow.cn/v1" api_key = self.envs["SILICON_API_KEY"] if not model: model = self.envs["SILICON_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) - self.prompttext=prompt + self.prompttext = prompt class GeminiTranslator(OpenAITranslator): @@ -364,14 +386,15 @@ class GeminiTranslator(OpenAITranslator): "GEMINI_MODEL": "gemini-1.5-flash", } - def __init__(self, lang_in, lang_out, model, envs=None,prompt=None): + def __init__(self, lang_in, lang_out, model, envs=None, prompt=None): self.set_envs(envs) base_url = "https://generativelanguage.googleapis.com/v1beta/openai/" api_key = self.envs["GEMINI_API_KEY"] if not model: model = self.envs["GEMINI_MODEL"] super().__init__(lang_in, lang_out, model, base_url=base_url, api_key=api_key) - self.prompttext=prompt + self.prompttext = prompt + class AzureTranslator(BaseTranslator): # https://github.com/Azure/azure-sdk-for-python @@ -436,7 +459,7 @@ class AnythingLLMTranslator(BaseTranslator): "AnythingLLM_APIKEY": None, } - def __init__(self, lang_out, lang_in, model, envs=None,prompt=None): + def __init__(self, lang_out, lang_in, model, envs=None, prompt=None): self.set_envs(envs) super().__init__(lang_out, lang_in, model) self.api_url = self.envs["AnythingLLM_URL"] @@ -446,10 +469,10 @@ def __init__(self, lang_out, lang_in, model, envs=None,prompt=None): "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", } - self.prompttext=prompt + self.prompttext = prompt def translate(self, text): - messages = self.prompt(text,self.prompttext) + messages = self.prompt(text, self.prompttext) payload = { "message": messages, "mode": "chat", From 7d1caa730a724fd29b0b6aa57b601a8447b60254 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Wed, 18 Dec 2024 14:58:42 +0800 Subject: [PATCH 091/114] =?UTF-8?q?readme=E8=B0=83=E6=95=B4=EF=BC=8C?= =?UTF-8?q?=E5=BC=80=E5=85=B3=E8=B0=83=E6=95=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README_zh-CN.md | 9 ++++++--- pdf2zh/pdf2zh.py | 2 -- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README_zh-CN.md b/README_zh-CN.md index 2a9db04e..78bba74a 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -254,13 +254,16 @@ pdf2zh example.pdf -f "(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-| pdf2zh example.pdf -t 1 ```

自定义大模型prompt

-使用 `-pr` 或 `--prompt` 指定使用大模型翻译时使用的prompt文件。 + +使用 `-pr` 或 `--prompt` 指定使用大模型翻译时使用的 Prompt 文件。 + ```bash pdf2zh example.pdf -pr prompt.txt ``` -示例prompt.txt文件 +示例 `prompt.txt` 文件 + ``` [ { @@ -275,7 +278,7 @@ pdf2zh example.pdf -pr prompt.txt ``` -自定义prompt文件中，可以使用三个内置变量用来传递参数。 +自定义 Prompt 文件中，可以使用三个内置变量用来传递参数。 |**变量名**|**说明**| |-|-| |`lang_in`|输入的语言| diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index ef86bb97..bdd9ab22 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -118,14 +118,12 @@ def create_parser() -> argparse.ArgumentParser: ) parse_params.add_argument( "--authorized", - "-a", type=str, nargs="+", help="user name and password.", ) parse_params.add_argument( "--prompt", - "-pr", type=str, help="user custom prompt.", ) From 390f1bf98176674e46bbf1f353ef9addacfd9c92 Mon Sep 17 00:00:00 2001 From: hellofinch Date: Wed, 18 Dec 2024 15:02:08 +0800 Subject: [PATCH 092/114] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=BC=80=E5=85=B3?= =?UTF-8?q?=E6=8F=8F=E8=BF=B0=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +++--- README_ja-JP.md | 6 +++--- README_zh-CN.md | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d29d62b2..f2e79e4c 100644 --- a/README.md +++ b/README.md @@ -172,8 +172,8 @@ In the following table, we list all advanced options for reference: | `-o` | Output dir | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [Exceptions](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | | `--share` | [Get gradio public link] | `pdf2zh -i --share` | -| `-a` | [add authorization and custom login page] | `pdf2zh -i -a users.txt [auth.html]` | -| `-pr` | [custom llm prompt] | `pdf2zh -pr [prompt.txt]` | +| `--authorized` | [add authorization and custom login page] | `pdf2zh -i --authorized users.txt [auth.html]` | +| `--prompt` | [custom llm prompt] | `pdf2zh --prompt [prompt.txt]` |

Full / partial document translation

@@ -254,7 +254,7 @@ Use `-t` to specify how many threads to use in translation: pdf2zh example.pdf -t 1 ```

custom prompt

-Use `-pr` or `--prompt` to specify which prompt to use in llm: +Use `--prompt` to specify which prompt to use in llm: ```bash pdf2zh example.pdf -pr prompt.txt ``` diff --git a/README_ja-JP.md b/README_ja-JP.md index fd3efd07..c107aa95 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -173,8 +173,8 @@ Python環境を事前にインストールする必要はありません | `-o` | 出力ディレクトリ | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [例外](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | | `--share` | [gradio公開リンクを取得] | `pdf2zh -i --share` | -| `-a` | [ウェブ認証とカスタム認証ページの追加] | `pdf2zh -i -a users.txt [auth.html]` | -| `-pr` | [カスタムビッグモデルのプロンプトを使用する] | `pdf2zh -pr [prompt.txt]` | +| `--authorized` | [ウェブ認証とカスタム認証ページの追加] | `pdf2zh -i --authorized users.txt [auth.html]` | +| `--prompt` | [カスタムビッグモデルのプロンプトを使用する] | `pdf2zh --prompt [prompt.txt]` |

全文または部分的なドキュメント翻訳

@@ -257,7 +257,7 @@ pdf2zh example.pdf -t 1

custom prompt

(need Japenese translation) -Use `-pr` or `--prompt` to specify which prompt to use in llm: +Use `--prompt` to specify which prompt to use in llm: ```bash pdf2zh example.pdf -pr prompt.txt ``` diff --git a/README_zh-CN.md b/README_zh-CN.md index 78bba74a..dfb12ead 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -172,8 +172,8 @@ USE_MODELSCOPE=1 pdf2zh | `-o` | 输出目录 | `pdf2zh example.pdf -o output` | | `-f`, `-c` | [例外规则](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | | `--share` | [获取 gradio 公开链接] | `pdf2zh -i --share` | -| `-a` | [添加网页认证和自定义认证页] | `pdf2zh -i -a users.txt [auth.html]` | -| `-pr` | [使用自定义的大模型prompt] | `pdf2zh -pr [prompt.txt]` | +| `--authorized` | [添加网页认证和自定义认证页] | `pdf2zh -i --authorized users.txt [auth.html]` | +| `--prompt` | [使用自定义的大模型prompt] | `pdf2zh --prompt [prompt.txt]` |

全文或部分文档翻译

@@ -255,7 +255,7 @@ pdf2zh example.pdf -t 1 ```

自定义大模型prompt

-使用 `-pr` 或 `--prompt` 指定使用大模型翻译时使用的 Prompt 文件。 +使用 `--prompt` 指定使用大模型翻译时使用的 Prompt 文件。 ```bash pdf2zh example.pdf -pr prompt.txt From e73658bc0461c326c853765b3ccfb7009d33141e Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Wed, 18 Dec 2024 17:51:41 +0800 Subject: [PATCH 093/114] doc: badge --- README.md | 2 ++ README_ja-JP.md | 2 ++ README_zh-CN.md | 2 ++ 3 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 7f7cb3eb..21467611 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ English | [简体中文](README_zh-CN.md) | [日本語](README_ja-JP.md) + +

diff --git a/README_ja-JP.md b/README_ja-JP.md index 101a02c2..19befd06 100644 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -23,6 +23,8 @@ + +

diff --git a/README_zh-CN.md b/README_zh-CN.md index 838f1718..3d52c940 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -23,6 +23,8 @@ + +

From 549e7962403dbdf46e67268dfa078978bc21b9d5 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Wed, 18 Dec 2024 18:03:30 +0800 Subject: [PATCH 094/114] chore: format --- pdf2zh/converter.py | 2 +- pdf2zh/pdf2zh.py | 2 +- pdf2zh/pdfinterp.py | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pdf2zh/converter.py b/pdf2zh/converter.py index 551db0ce..2eb6d9cd 100644 --- a/pdf2zh/converter.py +++ b/pdf2zh/converter.py @@ -152,7 +152,7 @@ def __init__( for translator in [GoogleTranslator, BingTranslator, DeepLTranslator, DeepLXTranslator, OllamaTranslator, AzureOpenAITranslator, OpenAITranslator, ZhipuTranslator, ModelScopeTranslator, SiliconTranslator, GeminiTranslator, AzureTranslator, TencentTranslator, DifyTranslator, AnythingLLMTranslator]: if service_name == translator.name: - self.translator = translator(lang_in, lang_out, service_model, envs=envs,prompt=prompt) + self.translator = translator(lang_in, lang_out, service_model, envs=envs, prompt=prompt) if not self.translator: raise ValueError("Unsupported translation service") diff --git a/pdf2zh/pdf2zh.py b/pdf2zh/pdf2zh.py index bdd9ab22..e2f06b67 100644 --- a/pdf2zh/pdf2zh.py +++ b/pdf2zh/pdf2zh.py @@ -178,7 +178,7 @@ def main(args: Optional[List[str]] = None) -> int: with open(parsed_args.prompt, "r", encoding="utf-8") as file: content = file.read() parsed_args.prompt = Template(content) - except Exception as e: + except Exception: raise ValueError("prompt error.") translate(**vars(parsed_args)) diff --git a/pdf2zh/pdfinterp.py b/pdf2zh/pdfinterp.py index adbef1ee..62ff4ed3 100644 --- a/pdf2zh/pdfinterp.py +++ b/pdf2zh/pdfinterp.py @@ -236,9 +236,9 @@ def do_Do(self, xobjid_arg: PDFStackT) -> None: pos_inv = -np.mat(ctm[4:]) * ctm_inv a, b, c, d = ctm_inv.reshape(4).tolist() e, f = pos_inv.tolist()[0] - self.obj_patch[ - self.xobjmap[xobjid].objid - ] = f"q {ops_base}Q {a} {b} {c} {d} {e} {f} cm {ops_new}" + self.obj_patch[self.xobjmap[xobjid].objid] = ( + f"q {ops_base}Q {a} {b} {c} {d} {e} {f} cm {ops_new}" + ) except Exception: pass elif subtype is LITERAL_IMAGE and "Width" in xobj and "Height" in xobj: @@ -269,9 +269,9 @@ def process_page(self, page: PDFPage) -> None: self.device.fontmap = self.fontmap ops_new = self.device.end_page(page) # 上面渲染的时候会根据 cropbox 减掉页面偏移得到真实坐标，这里输出的时候需要用 cm 把页面偏移加回来 - self.obj_patch[ - page.page_xref - ] = f"q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}" # ops_base 里可能有图，需要让 ops_new 里的文字覆盖在上面，使用 q/Q 重置位置矩阵 + self.obj_patch[page.page_xref] = ( + f"q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}" # ops_base 里可能有图，需要让 ops_new 里的文字覆盖在上面，使用 q/Q 重置位置矩阵 + ) for obj in page.contents: self.obj_patch[obj.objid] = "" From 4e2e12e902c3d68676187f243976aaa6ecb23f11 Mon Sep 17 00:00:00 2001 From: Rongxin Date: Wed, 18 Dec 2024 20:19:33 +0800 Subject: [PATCH 095/114] feat (workflow): auto translate issues --- .github/workflows/issue-translator.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/issue-translator.yml diff --git a/.github/workflows/issue-translator.yml b/.github/workflows/issue-translator.yml new file mode 100644 index 00000000..01bf7609 --- /dev/null +++ b/.github/workflows/issue-translator.yml @@ -0,0 +1,15 @@ +name: 'issue-translator' +on: + issue_comment: + types: [created] + issues: + types: [opened] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: usthe/issues-translate-action@v2.7 + with: + IS_MODIFY_TITLE: true + CUSTOM_BOT_NOTE: The issue has been automatically translated into English. \ No newline at end of file From 550025134a08b18cf507839fa698404179698990 Mon Sep 17 00:00:00 2001 From: Rongxin Date: Wed, 18 Dec 2024 21:10:19 +0800 Subject: [PATCH 096/114] ci (test): cli and gui --- .github/workflows/issue-translator.yml | 2 +- .github/workflows/python-build.yml | 18 +++++++++++++++--- test/file/translate.cli.plain.text.pdf | Bin 0 -> 14481 bytes 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 test/file/translate.cli.plain.text.pdf diff --git a/.github/workflows/issue-translator.yml b/.github/workflows/issue-translator.yml index 01bf7609..948bf260 100644 --- a/.github/workflows/issue-translator.yml +++ b/.github/workflows/issue-translator.yml @@ -1,4 +1,4 @@ -name: 'issue-translator' +name: 'Issue Translator' on: issue_comment: types: [created] diff --git a/.github/workflows/python-build.yml b/.github/workflows/python-build.yml index e2e330d9..d4dd3c0c 100644 --- a/.github/workflows/python-build.yml +++ b/.github/workflows/python-build.yml @@ -1,4 +1,4 @@ -name: Build Python Package +name: Test and Build Python Package on: push: @@ -22,10 +22,22 @@ jobs: python -m pip install --upgrade pip pip install build flake8 black - - name: Check code format + - name: Test - Code format run: | black --check --diff --color pdf2zh/*.py flake8 --ignore E203,E261,E501,W503,E741 - - name: Build package + - name: Test - Local installation + run: + python -m pip install -e . + + - name: Test - Translate a PDF file with plain text only + run: + pdf2zh ./test/file/translate.cli.plain.text.pdf + + - name: Test - Start GUI and exit + run: + timeout 10 pdf2zh -i || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi + + - name: Build as a package run: python -m build diff --git a/test/file/translate.cli.plain.text.pdf b/test/file/translate.cli.plain.text.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e5364f4f23bf369bb8c2f926d61e761f29c25e3a GIT binary patch literal 14481 zcmeHu2UJsA^RIwNQ&CiU2_RJ2ucU(9YLfhDjlg(qzTdm z=}o#KMJXcnoy6{`42a9pF z#Oh*@NNqG43xWva`_;@b4$3HNG@t|5L|TCWHl&mkSPh9aw*nM`prUxj>vl+YBt{d1 zv_@i(C`;S`a9oF!)G2Z#$_j4?Zpgn`gtr7Nhjv8)1A(r4J8}0K)=?Dxy(X+z$lweGZCH5ClJ4z-$E%>=hlXITi^pT+x9Df`Bg$ z2cQiB0bjhuU`-6#QU{4O0t1+Wb&(!eu(A^Xn(X%^@o^%#sH}-xW<2jWmKij3YQty6C=Ax)9P59+me=}JgpSzgWD`A;js@_InL3JQXOP90AdEYb2!i5`u!2f!)Z?QU+z?hy)3OWpL0bwmH0s%H8kM9={5r7egn%;L#7yyy}A5u_ZL0}Of zovs5Scph2&V&xYW6vPhzEP6N;H!QGbaVmh7goJ{i_*MHeYWRKPZ?pY#D*?-@fz{C% zCv!)zr8x-lH>uw?;eV2nllevJw~Z1X5P%Ej)_ zpvNiu%S|8%KCr=xb{H3|oUJ(q=Nnb?zuI_=f$bc>ApSPF-}dFdYyMjR{AJhwuL1D$ zekwinqqWJlqObDQLJ!KQ%nIn_>$6n${Mk9O?Z)Z0uD> z+S+N;3)N)?T+wkY0b=6|Ou0yPy-J>v(a+ogV6qDG#PS$D8qO|$4d?#cgP@{<)t8VF z)YnBhlfHxDh9N^Q^i&V{`S?HHpYV5#A6v%1b^{cK-&Ox|gGrQ{eUspsfUjaLeEkYh zR9sR1XUoWl!Mk!3%$E~d3qT3?_`tXJ?$1TZl3d(L@{AO1>gme4DMD}*Y+8}@u%cT& zZJ$wCHt4RaBUgj|1C;J<$)^qQkTK*{xJbd*=LLSg+Addb`gO&AW|9&-WIm<1d}8=k zhS7}M>Gz@R)yMsn$xe7fy{^4iiy(d3RvwXAr`C}G^F3##a(S3$GT*#R6i%E2;S~iv zBaXkf8*mkhT`;RRyk=79tJ~SXcTSDsCj5=+<3-umPkSw2yv>A9z(#}$Z&N;Bq~&p_ z(jZHoH3<58?gUE-Z@rmNF*fC&q5FBU{cGq%Au!QD4muDJR1hNkOF*yOqQEk|_FU~; zB|NSiMVMCsBBRJBRXIVXovjp>%_o}e_uj>=xal4ie z?Rm2c*K3A-Z4SRJ`u0fII2hiel1(_K)a>hARP5Q~D`CTNFP*^6%Gq|YH^XF4J3Xp4 zL-A-*(^zekgqQIiU;Uj!5?LL(k%Qv-46eC^$9=x8dT$R{k1*< zR$-boRG47n)Y1Eb%8!<0Azsq?ATl#{k^Z31h?a|j@NLIkKi)>z_@UwHGXZ+G>+ zDp$3QCOd#LoU6C}V39zjb&JUGR^afk!XX2*cFNLP#p$&zbEch^A}ukE?b@-*9h&Uz z#hi|#O}XmCG_QKsO3(dGxt)ui3kQSmec!TjwoUKlH+9V}+}x=29G5^bi$<4BbKChn4W|<cD_iGrY)HF@BHH`9GSTn8xH_pOa7DpW@>s`APzv;elyV!g2(fhi_BDUph=|qYis^Zfg^)3lt z0(177Wb~@c;}@)U)?No2A*ueTJ8NlwYnA~0d5qnISK zXB+siFY1w8#%mM49V`%JWl%B5lXc*iX7rdnwEq+3@oC6YF&9(|Kgr$1jlz87Kq$k@ z<8OtZ^-=^`A8a;G_cdG~DVb~3oF?`opI|q6H4;HfVH|xHd;bD$0;|ClYo?M!HYk`> zM1Ae$4A&eZOFZ2Ft6Y))tRpcaM`nk=p9`(OgmDy0GE+4l9s42Si{ois4G)B#h$at+ zgu{p1I^-L~rth{AfLWo3euOZBh@Q~5}Q5e&yEy>bGfdX@UL_JA1zC(TzKO_u4MwA_80~j> z*ih-h_r$+qRZrJMyE?6u>O-TKfr7bq4msFOp*dqzqQr8dguN2- zzy$StFAb6XY4ieLav%F5ww|7zKtUtwoFxb0NY)b!t!ZG@`uIctg}yAu?%ux3-OSe; z$t6C0UDQoliIbDD5ulx6etqkSBK=4Hjd$&Q>V!rcu9=WK9Z)(ZEyYvEjUI&EoDDl` z`H;7T$2$pr`|*DISk~q}nO6eB&%@f_6^y6pdE|CICw4;Q%kC*Dsv|6964Q(UCd){<%~^%8zC`QgFuE)EEkp*cju!YNEv)EKx{!j9kYCzTd9@rT5fx4O^AuZ_R0w&v6~B+ ztLl$6iwkI)Z>;s@r%Smtfj?Z+JtbrH#p>Fui+85L*VD5LpR~qhqDWoG6P+2Kwr~?= zGL~pb`@OTyd)EGLVQ!o8=^)v8)OE(P@xqN?&B=UXV~RVHsb( zavd?^Xh0K4Xzj7HFD3HMfHVnmiq_O^qik;Xfern~@8>V(|%x!Z0 zBXgEm(evzD@(&89lTVfSFUeQMLN6DU#fiHTOjxLR%76xP9W5&`$g$^_J2Y1eAQ;^d zttFSt=%Qk+rVguA7jbV`$(PHC8^h|wZ(-+);5i@-x=5)N;nLwdI+HUyFr!!`vBv1I zegWL}{iocB@%1FVOw5pVNCft_fU)5th9s9b$ep{qpsba}8dy zRg9OMO1YknkiYP@zf%}szoJ<(F+f#UMe}r1s88hndlSA2Xzu;9<<(bD!Y}WoM32VP zVlSr-rJcBNTW=jCJ}})hgnZ(-ysDoyXME5YtZ>!aPFY?&|F8^^kYB>b`;D0)&h&{PL6Kws~HP zt&I4iOCHBK_TR@GcL_VYYtT$oTu}Wv*a=b|n$^XvcVX?Moj_11sDn>;U)Q3tp`LX!5DcqE8=a(Z@pNa>v@;W|2yeDWM_~C0DDKg3L5NaeKkI zSclJ#GNs!l9PaZl2APb1gyba=Q(D}l$EteQM2(4)autb|ggSMTZzt$^yz+XxhiZbR zM)Kxl=yq2oU)pSXbCQ|5`hrP9WY$H(DsLyj#3r{Co#E7h_jd?J&g7eQq@Ntt30Gb= z4s$R2R^Y?0BD#e+Yhy2uNO!h?QP4-aj zHQQca3M82epP^?fm%AhMOlm|IOln;{^)x0Fh6Fg)K2AotyH*ueyH#%p)bBl7KK<>f zLr|Lggn;zwsDNm-*wABRrsa(SH@(RhU9R1(W7ToboJ=O)Rj)pESRQ>=knqA&Z(}s9 zfM1O#vGCAMPtNC!>+(X4v1`r^acZfC)t zy=t2!Jo`U+*Otv)tQ?N?hzNwBVp3NuUK@dgKT{}PKdV9`uIzLO-k3Uf6g_Yz#<$eD zOEUh5tK;gnaL`tJ?VJ?VYlK=O++YtCb5X}C%J!m)g%#uZb9d}8p|_Hs%yBD6vucLN zdS5;`6LO>V{IuPsZ%BC5ari`2JufkwNUF|^vso9$fw#}4Ddt_%|SMq`F4dVFTqFDf0z2m0NNjEt7AN0z**Hs%SBGss8I zFt|Ju6%lqCA0C&e&>6qEQmPv(CN4gD;xy$0RVnYOgZjxSpRp6|C5tO6Q;y@6#nC1j zDjBugO@1MPyAw_iNQe$>z~kb5ER!E6#@^Ju9$9BRl3>$RnZTWNwb#eY@4~ zn(EfIo^5harau9b#mBuB3JP(*v^OTp1dER{RKC8qLY*3+bjAP`>SzsREpDwHcQUQ^ z_~fMt_c_pdk5Cyp=W_SRp?mLM`|B?!clcNZ>gD(fsaD)B75gXVw>>MbjrK&C5FOa5 zL|9C2i7tumEt%Dd)mW~U=T+r0B@{SJozts72;B^Ekf!FBX0{mrIzJUxZWO4*Zm?FE zUsy8WXjp$ME22^_toOzEbdTI{>8n{gCxdbe1r?#lltt={J02xP`T7NApSdlt^Iim6 zW#oG-dwZ0%*qOZ9CQ3?IpUpZhX?N&$0OizC{GODW@6jip$km;;Z;#ESZy#0I6Z!5m zH&1l&&jxu#y48q2xOFl8aJZp&EnN-yTJ8bK!JB+`jd7o&mL^q==Vv~bN(8DL%p=R= zuAPzh%hT(haTUnNI$d{U7Ti(C_F;w@0 za(-S}d`^Ox5{G4P=WWye%S{jCg`H>kuP^g=IaR##Q0-qz?avU)Gm&EOeEPX+ihF8z zY>$gMZ)*SH_VhQ(!Q8ry@wNS~50)gy_r?k(O2l_qokSlnCwa_&OEmRc=(!w0F{7s} zY_M9$N0X1_u~&;GFINR6Uvc8dF6RrPbuj1>Vkt!(zWY=X?-NT@vePm(BD$@m;eN2i zYTfa$7qm}XY^WkplylXzqT9*H!ib}Qbvq}!c;2P!(wKwcbW|?CxSA2lsOaT&KCt0C z`+3fl4`Q{;exez*%iTVfk!D_-ZCOo}iB3M1#YaS|(w=o1w!>m|XbX$jUN@_u)w@;R zw>~Uw9Wm7V41KukF2T1dInYNOA(ec@T1&c?5IfH&(ER>M)T0Y8I?JhZUq8HJ8vsru zLt3b1*`qjT~4?cUU2u7vV=a z*}FO9A!$z&ZsgIR#>gm)jPzTaX^b}awmEuq^}3jf#M4c*d0N#!K2}dl+G)0Zug-mt zFdsu(Qg{FA_0MdXk4f$1m)r%MUbWenrFIwz^gXTeoy~Zs5koPTchtBMpRxB%<7kU( zqH0OvGll12>rv3+a(&X{8}*)(66kc%p07*ehw15~T^l|Ko7;6<=_HbyVf$-{;hJ?3 z*-jA->dhzL5?PQlcBT9Y(>XEa0RzD{o#ii%n<Egpo!Tbp!fVxTDUTR1yxrlL82e&-vD1x%uB0ASzLK41pEFWG1=d{=8GAQZG^=7g zrqx()Z7Mp9D$Elaqt@0cZNZNhb4~bxwYT^ZiLV9m%rL2 ze=b(a(}s_eQvNl2a+$=#oEHOe)Z6WHXVoSJGjlja+vr?jx$Pkk#JP~%`Qm0P#q{g_ z82!|@79rWx30biiZ8hYaqnH>%;2B0N_Eb+>me7~(cIS_0t3(+>v*(*HE;wMWhS}+S zvF_rdP|ESBqBVOkTBhWQ??C8__c3K8x0UCF4MA$w;sE@r3#WoL(hdr1c$E zQAK!ld3hBfI6NB`B)WIQ2*cUZ>DOm8s^YrdeqhMUtoGQv{G<+>;nlkdvplZQtbpbD z;<(m#dKRVa*-`ce&4Z z1SX#jc&`s*7UppNN`Lcszau@hpycwr2OA@krF%qbeCPevM<0}G8Sg^}vqnxv*#(}o zi*=;_WUHDzR~FcOEQot^p}gDSwOvx&MHkyv?Ab(dCgo-t!^RlSu-+VY`iyhx4M^h| z9{xCUT1BaZ;1;U>&N594C?k?ON8E${X;uhNM5@I-S7@|fZs3W|qxNqPQ{+AblfSzn z+;65B`jlfZJ82Vp;$~#ymOi=hxWy$O@z`Yfln;e(lH=$HXYB1CU53>swd#a8#w@C_ zrR#FBHi$lqu(7oNC^+6YV81Qw^g&|y?Vy-6&jU41OD)^h!B~ycoFy3J2j*Ir46L2i zxO4LOq=K(ObGvEpYvewn4ak5hfwR--P9$L4Y;qsasn9)R?_LpRuv9dC*%WBSd$v0e z8caSYy)amcASQ60#9pKi<>Ng@yx8rKkrdw5mij0a$uIRZK&9=~`-!soIf;Cg63JSS zmxAeoaiY{mQ89fp&qH$_MbWFPYP3?>dt`pqZMtR{njCwFyTOq&#P#mS7AP_7EPsOW zl2)8NpZDD~;(PP@yk|r1D`u7!YUUKZ?6329b@!zq(mIArH`dB3_pEYi7h|7(BXdZg zQ1o48D^84)pjX9UqSjfKFW&N;UD;$q*UPqJxyho>D@VZ!ATb054Eg7kF;583DLBOJ z?67FGI=`_xR(X@^m43=s*toE!20fC&^yr8AxNUlw=bgC}M_9aXv#gkg z2Y?`5^BO5%l0|p~g4%WbDC2GQd+R{~?e__rfd7X{wH-_3WXe9f2ydnB;mh7@;F6hX zc$#~f-tJhB|LKS@57D@F)ZFc>$3*<^zFDy-i?Cw(Aa~EJjQ~oz)MT6FxFEH*a65(I z8mpLMc16^7#uAB0VTQ~74S05!ZHKik+bvZgA4Ip1Ma(i4=az)MBX-g#{8CsK+<9xf zo{TwJSxUOSDo2g1v-)I(2S(w~aJaseVUIVFV zveU`)M}BSVO*12q%~?i0`=(pl2?bZ=`uWUaw>S>1R^X$mZ}f``qY3^9BwsJmDY$lq=eXibgmds*8* zZuHqbLOJuTRoAt|g}4n&BO|nXLnNJ0A-F@>mVDfxHuSTwm`S){@bdWuFGaNjv8zv6 zpWP-i?O({!t~TWI>qwH~-;!ZHT0GFK)~4ycx^cnQ!)Kt(ORE0rvnRXfEMj>_(-*#p zv{x}0pg)ms3?7u}?`pcojJtezbaYLw^3?}L@qYN4&8wSrlSN5fRSAK_ zQgtj58%I$ca2Ao$oauaP&7zcH<pPq%4z;vEBw;pN*uQXP(rg|H_ zym&-y(Mml~nvu?>JW07>B4Am7oc-1uWn>mtI!tJZR855h?;p>&kgiEx_?+QedRNHj zLC#)u>x21y+v3%iu{D`ZJs-zBS2t?fBfh3H7-&`(@((;Ky-1S85y(#`P`Lg)X~z3w zxPkKHs(G@y?kWDS1H=Y7pDQn_CP6YTTe)zlrtD>YcGQ33c_M>!$|+_oC975EQ#M_) z=W4(O?I)#tzHGO8!cuN;*UlRBcvLGLsctn4lO^$AX{aqCVwAbm`9iGi8RxA1z=TZM z+6g2}VO#4iYm4t9_YrK*Jr(7lxzT;t#eEY-Pm#Lf3>QJ%-;1p@>{m=Z=(w8yiNl&Z zI{3Y?$1}7;O<|r+_kE6(*H9)*7@A`F{%h-)=*_w%+o17;$9xNH|t82kz6LfTIOJ{l(3Ki*u5v)V(ZIlT&~(chfd!KB5|?F8z-}M ztl7n1F-Z4cTqU^bxL@dyaI-R=G{kBbHLBbOn~Z%TzXUUle-c2d^^&b`W<y^A^tH z2S;YsgvhCG(mdmX^EcPmBIDjKS>AZi=$hPd25yp6p)kqGSvN|0NAS)zsi1>%aXtS- zeZ^1WmLv((9OBN}GgrPnoTD|LqV9<{Be<1je0=|W72yuaoLYk~E1KX?d^4x)B!)Xb z+pWRAnyy!cl92fk>*a`a;(bsT!6EeE{Pe~q@seAJsq7xqhi*M`iG`rDC!G8SYHwPMz34eIVXY!JXaj`&jz;{@Jc`yuq?`H>xzr^t%op2xeZ_ zCt0FG5kIH7bthj|eA?~3TLa2mdYbk9ndygu$367ngtX`H6W%9Xk3K`KPS6?vzoD%} zs7|qXjD?Ds@Rt0}wT4LYda|A8>tqd!0R{~k6d5P>!`JB=5{ZX7_KVkfrx$UJy!FI8 zX4G|rJJ;SnyJJREcX~&(QIw#bzd`YRO`|m7Vd*;f4w|e;cPG>Nl(TvgyY9WbJ4m)$ z0@uUd-8tPypnqwQu;7f|u>vYRf&!W$BGeg%W2w!Ym+LFf6&Zcv)E#AI9n_8-Ao57W z(oDb3i6pwM?d2EzHtem6ty#<={Ys&-ncSZD+bNTja<{)}w;Tc4&By=A=Q zc3*DRN1L%N)T#z=2EAQwW;-DB&@E<+JG{I*engb?Ajs^WUn2gQAOs|58G&G;5EBpx z2DF5QMSvDW2mv%j@J*N~t{=z&0U_YRM!>5?ArsKg={)%42mGttKRW-)^ug!9{dmiZ zPs{j`8uw=^lD?Yik8~^{K|#@xcnu92v|8 z!n0rnsKaySuv!xC;rOF*aZb0)YSv zLcvfdKfuB7;)%kVd+?)N*nepFkvaloGk|a!%`MQbSV=ZE&>wo8f6|L`5x^TJV2O4D z107(9fDjm`2#k~bNz)(J{Y3dk-By-=n(XX~al{*JWeG-pPfv0I)I;#b;E0OaotJR|sL@&Bf!fP?-t^=~M600~_X0YSW(0F&4sDI>aQ zw4>DD(?%q~zxGS~NkHP#M5J(OB>WI4KLoA|fr`OkV)$GVF+stfM89+5C4uY`J8RGX z6Um=A|GTaNOjT3+uOV=9`hkL)nwUJ=(iN9urYtYz>S|{tCaVCIRg^=B@+07~5PldO zBFqmL6oK)J$UqPZB5+YbArVFVH2*G(nN(gX{ZkA2=!lr@}wa+ERa=)qy$z zNj7)jB>vCO9Y1H-%iFm)JDPg}ryE^6ERgB`y?o}U$A{mt==HUc)?ghwFQDWA0oF4D zL4XnxTt&(EvK)L`KR)drg3nP0E6W>6AVd}Apt7C zzswYd2nzrF(BhHC-_>#I^5ruprbiw-&psr+c|B6|ZY(_`<3(Q0`%f-kfEBqNO))O0 zEbvgVAHU$j69aEMH9)jU(J6oZhEd}qhBkHe>5-0B54OqY3g_hbZpugdzI`lN*CncH z{4h^R!@8HBjio+}@_RkSkHlC2bo^tM2T)^lid+yRBnbTd;{#MHi3ke|34?GwKWIP= z2JQnw{Yis}04eLg(4Y{&1HaHSF zOn#vWiT=(9gZ^e81P1?Y9uN`X-^Yc+e;Y$k5F+xASpAM4{2%xbztf;V+0(DsLm~e_ zgZ@qfyz`4J6!!Z(pumggU$DZJ!TgFnKokAdZxBI<;IDpz2tg3P)1X2>;f%$Y19c!6 zeC-^5DGCYxUtfRUSC}~l`#r!RLc%a%pqP(CK|>L!+xjEuftde)VH;2phjat}V*;22 X#_dl)l>u-CK=!NAFd;$cA4dN#r5V6* literal 0 HcmV?d00001 From 1fa17241105bad1041e423e4681719ffdf0197cc Mon Sep 17 00:00:00 2001 From: Rongxin Date: Wed, 18 Dec 2024 22:53:11 +0800 Subject: [PATCH 097/114] chore: remove unnecessary files into sub-folders --- README.md | 4 ++-- README_ja-JP.md => docs/README_ja-JP.md | 4 ++-- README_zh-CN.md => docs/README_zh-CN.md | 4 ++-- Dockerfile.Demo => script/Dockerfile.Demo | 0 setup.bat => script/setup.bat | 0 5 files changed, 6 insertions(+), 6 deletions(-) rename README_ja-JP.md => docs/README_ja-JP.md (98%) rename README_zh-CN.md => docs/README_zh-CN.md (99%) rename Dockerfile.Demo => script/Dockerfile.Demo (100%) rename setup.bat => script/setup.bat (100%) diff --git a/README.md b/README.md index c8cb9427..8e58fc2e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
-English | [简体中文](README_zh-CN.md) | [日本語](README_ja-JP.md) +English | [简体中文](docs/README_zh-CN.md) | [日本語](docs/README_ja-JP.md) PDF2ZH @@ -98,7 +98,7 @@ USE_MODELSCOPE=1 pdf2zh No need to pre-install Python environment -Download [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/setup.bat) and double-click to run +Download [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/script/setup.bat) and double-click to run

Method III. GUI

diff --git a/README_ja-JP.md b/docs/README_ja-JP.md similarity index 98% rename from README_ja-JP.md rename to docs/README_ja-JP.md index f76c5d2e..8e36893b 100644 --- a/README_ja-JP.md +++ b/docs/README_ja-JP.md @@ -1,6 +1,6 @@
-[English](README.md) | [简体中文](README_zh-CN.md) | 日本語 +[English](../README.md) | [简体中文](README_zh-CN.md) | 日本語 PDF2ZH @@ -99,7 +99,7 @@ USE_MODELSCOPE=1 pdf2zh Python環境を事前にインストールする必要はありません -[setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/setup.bat) をダウンロードしてダブルクリックして実行します +[setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/script/setup.bat) をダウンロードしてダブルクリックして実行します

方法3. GUI

diff --git a/README_zh-CN.md b/docs/README_zh-CN.md similarity index 99% rename from README_zh-CN.md rename to docs/README_zh-CN.md index 44890d3b..0d7b23b8 100644 --- a/README_zh-CN.md +++ b/docs/README_zh-CN.md @@ -1,6 +1,6 @@
-[English](README.md) | 简体中文 | [日本語](README_ja-JP.md) +[English](../README.md) | 简体中文 | [日本語](README_ja-JP.md) PDF2ZH @@ -98,7 +98,7 @@ USE_MODELSCOPE=1 pdf2zh 无需预先安装 Python 环境 -下载 [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/setup.bat) 并双击运行 +下载 [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/script/setup.bat) 并双击运行

方法三、图形交互界面

diff --git a/Dockerfile.Demo b/script/Dockerfile.Demo similarity index 100% rename from Dockerfile.Demo rename to script/Dockerfile.Demo diff --git a/setup.bat b/script/setup.bat similarity index 100% rename from setup.bat rename to script/setup.bat From 40bbf7e9313113ed3d05808bfa77854251fe348d Mon Sep 17 00:00:00 2001 From: Yadomin Jinta Date: Thu, 19 Dec 2024 01:00:26 +0800 Subject: [PATCH 098/114] fix: gui launch error --- pdf2zh/gui.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index cc582148..bb882b77 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -445,6 +445,8 @@ def on_select_filetype(file_type): def readuserandpasswd(file_path): tuple_list = [] content = "" + if file_path is None or len(file_path) == 0: + return tuple_list, content if len(file_path) == 2: try: with open(file_path[1], "r", encoding="utf-8") as file: From a31cdc46bb893184e1b8177fdb38899581a9e8e4 Mon Sep 17 00:00:00 2001 From: Reynard Date: Thu, 19 Dec 2024 01:07:35 +0800 Subject: [PATCH 099/114] doc: simplified readme, step 1 --- README.md | 267 +++++++++++------------------------------------ docs/ADVANCED.md | 147 ++++++++++++++++++++++++++ docs/APIS.md | 92 ++++++++++++++++ 3 files changed, 302 insertions(+), 204 deletions(-) create mode 100644 docs/ADVANCED.md create mode 100644 docs/APIS.md diff --git a/README.md b/README.md index 8e58fc2e..906a6618 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ English | [简体中文](docs/README_zh-CN.md) | [日本語](docs/README_ja-JP.md) -PDF2ZH +PDF2ZH

PDFMathTranslate

@@ -14,19 +14,19 @@ English | [简体中文](docs/README_zh-CN.md) | [日本語](docs/README_ja-JP.m - - - + + - - + + +

Byaidu%2FPDFMathTranslate | Trendshift @@ -35,7 +35,7 @@ English | [简体中文](docs/README_zh-CN.md) | [日本語](docs/README_ja-JP.m PDF scientific paper translation and bilingual comparison. -- 📊 Preserve formulas, charts, table of contents, and annotations *([preview](#preview))*. +- 📊 Preserve formulas, charts, table of contents, and annotations _([preview](#preview))_. - 🌐 Support [multiple languages](#language), and diverse [translation services](#services). - 🤖 Provides [commandline tool](#usage), [interactive user interface](#gui), and [Docker](#docker) @@ -43,15 +43,9 @@ Feel free to provide feedback in [GitHub Issues](https://github.com/Byaidu/PDFMa

Updates

-- [Nov. 26 2024] CLI now supports online file(s) *(by [@reycn](https://github.com/reycn))* -- [Nov. 24 2024] [ONNX](https://github.com/onnx/onnx) support to reduce dependency sizes *(by [@Wybxc](https://github.com/Wybxc))* -- [Nov. 23 2024] 🌟 [Public Service](#demo) online! *(by [@Byaidu](https://github.com/Byaidu))* -- [Nov. 23 2024] Firewall for preventing web bots *(by [@Byaidu](https://github.com/Byaidu))* -- [Nov. 22 2024] GUI now supports Italian, and has been improved *(by [@Byaidu](https://github.com/Byaidu), [@reycn](https://github.com/reycn))* -- [Nov. 22 2024] You can now share your deployed service to others *(by [@Zxis233](https://github.com/Zxis233))* -- [Nov. 22 2024] Now supports Tencent Translation *(by [@hellofinch](https://github.com/hellofinch))* -- [Nov. 21 2024] GUI now supports downloading dual-document *(by [@reycn](https://github.com/reycn))* -- [Nov. 20 2024] 🌟 [Demo](#demo) online! *(by [@reycn](https://github.com/reycn))* +- [Nov. 26 2024] CLI now supports online file(s) _(by [@reycn](https://github.com/reycn))_ +- [Nov. 24 2024] [ONNX](https://github.com/onnx/onnx) support to reduce dependency sizes _(by [@Wybxc](https://github.com/Wybxc))_ +- [Nov. 23 2024] 🌟 [Public Service](#demo) online! _(by [@Byaidu](https://github.com/Byaidu))_

Preview

@@ -59,15 +53,14 @@ Feel free to provide feedback in [GitHub Issues](https://github.com/Byaidu/PDFMa
-

Public Service 🌟

- -### Free Service () +

Online Service 🌟

-You can try our [public service](https://pdf2zh.com/) online without installation. +You can try our application out using either of the following demos: -### Demos +- [Public free service](https://pdf2zh.com/) online without installation _(recommended)_. +- [Demo hosted on HuggingFace](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker) +- [Demo hosted on ModelScope](https://www.modelscope.cn/studios/AI-ModelScope/PDFMathTranslate) without installation. -You can try [our demo on HuggingFace](https://huggingface.co/spaces/reycn/PDFMathTranslate-Docker) or [our demo on ModelScope](https://www.modelscope.cn/studios/AI-ModelScope/PDFMathTranslate) without installation. Note that the computing resources of the demo are limited, so please avoid abusing them.

Installation and Usage

@@ -75,24 +68,25 @@ Note that the computing resources of the demo are limited, so please avoid abusi We provide four methods for using this project: [Commandline](#cmd), [Portable](#portable), [GUI](#gui), and [Docker](#docker). pdf2zh needs an extra model(`wybxc/DocLayout-YOLO-DocStructBench-onnx`), which can be found in modelscope. if you have a problem with downloading this model, try this environment variable: + ```shell USE_MODELSCOPE=1 pdf2zh ```

Method I. Commandline

- 1. Python installed (3.8 <= version <= 3.12) - 2. Install our package: +1. Python installed (3.8 <= version <= 3.12) +2. Install our package: - ```bash - pip install pdf2zh - ``` + ```bash + pip install pdf2zh + ``` - 3. Execute translation, files generated in [current working directory](https://chatgpt.com/share/6745ed36-9acc-800e-8a90-59204bd13444): +3. Execute translation, files generated in [current working directory](https://chatgpt.com/share/6745ed36-9acc-800e-8a90-59204bd13444): - ```bash - pdf2zh document.pdf - ``` + ```bash + pdf2zh document.pdf + ```

Method II. Portable

@@ -105,23 +99,23 @@ Download [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/r 1. Python installed (3.8 <= version <= 3.12) 2. Install our package: - ```bash - pip install pdf2zh - ``` + ```bash + pip install pdf2zh + ``` 3. Start using in browser: - ```bash - pdf2zh -i - ``` + ```bash + pdf2zh -i + ``` 4. If your browswer has not been started automatically, goto - ```bash - http://localhost:7860/ - ``` + ```bash + http://localhost:7860/ + ``` - + See [documentation for GUI](./docs/README_GUI.md) for more details. @@ -129,16 +123,16 @@ See [documentation for GUI](./docs/README_GUI.md) for more details. 1. Pull and run: - ```bash - docker pull byaidu/pdf2zh - docker run -d -p 7860:7860 byaidu/pdf2zh - ``` + ```bash + docker pull byaidu/pdf2zh + docker run -d -p 7860:7860 byaidu/pdf2zh + ``` 2. Open in browser: - ``` - http://localhost:7860/ - ``` + ``` + http://localhost:7860/ + ``` For docker deployment on cloud service: @@ -157,171 +151,36 @@ For docker deployment on cloud service: Execute the translation command in the command line to generate the translated document `example-mono.pdf` and the bilingual document `example-dual.pdf` in the current working directory. Use Google as the default translation service. -cmd +cmd In the following table, we list all advanced options for reference: -| Option | Function | Example | -| -------- | ------- |------- | -| files | Local files | `pdf2zh ~/local.pdf` | -| links | Online files | `pdf2zh http://arxiv.org/paper.pdf` | -| `-i` | [Enter GUI](#gui) | `pdf2zh -i` | -| `-p` | [Partial document translation](#partial) | `pdf2zh example.pdf -p 1` | -| `-li` | [Source language](#languages) | `pdf2zh example.pdf -li en` | -| `-lo` | [Target language](#languages) | `pdf2zh example.pdf -lo zh` | -| `-s` | [Translation service](#services) | `pdf2zh example.pdf -s deepl` | -| `-t` | [Multi-threads](#threads) | `pdf2zh example.pdf -t 1` | -| `-o` | Output dir | `pdf2zh example.pdf -o output` | -| `-f`, `-c` | [Exceptions](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | -| `--share` | [Get gradio public link] | `pdf2zh -i --share` | +| Option | Function | Example | +| -------------- | ----------------------------------------- | ---------------------------------------------- | +| files | Local files | `pdf2zh ~/local.pdf` | +| links | Online files | `pdf2zh http://arxiv.org/paper.pdf` | +| `-i` | [Enter GUI](#gui) | `pdf2zh -i` | +| `-p` | [Partial document translation](#partial) | `pdf2zh example.pdf -p 1` | +| `-li` | [Source language](#languages) | `pdf2zh example.pdf -li en` | +| `-lo` | [Target language](#languages) | `pdf2zh example.pdf -lo zh` | +| `-s` | [Translation service](#services) | `pdf2zh example.pdf -s deepl` | +| `-t` | [Multi-threads](#threads) | `pdf2zh example.pdf -t 1` | +| `-o` | Output dir | `pdf2zh example.pdf -o output` | +| `-f`, `-c` | [Exceptions](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | +| `--share` | [Get gradio public link] | `pdf2zh -i --share` | | `--authorized` | [add authorization and custom login page] | `pdf2zh -i --authorized users.txt [auth.html]` | -| `--prompt` | [custom llm prompt] | `pdf2zh --prompt [prompt.txt]` | - -

Full / partial document translation

- -- Entire document - - ```bash - pdf2zh example.pdf - ``` - -- Part of the document - - ```bash - pdf2zh example.pdf -p 1-3,5 - ``` - -

Specify source and target languages

- -See [Google Languages Codes](https://developers.google.com/admin-sdk/directory/v1/languages), [DeepL Languages Codes](https://developers.deepl.com/docs/resources/supported-languages) - -```bash -pdf2zh example.pdf -li en -lo ja -``` - -

Translate with Different Services

- -The table below outlines the required [environment variables](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4) for each translation service. Make sure to set them before using the respective service. - -|**Translator**|**Service**|**Environment Variables**|**Default Values**|**Notes**| -|-|-|-|-|-| -|**Google (Default)**|`google`|None|N/A|None| -|**Bing**|`bing`|None|N/A|None| -|**DeepL**|`deepl`|`DEEPL_AUTH_KEY`|`[Your Key]`|See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API)| -|**DeepLX**|`deeplx`|`DEEPLX_ENDPOINT`|`https://api.deepl.com/translate`|See [DeepLX](https://github.com/OwO-Network/DeepLX)| -|**Ollama**|`ollama`|`OLLAMA_HOST`, `OLLAMA_MODEL`|`http://127.0.0.1:11434`, `gemma2`|See [Ollama](https://github.com/ollama/ollama)| -|**OpenAI**|`openai`|`OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL`|`https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini`|See [OpenAI](https://platform.openai.com/docs/overview)| -|**AzureOpenAI**|`azure-openai`|`AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL`|`[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini`|See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python)| -|**Zhipu**|`zhipu`|`ZHIPU_API_KEY`, `ZHIPU_MODEL`|`[Your Key]`, `glm-4-flash`|See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk)| -| **ModelScope** | `ModelScope` |`MODELSCOPE_API_KEY`, `MODELSCOPE_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-Coder-32B-Instruct`| See [ModelScope](https://www.modelscope.cn/docs/model-service/API-Inference/intro)| -|**Silicon**|`silicon`|`SILICON_API_KEY`, `SILICON_MODEL`|`[Your Key]`, `Qwen/Qwen2.5-7B-Instruct`|See [SiliconCloud](https://docs.siliconflow.cn/quickstart)| -|**Gemini**|`gemini`|`GEMINI_API_KEY`, `GEMINI_MODEL`|`[Your Key]`, `gemini-1.5-flash`|See [Gemini](https://ai.google.dev/gemini-api/docs/openai)| -|**Azure**|`azure`|`AZURE_ENDPOINT`, `AZURE_API_KEY`|`https://api.translator.azure.cn`, `[Your Key]`|See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview)| -|**Tencent**|`tencent`|`TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY`|`[Your ID]`, `[Your Key]`|See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104)| -|**Dify**|`dify`|`DIFY_API_URL`, `DIFY_API_KEY`|`[Your DIFY URL]`, `[Your Key]`|See [Dify](https://github.com/langgenius/dify),Three variables, lang_out, lang_in, and text, need to be defined in Dify's workflow input.| -|**AnythingLLM**|`anythingllm`|`AnythingLLM_URL`, `AnythingLLM_APIKEY`|`[Your AnythingLLM URL]`, `[Your Key]`|See [anything-llm](https://github.com/Mintplex-Labs/anything-llm)| - -Use `-s service` or `-s service:model` to specify service: - -```bash -pdf2zh example.pdf -s openai:gpt-4o-mini -``` - -Or specify model with environment variables: - -```bash -set OPENAI_MODEL=gpt-4o-mini -pdf2zh example.pdf -s openai -``` +| `--prompt` | [custom llm prompt] | `pdf2zh --prompt [prompt.txt]` | -

Translate wih exceptions

+For detailed explanations, please refer to our document about [Advanced Usage](./docs/ADVANCED.md) for a full list of each option. -Use regex to specify formula fonts and characters that need to be preserved: - -```bash -pdf2zh example.pdf -f "(CM[^RT].*|MS.*|.*Ital)" -c "(\(|\||\)|\+|=|\d|[\u0080-\ufaff])" -``` - -Preserve `Latex`, `Mono`, `Code`, `Italic`, `Symbol` and `Math` fonts by default: - -```bash -pdf2zh example.pdf -f "(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-|rsfs|txsy|wasy|stmary|.*Mono|.*Code|.*Ital|.*Sym|.*Math)" -``` - -

Specify threads

- -Use `-t` to specify how many threads to use in translation: - -```bash -pdf2zh example.pdf -t 1 -``` -

custom prompt

-Use `--prompt` to specify which prompt to use in llm: -```bash -pdf2zh example.pdf -pr prompt.txt -``` - - -example prompt.txt -``` -[ - { - "role": "system", - "content": "You are a professional,authentic machine translation engine.", - }, - { - "role": "user", - "content": "Translate the following markdown source text to ${lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: ${text}\nTranslated Text:", - }, -] -``` +

Secondary Development (APIs)

+For downstream applications, please refer to our document about [API Details](./docs/APIS.md) for futher information about: -In custom prompt file, there are three variables can be used. -|**variables**|**comment**| -|-|-| -|`lang_in`|input language| -|`lang_out`|output language| -|`text`|text need to be translated| -

API

- -### Python - -```python -from pdf2zh import translate, translate_stream - -params = {"lang_in": "en", "lang_out": "zh", "service": "google", "thread": 4} -file_mono, file_dual = translate(files=["example.pdf"], **params)[0] -with open("example.pdf", "rb") as f: - stream_mono, stream_dual = translate_stream(stream=f.read(), **params) -``` - -### HTTP - -```bash -pip install pdf2zh[backend] -pdf2zh --flask -pdf2zh --celery worker -``` - -```bash -curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"lang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" -{"id":"d9894125-2f4e-45ea-9d93-1a9068d2045a"} - -curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a -{"info":{"n":13,"total":506},"state":"PROGRESS"} - -curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a -{"state":"SUCCESS"} - -curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/mono --output example-mono.pdf - -curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/dual --output example-dual.pdf - -curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a -X DELETE -``` +- [Python API](./docs/APIS.md#api-python), how to use the program in other Python programs +- [HTTP API](./docs/APIS.md#api-http), how to communicate with a server with the program installed -

TODO

+

TODOs

- [ ] Parse layout with DocLayNet based models, [PaddleX](https://github.com/PaddlePaddle/PaddleX/blob/17cc27ac3842e7880ca4aad92358d3ef8555429a/paddlex/repo_apis/PaddleDetection_api/object_det/official_categories.py#L81), [PaperMage](https://github.com/allenai/papermage/blob/9cd4bb48cbedab45d0f7a455711438f1632abebe/README.md?plain=1#L102), [SAM2](https://github.com/facebookresearch/sam2) diff --git a/docs/ADVANCED.md b/docs/ADVANCED.md new file mode 100644 index 00000000..c3e4f457 --- /dev/null +++ b/docs/ADVANCED.md @@ -0,0 +1,147 @@ +[**Documentation**](https://github.com/Byaidu/PDFMathTranslate) > **Advanced Usage** _(current)_ + +--- + +

Table of Contents

+ +- [Full / partial translation](#partial) +- [Specify source and target languages](#language) +- [Translate with different services](#services) +- [Translate wih exceptions](#exceptions) +- [Multi-threads](#threads) +- [Custom prompt](#prompt) + +--- + +

Full / partial translation

+ +- Entire document + + ```bash + pdf2zh example.pdf + ``` + +- Part of the document + + ```bash + pdf2zh example.pdf -p 1-3,5 + ``` + +[⬆️ Back to top](#toc) + +--- + +

Specify source and target languages

+ +See [Google Languages Codes](https://developers.google.com/admin-sdk/directory/v1/languages), [DeepL Languages Codes](https://developers.deepl.com/docs/resources/supported-languages) + +```bash +pdf2zh example.pdf -li en -lo ja +``` + +[⬆️ Back to top](#toc) + +--- + +

Translate with different services

+ +We've provided a detailed table on the required [environment variables](https://chatgpt.com/share/6734a83d-9d48-800e-8a46-f57ca6e8bcb4) for each translation service. Make sure to set them before using the respective service. + +| **Translator** | **Service** | **Environment Variables** | **Default Values** | **Notes** | +| -------------------- | -------------- | --------------------------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Google (Default)** | `google` | None | N/A | None | +| **Bing** | `bing` | None | N/A | None | +| **DeepL** | `deepl` | `DEEPL_AUTH_KEY` | `[Your Key]` | See [DeepL](https://support.deepl.com/hc/en-us/articles/360020695820-API-Key-for-DeepL-s-API) | +| **DeepLX** | `deeplx` | `DEEPLX_ENDPOINT` | `https://api.deepl.com/translate` | See [DeepLX](https://github.com/OwO-Network/DeepLX) | +| **Ollama** | `ollama` | `OLLAMA_HOST`, `OLLAMA_MODEL` | `http://127.0.0.1:11434`, `gemma2` | See [Ollama](https://github.com/ollama/ollama) | +| **OpenAI** | `openai` | `OPENAI_BASE_URL`, `OPENAI_API_KEY`, `OPENAI_MODEL` | `https://api.openai.com/v1`, `[Your Key]`, `gpt-4o-mini` | See [OpenAI](https://platform.openai.com/docs/overview) | +| **AzureOpenAI** | `azure-openai` | `AZURE_OPENAI_BASE_URL`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_MODEL` | `[Your Endpoint]`, `[Your Key]`, `gpt-4o-mini` | See [Azure OpenAI](https://learn.microsoft.com/zh-cn/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cjavascript-keyless%2Ctypescript-keyless%2Cpython&pivots=programming-language-python) | +| **Zhipu** | `zhipu` | `ZHIPU_API_KEY`, `ZHIPU_MODEL` | `[Your Key]`, `glm-4-flash` | See [Zhipu](https://open.bigmodel.cn/dev/api/thirdparty-frame/openai-sdk) | +| **ModelScope** | `ModelScope` | `MODELSCOPE_API_KEY`, `MODELSCOPE_MODEL` | `[Your Key]`, `Qwen/Qwen2.5-Coder-32B-Instruct` | See [ModelScope](https://www.modelscope.cn/docs/model-service/API-Inference/intro) | +| **Silicon** | `silicon` | `SILICON_API_KEY`, `SILICON_MODEL` | `[Your Key]`, `Qwen/Qwen2.5-7B-Instruct` | See [SiliconCloud](https://docs.siliconflow.cn/quickstart) | +| **Gemini** | `gemini` | `GEMINI_API_KEY`, `GEMINI_MODEL` | `[Your Key]`, `gemini-1.5-flash` | See [Gemini](https://ai.google.dev/gemini-api/docs/openai) | +| **Azure** | `azure` | `AZURE_ENDPOINT`, `AZURE_API_KEY` | `https://api.translator.azure.cn`, `[Your Key]` | See [Azure](https://docs.azure.cn/en-us/ai-services/translator/text-translation-overview) | +| **Tencent** | `tencent` | `TENCENTCLOUD_SECRET_ID`, `TENCENTCLOUD_SECRET_KEY` | `[Your ID]`, `[Your Key]` | See [Tencent](https://www.tencentcloud.com/products/tmt?from_qcintl=122110104) | +| **Dify** | `dify` | `DIFY_API_URL`, `DIFY_API_KEY` | `[Your DIFY URL]`, `[Your Key]` | See [Dify](https://github.com/langgenius/dify),Three variables, lang_out, lang_in, and text, need to be defined in Dify's workflow input. | +| **AnythingLLM** | `anythingllm` | `AnythingLLM_URL`, `AnythingLLM_APIKEY` | `[Your AnythingLLM URL]`, `[Your Key]` | See [anything-llm](https://github.com/Mintplex-Labs/anything-llm) | + +Use `-s service` or `-s service:model` to specify service: + +```bash +pdf2zh example.pdf -s openai:gpt-4o-mini +``` + +Or specify model with environment variables: + +```bash +set OPENAI_MODEL=gpt-4o-mini +pdf2zh example.pdf -s openai +``` + +[⬆️ Back to top](#toc) + +--- + +

Translate wih exceptions

+ +Use regex to specify formula fonts and characters that need to be preserved: + +```bash +pdf2zh example.pdf -f "(CM[^RT].*|MS.*|.*Ital)" -c "(\(|\||\)|\+|=|\d|[\u0080-\ufaff])" +``` + +Preserve `Latex`, `Mono`, `Code`, `Italic`, `Symbol` and `Math` fonts by default: + +```bash +pdf2zh example.pdf -f "(CM[^R]|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]|LINE|LCIRCLE|TeX-|rsfs|txsy|wasy|stmary|.*Mono|.*Code|.*Ital|.*Sym|.*Math)" +``` + +[⬆️ Back to top](#toc) + +--- + +

Multi-threads

+ +Use `-t` to specify how many threads to use in translation: + +```bash +pdf2zh example.pdf -t 1 +``` + +[⬆️ Back to top](#toc) + +--- + +

Custom prompt

+ +Use `--prompt` to specify which prompt to use in llm: + +```bash +pdf2zh example.pdf -pr prompt.txt +``` + +example prompt.txt + +``` +[ + { + "role": "system", + "content": "You are a professional,authentic machine translation engine.", + }, + { + "role": "user", + "content": "Translate the following markdown source text to ${lang_out}. Keep the formula notation {{v*}} unchanged. Output translation directly without any additional text.\nSource Text: ${text}\nTranslated Text:", + }, +] +``` + +In custom prompt file, there are three variables can be used. +|**variables**|**comment**| +|-|-| +|`lang_in`|input language| +|`lang_out`|output language| +|`text`|text need to be translated| + +[⬆️ Back to top](#toc) + +--- diff --git a/docs/APIS.md b/docs/APIS.md new file mode 100644 index 00000000..8f9ee170 --- /dev/null +++ b/docs/APIS.md @@ -0,0 +1,92 @@ +[**Documentation**](https://github.com/Byaidu/PDFMathTranslate) > **API Details** _(current)_ + +

Table of Content

+The present project supports two types of APIs; + +- [Functional calls in Python](#api-python) +- [HTTP protocols](#api-http) + +--- + +

Python

+ +As `pdf2zh` is an installed module in Python, we expose two methods for other programs to call in any Python scripts. + +For example, if you want translate a document from English to Chinese using Google Translate, you may use the following code: + +```python +from pdf2zh import translate, translate_stream + +params = { + 'lang_in': 'en', + 'lang_out': 'zh', + 'service': 'google', + 'thread': 4, + } +(file_mono, file_dual) = translate(files=['example.pdf'], **params)[0] + +with open('example.pdf', 'rb') as f: + (stream_mono, stream_dual) = translate_stream(stream=f.read(), + **params) + +``` + +[⬆️ Back to top](#toc) + +--- + +

HTTP

+ +In a more flexible way, you can communicate with the program using HTTP protocols, if: + +1. You have the backend installed & running + + ```bash + pip install pdf2zh[backend] + pdf2zh --flask + pdf2zh --celery worker + ``` + +2. Using HTTP protocols as follows: + + - Translate + + ```bash + curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"lang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" + {"id":"d9894125-2f4e-45ea-9d93-1a9068d2045a"} + ``` + + - Check Progress + + ```bash + curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a + {"info":{"n":13,"total":506},"state":"PROGRESS"} + ``` + + - Check Progress _(if finished)_ + + ```bash + curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a + {"state":"SUCCESS"} + ``` + + - Specifying output + + ```bash + curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/mono --output example-mono.pdf + ``` + + - Specifying the output as a bilingual file + + ```bash + curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/dual --output example-dual.pdf + ``` + + - Or delete it after the whole process + ```bash + curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a -X DELETE + ``` + +[⬆️ Back to top](#toc) + +--- From 63fa4b2daf8a3c9bd06d96120b6fedcad7d63b2e Mon Sep 17 00:00:00 2001 From: Reynard Date: Thu, 19 Dec 2024 01:10:17 +0800 Subject: [PATCH 100/114] doc: simplified readme, step 2 --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 906a6618..546e4c7e 100644 --- a/README.md +++ b/README.md @@ -155,21 +155,21 @@ Execute the translation command in the command line to generate the translated d In the following table, we list all advanced options for reference: -| Option | Function | Example | -| -------------- | ----------------------------------------- | ---------------------------------------------- | -| files | Local files | `pdf2zh ~/local.pdf` | -| links | Online files | `pdf2zh http://arxiv.org/paper.pdf` | -| `-i` | [Enter GUI](#gui) | `pdf2zh -i` | -| `-p` | [Partial document translation](#partial) | `pdf2zh example.pdf -p 1` | -| `-li` | [Source language](#languages) | `pdf2zh example.pdf -li en` | -| `-lo` | [Target language](#languages) | `pdf2zh example.pdf -lo zh` | -| `-s` | [Translation service](#services) | `pdf2zh example.pdf -s deepl` | -| `-t` | [Multi-threads](#threads) | `pdf2zh example.pdf -t 1` | -| `-o` | Output dir | `pdf2zh example.pdf -o output` | -| `-f`, `-c` | [Exceptions](#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | -| `--share` | [Get gradio public link] | `pdf2zh -i --share` | -| `--authorized` | [add authorization and custom login page] | `pdf2zh -i --authorized users.txt [auth.html]` | -| `--prompt` | [custom llm prompt] | `pdf2zh --prompt [prompt.txt]` | +| Option | Function | Example | +| -------------- | ------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | +| files | Local files | `pdf2zh ~/local.pdf` | +| links | Online files | `pdf2zh http://arxiv.org/paper.pdf` | +| `-i` | [Enter GUI](#gui) | `pdf2zh -i` | +| `-p` | [Partial document translation](https://github.com/Byaidu/PDFMathTranslate/blob/main/docs/ADVANCED.md#partial) | `pdf2zh example.pdf -p 1` | +| `-li` | [Source language](https://github.com/Byaidu/PDFMathTranslate/blob/main/docs/ADVANCED.md#languages) | `pdf2zh example.pdf -li en` | +| `-lo` | [Target language](https://github.com/Byaidu/PDFMathTranslate/blob/main/docs/ADVANCED.md#languages) | `pdf2zh example.pdf -lo zh` | +| `-s` | [Translation service](https://github.com/Byaidu/PDFMathTranslate/blob/main/docs/ADVANCED.md#services) | `pdf2zh example.pdf -s deepl` | +| `-t` | [Multi-threads](https://github.com/Byaidu/PDFMathTranslate/blob/main/docs/ADVANCED.md#threads) | `pdf2zh example.pdf -t 1` | +| `-o` | Output dir | `pdf2zh example.pdf -o output` | +| `-f`, `-c` | [Exceptions](https://github.com/Byaidu/PDFMathTranslate/blob/main/docs/ADVANCED.md#exceptions) | `pdf2zh example.pdf -f "(MS.*)"` | +| `--share` | Public link | `pdf2zh -i --share` | +| `--authorized` | Authorization | `pdf2zh -i --authorized users.txt [auth.html]` | +| `--prompt` | [Custom Prompt](https://github.com/Byaidu/PDFMathTranslate/blob/main/docs/ADVANCED.md#prompt) | `pdf2zh --prompt [prompt.txt]` | For detailed explanations, please refer to our document about [Advanced Usage](./docs/ADVANCED.md) for a full list of each option. From 7d45c7b76e4c984d203ece300bd0e7052080c748 Mon Sep 17 00:00:00 2001 From: Reynard Date: Thu, 19 Dec 2024 01:23:22 +0800 Subject: [PATCH 101/114] doc: simplified readme, step 3 --- README.md | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 546e4c7e..96549c30 100644 --- a/README.md +++ b/README.md @@ -65,15 +65,12 @@ Note that the computing resources of the demo are limited, so please avoid abusi

Installation and Usage

-We provide four methods for using this project: [Commandline](#cmd), [Portable](#portable), [GUI](#gui), and [Docker](#docker). +### Methods -pdf2zh needs an extra model(`wybxc/DocLayout-YOLO-DocStructBench-onnx`), which can be found in modelscope. if you have a problem with downloading this model, try this environment variable: +For different use cases, we provide four distinct methods to use our program: -```shell -USE_MODELSCOPE=1 pdf2zh -``` - -

Method I. Commandline

+
+ 1. Commandline 1. Python installed (3.8 <= version <= 3.12) 2. Install our package: @@ -88,20 +85,25 @@ USE_MODELSCOPE=1 pdf2zh pdf2zh document.pdf ``` -

Method II. Portable

+
-No need to pre-install Python environment +
+ 2. Portable (w/o Python installed) -Download [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/script/setup.bat) and double-click to run +1. Download [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/refs/heads/main/script/setup.bat) -

Method III. GUI

+2. Double-click to run. +
+ +
+ 3. Graphic user interface 1. Python installed (3.8 <= version <= 3.12) 2. Install our package: - ```bash - pip install pdf2zh - ``` +```bash +pip install pdf2zh +``` 3. Start using in browser: @@ -119,7 +121,10 @@ Download [setup.bat](https://raw.githubusercontent.com/Byaidu/PDFMathTranslate/r See [documentation for GUI](./docs/README_GUI.md) for more details. -

Method IV. Docker

+
+ +
+ 4. Docker 1. Pull and run: @@ -147,6 +152,18 @@ For docker deployment on cloud service: Deploy to Koyeb
+ + +### Unable to install? + +The present program needs an AI model(`wybxc/DocLayout-YOLO-DocStructBench-onnx`) before working and some users are not able to download due to network issues. If you have a problem with downloading this model, we provide a workaround using the following environment variable: + + ```shell + USE_MODELSCOPE=1 pdf2zh + ``` + +If the solution does not work to you / you encountered other issues, please refer to [frequently asked questions](https://github.com/Byaidu/PDFMathTranslate/wiki#-faq--%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98). +

Advanced Options

Execute the translation command in the command line to generate the translated document `example-mono.pdf` and the bilingual document `example-dual.pdf` in the current working directory. Use Google as the default translation service. From 8a201025089d0dc25c8bca02b23b4b0ce3022be9 Mon Sep 17 00:00:00 2001 From: awwaawwa <8493196+awwaawwa@users.noreply.github.com> Date: Thu, 19 Dec 2024 01:40:14 +0800 Subject: [PATCH 102/114] feat (docker): onnx model and font embedded (#276) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 将onnx模型和字体打包进docker镜像 * 调整docker构建顺序，先下模型后装包，并将预热更改为dockerfile RUN指令 * 移除无用import * 正确处理authorized为None * translate函数中更好的处理envs和prompt不存在的情况 * 移除docker镜像中无用文件 --------- Co-authored-by: Byaidu <909756245@qq.com> --- .dockerignore | 175 +++++++++++++++++++++++++++++++++++++++++++ .gitignore | 3 +- Dockerfile | 12 ++- pdf2zh/gui.py | 2 +- pdf2zh/high_level.py | 7 +- 5 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..f4bd0753 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,175 @@ +.github +docs +.git +.pre-commit-config.yaml +uv.lock +pdf2zh_files +gui/pdf2zh_files +gradio_files +tmp +gui/gradio_files +gui/tmp +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ +.vscode +.DS_Store diff --git a/.gitignore b/.gitignore index eb081f82..264d5513 100644 --- a/.gitignore +++ b/.gitignore @@ -165,6 +165,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ .vscode .DS_Store +uv.lock diff --git a/Dockerfile b/Dockerfile index bd9e25b3..91b8920e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,19 @@ -FROM python:3.12 +FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim WORKDIR /app -COPY . . EXPOSE 7860 ENV PYTHONUNBUFFERED=1 +ADD "https://github.com/satbyy/go-noto-universal/releases/download/v7.0/GoNotoKurrent-Regular.ttf" /app +RUN apt-get update && \ + apt-get install --no-install-recommends -y libgl1 && \ + rm -rf /var/lib/apt/lists/* && uv pip install --system --no-cache huggingface-hub && \ + python3 -c "from huggingface_hub import hf_hub_download; hf_hub_download('wybxc/DocLayout-YOLO-DocStructBench-onnx','doclayout_yolo_docstructbench_imgsz1024.onnx');" -RUN apt-get update && apt-get install -y libgl1 +COPY . . -RUN pip install . +RUN uv pip install --system --no-cache . CMD ["pdf2zh", "-i"] \ No newline at end of file diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index bb882b77..c643a785 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -445,7 +445,7 @@ def on_select_filetype(file_type): def readuserandpasswd(file_path): tuple_list = [] content = "" - if file_path is None or len(file_path) == 0: + if file_path is None: return tuple_list, content if len(file_path) == 2: try: diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index d6ebd79c..0e86e5cb 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -187,7 +187,10 @@ def translate_stream( font_list.append((resfont, None)) elif lang_out.lower() in noto_list: # noto resfont = "noto" - ttf_path = os.path.join(tempfile.gettempdir(), "GoNotoKurrent-Regular.ttf") + # docker + ttf_path = '/app/GoNotoKurrent-Regular.ttf' + if not os.path.exists(ttf_path): + ttf_path = os.path.join(tempfile.gettempdir(), "GoNotoKurrent-Regular.ttf") if not os.path.exists(ttf_path): print("Downloading Noto font...") urllib.request.urlretrieve( @@ -294,7 +297,7 @@ def translate( doc_raw = open(file, "rb") s_raw = doc_raw.read() s_mono, s_dual = translate_stream( - s_raw, envs=kwarg.get("envs"), prompt=kwarg["prompt"], **locals() + s_raw, envs=kwarg.get("envs", {}), prompt=kwarg.get("prompt", []), **locals() ) file_mono = Path(output) / f"{filename}-mono.pdf" file_dual = Path(output) / f"{filename}-dual.pdf" From 10d9cf19e837f47eb6de15045ac2b02c9cc0a5ef Mon Sep 17 00:00:00 2001 From: hellofinch <42131198+hellofinch@users.noreply.github.com> Date: Thu, 19 Dec 2024 01:42:24 +0800 Subject: [PATCH 103/114] feat (gui): add custom prompt (#275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gui支持自定义页码。 * 修改页数显示bug * GUI支持自定义prompt。 * format --------- Co-authored-by: Byaidu <909756245@qq.com> --- pdf2zh/gui.py | 47 ++++++++++++++++++++++++++++++++++++++++---- pdf2zh/translator.py | 9 +++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index c643a785..c54b836e 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -64,6 +64,7 @@ "All": None, "First": [0], "First 5 pages": list(range(0, 5)), + "Others": None, } flag_demo = False @@ -125,6 +126,9 @@ def translate_file( lang_from, lang_to, page_range, + page_input, + prompt, + threads, recaptcha_response, state, progress=gr.Progress(), @@ -161,7 +165,16 @@ def translate_file( file_dual = output / f"{filename}-dual.pdf" translator = service_map[service] - selected_page = page_map[page_range] + if page_range != "Others": + selected_page = page_map[page_range] + else: + selected_page = [] + for p in page_input.split(","): + if "-" in p: + start, end = p.split("-") + selected_page.extend(range(int(start) - 1, int(end))) + else: + selected_page.append(int(p) - 1) lang_from = lang_map[lang_from] lang_to = lang_map[lang_to] @@ -181,10 +194,11 @@ def progress_bar(t: tqdm.tqdm): "lang_out": lang_to, "service": f"{translator.name}", "output": output, - "thread": 4, + "thread": int(threads), "callback": progress_bar, "cancellation_event": cancellation_event_map[session_id], "envs": _envs, + "prompt": prompt, } try: translate(**param) @@ -319,15 +333,30 @@ def progress_bar(t: tqdm.tqdm): value=list(page_map.keys())[0], ) + page_input = gr.Textbox( + label="Page range", + visible=False, + interactive=True, + ) + + with gr.Accordion("Open for More Experimental Options!", open=False): + gr.Markdown("#### Experimental") + threads = gr.Textbox(label="number of threads", interactive=True) + prompt = gr.Textbox( + label="Custom Prompt for llm", interactive=True, visible=False + ) + envs.append(prompt) + def on_select_service(service, evt: gr.EventData): translator = service_map[service] _envs = [] - for i in range(3): + for i in range(4): _envs.append(gr.update(visible=False, value="")) for i, env in enumerate(translator.envs.items()): _envs[i] = gr.update( visible=True, label=env[0], value=os.getenv(env[0], env[1]) ) + _envs[-1] = gr.update(visible=translator.CustomPrompt) return _envs def on_select_filetype(file_type): @@ -336,6 +365,12 @@ def on_select_filetype(file_type): gr.update(visible=file_type == "Link"), ) + def on_select_page(choice): + if choice == "Others": + return gr.update(visible=True) + else: + return gr.update(visible=False) + output_title = gr.Markdown("## Translated", visible=False) output_file_mono = gr.File( label="Download Translation (Mono)", visible=False @@ -358,6 +393,7 @@ def on_select_filetype(file_type): """, elem_classes=["secondary-text"], ) + page_range.select(on_select_page, page_range, page_input) service.select( on_select_service, service, @@ -422,6 +458,9 @@ def on_select_filetype(file_type): lang_from, lang_to, page_range, + page_input, + prompt, + threads, recaptcha_response, state, *envs, @@ -445,7 +484,7 @@ def on_select_filetype(file_type): def readuserandpasswd(file_path): tuple_list = [] content = "" - if file_path is None: + if not file_path: return tuple_list, content if len(file_path) == 2: try: diff --git a/pdf2zh/translator.py b/pdf2zh/translator.py index fa2cd44f..5046f6e3 100644 --- a/pdf2zh/translator.py +++ b/pdf2zh/translator.py @@ -26,6 +26,7 @@ class BaseTranslator: name = "base" envs = {} lang_map = {} + CustomPrompt = False def __init__(self, lang_in, lang_out, model): lang_in = self.lang_map.get(lang_in.lower(), lang_in) @@ -200,6 +201,7 @@ class OllamaTranslator(BaseTranslator): "OLLAMA_HOST": "http://127.0.0.1:11434", "OLLAMA_MODEL": "gemma2", } + CustomPrompt = True def __init__(self, lang_in, lang_out, model, envs=None, prompt=None): self.set_envs(envs) @@ -230,6 +232,7 @@ class OpenAITranslator(BaseTranslator): "OPENAI_API_KEY": None, "OPENAI_MODEL": "gpt-4o-mini", } + CustomPrompt = True def __init__( self, @@ -265,6 +268,7 @@ class AzureOpenAITranslator(BaseTranslator): "AZURE_OPENAI_API_KEY": None, "AZURE_OPENAI_MODEL": "gpt-4o-mini", } + CustomPrompt = True def __init__( self, @@ -306,6 +310,7 @@ class ModelScopeTranslator(OpenAITranslator): "MODELSCOPE_API_KEY": None, "MODELSCOPE_MODEL": "Qwen/Qwen2.5-32B-Instruct", } + CustomPrompt = True def __init__( self, @@ -333,6 +338,7 @@ class ZhipuTranslator(OpenAITranslator): "ZHIPU_API_KEY": None, "ZHIPU_MODEL": "glm-4-flash", } + CustomPrompt = True def __init__(self, lang_in, lang_out, model, envs=None, prompt=None): self.set_envs(envs) @@ -367,6 +373,7 @@ class SiliconTranslator(OpenAITranslator): "SILICON_API_KEY": None, "SILICON_MODEL": "Qwen/Qwen2.5-7B-Instruct", } + CustomPrompt = True def __init__(self, lang_in, lang_out, model, envs=None, prompt=None): self.set_envs(envs) @@ -385,6 +392,7 @@ class GeminiTranslator(OpenAITranslator): "GEMINI_API_KEY": None, "GEMINI_MODEL": "gemini-1.5-flash", } + CustomPrompt = True def __init__(self, lang_in, lang_out, model, envs=None, prompt=None): self.set_envs(envs) @@ -458,6 +466,7 @@ class AnythingLLMTranslator(BaseTranslator): "AnythingLLM_URL": None, "AnythingLLM_APIKEY": None, } + CustomPrompt = True def __init__(self, lang_out, lang_in, model, envs=None, prompt=None): self.set_envs(envs) From 374284af7f73ea8f60475c9cb8d8b07bcf3702b9 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Thu, 19 Dec 2024 01:52:34 +0800 Subject: [PATCH 104/114] chore: format --- pdf2zh/high_level.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pdf2zh/high_level.py b/pdf2zh/high_level.py index 0e86e5cb..41d760d4 100644 --- a/pdf2zh/high_level.py +++ b/pdf2zh/high_level.py @@ -188,7 +188,7 @@ def translate_stream( elif lang_out.lower() in noto_list: # noto resfont = "noto" # docker - ttf_path = '/app/GoNotoKurrent-Regular.ttf' + ttf_path = "/app/GoNotoKurrent-Regular.ttf" if not os.path.exists(ttf_path): ttf_path = os.path.join(tempfile.gettempdir(), "GoNotoKurrent-Regular.ttf") if not os.path.exists(ttf_path): @@ -297,7 +297,10 @@ def translate( doc_raw = open(file, "rb") s_raw = doc_raw.read() s_mono, s_dual = translate_stream( - s_raw, envs=kwarg.get("envs", {}), prompt=kwarg.get("prompt", []), **locals() + s_raw, + envs=kwarg.get("envs", {}), + prompt=kwarg.get("prompt", []), + **locals(), ) file_mono = Path(output) / f"{filename}-mono.pdf" file_dual = Path(output) / f"{filename}-dual.pdf" From a4f0415ab3cac7a4abe72cd54d647585d2071dfa Mon Sep 17 00:00:00 2001 From: Reynard Date: Thu, 19 Dec 2024 01:54:30 +0800 Subject: [PATCH 105/114] ci (format): add auto-black formatter --- .github/workflows/black.format.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/black.format.yml diff --git a/.github/workflows/black.format.yml b/.github/workflows/black.format.yml new file mode 100644 index 00000000..661a0719 --- /dev/null +++ b/.github/workflows/black.format.yml @@ -0,0 +1,10 @@ +name: Format Code with Black + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: psf/black@stable \ No newline at end of file From f495797510ffa40a04e55893f05ddcd4e6578a21 Mon Sep 17 00:00:00 2001 From: Yadomin Jinta Date: Thu, 19 Dec 2024 00:42:14 +0800 Subject: [PATCH 106/114] feat: add unit test --- .github/workflows/python-build.yml | 12 +-- pyproject.toml | 4 +- test/test_converter.py | 132 +++++++++++++++++++++++++++++ test/test_doclayout.py | 105 +++++++++++++++++++++++ 4 files changed, 246 insertions(+), 7 deletions(-) create mode 100644 test/test_converter.py create mode 100644 test/test_doclayout.py diff --git a/.github/workflows/python-build.yml b/.github/workflows/python-build.yml index d4dd3c0c..a82fa28c 100644 --- a/.github/workflows/python-build.yml +++ b/.github/workflows/python-build.yml @@ -19,17 +19,17 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install build flake8 black + pip3 install -U pip + pip3 install -e .[dev] - name: Test - Code format run: | black --check --diff --color pdf2zh/*.py flake8 --ignore E203,E261,E501,W503,E741 - - - name: Test - Local installation - run: - python -m pip install -e . + + - name: Test - Unit Test + run: | + pytest . - name: Test - Translate a PDF file with plain text only run: diff --git a/pyproject.toml b/pyproject.toml index c8bdefa5..82e334fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,9 @@ dependencies = [ dev = [ "black", "flake8", - "pre-commit" + "pre-commit", + "pytest", + "build" ] backend = [ "flask", diff --git a/test/test_converter.py b/test/test_converter.py new file mode 100644 index 00000000..dbc57f95 --- /dev/null +++ b/test/test_converter.py @@ -0,0 +1,132 @@ +import unittest +from unittest.mock import Mock, patch, MagicMock +from pdfminer.layout import LTPage, LTChar, LTLine +from pdfminer.pdfinterp import PDFResourceManager +from pdf2zh.converter import PDFConverterEx, TranslateConverter + + +class TestPDFConverterEx(unittest.TestCase): + def setUp(self): + self.rsrcmgr = PDFResourceManager() + self.converter = PDFConverterEx(self.rsrcmgr) + + def test_begin_page(self): + mock_page = Mock() + mock_page.pageno = 1 + mock_page.cropbox = (0, 0, 100, 200) + mock_ctm = [1, 0, 0, 1, 0, 0] + self.converter.begin_page(mock_page, mock_ctm) + self.assertIsNotNone(self.converter.cur_item) + self.assertEqual(self.converter.cur_item.pageid, 1) + + def test_render_char(self): + mock_matrix = (1, 2, 3, 4, 5, 6) + mock_font = Mock() + mock_font.to_unichr.return_value = "A" + mock_font.char_width.return_value = 10 + mock_font.char_disp.return_value = (0, 0) + graphic_state = Mock() + self.converter.cur_item = Mock() + result = self.converter.render_char( + mock_matrix, + mock_font, + fontsize=12, + scaling=1.0, + rise=0, + cid=65, + ncs=None, + graphicstate=graphic_state, + ) + self.assertEqual(result, 120.0) # Expected text width + + +class TestTranslateConverter(unittest.TestCase): + def setUp(self): + self.rsrcmgr = PDFResourceManager() + self.layout = {1: Mock()} + self.translator_class = Mock() + self.converter = TranslateConverter( + self.rsrcmgr, + layout=self.layout, + lang_in="en", + lang_out="zh", + service="google", + ) + + def test_translator_initialization(self): + self.assertIsNotNone(self.converter.translator) + self.assertEqual(self.converter.translator.lang_in, "en") + self.assertEqual(self.converter.translator.lang_out, "zh-CN") + + @patch("pdf2zh.converter.TranslateConverter.receive_layout") + def test_receive_layout(self, mock_receive_layout): + mock_page = LTPage(1, (0, 0, 100, 200)) + mock_font = Mock() + mock_font.fontname.return_value = "mock_font" + mock_page.add( + LTChar( + matrix=(1, 2, 3, 4, 5, 6), + font=mock_font, + fontsize=12, + scaling=1.0, + rise=0, + text="A", + textwidth=10, + textdisp=(1.0, 1.0), + ncs=Mock(), + graphicstate=Mock(), + ) + ) + self.converter.receive_layout(mock_page) + mock_receive_layout.assert_called_once_with(mock_page) + + @patch("concurrent.futures.ThreadPoolExecutor") + @patch("pdf2zh.cache") + def test_translation(self, mock_cache, mock_executor): + mock_executor.return_value.__enter__.return_value.map.return_value = [ + "你好", + "{v1}", + ] + mock_cache.deterministic_hash.return_value = "test_hash" + mock_cache.load_paragraph.return_value = None + mock_cache.write_paragraph.return_value = None + + sstk = ["Hello", "{v1}"] + self.converter.thread = 2 + results = [] + with patch.object(self.converter, "translator") as mock_translator: + mock_translator.translate.side_effect = lambda x: ( + "你好" if x == "Hello" else x + ) + for s in sstk: + results.append(self.converter.translator.translate(s)) + self.assertEqual(results, ["你好", "{v1}"]) + + def test_receive_layout_with_complex_formula(self): + ltpage = LTPage(1, (0, 0, 500, 500)) + ltchar = Mock() + ltchar.fontname.return_value = "mock_font" + ltline = LTLine(0.1, (0, 0), (10, 20)) + ltpage.add(ltchar) + ltpage.add(ltline) + mock_layout = MagicMock() + mock_layout.shape = (100, 100) + mock_layout.__getitem__.return_value = -1 + self.converter.layout = [None, mock_layout] + self.converter.thread = 1 + result = self.converter.receive_layout(ltpage) + self.assertIsNotNone(result) + + def test_invalid_translation_service(self): + with self.assertRaises(ValueError): + TranslateConverter( + self.rsrcmgr, + layout=self.layout, + lang_in="en", + lang_out="zh", + service="InvalidService", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_doclayout.py b/test/test_doclayout.py new file mode 100644 index 00000000..3add5618 --- /dev/null +++ b/test/test_doclayout.py @@ -0,0 +1,105 @@ +import unittest +from unittest.mock import patch, MagicMock +import numpy as np +from pdf2zh.doclayout import ( + OnnxModel, + YoloResult, + YoloBox, +) + + +class TestOnnxModel(unittest.TestCase): + @patch("onnx.load") + @patch("onnxruntime.InferenceSession") + def setUp(self, mock_inference_session, mock_onnx_load): + # Mock ONNX model metadata + mock_model = MagicMock() + mock_model.metadata_props = [ + MagicMock(key="stride", value="32"), + MagicMock(key="names", value="['class1', 'class2']"), + ] + mock_onnx_load.return_value = mock_model + + # Initialize OnnxModel with a fake path + self.model_path = "fake_model_path.onnx" + self.model = OnnxModel(self.model_path) + + def test_stride_property(self): + # Test that stride is correctly set from model metadata + self.assertEqual(self.model.stride, 32) + + def test_resize_and_pad_image(self): + # Create a dummy image (100x200) + image = np.ones((100, 200, 3), dtype=np.uint8) + resized_image = self.model.resize_and_pad_image(image, 1024) + + # Validate the output shape + self.assertEqual(resized_image.shape[0], 512) + self.assertEqual(resized_image.shape[1], 1024) + + # Check that padding has been added + padded_height = resized_image.shape[0] - image.shape[0] + padded_width = resized_image.shape[1] - image.shape[1] + self.assertGreater(padded_height, 0) + self.assertGreater(padded_width, 0) + + def test_scale_boxes(self): + img1_shape = (1024, 1024) # Model input shape + img0_shape = (500, 300) # Original image shape + boxes = np.array([[512, 512, 768, 768]]) # Example bounding box + + scaled_boxes = self.model.scale_boxes(img1_shape, boxes, img0_shape) + + # Verify the output is scaled correctly + self.assertEqual(scaled_boxes.shape, boxes.shape) + self.assertTrue(np.all(scaled_boxes <= max(img0_shape))) + + def test_predict(self): + # Mock model inference output + mock_output = np.random.random((1, 300, 6)) + self.model.model.run.return_value = [mock_output] + + # Create a dummy image + image = np.ones((500, 300, 3), dtype=np.uint8) + + results = self.model.predict(image) + + # Validate predictions + self.assertEqual(len(results), 1) + self.assertIsInstance(results[0], YoloResult) + self.assertGreater(len(results[0].boxes), 0) + self.assertIsInstance(results[0].boxes[0], YoloBox) + + +class TestYoloResult(unittest.TestCase): + def test_yolo_result(self): + # Example prediction data + boxes = [ + [100, 200, 300, 400, 0.9, 0], + [50, 100, 150, 200, 0.8, 1], + ] + names = ["class1", "class2"] + + result = YoloResult(boxes, names) + + # Validate the number of boxes and their order by confidence + self.assertEqual(len(result.boxes), 2) + self.assertGreater(result.boxes[0].conf, result.boxes[1].conf) + self.assertEqual(result.names, names) + + +class TestYoloBox(unittest.TestCase): + def test_yolo_box(self): + # Example box data + box_data = [100, 200, 300, 400, 0.9, 0] + + box = YoloBox(box_data) + + # Validate box properties + self.assertEqual(box.xyxy, box_data[:4]) + self.assertEqual(box.conf, box_data[4]) + self.assertEqual(box.cls, box_data[5]) + + +if __name__ == "__main__": + unittest.main() From a353e68cffae8cb70305e9f4ada2595de266eccb Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Thu, 19 Dec 2024 02:15:26 +0800 Subject: [PATCH 107/114] doc: HF_ENDPOINT --- README.md | 8 ++++---- docs/README_ja-JP.md | 2 +- docs/README_zh-CN.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 96549c30..492c1a45 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Note that the computing resources of the demo are limited, so please avoid abusi For different use cases, we provide four distinct methods to use our program: -
+
1. Commandline 1. Python installed (3.8 <= version <= 3.12) @@ -158,9 +158,9 @@ For docker deployment on cloud service: The present program needs an AI model(`wybxc/DocLayout-YOLO-DocStructBench-onnx`) before working and some users are not able to download due to network issues. If you have a problem with downloading this model, we provide a workaround using the following environment variable: - ```shell - USE_MODELSCOPE=1 pdf2zh - ``` +```shell +set HF_ENDPOINT=https://hf-mirror.com +``` If the solution does not work to you / you encountered other issues, please refer to [frequently asked questions](https://github.com/Byaidu/PDFMathTranslate/wiki#-faq--%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98). diff --git a/docs/README_ja-JP.md b/docs/README_ja-JP.md index 8e36893b..d88912e3 100644 --- a/docs/README_ja-JP.md +++ b/docs/README_ja-JP.md @@ -77,7 +77,7 @@ pdf2zhの実行には追加モデル（`wybxc/DocLayout-YOLO-DocStructBench-onnx`）が必要です。このモデルはModelScopeでも見つけることができます。起動時にこのモデルのダウンロードに問題がある場合は、以下の環境変数を使用してください： ```shell -USE_MODELSCOPE=1 pdf2zh +set HF_ENDPOINT=https://hf-mirror.com ```

方法1. コマンドライン

diff --git a/docs/README_zh-CN.md b/docs/README_zh-CN.md index 0d7b23b8..42e273b2 100644 --- a/docs/README_zh-CN.md +++ b/docs/README_zh-CN.md @@ -76,7 +76,7 @@ pdf2zh的运行依赖于额外模型(`wybxc/DocLayout-YOLO-DocStructBench-onnx`)，该模型在魔搭上也可以找到。如果你在启动时下载该模型遇到问题，请使用如下环境变量： ```shell -USE_MODELSCOPE=1 pdf2zh +set HF_ENDPOINT=https://hf-mirror.com ```

方法一、命令行工具

From 55b16c0d5599d4bde4e79239f3def0b63c65b15c Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Thu, 19 Dec 2024 02:23:38 +0800 Subject: [PATCH 108/114] doc: api --- docs/APIS.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/APIS.md b/docs/APIS.md index 8f9ee170..6ece971d 100644 --- a/docs/APIS.md +++ b/docs/APIS.md @@ -22,13 +22,16 @@ params = { 'lang_out': 'zh', 'service': 'google', 'thread': 4, - } +} +``` +Translate with files: +```python (file_mono, file_dual) = translate(files=['example.pdf'], **params)[0] - +``` +Translate with stream: +```python with open('example.pdf', 'rb') as f: - (stream_mono, stream_dual) = translate_stream(stream=f.read(), - **params) - + (stream_mono, stream_dual) = translate_stream(stream=f.read(), **params) ``` [⬆️ Back to top](#toc) @@ -39,7 +42,7 @@ with open('example.pdf', 'rb') as f: In a more flexible way, you can communicate with the program using HTTP protocols, if: -1. You have the backend installed & running +1. Install and run backend ```bash pip install pdf2zh[backend] @@ -49,7 +52,7 @@ In a more flexible way, you can communicate with the program using HTTP protocol 2. Using HTTP protocols as follows: - - Translate + - Submit translate task ```bash curl http://localhost:11008/v1/translate -F "file=@example.pdf" -F "data={\"lang_in\":\"en\",\"lang_out\":\"zh\",\"service\":\"google\",\"thread\":4}" @@ -70,19 +73,19 @@ In a more flexible way, you can communicate with the program using HTTP protocol {"state":"SUCCESS"} ``` - - Specifying output + - Save monolingual file ```bash curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/mono --output example-mono.pdf ``` - - Specifying the output as a bilingual file + - Save bilingual file ```bash curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a/dual --output example-dual.pdf ``` - - Or delete it after the whole process + - Interrupt if running and delete the task ```bash curl http://localhost:11008/v1/translate/d9894125-2f4e-45ea-9d93-1a9068d2045a -X DELETE ``` From ab2773e144fe9bcb5b88d992c72728f9ac719fce Mon Sep 17 00:00:00 2001 From: charles7668 Date: Thu, 19 Dec 2024 02:07:43 +0000 Subject: [PATCH 109/114] Fix an issue where translation would fail if the threads value was not provided --- pdf2zh/gui.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index c54b836e..b798ce15 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -187,6 +187,11 @@ def translate_file( def progress_bar(t: tqdm.tqdm): progress(t.n / t.total, desc="Translating...") + try: + threads = int(threads) + except ValueError: + threads = 1 + param = { "files": [str(file_raw)], "pages": selected_page, @@ -194,7 +199,7 @@ def progress_bar(t: tqdm.tqdm): "lang_out": lang_to, "service": f"{translator.name}", "output": output, - "thread": int(threads), + "thread": threads, "callback": progress_bar, "cancellation_event": cancellation_event_map[session_id], "envs": _envs, @@ -341,7 +346,9 @@ def progress_bar(t: tqdm.tqdm): with gr.Accordion("Open for More Experimental Options!", open=False): gr.Markdown("#### Experimental") - threads = gr.Textbox(label="number of threads", interactive=True) + threads = gr.Textbox( + label="number of threads", interactive=True, value="1" + ) prompt = gr.Textbox( label="Custom Prompt for llm", interactive=True, visible=False ) From 69b02ee735b3cbedfd0c306d550a28feb897dfcc Mon Sep 17 00:00:00 2001 From: Rongxin Date: Thu, 19 Dec 2024 12:33:53 +0800 Subject: [PATCH 110/114] fix (style): pep8 method name --- pdf2zh/gui.py | 43 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index c54b836e..34749179 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -1,35 +1,26 @@ +import asyncio +import cgi import os import shutil import uuid -import asyncio from asyncio import CancelledError from pathlib import Path -from pdf2zh import __version__ -from pdf2zh.high_level import translate -from pdf2zh.translator import ( - BaseTranslator, - GoogleTranslator, - BingTranslator, - DeepLTranslator, - DeepLXTranslator, - OllamaTranslator, - AzureOpenAITranslator, - OpenAITranslator, - ZhipuTranslator, - ModelScopeTranslator, - SiliconTranslator, - GeminiTranslator, - AzureTranslator, - TencentTranslator, - DifyTranslator, - AnythingLLMTranslator, -) import gradio as gr -from gradio_pdf import PDF -import tqdm import requests -import cgi +import tqdm +from gradio_pdf import PDF + +from pdf2zh import __version__ +from pdf2zh.high_level import translate +from pdf2zh.translator import (AnythingLLMTranslator, AzureOpenAITranslator, + AzureTranslator, BaseTranslator, BingTranslator, + DeepLTranslator, DeepLXTranslator, + DifyTranslator, GeminiTranslator, + GoogleTranslator, ModelScopeTranslator, + OllamaTranslator, OpenAITranslator, + SiliconTranslator, TencentTranslator, + ZhipuTranslator) service_map: dict[str, BaseTranslator] = { "Google": GoogleTranslator, @@ -481,7 +472,7 @@ def on_select_page(choice): ) -def readuserandpasswd(file_path): +def parse_user_passwd(file_path): tuple_list = [] content = "" if not file_path: @@ -503,7 +494,7 @@ def readuserandpasswd(file_path): def setup_gui(share=False, authfile=["", ""]): - userlist, html = readuserandpasswd(authfile) + userlist, html = parse_user_passwd(authfile) if flag_demo: demo.launch(server_name="0.0.0.0", max_file_size="5mb", inbrowser=True) else: From 5ce72a1fb88fcf0bbf18cc0463767fc7032075bc Mon Sep 17 00:00:00 2001 From: Rongxin Date: Thu, 19 Dec 2024 12:35:53 +0800 Subject: [PATCH 111/114] fix (style): black --- pdf2zh/gui.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 34749179..a040cc9f 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -13,14 +13,24 @@ from pdf2zh import __version__ from pdf2zh.high_level import translate -from pdf2zh.translator import (AnythingLLMTranslator, AzureOpenAITranslator, - AzureTranslator, BaseTranslator, BingTranslator, - DeepLTranslator, DeepLXTranslator, - DifyTranslator, GeminiTranslator, - GoogleTranslator, ModelScopeTranslator, - OllamaTranslator, OpenAITranslator, - SiliconTranslator, TencentTranslator, - ZhipuTranslator) +from pdf2zh.translator import ( + AnythingLLMTranslator, + AzureOpenAITranslator, + AzureTranslator, + BaseTranslator, + BingTranslator, + DeepLTranslator, + DeepLXTranslator, + DifyTranslator, + GeminiTranslator, + GoogleTranslator, + ModelScopeTranslator, + OllamaTranslator, + OpenAITranslator, + SiliconTranslator, + TencentTranslator, + ZhipuTranslator, +) service_map: dict[str, BaseTranslator] = { "Google": GoogleTranslator, From 709697f972c6e01f45d62d4a8066f6dd5e6725e5 Mon Sep 17 00:00:00 2001 From: Byaidu <909756245@qq.com> Date: Thu, 19 Dec 2024 13:04:37 +0800 Subject: [PATCH 112/114] Delete .github/workflows/issue-translator.yml --- .github/workflows/issue-translator.yml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .github/workflows/issue-translator.yml diff --git a/.github/workflows/issue-translator.yml b/.github/workflows/issue-translator.yml deleted file mode 100644 index 948bf260..00000000 --- a/.github/workflows/issue-translator.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: 'Issue Translator' -on: - issue_comment: - types: [created] - issues: - types: [opened] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: usthe/issues-translate-action@v2.7 - with: - IS_MODIFY_TITLE: true - CUSTOM_BOT_NOTE: The issue has been automatically translated into English. \ No newline at end of file From 259234391fb5e9c4b1ba8b3a99dcef5863295d55 Mon Sep 17 00:00:00 2001 From: Rongxin Date: Thu, 19 Dec 2024 13:53:19 +0800 Subject: [PATCH 113/114] chore (gui): add comments to all methods --- pdf2zh/gui.py | 151 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 119 insertions(+), 32 deletions(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index 7420cdf7..e91aa956 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -32,6 +32,7 @@ ZhipuTranslator, ) +# The following variables associate strings with translators service_map: dict[str, BaseTranslator] = { "Google": GoogleTranslator, "Bing": BingTranslator, @@ -49,6 +50,8 @@ "Dify": DifyTranslator, "AnythingLLM": AnythingLLMTranslator, } + +# The following variables associate strings with specific languages lang_map = { "Simplified Chinese": "zh", "Traditional Chinese": "zh-TW", @@ -61,6 +64,8 @@ "Spanish": "es", "Italian": "it", } + +# The following variable associate strings with page ranges page_map = { "All": None, "First": [0], @@ -68,7 +73,10 @@ "Others": None, } +# Check if this is a public demo, which has resource limits flag_demo = False + +# Limit resources if os.getenv("PDF2ZH_DEMO"): flag_demo = True service_map = { @@ -81,8 +89,18 @@ client_key = os.getenv("PDF2ZH_CLIENT_KEY") server_key = os.getenv("PDF2ZH_SERVER_KEY") +# Check if everything unconfigured +if os.getenv("PDF2ZH_INIT") is not False: + service_map = { + "Google": GoogleTranslator, + } + +# Public demo control def verify_recaptcha(response): + """ + This function verifies the reCAPTCHA response. + """ recaptcha_url = "https://www.google.com/recaptcha/api/siteverify" print("reCAPTCHA", server_key, response) data = {"secret": server_key, "response": response} @@ -91,7 +109,18 @@ def verify_recaptcha(response): return result.get("success") -def download_with_limit(url, save_path, size_limit): +def download_with_limit(url: str, save_path: str, size_limit: int) -> str: + """ + This function downloads a file from a URL and saves it to a specified path. + + Inputs: + - url: The URL to download the file from + - save_path: The path to save the file to + - size_limit: The maximum size of the file to download + + Returns: + - The path of the downloaded file + """ chunk_size = 1024 total_size = 0 with requests.get(url, stream=True, timeout=10) as response: @@ -111,7 +140,15 @@ def download_with_limit(url, save_path, size_limit): return save_path / filename -def stop_translate_file(state): +def stop_translate_file(state: dict) -> None: + """ + This function stops the translation process. + + Inputs: + - state: The state of the translation process + + Returns:- None + """ session_id = state["session_id"] if session_id is None: return @@ -135,10 +172,37 @@ def translate_file( progress=gr.Progress(), *envs, ): + """ + This function translates a PDF file from one language to another. + + Inputs: + - file_type: The type of file to translate + - file_input: The file to translate + - link_input: The link to the file to translate + - service: The translation service to use + - lang_from: The language to translate from + - lang_to: The language to translate to + - page_range: The range of pages to translate + - page_input: The input for the page range + - prompt: The custom prompt for the llm + - threads: The number of threads to use + - recaptcha_response: The reCAPTCHA response + - state: The state of the translation process + - progress: The progress bar + - envs: The environment variables + + Returns: + - The translated file + - The translated file + - The translated file + - The progress bar + - The progress bar + - The progress bar + """ session_id = uuid.uuid4() state["session_id"] = session_id cancellation_event_map[session_id] = asyncio.Event() - """Translate PDF content using selected service.""" + # Translate PDF content using selected service. if flag_demo and not verify_recaptcha(recaptcha_response): raise gr.Error("reCAPTCHA fail") @@ -200,7 +264,7 @@ def progress_bar(t: tqdm.tqdm): "lang_out": lang_to, "service": f"{translator.name}", "output": output, - "thread": threads, + "thread": int(threads), "callback": progress_bar, "cancellation_event": cancellation_event_map[session_id], "envs": _envs, @@ -243,14 +307,7 @@ def progress_bar(t: tqdm.tqdm): c950="#020B33", ) -cancellation_event_map = {} - -with gr.Blocks( - title="PDFMathTranslate - PDF Translation with preserved formats", - theme=gr.themes.Default( - primary_hue=custom_blue, spacing_size="md", radius_size="lg" - ), - css=""" +custom_css = """ .secondary-text {color: #999 !important;} footer {visibility: hidden} .env-warning {color: #dd5500 !important;} @@ -263,14 +320,15 @@ def progress_bar(t: tqdm.tqdm): } .progress-bar-wrap { - border-radius: 8px !important; + border-radius: 8px !important; } + .progress-bar { - border-radius: 8px !important; + border-radius: 8px !important; } - """, - head=( - """ + """ + +demo_recaptcha = """ """ - if flag_demo - else "" + +tech_details_string = f""" + Technical details + - GitHub: Byaidu/PDFMathTranslate
+ - GUI by: Rongxin
+ - Version: {__version__} + """ +cancellation_event_map = {} + + +# The following code creates the GUI +with gr.Blocks( + title="PDFMathTranslate - PDF Translation with preserved formats", + theme=gr.themes.Default( + primary_hue=custom_blue, spacing_size="md", radius_size="lg" ), + css=custom_css, + head=demo_recaptcha if flag_demo else "", ) as demo: gr.Markdown( "# [PDFMathTranslate @ GitHub](https://github.com/Byaidu/PDFMathTranslate)" @@ -393,12 +466,7 @@ def on_select_page(choice): translate_btn = gr.Button("Translate", variant="primary") cancellation_btn = gr.Button("Cancel", variant="secondary") tech_details_tog = gr.Markdown( - f""" - Technical details - - GitHub: Byaidu/PDFMathTranslate
- - GUI by: Rongxin
- - Version: {__version__} - """, + tech_details_string, elem_classes=["secondary-text"], ) page_range.select(on_select_page, page_range, page_input) @@ -489,7 +557,16 @@ def on_select_page(choice): ) -def parse_user_passwd(file_path): +def parse_user_passwd(file_path: str) -> tuple: + """ + Parse the user name and password from the file. + + Inputs: + - file_path: The file path to read. + Outputs: + - tuple_list: The list of tuples of user name and password. + - content: The content of the file + """ tuple_list = [] content = "" if not file_path: @@ -510,12 +587,22 @@ def parse_user_passwd(file_path): return tuple_list, content -def setup_gui(share=False, authfile=["", ""]): - userlist, html = parse_user_passwd(authfile) +def setup_gui(share: bool = False, auth_file: list = ["", ""]) -> None: + """ + Setup the GUI with the given parameters. + + Inputs: + - share: Whether to share the GUI. + - auth_file: The file path to read the user name and password. + + Outputs: + - None + """ + user_list, html = parse_user_passwd(auth_file) if flag_demo: demo.launch(server_name="0.0.0.0", max_file_size="5mb", inbrowser=True) else: - if len(userlist) == 0: + if len(user_list) == 0: try: demo.launch( server_name="0.0.0.0", debug=True, inbrowser=True, share=share @@ -540,7 +627,7 @@ def setup_gui(share=False, authfile=["", ""]): debug=True, inbrowser=True, share=share, - auth=userlist, + auth=user_list, auth_message=html, ) except Exception: @@ -553,7 +640,7 @@ def setup_gui(share=False, authfile=["", ""]): debug=True, inbrowser=True, share=share, - auth=userlist, + auth=user_list, auth_message=html, ) except Exception: @@ -564,7 +651,7 @@ def setup_gui(share=False, authfile=["", ""]): debug=True, inbrowser=True, share=True, - auth=userlist, + auth=user_list, auth_message=html, ) From 448eae51a0fdb4c0057c73d20c08abc5ff956fef Mon Sep 17 00:00:00 2001 From: hellofinch Date: Thu, 19 Dec 2024 14:18:45 +0800 Subject: [PATCH 114/114] =?UTF-8?q?feat=20(gui)=20:=20=E8=B0=83=E6=95=B4PD?= =?UTF-8?q?F=E9=A2=84=E8=A7=88=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pdf2zh/gui.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pdf2zh/gui.py b/pdf2zh/gui.py index e91aa956..6ce7450c 100644 --- a/pdf2zh/gui.py +++ b/pdf2zh/gui.py @@ -326,6 +326,10 @@ def progress_bar(t: tqdm.tqdm): .progress-bar { border-radius: 8px !important; } + + .pdf-canvas canvas { + width: 100%; + } """ demo_recaptcha = """ @@ -498,7 +502,7 @@ def on_select_page(choice): with gr.Column(scale=2): gr.Markdown("## Preview") - preview = PDF(label="Document Preview", visible=True) + preview = PDF(label="Document Preview", visible=True, height=2000) # Event handlers file_input.upload(