Skip to content

Commit

Permalink
集成有道翻译;
Browse files Browse the repository at this point in the history
修复协程计数变量问题;
  • Loading branch information
speauty committed Mar 2, 2023
1 parent 8f7572a commit 4bc29e9
Show file tree
Hide file tree
Showing 4 changed files with 249 additions and 24 deletions.
126 changes: 109 additions & 17 deletions src/logic/translate/translate.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,15 @@ import (
"gui.subtitle/src/srv/mt"
aliyun2 "gui.subtitle/src/srv/mt/aliyun"
"gui.subtitle/src/srv/mt/bd"
"gui.subtitle/src/srv/mt/youdao"
"gui.subtitle/src/util"
"gui.subtitle/src/util/lang"
"io"
"regexp"
"runtime"
"strings"
"sync"
"sync/atomic"
"time"
"unicode"
)
Expand Down Expand Up @@ -105,14 +107,16 @@ func Translate(ctx context.Context, mtEngine interface{}, contents []*Block, fro
}
wg := &sync.WaitGroup{}
var results []string
cntError := 0
cntError := new(atomic.Int32)
var lastError error
cntBlock := len(contents)
coroutineCtrlCtx, coroutineCtrlCtxCancelFunc := context.WithCancel(ctx)
defer coroutineCtrlCtxCancelFunc()

maxCoroutine := 10
cntBlockTranslated := 0
maxRetry := 3
cntBlockTranslated := new(atomic.Int32)
cntBlockTranslated.Store(0)
timeStart := carbon.Now()

switch mtEngine.(mt.MT).GetId() {
Expand Down Expand Up @@ -149,7 +153,7 @@ func Translate(ctx context.Context, mtEngine interface{}, contents []*Block, fro
case block, isOpen := <-blockChan:
if !isOpen {
results = append(results, fmt.Sprintf(
"[%s]%s结束, 协程序号: %d, 处理字幕行数: %d, 运行时长(s): %d, 原因: 数据通道关闭, 无数据, 主动退出当前协程",
"[%s]协程结束, 引擎: %s, 协程序号: %d, 处理字幕行数: %d, 运行时长(s): %d, 原因: 数据通道关闭, 无数据, 主动退出当前协程",
carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx,
localCoroutineCntTranslated, carbon.Now().DiffAbsInSeconds(localCoroutineTimeStart),
))
Expand All @@ -161,9 +165,9 @@ func Translate(ctx context.Context, mtEngine interface{}, contents []*Block, fro
args.ToLanguage = toLanguage.ToString()
translateResp, translateErr := mtEngine.(mt.MT).TextBatchTranslate(ctx, args)
if translateErr != nil {
msg := fmt.Sprintf("[%s]%s失败, 协程序号: %d, 错误: %s", carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx, translateErr.Error())
msg := fmt.Sprintf("[%s]%s翻译失败, 协程序号: %d, 错误: %s", carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx, translateErr.Error())
results = append(results, msg)
cntError++
cntError.Add(1)
lastError = fmt.Errorf(msg)
localCoroutineCtrlCtxCancelFunc()
runtime.Goexit()
Expand All @@ -174,11 +178,11 @@ func Translate(ctx context.Context, mtEngine interface{}, contents []*Block, fro
if blockTranslated.Idx == content.Idx {
if toLanguage == lang.ZH {
contents[contentIdx].TextZH = blockTranslated.StrTranslated
cntBlockTranslated++
cntBlockTranslated.Add(1)
localCoroutineCntTranslated++
} else if toLanguage == lang.EN {
contents[contentIdx].TextEN = blockTranslated.StrTranslated
cntBlockTranslated++
cntBlockTranslated.Add(1)
localCoroutineCntTranslated++
}
}
Expand All @@ -187,7 +191,7 @@ func Translate(ctx context.Context, mtEngine interface{}, contents []*Block, fro
default:
if util.IsCtxDone(localCoroutineCtrlCtx) {
results = append(results, fmt.Sprintf(
"[%s]%s失败, 协程序号: %d, 处理字幕行数: %d, 运行时长(s): %d, 错误: 协程出现中断信号, 强制退出",
"[%s]协程结束, 引擎: %s, 协程序号: %d, 处理字幕行数: %d, 运行时长(s): %d, 错误: 协程出现中断信号, 强制退出",
carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx,
localCoroutineCntTranslated, carbon.Now().DiffAbsInSeconds(localCoroutineTimeStart),
))
Expand Down Expand Up @@ -234,7 +238,7 @@ func Translate(ctx context.Context, mtEngine interface{}, contents []*Block, fro
case block, isOpen := <-blockChan:
if !isOpen { // 当前通道已关闭, 并且没有残留数据
results = append(results, fmt.Sprintf(
"[%s]%s结束, 协程序号: %d, 处理字幕行数: %d, 运行时长(s): %d, 原因: 数据通道关闭, 无数据, 主动退出当前协程",
"[%s]协程结束, 引擎: %s, 协程序号: %d, 处理字幕行数: %d, 运行时长(s): %d, 原因: 数据通道关闭, 无数据, 主动退出当前协程",
carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx,
localCoroutineCntTranslated, carbon.Now().DiffAbsInSeconds(localCoroutineTimeStart),
))
Expand All @@ -250,15 +254,15 @@ func Translate(ctx context.Context, mtEngine interface{}, contents []*Block, fro
for failIdx := 0; failIdx < currentCfg.AppVersion.GetRetryLimited(); failIdx++ {
translateResp, err = mtEngine.(mt.MT).TextTranslate(ctx, args)
if err != nil || translateResp == nil {
err = fmt.Errorf("[%s]%s失败, 协程序号: %d, 错误: %s", carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx, err)
err = fmt.Errorf("[%s]%s翻译失败, 协程序号: %d, 错误: %s", carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx, err)
time.Sleep(time.Millisecond * 100)
continue
}
break
}
if err != nil {
results = append(results, err.Error())
cntError++
cntError.Add(1)
lastError = err

if bd.ErrSign.IsExit(err) {
Expand All @@ -272,12 +276,14 @@ func Translate(ctx context.Context, mtEngine interface{}, contents []*Block, fro
if fromLanguage == lang.ZH {
if content.TextZH == translateRes.Idx {
contents[idx].TextEN = translateRes.StrTranslated
cntBlockTranslated++
cntBlockTranslated.Add(1)
localCoroutineCntTranslated++
}
} else if fromLanguage == lang.EN {
if content.TextEN == translateRes.Idx {
contents[idx].TextZH = translateRes.StrTranslated
cntBlockTranslated++
cntBlockTranslated.Add(1)
localCoroutineCntTranslated++
}
}
}
Expand All @@ -286,7 +292,93 @@ func Translate(ctx context.Context, mtEngine interface{}, contents []*Block, fro
default:
if util.IsCtxDone(localCoroutineCtrlCtx) {
results = append(results, fmt.Sprintf(
"[%s]%s失败, 协程序号: %d, 处理字幕行数: %d, 运行时长(s): %d, 错误: 协程出现中断信号, 强制退出",
"[%s]协程结束, 引擎: %s, 协程序号: %d, 处理字幕行数: %d, 运行时长(s): %d, 错误: 协程出现中断信号, 强制退出",
carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx,
localCoroutineCntTranslated, carbon.Now().DiffAbsInSeconds(localCoroutineTimeStart),
))
runtime.Goexit()
return
}
}
}
}(ctx, coroutineCtrlCtx, coroutineCtrlCtxCancelFunc, wg, coroutineIdx, blockChan)
}
case mt.IdYouDao:
blockChunked, cntBlockChunked := chunkBlocksForBaiDu(contents, 2e3, fromLanguage)
if maxCoroutine > cntBlockChunked {
maxCoroutine = cntBlockChunked
}
blockChan := make(chan string, maxCoroutine*3)
go func() {
for _, block := range blockChunked {
blockChan <- block
}
close(blockChan)
}()
for coroutineIdx := 0; coroutineIdx < maxCoroutine; coroutineIdx++ {
if util.IsCtxDone(coroutineCtrlCtx) {
results = append(results, fmt.Sprintf("[%s]%s失败, 错误: 协程出现中断信号, 停止继续创建协程", carbon.Now(), mtEngine.(mt.MT).GetName()))
break
}
wg.Add(1)
go func(localCtx context.Context, localCoroutineCtrlCtx context.Context, localCoroutineCtrlCtxCancelFunc context.CancelFunc, localWG *sync.WaitGroup, localCoroutineIdx int, localBlockChan chan string) {
defer localWG.Done()
localCoroutineCntTranslated := 0
localCoroutineTimeStart := carbon.Now()
for {
select {
case block, isOpen := <-blockChan:
if !isOpen { // 当前通道已关闭, 并且没有残留数据
results = append(results, fmt.Sprintf(
"[%s]协程结束, 引擎: %s, 协程序号: %d, 处理字幕行数: %d, 运行时长(s): %d, 原因: 数据通道关闭, 无数据, 主动退出当前协程",
carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx,
localCoroutineCntTranslated, carbon.Now().DiffAbsInSeconds(localCoroutineTimeStart),
))
runtime.Goexit()
return
}
args := new(youdao.TextTranslateArg).New(block)
args.FromLanguage = fromLanguage.ToString()
args.ToLanguage = toLanguage.ToString()
var err error
var translateResp []mt.TextTranslateResp

for failIdx := 0; failIdx < maxRetry; failIdx++ {
translateResp, err = mtEngine.(mt.MT).TextTranslate(ctx, args)
if err != nil || translateResp == nil {
err = fmt.Errorf("[%s]%s翻译失败, 协程序号: %d, 错误: %s", carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx, err)
continue
}
break
}
if err != nil {
results = append(results, err.Error())
cntError.Add(1)
lastError = err
runtime.Goexit()
return
}
for _, translateRes := range translateResp {
for idx, content := range contents {
if fromLanguage == lang.ZH {
if content.TextZH == translateRes.Idx {
contents[idx].TextEN = translateRes.StrTranslated
cntBlockTranslated.Add(1)
localCoroutineCntTranslated++
}
} else if fromLanguage == lang.EN {
if content.TextEN == translateRes.Idx {
contents[idx].TextZH = translateRes.StrTranslated
cntBlockTranslated.Add(1)
localCoroutineCntTranslated++
}
}
}
}
default:
if util.IsCtxDone(localCoroutineCtrlCtx) {
results = append(results, fmt.Sprintf(
"[%s]协程结束, 引擎: %s, 协程序号: %d, 处理字幕行数: %d, 运行时长(s): %d, 错误: 协程出现中断信号, 强制退出",
carbon.Now(), mtEngine.(mt.MT).GetName(), localCoroutineIdx,
localCoroutineCntTranslated, carbon.Now().DiffAbsInSeconds(localCoroutineTimeStart),
))
Expand All @@ -300,14 +392,14 @@ func Translate(ctx context.Context, mtEngine interface{}, contents []*Block, fro
}
wg.Wait()
resStr := "成功"
if cntBlockTranslated < cntBlock {
if cntBlockTranslated.Load() < int32(cntBlock) {
resStr = "失败"
}
results = append(results, fmt.Sprintf(
"[%s]翻译完成, 引擎: %s, 协程数量: %d, 翻译行数: %d, 结果: %s, 耗时(s): %d",
carbon.Now(), mtEngine.(mt.MT).GetName(), maxCoroutine, cntBlockTranslated, resStr, carbon.Now().DiffAbsInSeconds(timeStart),
carbon.Now(), mtEngine.(mt.MT).GetName(), maxCoroutine, cntBlockTranslated.Load(), resStr, carbon.Now().DiffAbsInSeconds(timeStart),
))
return results, cntError, lastError
return results, int(cntError.Load()), lastError
}

// preCheckBlocks 预检字幕块, 主要保证需要翻译的字幕块存在
Expand Down
7 changes: 5 additions & 2 deletions src/srv/mt/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ type MT interface {
type Id string

const (
IdALiYun Id = "ALi"
IdALiYun Id = "EngineALi"
IdBaiDu Id = "EngineBaiDu"
IdYouDao Id = "EngineYouDao"
)

type Engine int
Expand Down Expand Up @@ -48,9 +49,11 @@ const (
EngineALiYun Engine = iota
// EngineBaiDu 百度翻译
EngineBaiDu
// EngineYouDao 有道翻译
EngineYouDao
)

var engineZHMaps = []string{"阿里云", "百度"}
var engineZHMaps = []string{"阿里云", "百度", "有道"}

const BlockSep string = "\n"
const BlockIdxContentSep string = "@<"
Expand Down
126 changes: 126 additions & 0 deletions src/srv/mt/youdao/youdao.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package youdao

import (
"context"
"encoding/json"
"fmt"
"gui.subtitle/src/srv/mt"
"gui.subtitle/src/util/lang"
"io"
"net/http"
url2 "net/url"
"strings"
)

var api = "https://fanyi.youdao.com/translate?&doctype=json"

type MT struct {
}

func (youDaoMT *MT) GetId() mt.Id {
return mt.IdYouDao
}

func (youDaoMT *MT) GetName() string {
return mt.EngineYouDao.GetZH()
}

func (youDaoMT *MT) GetCfg() interface{} {
return nil
}

func (youDaoMT *MT) Init(_ context.Context, _ interface{}) error {
return nil
}

type TextTranslateArg struct {
SourceText string
ToLanguage string
FromLanguage string
}

func (arg *TextTranslateArg) New(text string) *TextTranslateArg {
arg.SourceText = text
arg.ToLanguage = lang.ZH.ToString()
arg.FromLanguage = lang.EN.ToString()
return arg
}

type youDaoMTResp struct {
Type string `json:"type"`
ErrorCode int `json:"errorCode"`
ElapsedTime int `json:"elapsedTime"`
TransResult [][]struct {
Src string `json:"src,omitempty"` // 原文
Tgt string `json:"tgt,omitempty"` // 译文
} `json:"translateResult"`
}

func (youDaoMT *MT) TextTranslate(ctx context.Context, args interface{}) ([]mt.TextTranslateResp, error) {
if _, ok := args.(*TextTranslateArg); !ok {
return nil, fmt.Errorf("the args for YoudaoMT.TextTranslateArg mismatched")
}
mtArgs := args.(*TextTranslateArg)
argType := youDaoMT.convertLanguage2Type(mtArgs.FromLanguage, mtArgs.ToLanguage)
mtArgs.SourceText = url2.QueryEscape(mtArgs.SourceText)
url := fmt.Sprintf("%s&type=%s&i=%s", api, argType, mtArgs.SourceText)
httpResp, err := http.DefaultClient.Get(url)
defer func() {
if httpResp.Body != nil {
_ = httpResp.Body.Close()
}
}()
if err != nil {
return nil, fmt.Errorf("网络请求[%s]出现异常, 错误: %s", url, err.Error())
}
respBytes, err := io.ReadAll(httpResp.Body)
if err != nil {
return nil, fmt.Errorf("读取报文出现异常, 错误: %s", err.Error())
}
youDaoResp := new(youDaoMTResp)
if err := json.Unmarshal(respBytes, youDaoResp); err != nil {
fmt.Println(string(respBytes))
return nil, fmt.Errorf("解析报文出现异常, 错误: %s", err.Error())
}
if youDaoResp.ErrorCode != 0 {
return nil, fmt.Errorf("翻译异常, 代码: %d", youDaoResp.ErrorCode)
}
var resp []mt.TextTranslateResp
for _, transBlockArray := range youDaoResp.TransResult {
for _, transBlock := range transBlockArray {
resp = append(resp, mt.TextTranslateResp{
Idx: transBlock.Src,
StrTranslated: transBlock.Tgt,
})
}
}
return resp, nil
}

func (youDaoMT *MT) TextBatchTranslate(_ context.Context, _ interface{}) ([]mt.TextTranslateResp, error) {
return nil, nil
}

// ZH_CN2EN 中文 » 英语
// ZH_CN2JA 中文 » 日语
// ZH_CN2KR 中文 » 韩语
// ZH_CN2FR 中文 » 法语
// ZH_CN2RU 中文 » 俄语
// ZH_CN2SP 中文 » 西语
// EN2ZH_CN 英语 » 中文
// JA2ZH_CN 日语 » 中文
// KR2ZH_CN 韩语 » 中文
// FR2ZH_CN 法语 » 中文
// RU2ZH_CN 俄语 » 中文
// SP2ZH_CN 西语 » 中文
func (youDaoMT *MT) convertLanguage2Type(fromLanguage string, toLanguage string) string {
fromLanguage = strings.ToUpper(fromLanguage)
toLanguage = strings.ToUpper(toLanguage)
if fromLanguage == "ZH" {
fromLanguage = "ZH_CN"
}
if toLanguage == "ZH" {
toLanguage = "ZH_CN"
}
return fmt.Sprintf("%s2%s", fromLanguage, toLanguage)
}
Loading

0 comments on commit 4bc29e9

Please sign in to comment.