From d236a0cc1a6ebc967011e928be749c27588eea33 Mon Sep 17 00:00:00 2001 From: bajins Date: Thu, 16 Jan 2020 17:19:13 +0800 Subject: [PATCH] =?UTF-8?q?add=20=E4=BD=BF=E7=94=A8form-data=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E4=B8=8B=E8=BD=BD=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- reptile/Netsarang.go | 310 ++++++++++++++++++++++++++++++++++++---- reptile/mail.go | 49 ++----- reptile/reptile_test.go | 61 ++++---- 3 files changed, 314 insertions(+), 106 deletions(-) diff --git a/reptile/Netsarang.go b/reptile/Netsarang.go index abc8a5f..8b3928f 100644 --- a/reptile/Netsarang.go +++ b/reptile/Netsarang.go @@ -12,12 +12,16 @@ package reptile import ( + "bytes" "context" + "encoding/base64" "errors" + "github.com/PuerkitoBio/goquery" "github.com/chromedp/cdproto/page" "github.com/chromedp/cdproto/target" "github.com/chromedp/chromedp" "log" + "net/http" "regexp" "strings" "time" @@ -40,12 +44,11 @@ func init() { func GetInfoUrl(product string) (string, error) { info := NetsarangMap[product] if NetsarangMap == nil || info.Url == "" || len(info.Url) == 0 || !utils.DateEqual(time.Now(), info.Time) { - ctx, cancel, mail, err := NetsarangGetMail() - defer cancel() + mail, err := NetsarangGetMail() if err != nil { return "", err } - url, err := NetsarangGetInfo(ctx, mail, product) + url, err := NetsarangGetInfo(mail, product) if err != nil { return "", err } @@ -55,38 +58,42 @@ func GetInfoUrl(product string) (string, error) { } // 获取可用mail -func NetsarangGetMail() (context.Context, context.CancelFunc, string, error) { - var mail string - ctx, cancel := ApplyRun() - err := chromedp.Run(ctx, GetMail24MailName(&mail)) +func NetsarangGetMail() (string, error) { + prefix := utils.RandomLowercaseAlphanumeric(9) + suffix, err := LinShiYouXiangSuffix() if err != nil { - return nil, nil, "", err + return "", err + } + res, err := LinShiYouXiangApply(prefix) + if err != nil { + return "", err } + log.Println(res) + mail := prefix + suffix log.Println("邮箱号：", mail) - return ctx, cancel, mail, nil + return mail, nil } // 获取所有链接信息 func NetsarangDownloadAll() { - ctx, cancel, mail, err := NetsarangGetMail() - defer cancel() + mail, err := NetsarangGetMail() if err != nil { log.Println(err) return } - _, err = NetsarangGetInfo(ctx, mail, "Xshell") + _, err = NetsarangGetInfo(mail, "Xshell") if err != nil { log.Println(err) } - _, err = NetsarangGetInfo(ctx, mail, "Xftp") + _, err = NetsarangGetInfo(mail, "Xftp") if err != nil { log.Println(err) } - _, err = NetsarangGetInfo(ctx, mail, "Xmanager") + _, err = NetsarangGetInfo(mail, "Xmanager") if err != nil { log.Println(err) } - _, err = NetsarangGetInfo(ctx, mail, "Xshell Plus") + _, err = NetsarangGetInfo(mail, "Xshell Plus") if err != nil { log.Println(err) } @@ -94,7 +101,251 @@ func NetsarangDownloadAll() { } // 获取链接信息 -func NetsarangGetInfo(ctx context.Context, mail, product string) (string, error) { +func NetsarangGetInfo(mail, product string) (string, error) { + if mail == "" || len(mail) == 0 { + return "", errors.New("mail不能为空") + } + if product == "" || len(product) == 0 { + return "", errors.New("product不能为空") + } + info := NetsarangMap[product] + // 如果数据不为空，并且日期为今天，这么做是为了避免消耗过多的性能，每天只查询一次 + if info.Url != "" && len(info.Url) > 1 && utils.DateEqual(time.Now(), info.Time) { + return "", nil + } + err := NetsarangSendMail(mail, product) + if err != nil { + return "", err + } + prefix := strings.Split(mail, "@")[0] + mailList, err := LinShiYouXiangList(prefix) + if err != nil { + return "", err + } + for i := 0; i < 30; { + if mailList != nil || err != nil { + break + } + time.Sleep(1 * time.Minute) + mailList, err = LinShiYouXiangList(prefix) + if err != nil { + return "", err + } + } + if len(mailList) == 0 || mailList == nil { + return "", errors.New("没有邮件") + } + mailbox := mailList[len(mailList)-1]["mailbox"].(string) + if mailbox == "" { + return "", errors.New("邮件前缀不存在") + } + mailId := mailList[len(mailList)-1]["id"].(string) + if mailId == "" { + return "", errors.New("邮件ID不存在") + } + // 获取最新一封邮件 + content, err := LinShiYouXiangGetMail(mailbox, mailId) + if err != nil { + return "", err + } + // 分割取内容 + text := strings.Split(content, "AmazonSES") + if len(text) < 2 { + return "", errors.New("邮件内容不正确") + } + // 解密，邮件协议Content-Transfer-Encoding指定了base64 + htmlText, err := base64.StdEncoding.DecodeString(text[1]) + if err != nil { + return "", err + } + // 解析HTML + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlText)) + if err != nil { + return "", err + } + href := doc.Find(`a[target="_blank"]`).Text() + + exp, err := regexp.Compile(`https://www\.netsarang\.com/(.*)/downloading/\?token=(.*)`) + if err != nil { + return "", err + } + hrf := exp.FindAllStringSubmatch(href, -1) + log.Println("token链接：", hrf[0][0]) + if hrf == nil || len(hrf) == 0 { + return "", errors.New("获取token链接为空") + } + url, err := NetsarangGetUrl(hrf[0][1], hrf[0][2]) + if err != nil { + return "", err + } + if url == nil || url["downlink"] == "" { + return "", errors.New("没有获取到url") + } + // 获取最终专业版产品下载地址 + // 在s字符串中，把old字符串替换为new字符串，n表示替换的次数，小于0表示全部替换 + ur := strings.Replace(url["downlink"].(string), ".exe", "r.exe", -1) + // 把产品信息存储到变量 + NetsarangMap[product] = NetsarangInfo{Time: time.Now(), Url: ur} + return ur, nil +} + +// 发送邮件 +func NetsarangSendMail(mail, product string) error { + if mail == "" || len(mail) == 0 { + return errors.New("邮箱号不能为空！") + } + if product == "" || len(product) == 0 { + return errors.New("产品不能为空！") + } + + productCode := "" + productName := "" + if strings.EqualFold(product, "Xshell") { + productCode = "4203" + productName = "xshell-download" + } else if strings.EqualFold(product, "Xftp") { + productCode = "4242" + productName = "xftp-download" + } else if strings.EqualFold(product, "Xmanager") { + productCode = "4066" + productName = "xmanager-power-suite-download" + } else if strings.EqualFold(product, "Xshell Plus") || product == "" { + productCode = "4132" + productName = "xshell-plus-download" + } + if productCode == "" || productName == "" { + return errors.New("产品不匹配") + } + data := map[string]string{ + "_wpcf7": "3016", + "_wpcf7_version": "5.1.1", + "_wpcf7_locale": "en_US", + "_wpcf7_unit_tag": "wpcf7-f3016-p" + productCode + "-o2", + "_wpcf7_container_post": productCode, + "g-recaptcha-response": "", + "md": "setDownload", + "language": "3", + "downloadType": "0", + "licenseType": "0", + "action": "/json/download/process.html", + "user-name": mail, + "email": mail, + "company": "", + "productName": productName, + } + httpClient := utils.HttpClient{ + Method: http.MethodPost, + UrlText: "https://www.netsarang.com/json/download/process.html", + ContentType: utils.ContentTypeMFD, + Params: data, + Header: nil, + } + js, err := httpClient.HttpReadBodyJsonMap() + if err != nil { + return err + } + if js == nil || !js["result"].(bool) || js["errorCounter"].(float64) != 0 { + return errors.New("邮箱发送失败！") + } + return nil +} + +// 获取下载产品信息 +func NetsarangGetUrl(lang, token string) (map[string]interface{}, error) { + if lang == "" || len(lang) == 0 { + return nil, errors.New("lang不能为空") + } + if token == "" || len(token) == 0 { + return nil, errors.New("token不能为空") + } + var language string + switch lang { + case "en": + language = "2" + break + case "ko": + language = "1" + break + case "zh": + language = "3" + break + case "ru": + language = "8" + break + case "pt": + language = "9" + break + default: + language = "en" + break + } + params := map[string]string{ + "md": "checkDownload", + "token": token, + "language": language, + } + return utils.HttpReadBodyJsonMap(http.MethodPost, "https://www.netsarang.com/json/download/process.html", utils.ContentTypeMFD, params, nil) +} + +// 通过ChromeDP获取所有链接信息 +func NetsarangDownloadAllDP() { + ctx, cancel, mail, err := NetsarangGetMailDP() + defer cancel() + if err != nil { + log.Println(err) + return + } + _, err = NetsarangGetInfoDP(ctx, mail, "Xshell") + if err != nil { + log.Println(err) + } + _, err = NetsarangGetInfoDP(ctx, mail, "Xftp") + if err != nil { + log.Println(err) + } + _, err = NetsarangGetInfoDP(ctx, mail, "Xmanager") + if err != nil { + log.Println(err) + } + _, err = NetsarangGetInfoDP(ctx, mail, "Xshell Plus") + if err != nil { + log.Println(err) + } + log.Println(NetsarangMap) +} + +// 获取单个产品信息 +func GetInfoUrlDP(product string) (string, error) { + info := NetsarangMap[product] + if NetsarangMap == nil || info.Url == "" || len(info.Url) == 0 || !utils.DateEqual(time.Now(), info.Time) { + ctx, cancel, mail, err := NetsarangGetMailDP() + defer cancel() + if err != nil { + return "", err + } + url, err := NetsarangGetInfoDP(ctx, mail, product) + if err != nil { + return "", err + } + info = NetsarangInfo{Time: time.Now(), Url: url} + } + return info.Url, nil +} + +// 通过ChromeDP获取可用mail +func NetsarangGetMailDP() (context.Context, context.CancelFunc, string, error) { + var mail string + ctx, cancel := ApplyRun() + err := chromedp.Run(ctx, GetMail24MailName(&mail)) + if err != nil { + return nil, nil, "", err + } + log.Println("邮箱号：", mail) + return ctx, cancel, mail, nil +} + +// 通过ChromeDP通过ChromeDP获取链接信息 +func NetsarangGetInfoDP(ctx context.Context, mail, product string) (string, error) { if ctx == nil { return "", errors.New("context不能为空") } @@ -109,44 +360,45 @@ func NetsarangGetInfo(ctx context.Context, mail, product string) (string, error) if info.Url != "" && len(info.Url) > 1 && utils.DateEqual(time.Now(), info.Time) { return "", nil } - err := NetsarangSendMail(ctx, mail, product) + err := NetsarangSendMailDP(ctx, mail, product) if err != nil { return "", err } - var mailList string - err = chromedp.Run(ctx, GetMail24LatestMail(&mailList)) + var mailContent string + err = chromedp.Run(ctx, GetMail24LatestMail(&mailContent)) if err != nil { return "", err } for i := 0; i < 30; { - if mailList != "" || err != nil { + if mailContent != "" || err != nil { break } - err = chromedp.Run(ctx, GetMail24LatestMail(&mailList)) + err = chromedp.Run(ctx, GetMail24LatestMail(&mailContent)) if err != nil { return "", err } } - exp, err := regexp.Compile(`https://www\.netsarang\.com/.*/downloading/\?token=.*`) + exp, err := regexp.Compile(`https://www\.netsarang\.com/(.*)/downloading/\?token=(.*)`) if err != nil { return "", err } - hrf := exp.FindString(mailList) + hrf := exp.FindString(mailContent) log.Println("token链接：", hrf) - if hrf == "" { + if hrf == "" || len(hrf) == 0 { return "", errors.New("获取token链接为空") } - url, err := NetsarangGetUrl(ctx, hrf) + url, err := NetsarangGetUrlDP(ctx, hrf) if err != nil { return "", err } + // 把产品信息存储到变量 NetsarangMap[product] = NetsarangInfo{Time: time.Now(), Url: url} return url, nil } -// 发送邮件 -func NetsarangSendMail(ctx context.Context, mail, product string) error { +// 通过ChromeDP发送邮件 +func NetsarangSendMailDP(ctx context.Context, mail, product string) error { if ctx == nil { return errors.New("context不能为空") } @@ -208,8 +460,8 @@ func NetsarangSendMail(ctx context.Context, mail, product string) error { return nil } -// 获取下载产品信息 -func NetsarangGetUrl(ctx context.Context, url string) (string, error) { +// 通过ChromeDP获取下载产品信息 +func NetsarangGetUrlDP(ctx context.Context, url string) (string, error) { if ctx == nil { return "", errors.New("context不能为空") } diff --git a/reptile/mail.go b/reptile/mail.go index 534a665..d059938 100644 --- a/reptile/mail.go +++ b/reptile/mail.go @@ -15,8 +15,8 @@ import ( "github.com/antchfx/htmlquery" "github.com/chromedp/cdproto/page" "github.com/chromedp/chromedp" - "io/ioutil" "math" + "net/http" "time" "tool-gin/utils" ) @@ -26,7 +26,7 @@ const LinShiYouXiang = "https://www.linshiyouxiang.net" // 获取邮箱号后缀 func LinShiYouXiangSuffix() (string, error) { var suffixArray []string - response, err := utils.HttpRequest("GET", LinShiYouXiang, "", nil, nil) + response, err := utils.HttpRequest(http.MethodGet, LinShiYouXiang, "", nil, nil) if err != nil { return "", err } @@ -51,31 +51,14 @@ func LinShiYouXiangApply(prefix string) (map[string]interface{}, error) { "mailbox": prefix, "_ts": utils.ToString(math.Round(float64(time.Now().Unix() / 1000))), } - response, err := utils.HttpRequest("GET", url, "", param, nil) - if err != nil { - return nil, err - } - result, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, err - } - stu, err := utils.JsonToMap(string(result)) - return stu, err + return utils.HttpReadBodyJsonMap(http.MethodGet, url, "", param, nil) } // 获取邮件列表 // prefix： 邮箱前缀 -func LinShiYouXiangList(prefix string) (string, error) { +func LinShiYouXiangList(prefix string) ([]map[string]interface{}, error) { url := LinShiYouXiang + "/api/v1/mailbox/" + prefix - response, err := utils.HttpRequest("GET", url, "", nil, nil) - if err != nil { - return "", err - } - result, err := ioutil.ReadAll(response.Body) - if err != nil { - return "", err - } - return string(result), nil + return utils.HttpReadBodyJsonArray(http.MethodGet, url, "", nil, nil) } // 获取邮件内容 @@ -91,31 +74,15 @@ func LinShiYouXiangList(prefix string) (string, error) { // doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlText)) func LinShiYouXiangGetMail(prefix, id string) (string, error) { url := LinShiYouXiang + "/mailbox/" + prefix + "/" + id + "/source" - response, err := utils.HttpRequest("GET", url, "", nil, nil) - if err != nil { - return "", err - } - result, err := ioutil.ReadAll(response.Body) - if err != nil { - return "", err - } - return string(result), nil + return utils.HttpReadBodyString(http.MethodGet, url, "", nil, nil) } // 删除邮件 // prefix： 邮箱前缀 // id: 邮件编号 -func LinShiYouXiangDelete(prefix, id string) (string, error) { +func LinShiYouXiangDelete(prefix, id string) (map[string]interface{}, error) { url := LinShiYouXiang + "/api/v1/mailbox/" + prefix + "/" + id - response, err := utils.HttpRequest("DELETE", url, "", nil, nil) - if err != nil { - return "", err - } - result, err := ioutil.ReadAll(response.Body) - if err != nil { - return "", err - } - return string(result), nil + return utils.HttpReadBodyJsonMap(http.MethodDelete, url, "", nil, nil) } const Mail24 = "http://24mail.chacuo.net" diff --git a/reptile/reptile_test.go b/reptile/reptile_test.go index a7ff056..1d1edc6 100644 --- a/reptile/reptile_test.go +++ b/reptile/reptile_test.go @@ -15,16 +15,37 @@ import ( "fmt" "github.com/chromedp/cdproto/network" "github.com/chromedp/chromedp" - "io/ioutil" "testing" "tool-gin/utils" ) +func TestLinShiYouXiangSuffix(t *testing.T) { + LinShiYouXiangSuffix() +} + +func TestLinShiYouXiangList(t *testing.T) { + list, _ := LinShiYouXiangList("5wij52emu") + t.Log(list) +} + +func TestGetMail24(t *testing.T) { + //GetMail24() + var test string + //ctx, cancel := ApplyDebug() + //defer cancel() + ctx, _ := ApplyDebug() + err := chromedp.Run(ctx, GetMail24MailName(&test)) + t.Log(err) + t.Log(test) + err = chromedp.Run(ctx, GetMail24LatestMail(&test)) + t.Log(err) + fmt.Println(test) +} + func TestApply(t *testing.T) { url := "https://www.netsarang.com/zh/downloading/?token=d1hXUC05Y3RVWXhJNWt6NF9rUHhDQUBaVjZXVkJRQU51VHEtRi1PVm1MQUFR" - response, err := utils.HttpRequest("GET", url, "", nil, nil) - result, err := ioutil.ReadAll(response.Body) - fmt.Println(string(result), err) + response, err := utils.HttpReadBodyString("GET", url, "", nil, nil) + fmt.Println(response, err) } func TestCDP(t *testing.T) { @@ -56,35 +77,3 @@ func Crawler(res *string) chromedp.Tasks { chromedp.Title(res), } } - -func TestLinShiYouXiangSuffix(t *testing.T) { - LinShiYouXiangSuffix() -} - -func TestLinShiYouXiangList(t *testing.T) { - list, _ := LinShiYouXiangList("5wij52emu") - t.Log(list) -} - -func TestDownloadNetsarang(t *testing.T) { - ctx, cancel, mail, err := NetsarangGetMail() - defer cancel() - if err != nil { - t.Error(err) - } - t.Log(NetsarangGetInfo(ctx, mail, "xshell")) -} - -func TestGetMail24(t *testing.T) { - //GetMail24() - var test string - //ctx, cancel := ApplyDebug() - //defer cancel() - ctx, _ := ApplyDebug() - err := chromedp.Run(ctx, GetMail24MailName(&test)) - t.Log(err) - t.Log(test) - err = chromedp.Run(ctx, GetMail24LatestMail(&test)) - t.Log(err) - fmt.Println(test) -}