From 58b1266831d7af76516cde3e6b6706cf04994437 Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Sat, 7 Dec 2019 15:54:35 +0800 Subject: [PATCH 01/14] =?UTF-8?q?feat:=20=E9=80=82=E9=85=8D=E6=96=B0?= =?UTF-8?q?=E6=96=87=E7=AB=A0=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/weibo.ts | 34 +++++++++++++++++++++------------- src/command/fetch/customer.ts | 35 +++++++++++++++++++++++++++++++---- src/type/namespace/weibo.d.ts | 2 +- 3 files changed, 53 insertions(+), 18 deletions(-) diff --git a/src/api/weibo.ts b/src/api/weibo.ts index 00fa107e..94a9bbeb 100644 --- a/src/api/weibo.ts +++ b/src/api/weibo.ts @@ -2,6 +2,7 @@ import _ from 'lodash' import Base from '~/src/api/base' import * as TypeWeibo from '~/src/type/namespace/weibo' import Util from '~/src/library/util/common' +import moment from 'moment' /** * 用户信息部分容器 id @@ -153,19 +154,26 @@ export default class Weibo extends Base { /** * 获取微博文章, 获取不到返回空对象 */ - static async asyncGetWeiboArticle(url: string) { - let responseHtml = await Base.http.get(url) - let json: TypeWeibo.TypeWeiboArticleRecord - try { - let scriptContent = responseHtml.split('')[1] - let rawJsContent = scriptContent.split('')[0] - let rawJson = jsContent.split('var $render_data = [')[1] - let jsonStr = rawJson.split('][0] || {};')[0] - json = JSON.parse(jsonStr) - } catch (e) { - json = {} - } + static async asyncGetWeiboArticle(articleId: string) { + let apiUrl = `https://card.weibo.com/article/m/aj/detail?id=${articleId}&_t=${moment().unix()}` + let articleUrl = `https://card.weibo.com/article/m/show/id/${articleId}` + + let response = await Base.http.get(apiUrl, { + headers: { + Referer: articleUrl, + }, + }) + let json: TypeWeibo.TypeWeiboArticleRecord = _.get(response, ['data'], {}) + // try { + // let scriptContent = responseHtml.split('')[1] + // let rawJsContent = scriptContent.split('')[0] + // let rawJson = jsContent.split('var $render_data = [')[1] + // let jsonStr = rawJson.split('][0] || {};')[0] + // json = JSON.parse(jsonStr) + // } catch (e) { + // json = {} + // } return json } diff --git a/src/command/fetch/customer.ts b/src/command/fetch/customer.ts index 818ceda2..86b85f86 100644 --- a/src/command/fetch/customer.ts +++ b/src/command/fetch/customer.ts @@ -13,6 +13,33 @@ import CommonUtil from '~/src/library/util/common' import * as TypeWeibo from '~/src/type/namespace/weibo' import Util from '~/src/library/util/common' +/** + * 解析微博文章id，方便构造api, 抓取文章内容 + * @param rawUrl + */ +function getArticleId(rawUrl = '') { + if (!rawUrl) { + return '' + } + let decodeUrl = unescape(rawUrl) + if (!decodeUrl) { + return '' + } + let rawArticleUrl = decodeUrl.split('url=')[1] + if (!rawArticleUrl) { + return '' + } + let baseArticleUrl = rawArticleUrl.split('?')[0] // url => 'https://card.weibo.com/article/m/show/id/2309404446645566701785' + if (!baseArticleUrl) { + return '' + } + let articleId = baseArticleUrl.split('show/id/')[1] + if (!articleId) { + return '' + } + return articleId +} + class FetchCustomer extends Base { fetchStartAtPageNo = 0 fetchEndAtPageNo = 10000 @@ -156,8 +183,8 @@ class FetchCustomer extends Base { ) { // 转发的是微博文章 let pageInfo = rawMblog.mblog.retweeted_status.page_info - let articleUrl = pageInfo.page_url - let articleRecord = await ApiWeibo.asyncGetWeiboArticle(articleUrl) + let articleId = getArticleId(pageInfo.page_url) + let articleRecord = await ApiWeibo.asyncGetWeiboArticle(articleId) if (_.isEmpty(articleRecord)) { // 文章详情获取失败, 不储存该记录 continue @@ -168,8 +195,8 @@ class FetchCustomer extends Base { if (rawMblog.mblog.page_info && rawMblog.mblog.page_info.type === 'article') { // 文章类型为微博文章 let pageInfo = rawMblog.mblog.page_info - let articleUrl = pageInfo.page_url - let articleRecord = await ApiWeibo.asyncGetWeiboArticle(articleUrl) + let articleId = getArticleId(pageInfo.page_url) + let articleRecord = await ApiWeibo.asyncGetWeiboArticle(articleId) if (_.isEmpty(articleRecord)) { // 文章详情获取失败, 不储存该记录 continue diff --git a/src/type/namespace/weibo.d.ts b/src/type/namespace/weibo.d.ts index c8da8462..bbb19c98 100644 --- a/src/type/namespace/weibo.d.ts +++ b/src/type/namespace/weibo.d.ts @@ -210,7 +210,7 @@ export type TypePageInfo = { page_pic: { url: 'https://wx4.sinaimg.cn/wap720/48c999f1gy1g6uza6a6qlj20d407egm6.jpg' } - page_url: 'https://media.weibo.cn/article?object_id=1022%3A2309404414789865570479&extparam=lmid--4414789868083942&luicode=10000011&lfid=2304131221171697_-_WEIBO_SECOND_PROFILE_WEIBO&id=2309404414789865570479' + page_url: 'https://m.weibo.cn/feature/applink?scheme=sinaweibo%3A%2F%2Farticlebrowser%3Fobject_id%3D1022%253A2309404446645566701785%26url%3Dhttps%253A%252F%252Fcard.weibo.com%252Farticle%252Fm%252Fshow%252Fid%252F2309404446645566701785%253F_wb_client_%253D1%26extparam%3Dlmid--4446645569803228&luicode=10000011&lfid=2304131913094142_-_WEIBO_SECOND_PROFILE_WEIBO' page_title: '香港反对派推促的美国反华法案对香港经济的影响？' content1: '香港反对派推促的美国反华法案对香港经济的影响？' content2: '' From ac3d0327bb2a498b9f325b236c86deccb57cfeab Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Sat, 7 Dec 2019 15:57:20 +0800 Subject: [PATCH 02/14] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0todo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 2126fca7..3b0e6ba9 100644 --- a/README.md +++ b/README.md @@ -157,3 +157,11 @@ ![老铁,谢谢了](http://ww1.sinaimg.cn/large/6671cfa8ly1g7y4zur0juj20ri16s41d.jpg) [致谢列表](https://www.easy-mock.com/mock/5d9b49fc896b9432186c1fa5/stablog/thank_you/list) + + +# 功能todo +1. 提供查看数据库内容的入口 + 1. 按账号查看/debug +2. 可以进行增量备份, 或者备份指定页码范围内的数据 + 1. 方便补全数据 +3. 解决转发微博的图片会显示两遍问题 \ No newline at end of file From fcb4ec8d5561c4ed4b3e0ca8c4e898d6c475fda1 Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Sun, 8 Dec 2019 17:49:33 +0800 Subject: [PATCH 03/14] =?UTF-8?q?feat:=20=E5=88=9D=E5=A7=8B=E5=8C=96?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=AE=A1=E7=90=86=E7=95=8C=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +++--- gui/src/view/App.vue | 6 ++++++ gui/src/view/manage/index.vue | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 gui/src/view/manage/index.vue diff --git a/README.md b/README.md index 3b0e6ba9..cb6f5130 100644 --- a/README.md +++ b/README.md @@ -160,8 +160,8 @@ # 功能todo -1. 提供查看数据库内容的入口 - 1. 按账号查看/debug +1. 解决转发微博的图片会显示两遍问题 2. 可以进行增量备份, 或者备份指定页码范围内的数据 1. 方便补全数据 -3. 解决转发微博的图片会显示两遍问题 \ No newline at end of file +3. 提供查看数据库内容的入口 + 1. 按账号查看/debug \ No newline at end of file diff --git a/gui/src/view/App.vue b/gui/src/view/App.vue index dff30ee5..7942cd46 100644 --- a/gui/src/view/App.vue +++ b/gui/src/view/App.vue @@ -6,6 +6,9 @@ + + + @@ -31,6 +34,7 @@ import Log from './log/index.vue' import Helper from './helper/index.vue' import Donate from './donate/index.vue' + import Manage from './manage/index.vue' import _ from 'lodash' export default { @@ -41,6 +45,7 @@ Login, Helper, Donate, + Manage, }, data(){ return { @@ -52,6 +57,7 @@ login:'login', helper:'helper', donate:'donate', + manage:'manage', } }, // 页面数据 diff --git a/gui/src/view/manage/index.vue b/gui/src/view/manage/index.vue new file mode 100644 index 00000000..821b1357 --- /dev/null +++ b/gui/src/view/manage/index.vue @@ -0,0 +1,22 @@ + + + + + From 17acb3e6774450e6cf66d8c6e7721349132a27f7 Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Tue, 10 Dec 2019 22:35:38 +0800 Subject: [PATCH 04/14] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0debug=E5=91=BD?= =?UTF-8?q?=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 4 ++-- package.json | 2 +- src/command/debug.ts | 23 +++++++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 src/command/debug.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index c9c0267e..891da5c1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -41,11 +41,11 @@ { "type": "node", "request": "launch", - "name": "debug", + "name": "debug-command", "program": "${workspaceFolder}/src/ace.ts", "args": [ // 抓取数据 - "Command:Demo" + "Debug" ], "outFiles": ["${workspaceFolder}/**/*.js"] } diff --git a/package.json b/package.json index a939864d..88f8dcf7 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "element-ui": "^2.12.0", "image-size": "^0.8.3", "json5": "^2.1.0", - "knex": "^0.19.5", + "knex": "^0.20.1", "lodash": "^4.17.5", "md5": "^2.2.1", "moment": "^2.24.0", diff --git a/src/command/debug.ts b/src/command/debug.ts new file mode 100644 index 00000000..d23dce2c --- /dev/null +++ b/src/command/debug.ts @@ -0,0 +1,23 @@ +import Base from '~/src/command/base' +import MBlog from '~/src/model/mblog' +import fs from 'fs' + +class CommandDemo extends Base { + static get signature() { + return ` + Debug + ` + } + + static get description() { + return '专业Debug' + } + + async execute() { + let result = await MBlog.asyncGetWeiboDistribution('1221171697') + this.log(`debug it`) + console.log(result) + } +} + +export default CommandDemo From 9024df0558de5c97517292fbe29d22c842d97d45 Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Tue, 10 Dec 2019 22:55:40 +0800 Subject: [PATCH 05/14] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0deug=E7=A8=8B?= =?UTF-8?q?=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 2 +- src/ace.ts | 1 + src/command/debug.ts | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 891da5c1..ded85f45 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -45,7 +45,7 @@ "program": "${workspaceFolder}/src/ace.ts", "args": [ // 抓取数据 - "Debug" + "debug" ], "outFiles": ["${workspaceFolder}/**/*.js"] } diff --git a/src/ace.ts b/src/ace.ts index 20bd404c..c7d5428c 100644 --- a/src/ace.ts +++ b/src/ace.ts @@ -12,6 +12,7 @@ import ace from '@adonisjs/ace' const registedCommandList = [ './command/demo', // 命令demo + './command/debug', // 专业debug命令 './command/init_env', // 初始化运行环境 './command/dispatch_task', // 分发任务 diff --git a/src/command/debug.ts b/src/command/debug.ts index d23dce2c..5d10090c 100644 --- a/src/command/debug.ts +++ b/src/command/debug.ts @@ -2,10 +2,10 @@ import Base from '~/src/command/base' import MBlog from '~/src/model/mblog' import fs from 'fs' -class CommandDemo extends Base { +class CommandDebug extends Base { static get signature() { return ` - Debug + debug ` } @@ -20,4 +20,4 @@ class CommandDemo extends Base { } } -export default CommandDemo +export default CommandDebug From d401f8b712330d3ee0aad95174f96eb15d3eab90 Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Wed, 11 Dec 2019 02:48:40 +0800 Subject: [PATCH 06/14] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E6=9F=A5?= =?UTF-8?q?=E7=9C=8B=E6=95=B0=E6=8D=AE=E5=BA=93/=E8=A7=A3=E5=86=B3reffer?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gui/public/index.html | 2 + gui/src/view/manage/index.vue | 280 ++++++++++++++++++++++++++++++++-- src/index.ts | 11 +- src/model/mblog.ts | 100 ++++++++++++ src/model/mblog_user.ts | 18 +++ 5 files changed, 399 insertions(+), 12 deletions(-) diff --git a/gui/public/index.html b/gui/public/index.html index bbc86a56..f0411de8 100644 --- a/gui/public/index.html +++ b/gui/public/index.html @@ -3,6 +3,8 @@ + + diff --git a/gui/src/view/manage/index.vue b/gui/src/view/manage/index.vue index 821b1357..c60f86a0 100644 --- a/gui/src/view/manage/index.vue +++ b/gui/src/view/manage/index.vue @@ -1,22 +1,282 @@ - - diff --git a/src/index.ts b/src/index.ts index aeba4601..e8f06023 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,8 @@ import ConfigHelperUtil from '~/src/library/util/config_helper' import PathConfig from '~/src/config/path' import Logger from '~/src/library/logger' import DispatchTaskCommand from '~/src/command/dispatch_task' +import MUser from '~/src/model/mblog_user' +import MBlog from '~/src/model/mblog' import fs from 'fs' import _ from 'lodash' @@ -48,10 +50,12 @@ function createWindow() { Menu.setApplicationMenu(null) } + const { screen } = Electron + const { width, height } = screen.getPrimaryDisplay().workAreaSize // Create the browser window. mainWindow = new BrowserWindow({ - width: 1366, - height: 768, + width, + height, // 自动隐藏菜单栏 autoHideMenuBar: true, // 窗口的默认标题 @@ -101,6 +105,9 @@ function createWindow() { }) global.pathConfig = PathConfig + // 向html代码注入MUser, 方便查询 + global.mUser = MUser + global.mBlog = MBlog } // This method will be called when Electron has finished diff --git a/src/model/mblog.ts b/src/model/mblog.ts index ae200cfb..4df4c071 100644 --- a/src/model/mblog.ts +++ b/src/model/mblog.ts @@ -11,6 +11,18 @@ type TypeMblogRecord = { post_publish_at: number } +type BlogDistributionMap = Map< + string, + { + date: string + key: string + type: 'year' | 'month' | 'day' + startAt: number + count: number + childrenMap: BlogDistributionMap + } +> + export default class Mblog extends Base { static TABLE_NAME = `total_mblog` static TABLE_COLUMN = [`id`, `author_uid`, `raw_json`] @@ -45,4 +57,92 @@ export default class Mblog extends Base { } return mblogRecordList } + + /** + * 获取数据库中的微博记录数分布 + * @param uid + */ + static async asyncGetWeiboDistribution(uid: string): Promise { + let postPublishAtList = >await this.db + .select(`post_publish_at`) + .from(this.TABLE_NAME) + .where('author_uid', '=', uid) + .orderBy(`post_publish_at`, 'desc') + .catch(() => { + return [] + }) + let distributionMap: Map>> = new Map() + + for (let item of postPublishAtList) { + let publishAt = item.post_publish_at + let YYYY = moment.unix(publishAt).format('YYYY') + let MM = moment.unix(publishAt).format('MM') + let DD = moment.unix(publishAt).format('DD') + + if (distributionMap.has(YYYY) === false) { + distributionMap.set(YYYY, new Map()) + } + let yearMap = distributionMap.get(YYYY)! + if (yearMap.has(MM) === false) { + yearMap.set(MM, new Map()) + } + let monthMap = yearMap.get(MM)! + if (monthMap.has(DD) === false) { + monthMap.set(DD, []) + } + let dayList = monthMap.get(DD)! + dayList.push(publishAt) + monthMap.set(DD, dayList) + yearMap.set(MM, monthMap) + distributionMap.set(YYYY, yearMap) + } + + let newYearMap: BlogDistributionMap = new Map() + for (let year of distributionMap.keys()) { + let yearCounter = 0 + let yearMap = distributionMap.get(year)! + let newMonthMap: BlogDistributionMap = new Map() + for (let month of yearMap.keys()) { + let monthCounter = 0 + let monthMap = yearMap.get(month)! + let newDayMap: BlogDistributionMap = new Map() + for (let day of monthMap.keys()) { + let dayCounter = monthMap.get(day)!.length + monthCounter += dayCounter + newDayMap.set(`${day}日`, { + date: `${day}日`, + key: `${year}-${month}-${day}`, + type: 'day', + startAt: moment(`${year}-${month}-${day}`, 'YYYY-MM-DD') + .startOf('day') + .unix(), + count: dayCounter, + childrenMap: new Map(), + }) + } + yearCounter += monthCounter + newMonthMap.set(`${month}月`, { + date: `${month}月`, + key: `${year}-${month}`, + type: 'month', + startAt: moment(`${year}-${month}-01`, 'YYYY-MM-DD') + .startOf('day') + .unix(), + count: monthCounter, + childrenMap: newDayMap, + }) + } + newYearMap.set(`${year}年`, { + date: `${year}年`, + key: `${year}`, + type: 'year', + startAt: moment(`${year}-01-01`, 'YYYY-MM-DD') + .startOf('day') + .unix(), + count: yearCounter, + childrenMap: newMonthMap, + }) + } + return newYearMap + } } diff --git a/src/model/mblog_user.ts b/src/model/mblog_user.ts index 82468d1c..c363222d 100644 --- a/src/model/mblog_user.ts +++ b/src/model/mblog_user.ts @@ -13,6 +13,24 @@ export default class MblogUser extends Base { static TABLE_NAME = `total_user` static TABLE_COLUMN = [`author_uid`, `raw_json`] + /** + * 获取数据库中的用户列表 + * @param author_uid + */ + static async asyncGetUserList(): Promise { + let rawRecordList = >await this.db + .select(this.TABLE_COLUMN) + .from(this.TABLE_NAME) + .catch(() => { + return [] + }) + let recordList = [] + for (let rawItem of rawRecordList) { + let record = JSON.parse(rawItem.raw_json) + recordList.push(record) + } + return recordList + } /** * 从数据库中获取微博记录列表 * @param id From 4a083f4d1815bd74783de3f2fe48ad81d3b0953c Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Thu, 12 Dec 2019 01:34:48 +0800 Subject: [PATCH 07/14] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E6=B5=8F?= =?UTF-8?q?=E8=A7=88=E6=95=B0=E6=8D=AE=E5=BA=93=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gui/src/view/customer_task/index.vue | 2 +- gui/src/view/manage/index.vue | 115 +++++++++++++++++++++++---- 2 files changed, 100 insertions(+), 17 deletions(-) diff --git a/gui/src/view/customer_task/index.vue b/gui/src/view/customer_task/index.vue index 92b99755..7e499cc9 100644 --- a/gui/src/view/customer_task/index.vue +++ b/gui/src/view/customer_task/index.vue @@ -245,7 +245,7 @@ export default Vue.extend({ mergeCount: 1000, fetchStartAtPageNo: 0, fetchEndAtPageNo: 100000, - outputStartAtMs: moment('2015-01-01 00:00:00').unix() * 1000, + outputStartAtMs: moment('2011-01-01 00:00:00').unix() * 1000, outputEndAtMs: moment() .add(1, 'year') diff --git a/gui/src/view/manage/index.vue b/gui/src/view/manage/index.vue index c60f86a0..42da495c 100644 --- a/gui/src/view/manage/index.vue +++ b/gui/src/view/manage/index.vue @@ -1,5 +1,5 @@ @@ -58,6 +102,7 @@ import { remote } from 'electron' import Vue from 'vue' import _ from 'lodash' +import moment from 'moment' import TypeWeibo from '~/gui/../src/type/namespace/weibo' type BlogDistributionItem = { @@ -93,8 +138,10 @@ export default { storageSelect: { year: string month: string + day: string yearList: string[] monthList: string[] + blogList: TypeWeibo.TypeMblog[] } loading: { distribution: boolean @@ -112,8 +159,10 @@ export default { storageSelect: { year: '', month: '', + day: '', yearList: [], monthList: [], + blogList: [], }, loading: { distribution: false, @@ -143,21 +192,36 @@ export default { this.database.blogDistributionMap = distributionMap this.forceUpdate++ }, + /** + * 获取当天微博数据列表 + */ + async getBlogListInRange(startAt: number, endAt: number) { + let MBlog = remote.getGlobal('mBlog') + let blogList = (await MBlog.asyncGetMblogList(this.state.selectUserId, startAt, endAt)) as TypeWeibo.TypeMblog[] + console.log(JSON.parse(JSON.stringify(blogList))) + return blogList + }, async handleSelectChange(currentRow: TypeWeibo.TypeWeiboUserInfo, oldCurrentRow: TypeWeibo.TypeWeiboUserInfo) { this.state.selectUserId = `${currentRow.id}` - this.state.storageSelect.month = '' - this.state.storageSelect.year = '' - this.state.storageSelect.yearList = [] - this.state.storageSelect.monthList = [] + this.resetSelect() await this.getDistribute() }, // 支持点选 - handleStorageSelectInTable(row: BlogDistributionItem) { + async handleStorageSelectInTable(row: BlogDistributionItem) { if (row.type === 'year') { this.state.storageSelect.year = row.date + this.state.storageSelect.blogList = [] } if (row.type === 'month') { this.state.storageSelect.month = row.date + this.state.storageSelect.blogList = [] + } + if (row.type === 'day') { + this.state.storageSelect.day = row.date + let startAt = row.startAt + let endAt = row.startAt + 86400 - 1 + let blogList = await this.getBlogListInRange(startAt, endAt) + this.state.storageSelect.blogList = blogList } }, weiboStorageYearList() { @@ -197,6 +261,19 @@ export default { clearStorageSelect() { this.state.storageSelect.year = '' this.state.storageSelect.month = '' + this.state.storageSelect.day = '' + this.state.storageSelect.blogList = [] + }, + resetSelect() { + this.state.storageSelect.month = '' + this.state.storageSelect.year = '' + this.state.storageSelect.day = '' + this.state.storageSelect.yearList = [] + this.state.storageSelect.monthList = [] + this.state.storageSelect.blogList = [] + }, + formatUnix(timestamp: number) { + return moment.unix(timestamp).format('YYYY-MM-DD HH:mm:ss') }, }, computed: { @@ -267,16 +344,22 @@ export default { } - From 03f6d75b05855afea756c7bcd176cc89c68dd726 Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Thu, 12 Dec 2019 23:34:58 +0800 Subject: [PATCH 08/14] =?UTF-8?q?feat:=20=E8=A7=A3=E5=86=B3=E6=8C=89?= =?UTF-8?q?=E5=8F=91=E5=B8=83=E6=97=B6=E9=97=B4=E6=8E=92=E5=BA=8F=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/command/generate/customer.ts | 9 +++------ src/model/mblog.ts | 10 ++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/command/generate/customer.ts b/src/command/generate/customer.ts index 71c277bc..7878010c 100644 --- a/src/command/generate/customer.ts +++ b/src/command/generate/customer.ts @@ -105,12 +105,9 @@ class GenerateCustomer extends Base { ) mblogList.sort((a, b) => { // 先进行排序 - let aSortBy = a.created_timestamp_at - let bSortBy = b.created_timestamp_at - if (a.created_timestamp_at === b.created_timestamp_at) { - aSortBy = parseInt(a.id) - bSortBy = parseInt(b.id) - } + // 根据接口 https://m.weibo.cn/feed/friends?max_id=4448802586999203 可以确认, id为确认时间线的关键 + let aSortBy = parseInt(a.idstr, 10) + let bSortBy = parseInt(b.idstr, 10) if (this.CUSTOMER_CONFIG_postAtOrderBy === 'asc') { return aSortBy! - bSortBy! } else { diff --git a/src/model/mblog.ts b/src/model/mblog.ts index 4df4c071..5fc40416 100644 --- a/src/model/mblog.ts +++ b/src/model/mblog.ts @@ -55,6 +55,16 @@ export default class Mblog extends Base { mblogRecordList.push(mblogRecord) } } + + // 按发布时间(id)排序 + mblogRecordList.sort((a, b) => { + // 先进行排序 + // 根据接口 https://m.weibo.cn/feed/friends?max_id=4448802586999203 可以确认, id为确认时间线的关键 + let aSortBy = parseInt(a.idstr, 10) + let bSortBy = parseInt(b.idstr, 10) + return aSortBy! - bSortBy! + }) + return mblogRecordList } From 987e65882d23dfd45e022fb94dc2b71e9e3c864e Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Fri, 13 Dec 2019 00:05:53 +0800 Subject: [PATCH 09/14] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dlog=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=E9=94=99=E5=88=AB=E5=AD=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/command/generate/customer.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/command/generate/customer.ts b/src/command/generate/customer.ts index 7878010c..3fdca31e 100644 --- a/src/command/generate/customer.ts +++ b/src/command/generate/customer.ts @@ -368,12 +368,12 @@ class GenerateCustomer extends Base { let dayIndex = 0 for (let weiboDayRecord of weiboDayList) { dayIndex++ - this.log(`正在处理第${dayIndex}/${weiboDayList.length}批微博记录`) + this.log(`将网页渲染为pdf, 正在处理第${dayIndex}/${weiboDayList.length}卷微博记录`) let weiboIndex = 0 for (let weiboRecord of weiboDayRecord.weiboList) { weiboIndex++ this.log( - `正在处理第${dayIndex}/${weiboDayList.length}批下,第${weiboIndex}/${weiboDayRecord.weiboList.length}条微博`, + `正在处理第${dayIndex}/${weiboDayList.length}卷中,第${weiboIndex}/${weiboDayRecord.weiboList.length}条微博`, ) let content = WeiboView.render([weiboRecord]) content = this.processContent(content) @@ -388,15 +388,15 @@ class GenerateCustomer extends Base { let imageBuffer = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: true, omitBackground: true }) if (imageBuffer.length < 1000) { // 图片渲染失败 - this.log(`第${dayIndex}/${weiboDayList.length}条微博渲染失败, 自动跳过`) + this.log(`第${weiboIndex}/${weiboDayRecord.weiboList.length}条微博渲染失败, 自动跳过`) continue } else { - this.log(`第${dayIndex}/${weiboDayList.length}条微博渲染成功`) + this.log(`第${weiboIndex}/${weiboDayRecord.weiboList.length}条微博渲染成功`) let size = await imageSize.imageSize(imageBuffer) let { width, height } = size this.log(`图片size=>`, { width, height }) if (!width || width <= 0 || !height || height <= 0) { - this.log(`第${dayIndex}/${weiboDayList.length}条微博截图捕获失败, 自动跳过`) + this.log(`第${weiboIndex}/${weiboDayRecord.weiboList.length}条微博截图捕获失败, 自动跳过`) continue } From 1b90b3ccd06488005fdaf5698ec41e39c29bf849 Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Fri, 13 Dec 2019 00:06:09 +0800 Subject: [PATCH 10/14] =?UTF-8?q?fix:=20=E8=A7=A3=E5=86=B3=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E6=8A=93=E5=8F=96=E5=BE=AE=E5=8D=9A=E6=96=87=E7=AB=A0?= =?UTF-8?q?=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/command/fetch/customer.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/command/fetch/customer.ts b/src/command/fetch/customer.ts index 86b85f86..80b62646 100644 --- a/src/command/fetch/customer.ts +++ b/src/command/fetch/customer.ts @@ -21,7 +21,8 @@ function getArticleId(rawUrl = '') { if (!rawUrl) { return '' } - let decodeUrl = unescape(rawUrl) + // 需要多次解析，才能将url完全解码成正常文本 + let decodeUrl = unescape(unescape(unescape(rawUrl))) if (!decodeUrl) { return '' } @@ -130,7 +131,7 @@ class FetchCustomer extends Base { } else { await this.fetchMblogListAndSaveToDb(uid, page, totalPageCount) // 微博的反爬虫措施太强, 只能用每5s抓一次的方式拿数据🤦‍♂️ - let sleep_s = 20 + let sleep_s = 3 this.log(`已抓取${page}/${totalPageCount}页记录, 休眠${sleep_s}s, 避免被封`) await Util.asyncSleep(sleep_s * 1000) } From 8d32af0df38044251876f29cfb8e20cb3f84f10a Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Fri, 13 Dec 2019 00:06:43 +0800 Subject: [PATCH 11/14] =?UTF-8?q?fix:=20=E7=AD=89=E5=BE=85=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E5=BA=94=E4=B8=BA20s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/command/fetch/customer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/command/fetch/customer.ts b/src/command/fetch/customer.ts index 80b62646..e2df247f 100644 --- a/src/command/fetch/customer.ts +++ b/src/command/fetch/customer.ts @@ -131,7 +131,7 @@ class FetchCustomer extends Base { } else { await this.fetchMblogListAndSaveToDb(uid, page, totalPageCount) // 微博的反爬虫措施太强, 只能用每5s抓一次的方式拿数据🤦‍♂️ - let sleep_s = 3 + let sleep_s = 20 this.log(`已抓取${page}/${totalPageCount}页记录, 休眠${sleep_s}s, 避免被封`) await Util.asyncSleep(sleep_s * 1000) } From c823e00e4c77ddfdbad388cd8a5d8a232b108f5d Mon Sep 17 00:00:00 2001 From: yaozeyuan Date: Fri, 13 Dec 2019 00:29:20 +0800 Subject: [PATCH 12/14] =?UTF-8?q?feat:=20=E5=87=86=E5=A4=87=E6=89=93?= =?UTF-8?q?=E5=8C=85=E5=8F=91=E5=B8=83=E6=96=B0=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 14 +----- changelog.md | 6 +++ gui/src/view/helper/index.vue | 19 ++++--- gui/src/view/log/index.vue | 93 ++++++++++++++++++----------------- package.json | 2 +- 5 files changed, 65 insertions(+), 69 deletions(-) create mode 100644 changelog.md diff --git a/README.md b/README.md index cb6f5130..9d0441bb 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,6 @@ ## 项目动机 -鉴于知乎世风日下, 还禁了项目前身[知乎助手](https://github.com/YaoZeyuan/zhihuhelp_with_node)的推广. 作者决定转战微博, 造福社会 - 严肃保护以[@Aioros先生](https://weibo.com/u/6646798696?refer_flag=0000015010_&from=feed&loc=nickname&sudaref=www.weibo.com&is_all=1)为代表的珍稀野生读物博主 @@ -154,14 +152,4 @@ # 支持作者 -![老铁,谢谢了](http://ww1.sinaimg.cn/large/6671cfa8ly1g7y4zur0juj20ri16s41d.jpg) - -[致谢列表](https://www.easy-mock.com/mock/5d9b49fc896b9432186c1fa5/stablog/thank_you/list) - - -# 功能todo -1. 解决转发微博的图片会显示两遍问题 -2. 可以进行增量备份, 或者备份指定页码范围内的数据 - 1. 方便补全数据 -3. 提供查看数据库内容的入口 - 1. 按账号查看/debug \ No newline at end of file +![老铁,谢谢了](http://ww1.sinaimg.cn/large/6671cfa8ly1g7y4zur0juj20ri16s41d.jpg) \ No newline at end of file diff --git a/changelog.md b/changelog.md new file mode 100644 index 00000000..fe6e4eb9 --- /dev/null +++ b/changelog.md @@ -0,0 +1,6 @@ +# 1.2.0 + +1. 支持浏览已备份微博数据 +2. 解决微博排序异常问题 +3. 解决无法展示微博文章问题 +4. 日志记录超过10w行, 自动清空 \ No newline at end of file diff --git a/gui/src/view/helper/index.vue b/gui/src/view/helper/index.vue index 6acaa259..a161f242 100644 --- a/gui/src/view/helper/index.vue +++ b/gui/src/view/helper/index.vue @@ -4,27 +4,26 @@ name="github-readme" id="github-readme" src="https://github.com/YaoZeyuan/stablog/blob/master/README.md" - style > diff --git a/gui/src/view/log/index.vue b/gui/src/view/log/index.vue index 0d73ea85..264fb244 100644 --- a/gui/src/view/log/index.vue +++ b/gui/src/view/log/index.vue @@ -1,61 +1,64 @@