- +
diff --git a/lib/app.js b/lib/app.js index c958f0f6f5994c..39b91f574e1c78 100644 --- a/lib/app.js +++ b/lib/app.js @@ -37,8 +37,8 @@ const app = new Koa(); app.proxy = true; // favicon -app.use(favicon(__dirname + '/favicon.png')); -app.use(serve(__dirname + '/static')); +app.use(favicon(__dirname + '/favicon.png', { maxAge: 31536000000 })); +app.use(serve(__dirname + '/static', { maxage: 31536000000 })); // global error handing app.use(onerror); diff --git a/lib/radar.js b/lib/radar.js index 09f24e6e8e99f4..29efd6ee842b3a 100644 --- a/lib/radar.js +++ b/lib/radar.js @@ -4,7 +4,7 @@ const toSource = require('tosource'); const { join } = require('path'); // Namespaces that do not require radar.js -const allowNamespace = ['discourse', 'ehentai', 'test']; +const allowNamespace = ['discourse', 'discuz', 'ehentai', 'test']; // Check if a radar.js file is exist under each folder of dirname for (const dir of fs.readdirSync(dirname)) { const dirPath = join(dirname, dir); diff --git a/lib/router.js b/lib/router.js index 4a818131d90578..f11c9016b6a2bb 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1446,9 +1446,9 @@ router.get('/gbcc/trust', lazyloadRouteHandler('./routes/gbcc/trust')); router.get('/cbc/topics/:topic?', lazyloadRouteHandler('./routes/cbc/topics')); // discuz -router.get('/discuz/:ver([7|x])/:cid([0-9]{2})/:link(.*)', lazyloadRouteHandler('./routes/discuz/discuz')); -router.get('/discuz/:ver([7|x])/:link(.*)', lazyloadRouteHandler('./routes/discuz/discuz')); -router.get('/discuz/:link(.*)', lazyloadRouteHandler('./routes/discuz/discuz')); +// router.get('/discuz/:ver([7|x])/:cid([0-9]{2})/:link(.*)', lazyloadRouteHandler('./routes/discuz/discuz')); +// router.get('/discuz/:ver([7|x])/:link(.*)', lazyloadRouteHandler('./routes/discuz/discuz')); +// router.get('/discuz/:link(.*)', lazyloadRouteHandler('./routes/discuz/discuz')); // China Dialogue 中外对话 router.get('/chinadialogue/topics/:topic', lazyloadRouteHandler('./routes/chinadialogue/topics')); diff --git a/lib/routes/discuz/discuz.js b/lib/routes/discuz/discuz.js deleted file mode 100644 index b936586c21fed8..00000000000000 --- a/lib/routes/discuz/discuz.js +++ /dev/null @@ -1,148 +0,0 @@ -const got = require('@/utils/got'); -const cheerio = require('cheerio'); -const iconv = require('iconv-lite'); -const dateUtil = require('@/utils/date'); -const config = require('@/config').value; - -// discuz 7.x 与 discuz x系列 通用文章内容抓取 -async function load(baseUrl, itemLink, ctx, charset, header) { - // 处理相对链接 - if (itemLink) { - if (baseUrl && !baseUrl.match(/^https?:\/\//)) { - if (baseUrl.match(/^\/\//)) { - baseUrl = 'http:' + baseUrl; - } else { - baseUrl = 'http://' + baseUrl; - } - } - itemLink = new URL(itemLink, baseUrl).href; - } - - const cache = await ctx.cache.get(itemLink); - if (cache) { - return cache; - } - - // 处理编码问题 - let responseData; - if (charset === 'utf-8') { - responseData = ( - await got({ - method: 'get', - url: itemLink, - headers: header, - }) - ).data; - } else { - responseData = iconv.decode( - ( - await got({ - method: 'get', - url: itemLink, - responseType: 'buffer', - headers: header, - }) - ).data, - charset - ); - } - if (!responseData) { - const description = '获取详细内容失败'; - return { description }; - } - const $ = cheerio.load(responseData); - // 只抓取论坛1楼消息 - const description = $('div#postlist div[id^=post] td[id^=postmessage]').slice(0, 1).html(); - ctx.cache.set(itemLink, description); - return { description }; -} - -module.exports = async (ctx) => { - let link = ctx.params.link; - const ver = ctx.params.ver ? ctx.params.ver.toUpperCase() : undefined; - const cid = ctx.params.cid; - link = link.replace(/:\/\//, ':/').replace(/:\//, '://'); - const cookie = cid === undefined ? '' : config.discuz.cookies[cid]; - if (cookie === undefined) { - throw Error('缺少对应论坛的cookie.'); - } - const header = { - Cookie: cookie, - Referer: link, - }; - const response = await got({ - method: 'get', - url: link, - headers: header, - }); - const contentType = response.headers['content-type'] || ''; - // 若没有指定编码,则默认utf-8 - let charset = 'utf-8'; - for (const attr of contentType.split(';')) { - if (attr.indexOf('charset=') >= 0) { - charset = attr.split('=').pop().toLowerCase(); - } - } - const responseData = - charset === 'utf-8' - ? response.data - : iconv.decode( - ( - await got({ - method: 'get', - url: link, - responseType: 'buffer', - headers: { - Cookie: cookie, - }, - }) - ).data, - charset - ); - const $ = cheerio.load(responseData); - const title = $('head > title').text(); - const version = ver ? 'DISCUZ! ' + ver : $('head > meta[name=generator]').attr('content'); - let process; - if (version.toUpperCase().startsWith('DISCUZ! 7')) { - // discuz 7.x 系列 - // 支持全文抓取,限制抓取页面5个 - const list = $('tbody[id^="normalthread"] > tr').slice(0, 5).get(); - process = await Promise.all( - list.map(async (item) => { - item = $(item); - const itemLink = item.find('span[id^=thread] a').attr('href'); - const single = { - title: item.find('span[id^=thread] a').text(), - link: itemLink, - pubDate: dateUtil(item.find('td.author em').text()), - }; - const detail = await load(link, itemLink, ctx, charset, header); - return Promise.resolve(Object.assign({}, single, detail)); - }) - ); - } else if (version.toUpperCase().startsWith('DISCUZ! X')) { - // discuz X 系列 - // 支持全文抓取,限制抓取页面5个 - const list = $('tbody[id^="normalthread"] > tr').slice(0, 5).get(); - process = await Promise.all( - list.map(async (item) => { - item = $(item); - const itemLink = item.find('a.xst').attr('href'); - const single = { - title: item.find('a.xst').text(), - link: itemLink, - pubDate: dateUtil(item.find('td.by:nth-child(3) em span').last().text()), - }; - const detail = await load(link, itemLink, ctx, charset, header); - return Promise.resolve(Object.assign({}, single, detail)); - }) - ); - } else { - throw Error('不支持当前Discuz版本.'); - } - ctx.state.data = { - title, - link, - item: process, - }; -}; diff --git a/lib/v2/abskoop/index.js b/lib/v2/abskoop/index.js index 8353bc1721c733..a6385aec1714a7 100644 --- a/lib/v2/abskoop/index.js +++ b/lib/v2/abskoop/index.js @@ -1,47 +1,37 @@ -const path = require('path'); const got = require('@/utils/got'); const cheerio = require('cheerio'); -const { art } = require('@/utils/render'); const { parseDate } = require('@/utils/parse-date'); module.exports = async (ctx) => { const response = await got({ method: 'get', - url: 'https://www.abskoop.com/page/1/', + url: 'https://www.ahhhhfs.com/wp-json/wp/v2/posts', + searchParams: { + per_page: ctx.query.limit ? parseInt(ctx.query.limit) : 10, + _embed: '', + }, }); - - const $list = cheerio.load(response.data); - const list = $list('.rizhuti_v2-widget-lastpost .scroll article') - .map(function () { - const link = $list(this).find('.entry-wrapper .entry-header a'); - return { - title: link.attr('title'), - link: link.attr('href'), - }; - }) - .get(); + const list = response.data.map((item) => ({ + title: item.title.rendered, + link: item.link, + pubDate: parseDate(item.date_gmt), + updated: parseDate(item.modified_gmt), + author: item._embedded.author[0].name, + category: [...new Set(item._embedded['wp:term'].flatMap((i) => i.map((j) => j.name)))], + })); const items = await Promise.all( - list.reverse().map((item) => + list.map((item) => ctx.cache.tryGet(item.link, async () => { const detailResponse = await got({ method: 'get', url: item.link }); const $detail = cheerio.load(detailResponse.data); - $detail('article .entry-content').find('#related_posts').remove(); - $detail('article .entry-content').find('#jp-relatedposts').remove(); - $detail('article .entry-content').find('.post-note').remove(); - $detail('article .entry-content').find('.entry-tags').remove(); - $detail('article .entry-content').find('.entry-share').remove(); - $detail('article .entry-content a').each(function () { - if ($detail(this).find('img').length > 0) { - $detail(this).replaceWith(``); - } - }); - const desc = []; - $detail('article .entry-content > *').each(function () { - desc.push($detail(this).html()); + $detail('article.post-content').find('.lwptoc').remove(); + $detail('article.post-content').find('#related_posts').remove(); + $detail('article.post-content').find('.entry-copyright').remove(); + $detail('article.post-content img').each(function () { + $detail(this).replaceWith(``); }); - item.description = art(path.join(__dirname, 'templates/description.art'), { desc }); - item.pubDate = parseDate($detail('meta[property="article:published_time"]').attr('content')); + item.description = $detail('article.post-content').html(); return item; }) ) @@ -49,7 +39,7 @@ module.exports = async (ctx) => { ctx.state.data = { title: 'ahhhhfs-A姐分享', - link: 'https://www.abskoop.com', + link: 'https://www.ahhhhfs.com', description: 'A姐分享,分享各种网络云盘资源、BT种子、高清电影电视剧和羊毛福利,收集各种有趣实用的软件和APP的下载、安装、使用方法,发现一些稀奇古怪的的网站,折腾一些有趣实用的教程,关注谷歌苹果等互联网最新的资讯动态,探索新领域,发现新美好,分享小快乐。', item: items, diff --git a/lib/v2/abskoop/nsfw.js b/lib/v2/abskoop/nsfw.js index fa59f818652801..100695a0f9b6ca 100644 --- a/lib/v2/abskoop/nsfw.js +++ b/lib/v2/abskoop/nsfw.js @@ -1,52 +1,31 @@ -const path = require('path'); const got = require('@/utils/got'); -const cheerio = require('cheerio'); -const { art } = require('@/utils/render'); const { parseDate } = require('@/utils/parse-date'); module.exports = async (ctx) => { const response = await got({ method: 'get', - url: 'https://nsfw.ahhhhfs.com/articles-archive/', + url: 'https://nsfw.abskoop.com/wp-json/wp/v2/posts', + searchParams: { + per_page: ctx.query.limit ? parseInt(ctx.query.limit) : 10, + _embed: '', + }, }); - const $list = cheerio.load(response.data); - const list = $list('article li') - .slice(0, 20) - .map(function () { - return { - link: $list(this).find('a').attr('href'), - }; - }) - .get(); - - const items = await Promise.all( - list.map((item) => - ctx.cache.tryGet(item.link, async () => { - const detailResponse = await got({ method: 'get', url: item.link }); - const $detail = cheerio.load(detailResponse.data); - $detail('article .entry-content a').each(function () { - if ($detail(this).find('img').length > 0) { - $detail(this).replaceWith(``); - } - }); - const desc = []; - $detail('article .entry-content > p').each(function () { - desc.push($detail(this).html()); - }); - item.title = $detail('article h1.entry-title').text(); - item.description = art(path.join(__dirname, 'templates/description.art'), { desc }); - item.pubDate = parseDate($detail('meta[property="article:published_time"]').attr('content')); - return item; - }) - ) - ); + const list = response.data.map((item) => ({ + title: item.title.rendered, + description: item.content.rendered, + link: item.link, + pubDate: parseDate(item.date_gmt), + updated: parseDate(item.modified_gmt), + author: item._embedded.author[0].name, + category: [...new Set(item._embedded['wp:term'].flatMap((i) => i.map((j) => j.name)))], + })); ctx.state.data = { title: 'ahhhhfs-A姐分享NSFW', - link: 'https://nsfw.ahhhhfs.com', + link: 'https://nsfw.ahhhhfs.com/articles-archive', description: 'A姐分享NSFW,分享各种网络云盘资源、BT种子、磁力链接、高清电影电视剧和羊毛福利,收集各种有趣实用的软件和APP的下载、安装、使用方法,发现一些稀奇古怪的的网站,折腾一些有趣实用的教程,关注谷歌苹果等互联网最新的资讯动态,探索新领域,发现新美好,分享小快乐。', - item: items, + item: list, }; }; diff --git a/lib/v2/abskoop/radar.js b/lib/v2/abskoop/radar.js index 8cd53cd15b1afc..60544a7a3761f6 100644 --- a/lib/v2/abskoop/radar.js +++ b/lib/v2/abskoop/radar.js @@ -1,11 +1,22 @@ module.exports = { 'abskoop.com': { + _name: 'A姐分享', + nsfw: [ + { + title: 'NSFW 存档列表', + docs: 'https://docs.rsshub.app/routes/multimedia#a-jie-fen-xiang', + source: ['/articles-archive', '/'], + target: '/abskoop', + }, + ], + }, + 'ahhhhfs.com': { _name: 'A姐分享', '.': [ { title: '存档列表', - docs: 'https://docs.rsshub.app/routes/multimedia#abskoop', - source: ['/archives'], + docs: 'https://docs.rsshub.app/routes/multimedia#a-jie-fen-xiang', + source: ['/'], target: '/abskoop', }, ], diff --git a/lib/v2/abskoop/templates/description.art b/lib/v2/abskoop/templates/description.art deleted file mode 100644 index 4594e979931118..00000000000000 --- a/lib/v2/abskoop/templates/description.art +++ /dev/null @@ -1,3 +0,0 @@ -{{each desc}} -
{{@ $value}}
-{{/each}} diff --git a/lib/v2/bilibili/followings_dynamic.js b/lib/v2/bilibili/followings_dynamic.js index 777fb9df001f15..244e0b0c872028 100644 --- a/lib/v2/bilibili/followings_dynamic.js +++ b/lib/v2/bilibili/followings_dynamic.js @@ -117,7 +117,7 @@ module.exports = async (ctx) => { }); } // 作者信息 - let author = '哔哩哔哩番剧'; + let author = ''; if (item.desc?.user_profile) { author = item.desc.user_profile.info.uname; } diff --git a/lib/v2/discuz/discuz.js b/lib/v2/discuz/discuz.js new file mode 100644 index 00000000000000..af83e0877d31e7 --- /dev/null +++ b/lib/v2/discuz/discuz.js @@ -0,0 +1,156 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const iconv = require('iconv-lite'); +const { parseDate } = require('@/utils/parse-date'); +const config = require('@/config').value; + +function fixUrl(itemLink, baseUrl) { + // 处理相对链接 + if (itemLink) { + if (baseUrl && !baseUrl.match(/^https?:\/\//)) { + if (baseUrl.match(/^\/\//)) { + baseUrl = 'http:' + baseUrl; + } else { + baseUrl = 'http://' + baseUrl; + } + } + itemLink = new URL(itemLink, baseUrl).href; + } + return itemLink; +} + +// discuz 7.x 与 discuz x系列 通用文章内容抓取 +async function load(itemLink, charset, header) { + // 处理编码问题 + const response = await got({ + method: 'get', + url: itemLink, + responseType: 'buffer', + headers: header, + }); + + const responseData = iconv.decode(response.data, charset ?? 'utf-8'); + if (!responseData) { + const description = '获取详细内容失败'; + return { description }; + } + + const $ = cheerio.load(responseData); + + const post = $('div#postlist div[id^=post] td[id^=postmessage]').first(); + + // fix lazyload image + post.find('img').each((_, img) => { + img = $(img); + if (img.attr('src')?.endsWith('none.gif') && img.attr('file')) { + img.attr('src', img.attr('file') || img.attr('zoomfile')); + img.removeAttr('file'); + img.removeAttr('zoomfile'); + } + }); + + // 只抓取论坛1楼消息 + const description = post.html(); + + return { description }; +} + +module.exports = async (ctx) => { + let link = ctx.params.link; + const ver = ctx.params.ver ? ctx.params.ver.toUpperCase() : undefined; + const cid = ctx.params.cid; + link = link.replace(/:\/\//, ':/').replace(/:\//, '://'); + + const cookie = cid === undefined ? '' : config.discuz.cookies[cid]; + if (cookie === undefined) { + throw Error('缺少对应论坛的cookie.'); + } + + const header = { + Cookie: cookie, + }; + + const response = await got({ + method: 'get', + url: link, + responseType: 'buffer', + headers: header, + }); + + const responseData = response.data; + // 若没有指定编码,则默认utf-8 + const contentType = response.headers['content-type'] || ''; + let $ = cheerio.load(iconv.decode(responseData, 'utf-8')); + const charset = contentType.match(/charset=([^;]*)/)?.[1] ?? $('meta[charset]').attr('charset') ?? $('meta[http-equiv="Content-Type"]').attr('content')?.split('charset=')?.[1]; + if (charset?.toLowerCase() !== 'utf-8') { + $ = cheerio.load(iconv.decode(responseData, charset ?? 'utf-8')); + } + + const version = ver ? `DISCUZ! ${ver}` : $('head > meta[name=generator]').attr('content'); + + let items; + if (version.toUpperCase().startsWith('DISCUZ! 7')) { + // discuz 7.x 系列 + // 支持全文抓取,限制抓取页面5个 + const list = $('tbody[id^="normalthread"] > tr') + .slice(0, ctx.query.limit ? parseInt(ctx.query.limit, 10) : 5) + .toArray() + .map((item) => { + item = $(item); + const a = item.find('span[id^=thread] a'); + return { + title: a.text().trim(), + link: fixUrl(a.attr('href'), link), + pubDate: item.find('td.author em').length ? parseDate(item.find('td.author em').text().trim()) : undefined, + author: item.find('td.author cite a').text().trim(), + }; + }); + + items = await Promise.all( + list.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { description } = await load(item.link, charset, header); + + item.description = description; + return item; + }) + ) + ); + } else if (version.toUpperCase().startsWith('DISCUZ! X')) { + // discuz X 系列 + // 支持全文抓取,限制抓取页面5个 + const list = $('tbody[id^="normalthread"] > tr') + .slice(0, ctx.query.limit ? parseInt(ctx.query.limit, 10) : 5) + .toArray() + .map((item) => { + item = $(item); + const a = item.find('a.xst'); + return { + title: a.text(), + link: fixUrl(a.attr('href'), link), + pubDate: item.find('td.by:nth-child(3) em span').last().length ? parseDate(item.find('td.by:nth-child(3) em span').last().text().trim()) : undefined, + author: item.find('td.by:nth-child(3) cite a').text().trim(), + }; + }); + + items = await Promise.all( + list.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { description } = await load(item.link, charset, header); + + item.description = description; + return item; + }) + ) + ); + } else { + throw Error('不支持当前Discuz版本.'); + } + + ctx.state.data = { + title: $('head > title').text(), + description: $('head > meta[name=description]').attr('content'), + link, + item: items, + }; +}; diff --git a/lib/v2/discuz/maintainer.js b/lib/v2/discuz/maintainer.js new file mode 100644 index 00000000000000..cc199b2941c858 --- /dev/null +++ b/lib/v2/discuz/maintainer.js @@ -0,0 +1,5 @@ +module.exports = { + '/:link': ['junfengP'], + '/:ver/:link': ['junfengP'], + '/:ver/:cid/:link': ['junfengP'], +}; diff --git a/lib/v2/discuz/radar.js b/lib/v2/discuz/radar.js new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/lib/v2/discuz/router.js b/lib/v2/discuz/router.js new file mode 100644 index 00000000000000..fff35e3c42650e --- /dev/null +++ b/lib/v2/discuz/router.js @@ -0,0 +1,5 @@ +module.exports = (router) => { + router.get('/:ver([7|x])/:cid([0-9]{2})/:link(.*)', require('./discuz')); + router.get('/:ver([7|x])/:link(.*)', require('./discuz')); + router.get('/:link(.*)', require('./discuz')); +}; diff --git a/lib/v2/rsshub/routes.js b/lib/v2/rsshub/routes.js index a8ec2b72ebd801..4e1e6389bdf339 100644 --- a/lib/v2/rsshub/routes.js +++ b/lib/v2/rsshub/routes.js @@ -2,7 +2,9 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); module.exports = async (ctx) => { - const lang = ctx.params.lang === 'en' ? 'en/' : ''; + const isEnglish = ctx.params.lang !== 'zh'; + + const lang = isEnglish ? '' : 'zh/'; const types = [ 'social-media', 'new-media', @@ -43,10 +45,10 @@ module.exports = async (ctx) => { const list = all.flatMap(({ page, item, type }) => item.map((item) => ({ page, item, type }))); ctx.state.data = { - title: lang === 'en/' ? 'RSSHub has new routes' : 'RSSHub 有新路由啦', + title: isEnglish ? 'RSSHub has new routes' : 'RSSHub 有新路由啦', link: 'https://docs.rsshub.app', - description: lang === 'en/' ? 'Everything is RSSible' : '万物皆可 RSS', - language: lang === 'en/' ? 'en-us' : 'zh-cn', + description: isEnglish ? 'Everything is RSSible' : '万物皆可 RSS', + language: isEnglish ? 'en-us' : 'zh-cn', item: list.map(({ page, item, type }) => { const $ = cheerio.load(page); item = $(item); diff --git a/lib/v2/ustc/eeis.js b/lib/v2/ustc/eeis.js index bd9eb36209dd2a..68d2528d860098 100644 --- a/lib/v2/ustc/eeis.js +++ b/lib/v2/ustc/eeis.js @@ -21,13 +21,14 @@ module.exports = async (ctx) => { const response = await got(`${host}/${id}/list.htm`); const $ = cheerio.load(response.data); const list = $('div[portletmode=simpleList]') - .find('div.newslist') + .find('article') .toArray() .map((item) => { item = $(item); - const title = item.find('a').attr('title').trim(); - const link = item.find('a').attr('href').startsWith('/') ? host + item.find('a').attr('href') : item.find('a').attr('href'); - const pubDate = timezone(parseDate(item.find('.text-secondary').text() + '/' + item.find('.text-primary').text(), 'YYYY/MM/DD'), +8); + const title = item.find('h4 > a').eq(1).attr('title').trim(); + let link = item.find('h4 > a').attr('href'); + link = link.startsWith('/') ? host + link : link; + const pubDate = timezone(parseDate(item.find('.post-date > time').text().replace('发布时间:', ''), 'YYYY-MM-DD'), +8); return { title, pubDate, diff --git a/lib/v2/ustc/sist.js b/lib/v2/ustc/sist.js index da13224bd1e785..93a1e7738df903 100644 --- a/lib/v2/ustc/sist.js +++ b/lib/v2/ustc/sist.js @@ -20,14 +20,15 @@ module.exports = async (ctx) => { const response = await got(`${host}/${id}/list.htm`); const $ = cheerio.load(response.data); - let items = $('div#wp_news_w12') - .find('li') + let items = $('div[portletmode=simpleList]') + .find('div.card') .toArray() .map((item) => { item = $(item); - const title = item.find('a').attr('title').trim(); - const link = item.find('a').attr('href').startsWith('/') ? host + item.find('a').attr('href') : item.find('a').attr('href'); - const pubDate = timezone(parseDate(item.find('span').text(), 'YYYY-MM-DD'), +8); + const title = item.find('.card-title > a').attr('title').trim(); + let link = item.find('.card-title > a').attr('href'); + link = link.startsWith('/') ? host + link : link; + const pubDate = timezone(parseDate(item.find('time').text().replace('发布时间:', ''), 'YYYY-MM-DD'), +8); return { title, pubDate, diff --git a/lib/views/welcome.art b/lib/views/welcome.art index f97703118baa90..0ff6ba6393e238 100644 --- a/lib/views/welcome.art +++ b/lib/views/welcome.art @@ -79,7 +79,7 @@- +