Merge pull request #64 from DIYgod/master

[pull] master from diygod:master

lib/v2/ainvest/article.js

@@ -0,0 +1,42 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { getHeaders, randomString, encryptAES, decryptAES } = require('./utils');
+
+module.exports = async (ctx) => {
+    const key = randomString(16);
+
+    const { data: response } = await got.post('https://api.ainvest.com/gw/socialcenter/v1/edu/article/listArticle', {
+        headers: getHeaders(key),
+        searchParams: {
+            timestamp: Date.now(),
+        },
+        data: encryptAES(
+            JSON.stringify({
+                batch: ctx.query.limit ? parseInt(ctx.query.limit, 10) : 30,
+                startId: null,
+                tags: {
+                    in: ['markettrends', 'premarket', 'companyinsights', 'macro'],
+                    and: ['web', 'creationplatform'],
+                },
+            }),
+            key
+        ),
+    });
+
+    const { data } = JSON.parse(decryptAES(response, key));
+
+    const items = data.map((item) => ({
+        title: item.title,
+        description: item.content,
+        link: item.sourceUrl,
+        pubDate: parseDate(item.postDate, 'x'),
+        category: [item.nickName, ...item.tags.map((tag) => tag.code)],
+    }));
+
+    ctx.state.data = {
+        title: 'AInvest - Latest Articles',
+        link: 'https://www.ainvest.com/news',
+        language: 'en',
+        item: items,
+    };
+};

lib/v2/ainvest/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/article': ['TonyRL'],
+    '/news': ['TonyRL'],
+};

lib/v2/ainvest/news.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { getHeaders, randomString, decryptAES } = require('./utils');
+
+module.exports = async (ctx) => {
+    const key = randomString(16);
+
+    const { data: response } = await got('https://api.ainvest.com/gw/news_f10/v1/newsFlash/getNewsData', {
+        headers: getHeaders(key),
+        searchParams: {
+            terminal: 'web',
+            tab: 'all',
+            page: 1,
+            size: ctx.query.limit ? parseInt(ctx.query.limit, 10) : 50,
+            lastId: '',
+            timestamp: Date.now(),
+        },
+    });
+
+    const { data } = JSON.parse(decryptAES(response, key));
+
+    const items = data.content.map((item) => ({
+        title: item.title,
+        description: item.content,
+        link: item.sourceUrl,
+        pubDate: parseDate(item.publishTime, 'x'),
+        category: item.tagList.map((tag) => tag.nameEn),
+        author: item.userInfo.nickname,
+        upvotes: item.likeCount,
+        comments: item.commentCount,
+    }));
+
+    ctx.state.data = {
+        title: 'AInvest - Latest News',
+        link: 'https://www.ainvest.com/news',
+        language: 'en',
+        item: items,
+    };
+};

lib/v2/ainvest/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'ainvest.com': {
+        _name: 'AInvest',
+        '.': [
+            {
+                title: 'Latest Article',
+                docs: 'https://docs.rsshub.app/finance#ainvest',
+                source: ['/news'],
+                target: '/ainvest/article',
+            },
+            {
+                title: 'Latest News',
+                docs: 'https://docs.rsshub.app/finance#ainvest',
+                source: ['/news'],
+                target: '/ainvest/news',
+            },
+        ],
+    },
+};

lib/v2/ainvest/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/article', require('./article'));
+    router.get('/news', require('./news'));
+};

lib/v2/ainvest/utils.js

@@ -0,0 +1,73 @@
+const crypto = require('crypto');
+const CryptoJS = require('crypto-js');
+const { KJUR, KEYUTIL, hextob64 } = require('jsrsasign');
+
+const publicKey =
+    'MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCARnxLlrhTK28bEV7s2IROjT73KLSjfqpKIvV8L+Yhe4BrF0Ut4oOH728HZlbSF0C3N0vXZjLAFesoS4v1pYOjVCPXl920Lh2seCv82m0cK78WMGuqZTfA44Nv7JsQMHC3+J6IZm8YD53ft2d8mYBFgKektduucjx8sObe7eRyoQIDAQAB';
+
+const randomString = (length) => {
+    if (length > 32) {
+        throw Error('Max length is 32.');
+    }
+    return uuidv4().replace(/-/g, '').substring(0, length);
+};
+
+const uuidv4 = () => crypto.randomUUID();
+
+/**
+ * @param {string} str
+ * @returns {CryptoJS.lib.WordArray}
+ */
+const MD5 = (str) => CryptoJS.MD5(str);
+
+const encryptAES = (data, key) => {
+    if (typeof key === 'string') {
+        key = MD5(key);
+    }
+    return CryptoJS.AES.encrypt(CryptoJS.enc.Utf8.parse(data), key, {
+        mode: CryptoJS.mode.ECB,
+        padding: CryptoJS.pad.Pkcs7,
+    }).toString();
+};
+
+const decryptAES = (data, key) => {
+    if (typeof key === 'string') {
+        key = MD5(key);
+    }
+    return CryptoJS.AES.decrypt(data, key, {
+        mode: CryptoJS.mode.ECB,
+        padding: CryptoJS.pad.Pkcs7,
+    }).toString(CryptoJS.enc.Utf8);
+};
+
+const encryptRSA = (data) => {
+    // Original code:
+    // var n = new JSEncrypt();
+    // n.setPublicKey(pubKey);
+    // return n.encrypt(message);
+    // Note: Server will reject the public key if it's encrypted using crypto.publicEncrypt().
+    let pubKey = `-----BEGIN PUBLIC KEY-----${publicKey}-----END PUBLIC KEY-----`;
+    pubKey = KEYUTIL.getKey(pubKey);
+    return hextob64(KJUR.crypto.Cipher.encrypt(data, pubKey));
+};
+
+const getHeaders = (key) => {
+    const fingerPrint = uuidv4();
+
+    return {
+        'content-type': 'application/json',
+        'ovse-trace': uuidv4(),
+        callertype: 'USER',
+        fingerprint: encryptAES(fingerPrint, MD5(key)),
+        onetimeskey: encryptRSA(key),
+        timestamp: encryptAES(Date.now(), key),
+        referer: 'https://www.ainvest.com/',
+    };
+};
+
+module.exports = {
+    randomString,
+    encryptAES,
+    decryptAES,
+    getHeaders,
+};

lib/v2/kamen-rider-official/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:category?': ['nczitzk'],
+};

lib/v2/kamen-rider-official/news.js

@@ -0,0 +1,110 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const { category } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 50;
+
+    const rootUrl = 'https://www.kamen-rider-official.com';
+    const apiUrl = new URL('api/v1/news_articles', rootUrl).href;
+    const currentUrl = new URL(`news_articles/${category ? `?category=${category}` : ''}`, rootUrl).href;
+
+    const { data: currentResponse } = await got(currentUrl);
+
+    const buildId = currentResponse.match(/"buildId":"(\w+)"/)[1];
+
+    const apiCategoryUrl = new URL(`_next/data/${buildId}/news_articles.json`, rootUrl).href;
+
+    const { data: categoryResponse } = await got(apiCategoryUrl);
+
+    const id = categoryResponse.pageProps.categoryIds[category];
+
+    const { data: response } = await got(apiUrl, {
+        searchParams: {
+            category_id: id,
+            limit,
+            offset: 0,
+        },
+    });
+
+    let items = response.news_articles.slice(0, limit).map((item) => ({
+        title: item.list_title,
+        link: new URL(item.path, rootUrl).href,
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            image: item.list_image_path
+                ? {
+                      src: new URL(item.list_image_path, rootUrl).href,
+                      alt: item.list_title,
+                  }
+                : undefined,
+        }),
+        author: item.author,
+        category: [item.category_name, item.category_2_name].filter((c) => c),
+        guid: `kamen-rider-official-${item.id}`,
+        pubDate: parseDate(item.release_date),
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: detailResponse } = await got(item.link);
+
+                const content = cheerio.load(detailResponse);
+
+                content('a.c-button').each(function () {
+                    content(this).parent().remove();
+                });
+
+                content('img').each(function () {
+                    content(this).replaceWith(
+                        art(path.join(__dirname, 'templates/description.art'), {
+                            image: {
+                                src: content(this).prop('src'),
+                            },
+                        })
+                    );
+                });
+
+                item.title = content('h1.p-post__title').text() || item.title;
+                item.description = content('main.p-post__main').html();
+                item.author = content('div.p-post__responsibility p')
+                    .toArray()
+                    .map((a) => content(a).text())
+                    .join(' / ');
+                item.category = Array.from(
+                    new Set(
+                        [
+                            ...item.category,
+                            ...content('ul.p-post__categories li a.p-post__category')
+                                .toArray()
+                                .map((c) => content(c).text().trim()),
+                        ].filter((c) => c)
+                    )
+                );
+
+                return item;
+            })
+        )
+    );
+
+    const $ = cheerio.load(currentResponse);
+
+    const icon = new URL($('link[rel="icon"]').prop('href'), rootUrl).href;
+
+    ctx.state.data = {
+        item: items,
+        title: `${$('title').text().split(/ー/)[0]}${category ? ` - ${category}` : ''}`,
+        link: currentUrl,
+        description: $('meta[property="og:description"]').prop('content'),
+        language: $('html').prop('lang'),
+        image: $('meta[property="og:image"]').prop('content'),
+        icon,
+        logo: icon,
+        subtitle: $('meta[property="keywords"]').prop('content'),
+        author: $('meta[property="og:site_name"]').prop('content'),
+        allowEmpty: true,
+    };
+};

lib/v2/kamen-rider-official/radar.js

@@ -0,0 +1,18 @@
+module.exports = {
+    'kamen-rider-official.com': {
+        _name: '仮面ライダ',
+        '.': [
+            {
+                title: '最新情報',
+                docs: 'https://docs.rsshub.app/routes/new-media#fan-mian-%E3%83%A9%E3%82%A4%E3%83%80-zui-xin-qing-bao',
+                source: ['/news_articles'],
+                target: (params, url) => {
+                    url = new URL(url);
+                    const category = url.searchParams.get('category');
+
+                    return `/kamen-rider-official/news${category ? `/${category}` : ''}`;
+                },
+            },
+        ],
+    },
+};

lib/v2/kamen-rider-official/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/news/:category?', require('./news'));
+};

lib/v2/kamen-rider-official/templates/description.art

@@ -0,0 +1,10 @@
+{{ if image }}
+<figure>
+  <img
+    src="{{ image.src }}"
+    {{ if image.alt }}
+      alt="{{ image.alt }}"
+    {{ /if }}
+  >
+</figure>
+{{ /if }}

lib/v2/reuters/common.js

@@ -55,8 +55,11 @@ module.exports = async (ctx) => {
     let items = response.result.articles.map((e) => ({
         title: e.title,
         link: new URL(e.canonical_url, rootUrl).href,
+        guid: e.id,
     }));
 
+    items = items.filter((e, i) => items.findIndex((f) => e.guid === f.guid) === i);
+
     items = await Promise.all(
         items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {

lib/v2/udn/breaking-news.js

@@ -55,7 +55,7 @@ module.exports = async (ctx) => {
                     author: data.author.name,
                     description,
                     pubDate: timezone(parseDate(item.time.date, 'YYYY-MM-DD HH:mm'), +8),
-                    category: [data.articleSection, ...data.keywords.split(',')],
+                    category: [data.articleSection, $("meta[name='subsection']").attr('content'), ...data.keywords.split(',')],
                     link,
                 };
             });

lib/v2/xinpianchang/index.js

@@ -0,0 +1,19 @@
+const { rootUrl, getData, processItems } = require('./util');
+
+module.exports = async (ctx) => {
+    const { params = 'article-0-0-all-all-0-0-score' } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 60;
+
+    const currentUrl = new URL(`discover/${params}`, rootUrl).href;
+
+    const { data, response } = await getData(currentUrl, ctx.cache.tryGet);
+
+    let items = JSON.parse(response.match(/"list":(\[.*?\]),"total"/)[1]);
+
+    items = await processItems(items.slice(0, limit), ctx.cache.tryGet);
+
+    ctx.state.data = {
+        ...data,
+        item: items,
+    };
+};

lib/v2/xinpianchang/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/discover/:params?': ['nczitzk'],
+    '/rank/:category?': ['nczitzk'],
+};