feat(route): Add 微博热搜内容摘要 (#13764)

* fuliba added * style: auto format * doc refine for fuliba * update per lint info * fix some title and style * update per SSRF concern comment * update per SSRF concern during review * Update lib/v2/fuliba/radar.js Per comment on naming of title Co-authored-by: Tony <[email protected]> * Update website/docs/routes/new-media.mdx Co-authored-by: Tony <[email protected]> * turn to wordpress api * Update website/docs/routes/new-media.mdx Co-authored-by: Tony <[email protected]> * wip for fulltext weibo hot * refine details and doc * weibo content for hot * remove emoji img as not friendly while being handled by some reader tool * format refinement * fix single card link address * update per review comment * doc update * fetch picture with additional flag * minor format adjustment * Update doc with refined example Co-authored-by: Tony <[email protected]> * get full picture opition as well --------- Co-authored-by: shinemoon <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
DIYgod · Nov 30, 2023 · f3bf25f · f3bf25f
1 parent f6938ac
commit f3bf25f
Show file tree

Hide file tree

Showing 6 changed files with 146 additions and 11 deletions.
diff --git a/lib/v2/weibo/maintainer.js b/lib/v2/weibo/maintainer.js
@@ -2,7 +2,7 @@ module.exports = {
     '/group/:gid/:gname?/:routeParams?': ['monologconnor', 'Rongronggg9'],
     '/keyword/:keyword/:routeParams?': ['DIYgod', 'Rongronggg9'],
     '/oasis/user/:userid': ['kt286'],
-    '/search/hot': ['xyqfer'],
+    '/search/hot/:fulltext?': ['xyqfer', 'shinemoon'],
     '/super_index/:id/:type?/:routeParams?': ['zengxs', 'Rongronggg9'],
     '/timeline/:uid/:feature?/:routeParams?': ['zytomorrow', 'DIYgod', 'Rongronggg9'],
     '/user/:uid/:routeParams?': ['DIYgod', 'iplusx', 'Rongronggg9'],

diff --git a/lib/v2/weibo/radar.js b/lib/v2/weibo/radar.js
@@ -30,7 +30,7 @@ module.exports = {
                 title: '热搜榜',
                 docs: 'https://docs.rsshub.app/routes/social-media#wei-bo',
                 source: '/top/summary',
-                target: '/weibo/search/hot',
+                target: '/weibo/search/hot/:fulltext?',
             },
         ],
     },

diff --git a/lib/v2/weibo/router.js b/lib/v2/weibo/router.js
@@ -2,7 +2,7 @@ module.exports = (router) => {
     router.get('/group/:gid/:gname?/:routeParams?', require('./group'));
     router.get('/keyword/:keyword/:routeParams?', require('./keyword'));
     router.get('/oasis/user/:userid', require('./oasis/user'));
-    router.get('/search/hot', require('./search/hot'));
+    router.get('/search/hot/:fulltext?', require('./search/hot'));
     router.get('/super_index/:id/:type?/:routeParams?', require('./super_index'));
     router.get('/timeline/:uid/:feature?/:routeParams?', require('./timeline'));
     router.get('/user/:uid/:routeParams?', require('./user'));

diff --git a/lib/v2/weibo/search/hot.js b/lib/v2/weibo/search/hot.js
@@ -1,7 +1,17 @@
 const got = require('@/utils/got');
+const config = require('@/config').value;
+const { art } = require('@/utils/render');
+const cheerio = require('cheerio');
+const path = require('path');
 // const weiboUtils = require('../utils');
 
+// Default hide all picture
+let wpic = 'false';
+let fullpic = 'false';
+
 module.exports = async (ctx) => {
+    wpic = ctx.query.pic ?? 'false';
+    fullpic = ctx.query.fullpic ?? 'false';
     const {
         data: { data },
     } = await got({
@@ -15,21 +25,122 @@ module.exports = async (ctx) => {
         },
     });
 
-    ctx.state.data = {
-        title: '微博热搜榜',
-        link: 'https://s.weibo.com/top/summary?cate=realtimehot',
-        description: '实时热点，每分钟更新一次',
-        item: data.cards[0].card_group.map((item) => {
+    let resultItems = null;
+    if (ctx.params.fulltext === 'fulltext') {
+        const cardslist = data.cards[0].card_group;
+        // Topic List
+        const tlist = cardslist.map((item) => {
             const title = item.desc;
             const link = `https://m.weibo.cn/search?containerid=100103type%3D1%26q%3D${encodeURIComponent(item.desc)}`;
-            const description = item.desc;
+            const plink = `https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D1%26q%3D${encodeURIComponent(item.desc)}`;
+            return {
+                title,
+                link,
+                plink,
+            };
+        });
 
+        resultItems = await Promise.all(
+            tlist.map((i) =>
+                ctx.cache.tryGet(i.plink, async () => {
+                    const pInfo = await fetchContent(i.plink);
+                    i.description = pInfo.content;
+                    return i;
+                })
+            )
+        );
+    } else {
+        resultItems = data.cards[0].card_group.map((item) => {
+            const title = item.desc;
+            const link = `https://m.weibo.cn/search?containerid=100103type%3D1%26q%3D${encodeURIComponent(item.desc)}`;
+            const description = item.desc;
             return {
                 title,
                 description,
                 link,
             };
-        }),
+        });
+    }
+
+    // Update ctx
+    ctx.state.data = {
+        title: '微博热搜榜',
+        link: 'https://s.weibo.com/top/summary?cate=realtimehot',
+        description: '实时热点，每分钟更新一次',
+        item: resultItems,
     };
     // ctx.state.data = weiboUtils.sinaimgTvax(ctx.state.data); // no image in the route
 };
+
+async function fetchContent(url) {
+    // Fetch the subpageinof
+    const cookieString = config.weibo.cookies ? config.weibo.cookies : '';
+    const subres = await got(url, {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    let demostr = '';
+    try {
+        const rdata = subres.data;
+        const cards = rdata.data.cards;
+        // Need to find one cards with 'type ==9'
+        demostr = seekContent(cards);
+    } catch (e) {
+        // console.log(e);
+        // console.log(url);
+    }
+    const ret = demostr;
+    return {
+        content: ret,
+    };
+}
+
+function seekContent(clist) {
+    const $ = cheerio.load('<div id="wbcontent"></div>');
+    const stub = $('#wbcontent');
+
+    // To for..of per reviewers comment
+    // Need to find one clist with 'type ==9'
+    for (const curitem of clist) {
+        if (curitem.card_type === 9) {
+            const tbpic = curitem.mblog.thumbnail_pic ?? '';
+            const index = tbpic.lastIndexOf('/');
+            const thumbfolder = tbpic.substring(0, index + 1);
+
+            const curcontent = cheerio.load(curitem.mblog.text);
+            if (wpic === 'true') {
+                curcontent('img').attr('width', '1em').attr('height', '1em');
+            } else {
+                curcontent('img').remove();
+            }
+            const section = art(path.join(__dirname, 'template/digest.art'), {
+                author: {
+                    link: curitem.mblog.user.profile_url,
+                    name: curitem.mblog.user.screen_name,
+                },
+                msg: curcontent.html(),
+                link: curitem.scheme,
+                postinfo: curitem.mblog.created_at,
+                picnum: wpic === 'true' ? curitem.mblog.pic_num : 0,
+                pics:
+                    wpic === 'true' && curitem.mblog.pic_num > 0
+                        ? curitem.mblog.pics.map((item) => {
+                              // Get thumbnail_pic instead of orginal ones
+                              const pid = item.pid;
+                              if (fullpic === 'false') {
+                                  return { url: thumbfolder + pid + '.jpg', rurl: item.url };
+                              } else {
+                                  return { url: item.url, rurl: item.url };
+                              }
+                          })
+                        : [],
+            });
+            stub.append(section);
+        }
+        if (curitem.card_type === 11) {
+            stub.append(seekContent(curitem.card_group));
+        }
+    }
+    return stub.html();
+}
diff --git a/lib/v2/weibo/search/template/digest.art b/lib/v2/weibo/search/template/digest.art
@@ -0,0 +1,17 @@
+<div class="quoted">
+    <a style="text-decoration: none;" href="{{ author.link }}">{{ author.name }}</a>
+    <span><a href="{{link }}"> | {{ postinfo }} </a></span>
+</div>
+<div class="content">
+{{@ msg }}
+</div>
+
+{{if picnum > 0 }}
+    <br>
+    <div class='pic-row'>
+    {{each pics}}
+        <a href="{{$value.rurl}}"><img src="{{$value.url}}" ></a>
+    {{/each}}
+    </div>
+{{/if}}
+<hr>
diff --git a/website/docs/routes/social-media.mdx b/website/docs/routes/social-media.mdx
@@ -1736,7 +1736,14 @@ rule
 
 ### 热搜榜 {#wei-bo-re-sou-bang}
 
-<Route author="xyqfer" example="/weibo/search/hot" path="/weibo/search/hot" anticrawler="1" radar="1" rssbud="1"/>
+<Route author="xyqfer shinemoon" example="/weibo/search/hot" path="/weibo/search/hot/:fulltext?" anticrawler="1" radar="1" rssbud="1"/>
+
+- 使用`/weibo/search/hot`可以获取热搜条目列表；
+- 使用`/weibo/search/hot/fulltext`可以进一步获取热搜条目下的摘要信息（不含图片视频）；
+    - 使用`/weibo/search/hot/fulltext?pic=true`可以获取图片缩略（但需要配合额外的手段，例如浏览器上的Header Editor等来修改referer参数为`https://weibo.com'，以规避微博的外链限制,否则图片无法显示。)
+    - 使用`/weibo/search/hot/fulltext?pic=true&fullpic=true`可以获取Original图片（但需要配合额外的手段，例如浏览器上的Header Editor等来修改referer参数为`https://weibo.com'，以规避微博的外链限制,否则图片无法显示。)
+
+
 
 ### 超话 {#wei-bo-chao-hua}