网易云评论爬取

烟熏火燎 · 发表于 2022-9-21 15:30:25

# 网易云评论爬取
# 加密算法破解
import sys
import re  # 正则表达式，进行文字匹配
from bs4 import BeautifulSoup  # (网页解析，获取数据)
# from urllib import request
# from urllib import error
import requests
from Crypto.Cipher import AES
from base64 import b64encode
# from urllib import parse
import json
url = &#34;https://music.163.com/weapi/comment/resource/comments/get?csrf_token=&#34;

#https://music.163.com/weapi/comment/resource/comments/get?csrf_token=
&#39;&#39;&#39; var bKB1x = window.asrsea(JSON.stringify(i9b), buV0x([&#34;流泪&#34;, &#34;强&#34;]), buV0x(Rg4k.md), buV0x([&#34;爱心&#34;, &#34;女孩&#34;, &#34;惊恐&#34;, &#34;大笑&#34;]));
         e9f.data = j9a.cr0x({          #(d,e,f,g)
            params: bKB1x.encText,
            encSecKey: bKB1x.encSecKey

            #buV0x([&#34;流泪&#34;, &#34;强&#34;]) 放入控制台跑代码得到固定值
            #&#39;010001&#39;
            #buV0x(Rg4k.md) 同上
            #00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7
            #buV0x([&#34;爱心&#34;, &#34;女孩&#34;, &#34;惊恐&#34;, &#34;大笑&#34;])
            #0CoJUm6Qyw8W8jud
         })
      }&#39;&#39;&#39;

op=0
headers={&#39;user-agent&#39;:&#39;自己的ua&#39;}

def remove_duplicates():
f_read = open(r&#39;./网易云音乐评论.txt&#39;, &#39;r&#39;, encoding=&#39;utf-8&#39;)  # 将需要去除重复值的txt文本重命名text.txt
f_write = open(r&#39;./去除重复值后的文本.txt&#39;, &#39;w&#39;, encoding=&#39;utf-8&#39;)  # 去除重复值之后，生成新的txt文本 --“去除重复值后的文本.txt”
data1 = set()
for a in [a.strip(&#39;\n&#39;) for a in list(f_read)]:
      if a not in data1:
         f_write.write(a + &#39;\n&#39;)
         data1.add(a)
f_read.close()
f_write.close()

remove_duplicates()
print(&#39;去重完成&#39;)

i = &#34;c0Aitq6E14zqzMfy&#34;
e = &#34;010001&#34;
g = &#34;0CoJUm6Qyw8W8jud&#34;
f = &#34;00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7&#34;

# 加密算法
def get_encSecKey():
return &#34;2c8ec4bc37aa7d9aa8bb8bc9494ab7639961a1b2e3607b31241630e0902b009c61e88f5a97fc67ca92f7bdf1b1b193d131eb02268f29a83a62aae22e1e55ed9a514079ef39d5974a03e8cffdfaf44eaa78779a9e8edbf47a3ad0238aee6e7374d9fe9668644b19197b3efc1869ad207b663d35b233c5f643e394df489ba35a3f&#34;
def get_params(data):
first = enc_params(data,g)
second = enc_params(first,i)
return second
def enc_params(data,key):#加密
iv = &#34;0102030405060708&#34;
data = to_16(data)
aes = AES.new(key=key.encode(&#34;utf-8&#34;),IV=iv.encode(&#34;utf-8&#34;),mode=AES.MODE_CBC)#创建加密器
bs = aes.encrypt(data.encode(&#34;utf-8&#34;))
return str(b64encode(bs),&#34;utf-8&#34;)
def to_16(data):#转换成16的倍数
pad = 16-len(data) % 16
data += chr(pad)*pad
return data
if __name__ == &#39;__main__&#39;:
page = int(input(&#34;输入爬取页数：&#34;))
rid = int(input(&#34;输入音乐id:&#34;))
fp = open(&#39;./网易云音乐评论.txt&#39;, &#39;w&#39;, encoding=&#39;utf-8&#39;)#可添加随意歌曲

for u in range(1,page+1):
      pum = u*20
      data = {
         &#34;csrf_token&#34;:&#34;&#34;,
         &#34;cursor&#34;: &#34;-1&#34;,
         &#34;offset&#34;: &#34;0&#34;,
         &#34;orderType&#34;: &#34;1&#34;,
         &#34;pageNo&#34;: &#34;1&#34;,
         &#34;pageSize&#34;: &#34;{}&#34;.format(pum),
         &#34;rid&#34;: &#34;R_SO_4_{}&#34;.format(rid),
         &#34;threadId&#34;: &#34;R_SO_4_{}&#34;.format(rid)
         #1922888354
      }
      response = requests.post(url,data={
         &#34;params&#34;:get_params(json.dumps(data)),
         &#34;encSecKey&#34;:get_encSecKey()
      },headers=headers)
      response.encoding=&#39;utf-8&#39;
      html = response.text
# print(html)
      result = json.loads(response.content.decode(&#39;utf-8&#39;))
# for i in result[&#39;data&#39;][&#39;hotComments&#39;]:
#    print(i[&#39;content&#39;])
#    for c in result[&#39;data&#39;][&#39;comments&#39;]:
#       print(c[&#39;content&#39;])
      for hot in range(len(result[&#39;data&#39;][&#39;hotComments&#39;])):
         fp.write(&#39;hotComments&#39; + &#39; &#39;)
         fp.write(&#39;昵称：&#39; + result[&#39;data&#39;][&#39;hotComments&#39;][hot][&#39;user&#39;][&#39;nickname&#39;] + &#39;\n&#39;)
         fp.write(&#39;评论：&#39; + result[&#39;data&#39;][&#39;hotComments&#39;][hot][&#39;content&#39;] + &#39;\n&#39;)
         if result[&#39;data&#39;][&#39;hotComments&#39;][hot][&#39;user&#39;][&#39;vipRights&#39;] == None:
            fp.write(&#39;vip:yes&#39; + &#39;\n&#39;)
         else:
            fp.write(&#39;vip:no&#39; + &#39;\n&#39;)
         fp.write(&#39;点赞数&#39; + str(result[&#39;data&#39;][&#39;hotComments&#39;][hot][&#39;likedCount&#39;]) + &#39;\n&#39;)
         fp.write(&#39;-------------------------------------&#39; + &#39;\n&#39;)

         # print(result[&#39;data&#39;][&#39;hotComments&#39;][1][&#39;user&#39;][&#39;nickname&#39;])

         # comments
      for r in range(20):
         fp.write(&#39;comments&#39;)
         fp.write(&#39;昵称：&#39; + result[&#39;data&#39;][&#39;comments&#39;][r][&#39;user&#39;][&#39;nickname&#39;] + &#39;\n&#39;)
         fp.write(&#39;评论：&#39; + result[&#39;data&#39;][&#39;comments&#39;][r][&#39;content&#39;] + &#39;\n&#39;)
         fp.write(&#39;头像：&#39;+result[&#39;data&#39;][&#39;comments&#39;][r][&#39;user&#39;][&#39;avatarUrl&#39;]+&#39;\n&#39;)
         if result[&#39;data&#39;][&#39;comments&#39;][r][&#39;user&#39;][&#39;vipRights&#39;] == None:
            fp.write(&#39;vip:yes&#39; + &#39;\n&#39;)
         else:
            fp.write(&#39;vip:no&#39; + &#39;\n&#39;)
         fp.write(&#39;点赞数&#39; + str(result[&#39;data&#39;][&#39;comments&#39;][r][&#39;likedCount&#39;]) + &#39;\n&#39;)
         fp.write(&#39;-------------------------------------------------------&#39; + &#39;\n&#39;)
      print(&#39;第{}页爬取完毕&#39;.format(u))
      if(u==page):
         op = 1
fp.close()
if (op==1):
      remove_duplicates()

&#39;&#39;&#39;
# chinese = re.findall(&#39;[\u4e00-\u9fa5]&#39;,html)
# comments = &#34;&#34;
# for i in chinese:
#    comments+=i
# print(comments)
&#39;&#39;&#39;
&#39;&#39;&#39;function a(a) {
      var d, e, b = &#34;abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789&#34;, c = &#34;&#34;;
      for (d = 0; a > d; d += 1)
         e = Math.random() * b.length,
         e = Math.floor(e),
         c += b.charAt(e);
      return c
}
function b(a, b) {
      var c = CryptoJS.enc.Utf8.parse(b)
      , d = CryptoJS.enc.Utf8.parse(&#34;0102030405060708&#34;)
      , e = CryptoJS.enc.Utf8.parse(a)
      , f = CryptoJS.AES.encrypt(e, c, {
         iv: d,
         mode: CryptoJS.mode.CBC
      });
      return f.toString()
}
function c(a, b, c) {
      var d, e;
      return setMaxDigits(131),
      d = new RSAKeyPair(b,&#34;&#34;,c),
      e = encryptedString(d, a)
}
function d(d, e, f, g) {
      var h = {}
      , i = a(16);
      h.encText = b(d, g),#b就是加密算法

      h.encText = b(h.encText, i),#params的结果，两次加密

      h.encSecKey = c(i, e, f),
      return h
}
function e(a, b, d, e) {
      var f = {};
      return f.encText = c(a + e, b, d),
      f
}
window.asrsea = d,&#39;&#39;&#39;

注意：后面的保存借鉴了CSDN上一位作者的方法：(30条消息) python爬虫--爬取网易云音乐评论_南岸青栀*的博客-CSDN博客_python爬取网易云音乐评论
这位博主写的非常详细，但是不知道为什么运行的代码收集的信息总是有重复的部分，所以我添加了去重的功能，以供大家参考，如果有大佬能说明原因，不胜感激
另外，添加了输入音乐ID爬取的功能

北漂废人 · 发表于 2025-4-14 08:20:10

发发呆，回回帖，工作结束~