前言
最近在学习go，学习一门语言最好的方式就是实践，之前学习python也是从爬虫入手，现在使用go语言写一个网易云音乐的爬虫，下面会简单介绍开发的过程，代码是初学者的水平，欢迎吐槽。
本项目github地址https://github.com/zhujiajunup/yunyinyue
开发工具
- go1.11.2 windows/amd64
- Google Chrome 71.0.3578.98
- Fiddler v5.0.20182.28034
获取数据
不管用什么语言写爬虫，但步骤总是一致的，只是实现使用不应的语言而已，第一步当然是确认你想要什么，本次的目标是网易云音乐，我是想获取用户首页的听歌排行榜。
最先需要弄明白的是这些数据是怎么获取的，即云音乐是如何向服务器请求数据的，打开chrome的调试工具（F12）,点到Network，搜索“钟无艳”
可以看到数据是https://music.163.com/weapi/v1/play/record?csrf_token=的POST请求来获取的，再看看该请求发送了什么数据
可以看到提交了一个表单，参数为params和encSecKey。发送的数据应该是加了密的，下一步就需要知道云音乐是如何进行加密传输的。
从调试窗口可以看到，该请求是由https://s3.music.126.net/web/s/core_86994123ce247287ad52aafce6acdf9b.js?86994123ce247287ad52aafce6acdf9b发出的，加密逻辑应该就是在该js中处理的，将该js保存到本地并格式化后，并搜索encSecKey在哪里赋值的
v9m.bl9c = function(Y9P, e8e) {
var i8a = {},
e8e = NEJ.X({},
e8e),
mp3x = Y9P.indexOf("?");
if (window.GEnc && /(^|\.com)\/api/.test(Y9P) && !(e8e.headers && e8e.headers[eq1x.Bx8p] == eq1x.Iy0x) && !e8e.noEnc) {
if (mp3x != -1) {
i8a = k8c.hc2x(Y9P.substring(mp3x + 1));
Y9P = Y9P.substring(0, mp3x)
}
if (e8e.query) {
i8a = NEJ.X(i8a, k8c.fQ1x(e8e.query) ? k8c.hc2x(e8e.query) : e8e.query)
}
if (e8e.data) {
i8a = NEJ.X(i8a, k8c.fQ1x(e8e.data) ? k8c.hc2x(e8e.data) : e8e.data)
}
i8a["csrf_token"] = v9m.gO2x("__csrf");
Y9P = Y9P.replace("api", "weapi");
e8e.method = "post";
delete e8e.query;
var bUK2x = window.asrsea(JSON.stringify(i8a), brA4E(["流泪", "强"]), brA4E(WU5Z.md), brA4E(["爱心", "女孩", "惊恐", "大笑"]));
e8e.data = k8c.cz9q({
params: bUK2x.encText,
encSecKey: bUK2x.encSecKey
})
}
cwC9t(Y9P, e8e)
};可以看到是通过window.asrsea函数来获取的，接下来看window.asrsea是如何定义的
function() {
function a(a) {
var d, e, b = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
c = "";
for (d = 0; a > d; d += 1) e = Math.random() * b.length,
e = Math.floor(e),
c += b.charAt(e);
return c
}
function b(a, b) {
var c = CryptoJS.enc.Utf8.parse(b),
d = CryptoJS.enc.Utf8.parse("0102030405060708"),
e = CryptoJS.enc.Utf8.parse(a),
f = CryptoJS.AES.encrypt(e, c, {
iv: d,
mode: CryptoJS.mode.CBC
});
return f.toString()
}
function c(a, b, c) {
var d, e;
return setMaxDigits(131),
d = new RSAKeyPair(b, "", c),
e = encryptedString(d, a)
}
function d(d, e, f, g) {
var h = {},
i = a(16);
return h.encText = b(d, g),
h.encText = b(h.encText, i),
h.encSecKey = c(i, e, f),
h
}
function e(a, b, d, e) {
var f = {};
return f.encText = c(a + e, b, d),
f
}
window.asrsea = d,
window.ecnonasr = e
} ();加密算法就是这段代码，函数接收四个参数，进行了aes和ras加密，具体逻辑这里不进行详解，知道了处理逻辑，现在得获取这四个参数分别是什么，接下来使用Fiddler来将js替换为本地js,将参数打印出来即可。
Fiddler调试获取参数
- 配置代理
- 配置Https
- 修改core.js
将https://s3.music.126.net/web/s/core_86994123ce247287ad52aafce6acdf9b.js?86994123ce247287ad52aafce6acdf9b保存到本地，比如命名为core.js，编辑器打开编辑，在指定位置添加如下打印信息
v9m.bl9c = function(Y9P, e8e) {
var i8a = {},
e8e = NEJ.X({},
e8e),
mp3x = Y9P.indexOf("?");
if (window.GEnc && /(^|\.com)\/api/.test(Y9P) && !(e8e.headers && e8e.headers[eq1x.Bx8p] == eq1x.Iy0x) && !e8e.noEnc) {
if (mp3x != -1) {
i8a = k8c.hc2x(Y9P.substring(mp3x + 1));
Y9P = Y9P.substring(0, mp3x)
}
if (e8e.query) {
i8a = NEJ.X(i8a, k8c.fQ1x(e8e.query) ? k8c.hc2x(e8e.query) : e8e.query)
}
if (e8e.data) {
i8a = NEJ.X(i8a, k8c.fQ1x(e8e.data) ? k8c.hc2x(e8e.data) : e8e.data)
}
i8a["csrf_token"] = v9m.gO2x("__csrf");
Y9P = Y9P.replace("api", "weapi");
e8e.method = "post";
delete e8e.query;
var bUK2x = window.asrsea(JSON.stringify(i8a), brA4E(["流泪", "强"]), brA4E(WU5Z.md), brA4E(["爱心", "女孩", "惊恐", "大笑"]));
window.console.info(Y9P);
window.console.info(JSON.stringify(i8a));
window.console.info(JSON.stringify( brA4E(["流泪", "强"])));
window.console.info(JSON.stringify(brA4E(WU5Z.md)));
window.console.info(JSON.stringify(brA4E(["爱心", "女孩", "惊恐", "大笑"])));
e8e.data = k8c.cz9q({
params: bUK2x.encText,
encSecKey: bUK2x.encSecKey
})
}
cwC9t(Y9P, e8e)
};- 配置Fiddler Rule
一切准备就绪后，再打开浏览器，输入https://music.163.com/#/user/songs/rank?id=62947535，打开调试窗口得Console，就可以看到本地js中添加的输出日志了
不难发现除了第一个参数外，其他三个参数都是固定的，因此在后面的处理中，只需要处理第一个参数即可。
第一个参数就是加密前的请求参数
{
"uid": "62947535",
"type": "-1",
"limit": "1000",
"offset": "0",
"total": "true",
"csrf_token": ""
}那么已经弄清楚了数据获取的逻辑，接下来就是按照这个逻辑用go语言实现一遍了，最关键的就是加密算法了
go实现
项目结构
加密算法
来自https://studygolang.com/topics/5815
/*
Package encrypt provides encrypt algorithm such as rsa & aes
*/
package encrypt
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"encoding/base64"
"fmt"
"math/big"
"math/rand"
"time"
)
// generate string for given size
func RandomStr(size int) (result []byte) {
s := "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
strBytes := []byte(s)
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for i := 0; i < size; i++ {
result = append(result, strBytes[r.Intn(len(strBytes))])
}
return
}
func AesEncrypt(sSrc string, sKey string, aseKey string) (string, error) {
iv := []byte(aseKey)
block, err := aes.NewCipher([]byte(sKey))
if err != nil {
return "", err
}
padding := block.BlockSize() - len([]byte(sSrc))%block.BlockSize()
src := append([]byte(sSrc), bytes.Repeat([]byte{byte(padding)}, padding)...)
model := cipher.NewCBCEncrypter(block, iv)
cipherText := make([]byte, len(src))
model.CryptBlocks(cipherText, src)
return base64.StdEncoding.EncodeToString(cipherText), nil
}
func RsaEncrypt(key string, pubKey string, modulus string) string {
rKey := ""
for i := len(key) - 1; i >= 0; i-- { // reserve key
rKey += key[i : i+1]
}
hexRKey := ""
for _, char := range []rune(rKey) {
hexRKey += fmt.Sprintf("%x", int(char))
}
bigRKey, _ := big.NewInt(0).SetString(hexRKey, 16)
bigPubKey, _ := big.NewInt(0).SetString(pubKey, 16)
bigModulus, _ := big.NewInt(0).SetString(modulus, 16)
bigRs := bigRKey.Exp(bigRKey, bigPubKey, bigModulus)
hexRs := fmt.Sprintf("%x", bigRs)
return addPadding(hexRs, modulus)
}
func addPadding(encText string, modulus string) string {
ml := len(modulus)
for i := 0; ml > 0 && modulus[i:i+1] == "0"; i++ {
ml--
}
num := ml - len(encText)
prefix := ""
for i := 0; i < num; i++ {
prefix += "0"
}
return prefix + encText
}- Music163Spider
type Music163Spider struct {
// send request
client *http.Client
// request's header
headers map[string]string
}
func NewMusic164Spider() (spider Music163Spider) {
headers := make(map[string]string)
headers["Accept"] = "ext/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
// empty here
headers["Accept-Encoding"] = ""
headers["Content-Type"] = "application/x-www-form-urlencoded"
headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
headers["Host"] = constants.Music163Host
headers["Cache-Control"] = "no-cache"
headers["Connection"] = "keep-alive"
headers["Pragma"] = "no-cache"
headers["Origin"] = fmt.Sprintf("%s%s", constants.HttpsPrefix, constants.Music163Host)
headers["Accept"] = "ext/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
return Music163Spider{
client: &http.Client{},
headers: headers,
}
}加密函数，实现window.asrsea的加密功能
func (spider Music163Spider) dataEncrypt(dataBytes []byte) (content map[string]string) {
content = make(map[string]string)
randomBytes := encrypt.RandomStr(16)
params, err := encrypt.AesEncrypt(string(dataBytes), constants.SrcretKey, constants.AseKey)
if err != nil {
fmt.Println(err)
}
params, err = encrypt.AesEncrypt(params, string(randomBytes), constants.AseKey)
if err != nil {
fmt.Println(err)
}
encSecKey := encrypt.RsaEncrypt(string(randomBytes), constants.PubKey, constants.Modulus)
if err != nil {
fmt.Println(err)
}
content["params"] = string(params)
content["encSecKey"] = string(encSecKey)
return content
}定义了发送post请求的方法
func (spider Music163Spider) httpPost(url string, headers map[string]string, params interface{}) (result []byte, err error) {
body := make(url2.Values)
jsonParams, err := json.Marshal(params)
if err != nil {
return nil, err
}
encryptResultMap := spider.dataEncrypt(jsonParams)
body["params"] = []string{encryptResultMap["params"]}
body["encSecKey"] = []string{encryptResultMap["encSecKey"]}
req, err := http.NewRequest("POST", url, strings.NewReader(body.Encode()))
for key, value := range headers {
req.Header.Add(key, value)
}
if err != nil {
return nil, err
}
resp, err := spider.client.Do(req)
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return data, nil
}发送参数
type BaseRequestBody struct {
Offset string `json:"offset"`
Total string `json:"totail"`
Limit string `json:"limit"`
CsrfToken string `json:"csrf_token"`
}
type PlayRecordRequestBody struct {
BaseRequestBody
Type string `json:"type"`
Uid string `json:"uid"`
}根据获取排行榜请求返回数据来创建相应的对象
type SongDetail struct {
Song common.Song `json:"song"`
// ignore other
}
type PlayRecord struct {
PlayCount int `json:"playCount"`
Score int `json:"score"`
SongDetail SongDetail `json:"song"`
}
type PlayRecordResp struct {
Code int `json:"code"`
AllData []PlayRecord `json:"allData"`
WeekData []PlayRecord `json:"weekData"`
}定义好对象后，就可以使用模拟发送请求了, spider的GetPlayRecord方法
func (spider Music163Spider) GetPlayRecord(userId string) (record response.PlayRecordResp, err error) {
playRecordReqBody := request.PlayRecordRequestBody{
Uid: userId,
Type: "-1",
BaseRequestBody: request.BaseRequestBody{
Offset: "0",
Total: "true",
Limit: "1000",
CsrfToken: "",
},
}
playRecordUrl := fmt.Sprintf("%s%s%s?csrf_token=", constants.HttpsPrefix, constants.Music163Host, constants.PlayRecord)
result, err := spider.httpPost(playRecordUrl, spider.headers, playRecordReqBody)
if err != nil {
return
}
playRecordResp := response.PlayRecordResp{}
json.Unmarshal([]byte(result), &playRecordResp)
return playRecordResp, nil
}测试一下
func main() {
musicSpider := spider.NewMusic164Spider()
record, _ := musicSpider.GetPlayRecord("62947535")
jsonData, _ := json.MarshalIndent(record, "", "\t")
fmt.Println(string(jsonData))
}结果输出
