大家好，我是辣條，
這是爬蟲系列的36篇，爬蟲之路永無止境，
爬取目標
網站：阿里巴巴1688.com - 全球領先的采購批發平臺,批發網
工具使用
開發工具：pycharm
開發環境：python3.7， Windows10
使用工具包：requests，urllib, time, re, execjs
重點學習的內容
-
JS混淆
-
正則運算式的使用
-
py執行js檔案的應用
-
網頁引數編碼
頁面分析
爬取:海量產地工廠，就上1688找工廠
h(d.token + "&" + i + "&" + g + "&" + c.data)
在這個JS代碼前面打上斷點，然后重繪頁面，進行js除錯，
d.token的生成 -> H5Request -> 在介面的請求頭里面 的cookie 包含 改引數
i = 時間戳
g = 固定引數
c.data =請求引數
h 為 JS加密方法 把 hjs代碼都扣下來 做一個單獨的js檔案，
function h(a) {
function b(a, b) {
return a << b | a >>> 32 - b
}
function c(a, b) {
var c, d, e, f, g;
return e = 2147483648 & a,
f = 2147483648 & b,
c = 1073741824 & a,
d = 1073741824 & b,
g = (1073741823 & a) + (1073741823 & b),
c & d ? 2147483648 ^ g ^ e ^ f : c | d ? 1073741824 & g ? 3221225472 ^ g ^ e ^ f : 1073741824 ^ g ^ e ^ f : g ^ e ^ f
}
function d(a, b, c) {
return a & b | ~a & c
}
function e(a, b, c) {
return a & c | b & ~c
}
function f(a, b, c) {
return a ^ b ^ c
}
function g(a, b, c) {
return b ^ (a | ~c)
}
function h(a, e, f, g, h, i, j) {
return a = c(a, c(c(d(e, f, g), h), j)),
c(b(a, i), e)
}
function i(a, d, f, g, h, i, j) {
return a = c(a, c(c(e(d, f, g), h), j)),
c(b(a, i), d)
}
function j(a, d, e, g, h, i, j) {
return a = c(a, c(c(f(d, e, g), h), j)),
c(b(a, i), d)
}
function k(a, d, e, f, h, i, j) {
return a = c(a, c(c(g(d, e, f), h), j)),
c(b(a, i), d)
}
function l(a) {
for (var b, c = a.length, d = c + 8, e = (d - d % 64) / 64, f = 16 * (e + 1), g = new Array(f - 1), h = 0, i = 0; c > i; )
b = (i - i % 4) / 4,
h = i % 4 * 8,
g[b] = g[b] | a.charCodeAt(i) << h,
i++;
return b = (i - i % 4) / 4,
h = i % 4 * 8,
g[b] = g[b] | 128 << h,
g[f - 2] = c << 3,
g[f - 1] = c >>> 29,
g
}
function m(a) {
var b, c, d = "", e = "";
for (c = 0; 3 >= c; c++)
b = a >>> 8 * c & 255,
e = "0" + b.toString(16),
d += e.substr(e.length - 2, 2);
return d
}
function n(a) {
a = a.replace(/\r\n/g, "\n");
for (var b = "", c = 0; c < a.length; c++) {
var d = a.charCodeAt(c);
128 > d ? b += String.fromCharCode(d) : d > 127 && 2048 > d ? (b += String.fromCharCode(d >> 6 | 192),
b += String.fromCharCode(63 & d | 128)) : (b += String.fromCharCode(d >> 12 | 224),
b += String.fromCharCode(d >> 6 & 63 | 128),
b += String.fromCharCode(63 & d | 128))
}
return b
}
var o, p, q, r, s, t, u, v, w, x = [], y = 7, z = 12, A = 17, B = 22, C = 5, D = 9, E = 14, F = 20, G = 4, H = 11, I = 16, J = 23, K = 6, L = 10, M = 15, N = 21;
for (a = n(a),
x = l(a),
t = 1732584193,
u = 4023233417,
v = 2562383102,
w = 271733878,
o = 0; o < x.length; o += 16)
p = t,
q = u,
r = v,
s = w,
t = h(t, u, v, w, x[o + 0], y, 3614090360),
w = h(w, t, u, v, x[o + 1], z, 3905402710),
v = h(v, w, t, u, x[o + 2], A, 606105819),
u = h(u, v, w, t, x[o + 3], B, 3250441966),
t = h(t, u, v, w, x[o + 4], y, 4118548399),
w = h(w, t, u, v, x[o + 5], z, 1200080426),
v = h(v, w, t, u, x[o + 6], A, 2821735955),
u = h(u, v, w, t, x[o + 7], B, 4249261313),
t = h(t, u, v, w, x[o + 8], y, 1770035416),
w = h(w, t, u, v, x[o + 9], z, 2336552879),
v = h(v, w, t, u, x[o + 10], A, 4294925233),
u = h(u, v, w, t, x[o + 11], B, 2304563134),
t = h(t, u, v, w, x[o + 12], y, 1804603682),
w = h(w, t, u, v, x[o + 13], z, 4254626195),
v = h(v, w, t, u, x[o + 14], A, 2792965006),
u = h(u, v, w, t, x[o + 15], B, 1236535329),
t = i(t, u, v, w, x[o + 1], C, 4129170786),
w = i(w, t, u, v, x[o + 6], D, 3225465664),
v = i(v, w, t, u, x[o + 11], E, 643717713),
u = i(u, v, w, t, x[o + 0], F, 3921069994),
t = i(t, u, v, w, x[o + 5], C, 3593408605),
w = i(w, t, u, v, x[o + 10], D, 38016083),
v = i(v, w, t, u, x[o + 15], E, 3634488961),
u = i(u, v, w, t, x[o + 4], F, 3889429448),
t = i(t, u, v, w, x[o + 9], C, 568446438),
w = i(w, t, u, v, x[o + 14], D, 3275163606),
v = i(v, w, t, u, x[o + 3], E, 4107603335),
u = i(u, v, w, t, x[o + 8], F, 1163531501),
t = i(t, u, v, w, x[o + 13], C, 2850285829),
w = i(w, t, u, v, x[o + 2], D, 4243563512),
v = i(v, w, t, u, x[o + 7], E, 1735328473),
u = i(u, v, w, t, x[o + 12], F, 2368359562),
t = j(t, u, v, w, x[o + 5], G, 4294588738),
w = j(w, t, u, v, x[o + 8], H, 2272392833),
v = j(v, w, t, u, x[o + 11], I, 1839030562),
u = j(u, v, w, t, x[o + 14], J, 4259657740),
t = j(t, u, v, w, x[o + 1], G, 2763975236),
w = j(w, t, u, v, x[o + 4], H, 1272893353),
v = j(v, w, t, u, x[o + 7], I, 4139469664),
u = j(u, v, w, t, x[o + 10], J, 3200236656),
t = j(t, u, v, w, x[o + 13], G, 681279174),
w = j(w, t, u, v, x[o + 0], H, 3936430074),
v = j(v, w, t, u, x[o + 3], I, 3572445317),
u = j(u, v, w, t, x[o + 6], J, 76029189),
t = j(t, u, v, w, x[o + 9], G, 3654602809),
w = j(w, t, u, v, x[o + 12], H, 3873151461),
v = j(v, w, t, u, x[o + 15], I, 530742520),
u = j(u, v, w, t, x[o + 2], J, 3299628645),
t = k(t, u, v, w, x[o + 0], K, 4096336452),
w = k(w, t, u, v, x[o + 7], L, 1126891415),
v = k(v, w, t, u, x[o + 14], M, 2878612391),
u = k(u, v, w, t, x[o + 5], N, 4237533241),
t = k(t, u, v, w, x[o + 12], K, 1700485571),
w = k(w, t, u, v, x[o + 3], L, 2399980690),
v = k(v, w, t, u, x[o + 10], M, 4293915773),
u = k(u, v, w, t, x[o + 1], N, 2240044497),
t = k(t, u, v, w, x[o + 8], K, 1873313359),
w = k(w, t, u, v, x[o + 15], L, 4264355552),
v = k(v, w, t, u, x[o + 6], M, 2734768916),
u = k(u, v, w, t, x[o + 13], N, 1309151649),
t = k(t, u, v, w, x[o + 4], K, 4149444226),
w = k(w, t, u, v, x[o + 11], L, 3174756917),
v = k(v, w, t, u, x[o + 2], M, 718787259),
u = k(u, v, w, t, x[o + 9], N, 3951481745),
t = c(t, p),
u = c(u, q),
v = c(v, r),
w = c(w, s);
var O = m(t) + m(u) + m(v) + m(w);
return O.toLowerCase()
}代碼實作
import requests
from urllib import parse
import time
import re
import execjs
?
?
time1 = int(time.time() * 1000)
with open('./567.js', 'r', encoding='utf-8') as f:
ctx = execjs.compile(f.read()) # 執行讀取的js代碼
data = {"cid":"FactoryRankServiceWidget:FactoryRankServiceWidget","methodName":"execute","params":"{\"extParam\":\"{\\\"methodName\\\":\\\"readRelatedRankEntries\\\",\\\"cateId\\\":7,\\\"size\\\":15}\"}"}
headers = {
'cookie': 'xlly_s=1; cna=s+HmGas+MCYCAXHwrb3t4n2g; ali_ab=240e:383:515:6f10:7c53:a1a0:ef97:a0ef.1635751613086.3; taklid=13ffc206f13c40139dcae830da1cd7a6; _m_h5_tk=9ef09cb41d7419fc32115e5eb51faa1a_1635776795184; _m_h5_tk_enc=90cb68510dccd56f830c770f0c4c9c16; alicnweb=touch_tb_at%3D1635768129101; unb=3346228600; cookie2=1bc1323f3cdd57911a40c00689bdd4cb; t=ad0b0773e5f83f3f46a8ad3ddbfb3d0b; _tb_token_=73b3e307beabe; __cn_logon__=false; UM_distinctid=17cdb6e180d368-0dd745c063c1f4-57b193e-1fa400-17cdb6e180e8ba; _csrf_token=1635769062782; keywordsHistory=%E7%94%B5%E8%84%91%3B%E7%BE%8E%E5%A6%86%3B%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91%3B%E7%94%B7%E7%94%9F%E5%A4%96%E5%A5%97%3B%E7%AB%A5%E8%A3%85%3B%E6%A3%89%E8%A2%AB; tfstk=cRB1Bw2q2V0_qEVqQGZEu20xQkv1ZXiWc5TGfsOGJ1eSpeS1iDDyPyvFiqd9KH1..; l=eBOC4p3VgzVguXHhXOfwourza77t7IRAguPzaNbMiOCP9T5w5HedW6UhmPTeCnGVh6zpR3WAVUVzBeYBqojidj4Kwp0OeOkmn; isg=BOjoTlOAHfQCkTFQpHAklfc6udb6EUwbw0DEqqIZLGNW_YhnSiBDq_m79ZUNTQTz',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36'
}
# token 生成
token = re.findall('_m_h5_tk=(.+?)_', headers['cookie'], re.S)[0]
appKey = "12574478"
p = (token + '&' + str(time1) + '&' + appKey + '&' + str(data))
sign = ctx.call('h', p)
url = f'https://h5api.m.1688.com/h5/mtop.taobao.widgetservice.getjsoncomponent/1.0/?jsv=2.4.11&appKey=12574478&t={str(time1)}&sign={sign}&api=mtop.taobao.widgetService.getJsonComponent&v=1.0&type=jsonp&isSec=0&timeout=20000&dataType=jsonp&callback=mtopjsonp9&'
# 需要轉碼 因為網頁的 urldata是 轉碼后的
urls = url + 'data=' + parse.quote(str(data))
# print(sign)
response = requests.get(urls, headers=headers).text
print(response)👇🏻 疑難解答、學習資料、路線圖可通過搜索下方 👇🏻
轉載請註明出處，本文鏈接：https://www.uj5u.com/ruanti/348443.html
標籤：其他
上一篇：[解題報告]《演算法零基礎100講》(第15講) 二分快速冪
下一篇：資料結構之線性表