一、业务背景
在日本代购数据采集中，User-Agent是反爬虫的第一道防线。很多网站会检查请求的User-Agent，如果看起来不像真实浏览器，就会拒绝响应或返回验证码。
二、User-Agent池的实现
python
import randomclass UserAgentPool: def init(self): self.agents = [ # Chrome (Windows) 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/119.0.0.0 Safari/537.36', # Chrome (Mac) 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36', # Firefox 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/121.0', # Safari 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 KHTML, like Gecko Version/17.1 Safari/605.1.15', # Edge 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Edge/120.0.0.0 Safari/537.36', ] self.current_index = 0 def get_random(self): """随机获取一个User-Agent""" return random.choice(self.agents) def get_round_robin(self): """轮询获取User-Agent""" agent = self.agents[self.current_index] self.current_index = (self.current_index + 1) % len(self.agents) return agent def get_for_platform(self, platform): """根据目标平台选择不同的User-Agent""" if platform == 'yahoo': # 雅虎拍卖偏好Chrome return self.agents[0] elif platform == 'mercari': # 煤炉偏好移动端 return 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15' else: return self.get_random()
三、浏览器指纹模拟
除了User-Agent，网站还会检测其他特征：
python
class BrowserFingerprint: def init(self): self.fingerprints = [ { 'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0', 'accept_language': 'zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7', 'accept_encoding': 'gzip, deflate, br', 'sec_ch_ua': '"Not_A Brand";v="8", "Chromium";v="120"', 'sec_ch_ua_platform': '"Windows"', }, # 更多指纹... ] def get_headers(self, platform=None): fingerprint = random.choice(self.fingerprints) return { 'User-Agent': fingerprint['user_agent'], 'Accept-Language': fingerprint['accept_language'], 'Accept-Encoding': fingerprint['accept_encoding'], 'Sec-Ch-Ua': fingerprint.get('sec_ch_ua', ''), 'Sec-Ch-Ua-Platform': fingerprint.get('sec_ch_ua_platform', ''), 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,/;q=0.8', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', }
四、请求间隔的随机化
固定间隔的请求容易被识别为爬虫：
python
import randomimport timeclass RequestDelayer: def init(self, min_delay=1, max_delay=3): self.min_delay = min_delay self.max_delay = max_delay def wait(self): """随机等待，模拟人类行为""" delay = random.uniform(self.min_delay, self.max_delay) time.sleep(delay) def wait_with_jitter(self, base_delay=2): """带抖动的等待""" # 正态分布，大部分在base_delay附近，偶尔有长间隔 jitter = random.gauss(0, base_delay 0.3) delay = max(0.5, base_delay + jitter) time.sleep(delay)
五、完整的请求封装
python
import requestsfrom requests.adapters import HTTPAdapterfrom urllib3.util.retry import Retryclass SmartRequester: def init(self): self.session = requests.Session() self.ua_pool = UserAgentPool() self.fingerprint = BrowserFingerprint() self.delayer = RequestDelayer() # 配置重试 retry = Retry( total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504] ) adapter = HTTPAdapter(max_retries=retry) self.session.mount('http://', adapter) self.session.mount('https://', adapter) def get(self, url, platform=None, kwargs): """带反爬策略的GET请求""" # 随机等待 self.delayer.wait() # 设置请求头 headers = self.fingerprint.get_headers(platform) headers['User-Agent'] = self.ua_pool.get_for_platform(platform) # 发送请求 response = self.session.get(url, headers=headers, timeout=10, kwargs) # 检查是否被限制 if response.status_code == 429: # 被限流，等待更长时间 time.sleep(30) return self.get(url, platform, *kwargs) return response
六、总结
User-Agent轮换和浏览器指纹模拟是反爬对抗的基础手段。核心原则：让每个请求看起来都像来自不同的真实用户——不同的UA、不同的请求头、不同的间隔时间。
