⭐前言
大家好，我是yma16，本文分享python_selenuim获取csdn新星赛道选手所在城市用echarts地图显示。
该系列文章：
python爬虫_django+vue3可视化csdn用户质量分
python爬虫_正则表达式获取天气预报并用echarts折线图显示
python爬虫_requests获取bilibili锻刀村系列的字幕并用分词划分可视化词云图展示
python爬虫_selenuim登录个人markdown博客站点
python爬虫_requests获取小黄人表情保存到文件夹
⭐selenuim获取新星赛道选手主页
💖 获取参赛选手主页思路分析
目标网址是个人新开赛道的报名页：https://bbs.csdn.net/topics/616574177
思路分解：
抓取用户id拼接出用户主页，通过点击next-btn翻页。
下一页的按钮样式分析：
临界条件，disabled为true翻页到底部：
python利用selenuim实现代码如下：
from selenium import webdriver import time,json dir_path='C:\\Users\MY\PycharmProjects\Spider_python\study2021\day07\dirver\msedgedriver.exe' driver=webdriver.Edge(executable_path=dir_path) url='https://bbs.csdn.net/topics/616574177' driver.get(url) now_url=driver.current_url userUrlObj={} # get uid def getUid(): cells=driver.find_elements_by_xpath('//a[@class="set-ellipsis def-color"]') for i in cells: uid=i.text userUrlObj[uid]=genUserProfile(uid) time.sleep(3) # user profile def genUserProfile(uid): return 'https://blog.csdn.net/{uid}'.format(uid=uid) # next def nextBtn(): try: nextBtnDom=driver.find_element_by_xpath('//button[@class="btn-next"]') print(nextBtnDom,nextBtnDom.text) disabled=nextBtnDom.get_attribute('disabled') print(disabled,'disabled') print(type(disabled),'disabled') print('str(disabled)',str(disabled)) if nextBtnDom and str(disabled)!='true': nextBtnDom.click() return True return False except Exception as e: print(e) return False def work(): getUid() nextFlag=nextBtn() # return if nextFlag is True: work() else: # end writeJson() def writeJson(): with open("./joinUserProfile.json", 'w', encoding='utf-8') as write_f: write_f.write(json.dumps(userUrlObj, indent=4, ensure_ascii=False)) if __name__=='__main__': work() driver.close()
抓取主页json如下：
💖 获取参赛选手ip属地思路分析
主页ip属地获取：通过类名
python获取ip属地：
from selenium import webdriver import time,json dir_path='C:\\Users\MY\PycharmProjects\Spider_python\study2021\day07\dirver\msedgedriver.exe' driver=webdriver.Edge(executable_path=dir_path) f = open('joinUserProfile.json', 'r') content = f.read() f.close() joinJson = json.loads(content) userIpInfo={} def getUserInfo(): for key in joinJson.keys(): print(key,'userIpInfo') requestUserInfo(key,joinJson[key]) time.sleep(3) writeJson() # open url def requestUserInfo(key,url): userIpInfoItem={} driver.get(url) nameDom=driver.find_element_by_xpath('//div[@class="user-profile-head-name"]') # first nickName=nameDom.find_element_by_tag_name('div').text ip=driver.find_element_by_xpath('//span[@class="address el-popover__reference"]').text userIpInfoItem['uid']=key userIpInfoItem['name']=nickName userIpInfoItem['ip']=ip userIpInfo[key]=userIpInfoItem print(userIpInfo) def writeJson(): with open("./joinUserProfile.json", 'w', encoding='utf-8') as write_f: write_f.write(json.dumps(userIpInfo, indent=4, ensure_ascii=False)) if __name__=='__main__': getUserInfo()
抓取过程：
💖 echarts可视化展示
前端展现层代码：
<template> <div> <div style="text-align: center;"> <a style="font-size: 24px;font-weight:bolder;">{{ state.title }}</a> </div> </div> <div> <a-table :scroll="{ x: 800, y: 600 }" :columns="state.columns" :data-source="state.dataSource" :loading="state.loading" :pagination="state.pagination" bordered style="border-bottom:1px solid #f0f0f0;"> <template #bodyCell="{ column, record }"> <template v-if="column.key === 'url'"> <a :href="record.url" target="_blank"> {{ record.url }} </a> </template> <template v-else-if="column.key === 'score'"> <span> <a-tag :color="record.score < 60 ? 'volcano' : record.score > 80 ? 'geekblue' : 'green'"> {{ record.score }} </a-tag> </span> </template> <template v-else-if="column.key === 'option'"> <a-tooltip placement="topLeft" :title="record.editUrl"> <a :href="record.editUrl" target="_blank"> 编辑文章 </a> </a-tooltip> </template> </template> </a-table> </div> <div id="barChartId" style="width:100vw;height:900px;margin: 0 auto"></div> </template> <script setup> import chinaJson from './chinaGeo.js'; import gameJson from './gameJson.js'; import { tableGameColumns } from './const.js' import * as echarts from 'echarts'; import { defineProps, reactive, watch, nextTick, onUnmounted, onMounted } from 'vue'; const props = defineProps({ tableData: [] }) const state = reactive({ title: '参赛选手所在城市', linesCoord: [], focusCity: '深圳', locationGis: [], centerLoction: [], aimPointData: [], airData: [], exportLoading: false, columns: tableGameColumns, dataSource: [], echartInstance: undefined, pagination: { total: 0, current: 1, pageSize: 10, pageSizeOptions: ['10', '50', '100'], showTotal: (total, range) => { return range[0] + '-' + range[1] + ' 共' + total + '个选手'; }, onShowSizeChange: changePageSize, // 改变每页数量时更新显示 onChange: changePage,//点击页码事件 } }) function initDataSource() { state.dataSource = [] state.total = 0 Object.keys(gameJson).forEach(uid => { state.dataSource.push({ uid: gameJson[uid].uid, name: gameJson[uid].name, ip: gameJson[uid].ip.split('：')[1] }) state.total += 1 }) // map } function initMap() { echarts.registerMap('chinaJson', chinaJson) let itemData = chinaJson.features let length = itemData.length state.aimPointData = [] for (let loc = 0; loc < length; ++loc) { let name = itemData[loc].properties.name state.aimPointData.push({ value: name }) let center = itemData[loc].properties.center // 中心位置 if (name.includes(state.focusCity)) { state.centerLoction = center } } for (let loc = 0; loc < length; ++loc) { let name = itemData[loc].properties.name let number = 0 let center = itemData[loc].properties.center Object.keys(gameJson).forEach(uid => { if (name.includes(gameJson[uid].ip.split('：')[1])) { number += 1 } }) state.locationGis.push({ value: center }) // eslint-disable-next-line eqeqeq if (!name.includes(state.focusCity)) { state.linesCoord.push([center, state.centerLoction]) } // eslint-disable-next-line eqeqeq if (name && name !== '') { let temp = { name: `${name}`, value: Number(number) } state.airData.push(temp) } continue } } // storage function changePage(page, pageSize) { state.pagination.current = page state.pagination.pageSize = pageSize } function changePageSize(current, pageSize) { state.pagination.current = current state.pagination.pageSize = pageSize } watch(() => props.tableData, (val) => { state.dataSource = val nextTick(() => { renderEchartBar() }) }, { deep: true, immediate: true }) function renderEchartBar() { // 基于准备好的dom，初始化echarts实例 const domInstance = document.getElementById('barChartId') if (domInstance) { domInstance.removeAttribute('_echarts_instance_') } else { return } const myChart = echarts.init(domInstance); const option = { title: { text: '中国地图', subtext: 'chinaJson' }, geo: { // 经纬度中心 center: state.centerLoction, type: 'map', map: 'chinaJson', // 这里的值要和上面registerMap的第一个参数一致 roam: true, // 拖拽 nameProperty: 'name', // 悬浮标签 label: { type: 'map', map: 'chinaJson', // 这里的值要和上面registerMap的第一个参数一致 // roam: false, // 拖拽 nameProperty: 'name', show: true, color: '#fff', backgroundColor: '#546de5', align: 'center', fontSize: 10, width: (function () { // let n = parseInt(Math.random() * 10) return 110 })(), height: 50, shadowColor: 'rgba(0,0,0,.7)', borderRadius: 10 }, zoom: 1.2 }, series: [ // 坐标点的热力数据 { data: state.airData, geoIndex: 0, // 将热力的数据和第0个geo配置关联在一起 type: 'map' }, { type: 'effectScatter', // 渲染显示 zlevel: 2, showEffectOn: 'render', data: state.locationGis, // 配置散点的坐标数据 coordinateSystem: 'geo', // 指明散点使用的坐标系统 rippleEffect: { // 缩放 scale: 4, // 涟漪的颜色 color: '#cf6a87', // 波纹数量 number: 2, // 扩散方式 stroke(线条) fill（区域覆盖） brushType: 'fill' }, // 形状 symbol: 'circle' }, ], visualMap: { min: 0, max: 100, inRange: { color: ['white', '#0984e3'] // 控制颜色渐变的范围 }, calculable: false // 出现滑块 } } // 使用刚指定的配置项和数据显示图表。 myChart.setOption(option, true); // 监听 state.echartInstance = myChart; myChart.on('click', function (params) { const findItem = state.dataSource.find(item => { return item.postTime == params.name }) if (params.name) { window.open(findItem.url, '_blank') } }); window.onresize = myChart.resize; } onUnmounted(() => { window.onresize = null }) onMounted(() => { initDataSource() initMap() }) </script>
效果：(链接直达：https://yma16.inscode.cc/)
⭐结束
本文分享到这结束，如有错误或者不足之处欢迎指出！
