基于Apify+node+react/vue搭建一个有点意思的爬虫平台

npm install apify --save

const Apify = require('apify');
Apify.main(async () => {
    const requestQueue = await Apify.openRequestQueue();
    await requestQueue.addRequest({ url: 'https://www.iana.org/' });
    const pseudoUrls = [new Apify.PseudoUrl('https://www.iana.org/[.*]')];
    const crawler = new Apify.PuppeteerCrawler({
        requestQueue,
        handlePageFunction: async ({ request, page }) => {
            const title = await page.title();
            console.log(`Title of ${request.url}: ${title}`);
            await Apify.utils.enqueueLinks({
                page,
                selector: 'a',
                pseudoUrls,
                requestQueue,
            });
        },
        maxRequestsPerCrawl: 100,
        maxConcurrency: 10,
    });
    await crawler.run();
});

// child.js
function computedTotal(arr, cb) {
    // 耗时计算任务
}
// 与主进程通信
// 监听主进程信号
process.on('message', (msg) => {
  computedTotal(bigDataArr, (flag) => {
    // 向主进程发送完成信号
    process.send(flag);
  })
});
// main.js
const { fork } = require('child_process');
app.use(async (ctx, next) => {
  if(ctx.url === '/fetch') {
    const data = ctx.request.body;
    // 通知子进程开始执行任务,并传入数据
    const res = await createPromisefork('./child.js', data)
  }
  // 创建异步线程
  function createPromisefork(childUrl, data) {
    // 加载子进程
    const res = fork(childUrl)
    // 通知子进程开始work
    data && res.send(data)
    return new Promise(reslove => {
        res.on('message', f => {
            reslove(f)
        })
    })  
  }
  await next()
})

// 异步队列
const queue = []
// 最大并发数
const max_parallel = 6
// 开始指针
let start = 0
for(let i = 0; i < urls.length; i++) {
  // 添加异步队列
  queue.push(fetchPage(browser, i, urls[i]))
  if(i && 
      (i+1) % max_parallel === 0 
        || i === (urls.length - 1)) {
    // 每隔6条执行一次, 实现异步分流执行, 控制并发数
    await Promise.all(queue.slice(start, i+1))
    start = i
  }
}

// 滚动高度
let scrollStep = 1080;
// 最大滚动高度, 防止无限加载的页面导致长效耗时任务
let max_height = 30000;
let m = {prevScroll: -1, curScroll: 0}
while (m.prevScroll !== m.curScroll && m.curScroll < max_height) {
    // 如果上一次滚动和本次滚动高度一样, 或者滚动高度大于设置的最高高度, 则停止截取
    m = await page.evaluate((scrollStep) => {
      if (document.scrollingElement) {
        let prevScroll = document.scrollingElement.scrollTop;
        document.scrollingElement.scrollTop = prevScroll + scrollStep; 
        let curScroll = document.scrollingElement.scrollTop
        return {prevScroll, curScroll}
      }
    }, scrollStep);
    // 等待3秒后继续滚动页面, 为了让页面加载充分
    await sleep(3000);
}
// 其他业务代码...
// 截取网页快照,并设置图片质量和保存路径
const screenshot = await page.screenshot({path: `static/${uid}.jpg`, fullPage: true, quality: 70});

const txt = await page.$eval('body', el => {
    // el即为dom节点, 可以对body的子节点进行提取,分析
    return {...}
})

const Koa  = require('koa');
const { resolve } = require('path');
const staticServer = require('koa-static');
const koaBody = require('koa-body');
const cors = require('koa2-cors');
const logger = require('koa-logger');
const glob = require('glob');
const { fork } = require('child_process');
const app = new Koa();
// 创建静态目录
app.use(staticServer(resolve(__dirname, './static')));
app.use(staticServer(resolve(__dirname, './db')));
app.use(koaBody());
app.use(logger());
const config = {
  imgPath: resolve('./', 'static'),
  txtPath: resolve('./', 'db')
}
// 设置跨域
app.use(cors({
  origin: function (ctx) {
      if (ctx.url.indexOf('fetch') > -1) {
        return '*'; // 允许来自所有域名请求
      }
      return ''; // 这样就能只允许 http://localhost 这个域名的请求了
  },
  exposeHeaders: ['WWW-Authenticate', 'Server-Authorization'],
  maxAge: 5,  //  该字段可选，用来指定本次预检请求的有效期，单位为秒
  credentials: true,
  allowMethods: ['GET', 'POST', 'PUT', 'DELETE'],
  allowHeaders: ['Content-Type', 'Authorization', 'Accept', 'x-requested-with'],
}))
// 创建异步线程
function createPromisefork(childUrl, data) {
  const res = fork(childUrl)
    data && res.send(data)
    return new Promise(reslove => {
      res.on('message', f => {
        reslove(f)
      })
    })  
}
app.use(async (ctx, next) => {
  if(ctx.url === '/fetch') {
    const data = ctx.request.body;
    const res = await createPromisefork('./child.js', data)
    // 获取文件路径
    const txtUrls = [];
    let reg = /.*?(\d+)\.\w*$/;
    glob.sync(`${config.txtPath}/*.*`).forEach(item => {
      if(reg.test(item)) {
        txtUrls.push(item.replace(reg, '$1'))
      } 
    })
    ctx.body = {
      state: res,
      data: txtUrls,
      msg: res ? '抓取完成' : '抓取失败,原因可能是非法的url或者请求超时或者服务器内部错误'
    }
  }
  await next()
})
app.listen(80)