spider-admin-pro 一个集爬虫Scrapy+Scrapyd爬虫项目查看 和 爬虫任务定时调度的可视化管理工具

$ pip3 install spider-admin-pro
$ python3 -m spider_admin_pro.run

$ git clone https://github.com/mouday/spider-admin-pro.git
$ python3 spider_admin_pro/run.py

yaml配置文件 >  env环境变量 > 默认配置

# flask 服务配置
PORT = 5002
HOST = '127.0.0.1'
# 登录账号密码
USERNAME = admin
PASSWORD = "123456"
JWT_KEY = FU0qnuV4t8rr1pvg93NZL3DLn6sHrR1sCQqRzachbo0=
# token过期时间，单位天
EXPIRES = 7
# scrapyd地址, 结尾不要加斜杆
SCRAPYD_SERVER = 'http://127.0.0.1:6800'
# 调度器 调度历史存储设置
# mysql or sqlite and other, any database for peewee support
SCHEDULE_HISTORY_DATABASE_URL = 'sqlite:///dbs/schedule_history.db'
# 调度器 定时任务存储地址
JOB_STORES_DATABASE_URL = 'sqlite:///dbs/apscheduler.db'
# 日志文件夹
LOG_DIR = 'logs'

# flask 服务配置
export SPIDER_ADMIN_PRO_PORT=5002
export SPIDER_ADMIN_PRO_HOST='127.0.0.1'
# 登录账号密码
export SPIDER_ADMIN_PRO_USERNAME='admin'
export SPIDER_ADMIN_PRO_PASSWORD='123456'
export SPIDER_ADMIN_PRO_JWT_KEY='FU0qnuV4t8rr1pvg93NZL3DLn6sHrR1sCQqRzachbo0='

$ source env.bash
$ python3 -m spider_admin_pro.run

# flask 服务配置
PORT: 5002
HOST: '127.0.0.1'
# 登录账号密码
USERNAME: admin
PASSWORD: "123456"
JWT_KEY: "FU0qnuV4t8rr1pvg93NZL3DLn6sHrR1sCQqRzachbo0="
# token过期时间，单位天
EXPIRES: 7
# scrapyd地址, 结尾不要加斜杆
SCRAPYD_SERVER: "http://127.0.0.1:6800"
# 日志文件夹
LOG_DIR: 'logs'

$ python -c 'import base64;import os;print(base64.b64encode(os.urandom(32)).decode())'

# 启动服务
$ gunicorn --config gunicorn.conf.py spider_admin_pro.run:app

# -*- coding: utf-8 -*-
"""
$ gunicorn --config gunicorn.conf.py spider_admin_pro.run:app
"""
import multiprocessing
import os
from gevent import monkey
monkey.patch_all()
# 日志文件夹
LOG_DIR = 'logs'
if not os.path.exists(LOG_DIR):
    os.mkdir(LOG_DIR)
def resolve_file(filename):
    return os.path.join(LOG_DIR, filename)
def get_workers():
    return multiprocessing.cpu_count() * 2 + 1
# daemon = True
daemon = False  # 使用supervisor不能是后台进程
# 进程名称
proc_name = "spider-admin-pro"
# 启动端口
bind = "127.0.0.1:5001"
# 日志文件
loglevel = 'debug'
pidfile = resolve_file("gunicorn.pid")
accesslog = resolve_file("access.log")
errorlog = resolve_file("error.log")
# 启动的进程数
# workers = get_workers()
workers = 2
worker_class = 'gevent'
# 启动时钩子
def on_starting(server):
    ip, port = server.address[0]
    print('server.address:', f'http://{ip}:{port}')

[program: spider-admin-pro]
directory=/spider-admin-pro
command=/usr/local/python3/bin/gunicorn --config gunicorn.conf.py spider_admin_pro.run:app
stdout_logfile=logs/out.log
stderr_logfile=logs/err.log
stdout_logfile_maxbytes = 20MB
stdout_logfile_backups = 0
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=0

server {
    listen 80;
    server_name _;
    access_log  /var/log/nginx/access.log;
    error_log  /var/log/nginx/error.log;
    location / {
        proxy_pass         http://127.0.0.1:5001/;
        proxy_redirect     off;
        proxy_set_header   Host                 $host;
        proxy_set_header   X-Real-IP            $remote_addr;
        proxy_set_header   X-Forwarded-For      $proxy_add_x_forwarded_for;
        proxy_set_header   X-Forwarded-Proto    $scheme;
    }
}

.
├── run.py        # 程序入口
├── api           # Controller层
├── service       # Sevice层
├── model         # Model层
├── exceptions    # 异常 
├── utils         # 工具类
└── web           # 静态web页

git clone https://github.com/mouday/spider-admin-pro.git
cd spider-admin-pro
python3 dev.py

pip3 install -U spider-admin-pro -i https://pypi.org/simple