rsyslog+ELK收集Cisco日志

[root@prod_rsyslog_160_128 ~]# yum install -y rsyslog  #安装rsyslog服务
[root@prod_rsyslog_160_128 ~]# egrep -v "^$|#" /etc/rsyslog.conf  
$ModLoad imudp                      # imupd是模块名，支持udp协议
$UDPServerRun 514                   #允许514端口接收使用UDP协议转发过来的日志
$ModLoad imtcp                      # imtcp是模块名，支持tcp协议
$InputTCPServerRun 514                 # 允许514端口接收使用TCP协议转发过来的日志
$WorkDirectory /var/lib/rsyslog
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat  #使用默认日志模板
$IncludeConfig /etc/rsyslog.d/*.conf
$OmitLocalLogging on   
$IMJournalStateFile imjournal.state
*.info;mail.none;authpriv.none;cron.none                /var/log/messages
authpriv.*                                              /var/log/secure
mail.*                                                  -/var/log/maillog
cron.*                                                  /var/log/cron
*.emerg                                                 :omusrmsg:*
uucp,news.crit                                          /var/log/spooler
local7.*                                                /var/log/boot.log
$template myformat,"%FROMHOST-IP%: %msg:2:$%\n"                    #定义日志模板，只记录日志信息不添加时间戳并删除日志开头空格并且将IP地址写入开头
$template cisco,"/net-log/cisco/%FROMHOST-IP%.log"                 #定义日志文件模板，Cisco为模板名称，以来源IP作为文件名
local4.* ?cisco;myformat                                           #定义级别为4的日志使用cisco myformat模板
[root@prod_rsyslog_160_128 ~]# systemctl restart rsyslog.service   #重启服务

[root@prod_rsyslog_160_128 ~]# tree /net-log/  #日志文件正常创建
/net-log/
└── cisco
    └── 192.169.5.249.log
1 directory, 1 file
[root@prod_rsyslog_160_128 ~]# tail -1  /net-log/cisco/192.169.5.249.log 
001068: Jun 18 12:29:49: %LINEPROTO-5-UPDOWN: Line protocol on Interface GigabitEthernet1/0/12, changed state to down  #日志格式如下

[root@prod_rsyslog_160_128 filebeat]# egrep -v "^$|#" filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  fields_under_root: true   #新增fields成为顶级目录，而不是将其放在fields目录下
  paths:
    - /net-log/cisco/*.log  #日志路径
  fields:
    type: net-cisco         #输出给kafka时作为topic使用
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 1
name: net-log               #主机名称
processors:
  - add_host_metadata: ~    
  - drop_fields:    #删除一些不需要字段
      fields: ["sort","beat","input_type","offset","source","agent"]
output.kafka:   #kafka集群信息
  enabled: true
  hosts: ["172.16.160.121:9092","172.16.160.123:9092","172.16.160.123:9092"]
  topic: '%{[type]}'  #如果 fields_under_root为false则需要修改为fields.type

[root@prod_rsyslog_160_128 filebeat]# cat /etc/systemd/system/filebeat.service   #注意替换文件中filebeat的路径
[Unit]
Description=filebeat server daemon
Documentation=/usrl/local/filebeat/filebeat -help  
Wants=network-online.target
After=network-online.target
[Service]
User=root
Group=root
ExecStart=/usr/local/filebeat/filebeat  -c /usr/local/filebeat/filebeat.yml --path.logs /usr/local/filebeat/logs   #不要添加-e，会使 --path.logs失效
Restart=always [Install] WantedBy=multi-user.target

[root@prod_logstash_160_124 config]# cat net-cisco.conf 
input{
    kafka {
        bootstrap_servers => "172.16.160.121:9092,172.16.160.123:9092,172.16.160.123:9092"
        consumer_threads => 5
        topics => "net-cisco"
        group_id =>  "net-log"
        decorate_events => true
        type => "net-cisco"
        codec => "json"
    }
}
filter{
       mutate {
        split => ["message",": "]  #使用": "对日志内容进行切割
        add_field => {
        "remote_ip" => "%{[message][0]}"
        "serial_num"  => "%{[message][1]}"
        "logdate" => "%{[message][2]}"
        "event"  => "%{[message][3]}"
        "messages"  => "%{[message][4]}"
        }
   }
        mutate {
            remove_field => ["message"]  #日志切割后删除
        }
        date {
            match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
            timezone => "Asia/Shanghai"
        }
}
output {
# stdout { codec => rubydebug }  #调试时使用
if [type] == "net-cisco" {
     elasticsearch {
        hosts => ["172.16.160.125:9200","172.16.160.126:9200","172.16.160.127:9200"]
        index => "net-cisco-%{+YYYY-MM-dd}"  #ES中索引名称，按天创建
    }
  }
}