本文是根据上一篇文章拓展的,观看时请结合上一篇文章:容器部署企业级日志分析平台ELK7.10.1(Elasisearch+Filebeat+Redis+Logstash+Kibana)https://blog.csdn.net/qq_44895681/article/details/122115093?spm=1001.2014.3001.5501
编写filebeat.yml文件
[root@es-master21 mnt]# cd filebeat/ [root@es-master21 filebeat]# vim filebeat.yml (使用时删除文件中带#的配置项,不然yml文件格式不对) filebeat.inputs: #inputs为复数,表名type可以有多个 - type: log #输入类型 access: enabled: true #启用这个type配置 #max_bytes: 20480 #单条日志的大小限制,建议限制(默认为10M,上面的设置表示超过20M就不会采集了) paths: - /var/log/nginx/access.log tags: ["nginx-access-log"] fields: #额外的字段(表示在filebeat收集Nginx的日志中多增加一个字段log_source,其值是nginx-access-21,用来在logstash的output输出到elasticsearch中判断日志的来源,从而建立相应的索引,也方便后期再Kibana中查看筛选数据) log_source: nginx-access-21 fields_under_root: true #设置为true,表示上面新增的字段是顶级参数。 - type: log access: enabled: true paths: - /var/log/nginx/error.log tags: ["nginx-error-log"] fields: log_source: nginx-error-21 #额外的字段(表示在filebeat收集Nginx的日志中多增加一个字段log_source,其值是nginx-error-21,用来在logstash的output输出到elasticsearch中判断日志的来源,从而建立相应的索引,也方便后期再Kibana中查看筛选数据,结尾有图) fields_under_root: true #设置为true,表示上面新增的字段是顶级参数。 setup.ilm.enabled: false output.redis: #输出到redis hosts: ["192.168.1.21:6379"] #redis地址及端口 password: "123456" #redis密码 db: 0 #redis的库 key: "nginx_log" #定义输入到redis的key名 [root@es-node22 mnt]# docker-compose -f docker-compose.yml up -d
编写logstash.conf文件
[root@es-master21 mnt]# cd logstash/ [root@es-master21 logstash]# vim config/logstash.conf (使用时删除文件中带#的配置项,不然yml文件格式不对) input { redis { port => "6379" host => "192.168.1.21" password => "123456" data_type => "list" db => "0" key => "nginx_log" } } output { #根据redis键 messages_secure 对应的列表值中,每一行数据的其中一个参数来判断日志来源 if [log_source] == 'access' { #注意判断条件的写法 elasticsearch { hosts => ["192.168.1.21:9200","192.168.1.22:9200","192.168.1.23:9200"] index => "nginx-access-%{+YYYY.MM.dd}" #user => "elastic" #password => "elastic123" } } if [log_source] == 'error' { #注意判断条件的写法 elasticsearch { hosts => ["192.168.1.21:9200","192.168.1.22:9200","192.168.1.23:9200"] index => "nginx-error-%{+YYYY.MM.dd}" } [root@es-node22 mnt]# docker-compose -f docker-compose.yml up -d [root@es-master21 mnt]# docker-compose ps Name Command State Ports --------------------------------------------------------------------------------------------------------- elasticsearch /tini -- /usr/local/bin/do ... Up 0.0.0.0:9200->9200/tcp,:::9200->9200/tcp, 0.0.0.0:9300->9300/tcp,:::9300->9300/tcp elasticsearch-head /bin/sh -c grunt server Up 0.0.0.0:9100->9100/tcp,:::9100->9100/tcp filebeat /usr/local/bin/docker-entr ... Up kibana /usr/local/bin/dumb-init - ... Up 0.0.0.0:5601->5601/tcp,:::5601->5601/tcp logstash /usr/local/bin/docker-entr ... Up 0.0.0.0:5044->5044/tcp,:::5044->5044/tcp, 9600/tcp redis docker-entrypoint.sh redis ... Up 0.0.0.0:6379->6379/tcp,:::6379->6379/tcp
或者也可以根据filebeat.yml中配置的tags做判断
... ... output { if "nginx-access-log" in [tags] { elasticsearch { hosts => [""192.168.1.21:9200","192.168.1.22:9200","192.168.1.23:9200"] index => "nginx-accesse-%{+YYYY.MM.dd}" } } if "nginx-error-log" in [tags] { elasticsearch { hosts => [""192.168.1.21:9200","192.168.1.22:9200","192.168.1.23:9200"] index => "nginx-error-%{+YYYY.MM.dd}" } } }