一、下载地址
官网：https://www.elastic.co/cn/downloads/logstash
百度云盘：
二、安装
tar zxvf logstash-6.2.1.tar.gz mv logstash-6.2.1 logstash
配置文件（配置文件放哪个目录都可以，在启动Logstash时可以指定配置文件位置）
vi k_es.conf
input{ kafka { bootstrap_servers => "10.10.6.225:9092" #kafka服务器地址，不是zookeeper client_id => "test" auto_offset_reset => "latest" consumer_threads => 5 decorate_events => true topics => ["test"] #控制kafka哪个topic，可以多个用逗号分割 codec => "json" } } filter{ json { source => "message" remove_field=>["message","beat","@version"] #删除没用的属性 add_field =>["customize","自定义字段"] #增加属性 } } output { elasticsearch { hosts => "localhost:9200" index => "logstash-nginxacclog-%{+YYYY.MM.dd}" #放到elasticsearch哪个index中 } }
启动（到bin目录下）
./logstash -f k_es.conf
后台运行
nohup /usr/local/logstash/bin/logstash -f /usr/local/logstash/bin/k_es.conf -w 8 -b 1000 > /dev/null 2>&1 &
配置文件备份
input{ kafka { bootstrap_servers => "10.100.2.210:9092" client_id => "nginxaccesslog" auto_offset_reset => "latest" consumer_threads => 5 decorate_events => true topics => ["nginx_access_log"] codec => "json" type => "nginx_log" } kafka { bootstrap_servers => "10.100.2.210:9092" client_id => "database" auto_offset_reset => "latest" consumer_threads => 5 decorate_events => true topics => ["t_resource_info","t_base_person","t_base_organization","t_base_student","t_base_parent","t_base_class"] codec => "json" type => "dsideal_db" } kafka { bootstrap_servers => "10.100.2.210:9092" client_id => "devops_real" auto_offset_reset => "latest" consumer_threads => 5 decorate_events => true topics => ["devopsrealinfo"] codec => "json" type => "devopsrealinfo" } kafka { bootstrap_servers => "10.100.2.210:9092" client_id => "devops_base" auto_offset_reset => "latest" consumer_threads => 5 decorate_events => true topics => ["devopsbaseinfo"] codec => "json" type => "devopsbaseinfo" } } filter{ mutate { gsub => ["message", "\\x22", '"'] } json { source => "message" remove_field=>["message","beat","@version"] } if [type] == "nginx_log" { geoip { source => "ip" target => "geoip" database => "/usr/local/GeoLite2-City.mmdb" } } } output { if [type] == "nginx_log" { elasticsearch { hosts => "10.100.2.210:9200" index => "nginx-access-log" } } if [type] == "dsideal_db" { elasticsearch { hosts => "10.100.2.210:9200" index => "%{table_name}" document_id => "%{id}" } } if [type] == "devopsbaseinfo" { elasticsearch { hosts => "localhost:9200" index => "devopsbaseinfo" document_id => "%{id}" } } if [type] == "devopsrealinfo" { elasticsearch { hosts => "localhost:9200" index => "devopsrealinfo" } } }
