一、环境准备
部署模式：单节点部署。
官网地址：Elasticsearch 平台 — 大规模查找实时答案 | Elastic
注意事项：
1. 部署及安装所用的用户不能是root
| 192.168.122.119 | Centos7.6 |
二、安装部署
2.1 下载安装包到指定文件夹，并解压
# 切换到非root用户，这里用的elasticsearch su - elasticsearch # 进入安装目录 cd /opt/module # 解压安装包 tar xf logstash-8.11.0.tar.gz # 给文件赋权 chown -R elasticsearch:elasticsearch /opt/module/logstash-8.11.0
2.2 复制证书文件
# 进入配置文件目录 cd logstash-8.11.0/config # 创建证书文件夹 mkdir certs # 将es的证书文件复制到certs文件夹下 cp /opt/module/elasticsearch-8.11.0/config/certs/http.p12 certs/
2.3 编辑配置文件
vim logstash.conf # 从redis里面拿日志数据，这里配置的也是哨兵集群的1主2从共三个节点。 input { redis { batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。 data_type => "list" #logstash redis插件工作方式 key => "ipu-cbs-server-dev-log" #监听的键值 host => "192.168.122.227" #redis地址 port => 6379 #redis端口号 password => "Redis@123456" #如果有安全认证，此项为密码 db => 0 #redis数据库的编号 threads => 1 #启用线程数量 tags => ["ipu-cbs-server-dev239"] } redis { batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。 data_type => "list" #logstash redis插件工作方式 key => "ipu-cbs-server-dev-log" #监听的键值 host => "192.168.122.237" #redis地址 port => 6379 #redis端口号 password => "Redis@123456" #如果有安全认证，此项为密码 db => 0 #redis数据库的编号 threads => 1 #启用线程数量 tags => ["ipu-cbs-server-dev239"] } redis { batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。 data_type => "list" #logstash redis插件工作方式 key => "ipu-cbs-server-dev-log" #监听的键值 host => "192.168.122.238" #redis地址 port => 6379 #redis端口号 password => "Redis@123456" #如果有安全认证，此项为密码 db => 0 #redis数据库的编号 threads => 1 #启用线程数量 tags => ["ipu-cbs-server-dev239"] } redis { batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。 data_type => "list" #logstash redis插件工作方式 key => "ipu-cbs-server-test-log" #监听的键值 host => "192.168.122.227" #redis地址 port => 6379 #redis端口号 password => "Redis@123456" #如果有安全认证，此项为密码 db => 0 #redis数据库的编号 threads => 1 #启用线程数量 tags => ["ipu-cbs-server-test251"] } redis { batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。 data_type => "list" #logstash redis插件工作方式 key => "ipu-cbs-server-test-log" #监听的键值 host => "192.168.122.237" #redis地址 port => 6379 #redis端口号 password => "Redis@123456" #如果有安全认证，此项为密码 db => 0 #redis数据库的编号 threads => 1 #启用线程数量 tags => ["ipu-cbs-server-test251"] } redis { batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。 data_type => "list" #logstash redis插件工作方式 key => "ipu-cbs-server-test-log" #监听的键值 host => "192.168.122.238" #redis地址 port => 6379 #redis端口号 password => "Redis@123456" #如果有安全认证，此项为密码 db => 0 #redis数据库的编号 threads => 1 #启用线程数量 tags => ["ipu-cbs-server-test251"] } } filter { # 去除message中日志颜色的转义符 mutate { gsub => [ "message", "\u001b\[32m", "", "message", "\u001b\[34m", "", "message", "\u001b\[35m", "", "message", "\u001b\[36m", "", "message", "\u001b\[0;39m", "" ] } grok { # match => { "message" => "%{DATESTAMP:logdate}" } # 将yy-MM-dd HH:mm:ss.SSS 格式的日期时间赋值为logdate match => { "message" => "%{TIMESTAMP_ISO8601:logdate} %{GREEDYDATA:log_message}" } } # 将logdate的值赋值给@timestamp date { match => [ "logdate", "YY-MM-dd HH:mm:ss.SSS" ] target => "@timestamp" timezone =>"+00:00" } mutate { # add_field => { "offset" => "%{[log][offset]}"} # add_field => { "logDateTime" => "%{logdate}"} # 删除不用的字段 replace => { "message" => "%{log_message}" } remove_field => ["event","input","host","ecs","log","@version","agent","logdate","log_message"] } # 将logDateTime转为日期类型 # date { # match => ["logDateTime", "yy-MM-dd HH:mm:ss.SSS"] # target => "logDateTime" # } } output { if "ipu-cbs-server-test251" in [tags] { elasticsearch { hosts => ["https://192.168.122.118:9200","https://192.168.122.119:9200","https://192.168.122.120:9200"] index => "ipu-cbs-server-test" # ssl => true # cacert => "/opt/module/logstash-8.11.0/config/certs/elasticsearch-ca.pem" user => "elastic" password => "elastic" ssl_certificate_verification => true truststore => "/opt/module/logstash-8.11.0/config/certs/http.p12" truststore_password => "123456" } } else if "ipu-cbs-server-dev239" in [tags] { elasticsearch { hosts => ["https://192.168.122.118:9200","https://192.168.122.119:9200","https://192.168.122.120:9200"] index => "ipu-cbs-server-dev" # ssl => true # # cacert => "/opt/module/logstash-8.11.0/config/certs/elasticsearch-ca.pem" user => "elastic" password => "elastic" ssl_certificate_verification => true truststore => "/opt/module/logstash-8.11.0/config/certs/http.p12" truststore_password => "123456" } } }
2.4 启动服务
# 编写启动命令文件 echo "nohup bin/logstash -f config/logstash.conf > ./log/logstash.log 2>&1 &" > start.sh # 赋予文件权限 chmod a+x start.sh # 启动服务 ./start.sh # 查看日志 tail -200f /opt/module/logstash-8.11.0/log/logstash.log
