Logstash、Filebeat安装与数据同步(+ES安装讲解)

# 查看可安装java版本
yum list java*
# 安装JDK11
yum install java-11-openjdk-demo.x86_64

https://www.elastic.co/cn/downloads/logstash

wget https://artifacts.elastic.co/downloads/logstash/logstash-8.3.3-linux-x86_64.tar.gz

mkdir /usr/local/logstash

tar -zxvf logstash-8.3.3-linux-x86_64.tar.gz -C /usr/local/logstash

cd logstash-8.3.3
bin/logstash -e 'input { stdin { } } output { stdout {} }'

input {
  heartbeat {
    interval => 10
    type => "heartbeat"
  }
}
output {
  stdout {
    codec => rubydebug
  }
}

bin/logstash -f heartbeat.conf

input {
  tcp {
    port => 9900
  }
}
output {
  stdout { }
}

bin/logstash -f weblog.conf

echo 'hello logstash' | nc localhost 9900

head -n 1 weblog-sample.log | nc localhost 9900

elasticsearch {
  hosts => ["http://172.28.70.231:9200"]
  index => "logstash"
  user => "elastic"
  password => "23hUMjScdjOkbKiqCYj9"
}

input {
  tcp {
    port => 9900
  }
}
filter {
  grok {
    match => { "message" => "%{COMBINEDAPACHELOG}" }
  }
  mutate {
    convert => {
      "bytes" => "integer"
    }
  }
  geoip {
    source => "clientip"
    target => "clientgeo"
  }
  useragent {
    source => "agent"
    target => "useragent"
  }
}
output {
  stdout { }
  elasticsearch {
    hosts => ["http://172.28.70.231:9200"]
    index => "logstash"
    user => "elastic"
    password => "23hUMjScdjOkbKiqCYj9"
  }
}

head -n 1 weblog-sample.log | nc localhost 9900

https://www.elastic.co/cn/downloads/beats/

wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-8.3.3-linux-x86_64.tar.gz

tar -zxvf filebeat-8.3.3-linux-x86_64.tar.gz

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /Users/liuxg/data/apache_logs/*
output.logstash:
  hosts: ["localhost:5044"]

$ ./filebeat -c filebeat_apache.yml

/Users/liuxg/data/apache_logs/*

[2022-08-04T11:05:00,966][ERROR][logstash.javapipeline    ][main] Pipeline error {:pipeline_id=>"main", :exception=>#<LogStash::ConfigurationError: GeoIP Filter in ECS-Compatiblity mode requires a `target` when `source` is not an `ip` sub-field,。

elasticsearch {
  hosts => ["http://172.28.70.231:9200"]
  index => "logstash"
  user => "elastic"
  password => "23hUMjScdjOkbKiqCYj9"
}

Exiting: error loading config file: open filebeat.yml: permission denied

Exiting: error loading config file: config file (“filebeat.yml”) can only be writable by the owner but the permissions are “-rwxrwxrwx” (to fix the permissions use: ‘chmod go-w /usr/share/filebeat/filebeat.yml’)

chmod 755 /home/filebeat/filebeat.yml