Elastic Stack日志收集系统笔记 （logstash部分）（下）

fields =>["city_name", "continent_code", "country_code2","country_code3", "country_name", "dma_code","ip", "latitude", "longitude","postal_code", "region_name", "timezone"]

NUMBER \d+

codec=>multiline  {
                        pattern =>"^\["
                        negate => true
                        what =>"previous"
}

vim logstash-nginx.conf
input {
        file {                                                   #使用file插件收集日志
          path =>"/var/log/nginx/*.log"           #指定日志路径
          start_position =>"beginning"            #指定日志读取的位置，默认是end，

}
}
filter {
        if [path] =~ "access" {                                            #判断是否是access日志
          mutate { replace => { type =>"nginx_access" } }     #如果是access日志则添加一个type字段
          grok {
            match => { "message"=> "%{COMBINEDAPACHELOG}" }     #使用grok插件过滤access日志，将其转化成多个字段，%{COMBINEDAPACHELOG}是grok定义好的表达式
          }
          date {
            match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]      #将收集到的日志日期作为时间戳
          }
        }
        else if [path] =~ "error"{                                        #判断是否是错误日志
          mutate { replace => { type =>"nginx_error" } }       #如果是错误日志，则添加一个type字段
          grok {
            match => { "message"=> "(?<datetime>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2})\[(?<errtype>\w+)\] \S+: \*\d+ (?<errmsg>[^,]+),(?<errinfo>.*$)" }           #使用grok插件过滤error日志，将其转化成多个字段，这里的表达式是自己定义的  
          }
          date {
            match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]      #将收集到的日志作为时间戳
          }
        }
}
output {
        if [type] =~ "access" {                   #如果收集到的日志为access类型，那么就将日志输出到access的索引
          elasticsearch {
            hosts =>["192.168.179.134:9200"]
            index =>"nginx_access-%{+YYYY.MM.dd}"
          }
        }
        else if [type] =~ "error"{               #如果收集到的日志为error类型，那么就将日志输出到error的索引
          elasticsearch {
            hosts =>["192.168.179.134:9200"]
            index =>"nginx_error-%{+YYYY.MM.dd}"
          }
        }
}

vim /etc/rsrslog.conf
*.* @@192.168.179.134:5514

vim rsyslog.conf
input {
        syslog {
           host =>"192.168.179.134"
          port => 5514
        }
}
output {
        stdout {
codec => rubydebug
        }
}

tar -zxfredis-4.0.11.tar.gz      #解压Redis源码
cp -r redis-4.0.11 /usr/local/   
mv redis-4.0.11/ redis
cd /usr/local/redis/
cp redis.conf bin/
make PREFIX=/usr/local/redis install     #编译安装Redis
echo "export PATH=$PATH:/usr/local/redis/bin" >> /etc/profile   将Redis加入环境变量

redis-server

vim /usr/local/redis/bin/redis.conf
daemonize yes

redis-cli shutdown

vim redis-consumer.conf
input {
        redis {         #定义redis消费者的配置
          data_type => "list"          #定义redis数据类型，redis支持三种数据类型，list，channel，pattern_channel,同的数据类型会导致实际采用不同的 Redis 命令操作，其中list，相当于队列；channel相当于发布订阅的某个特定的频道；pattern_channel相当于发布订阅某组频道。
          key =>"redis_logstash"       #定义redis列表或者频道的名称
          host =>"192.168.179.134"   #定义redis服务器的主机名
          port => 6379                      #定义redis服务器的端口
          db => 1                              #定义redis数据库编号
        }
}
output {
        elasticsearch {                           #输出到elastic上
          hosts =>"192.168.179.134"     #定义elasticsearch的地址
          index => "logstash_redis-%{+YYYY.MM.dd}"     #添加一个elasticsearch索引
        }
        stdout { codec => rubydebug }
}
vim redis-producer.conf 
 input { stdin { } }
output{                                #定义redis生产者的配置
        redis {
           host =>"192.168.179.134"     #同redis消费者的配置
           data_type => "list"
           db => 1
           port => 6379
           key => "logstash_redis"
        }
}

redis-server/usr/local/redis/bin/redis.conf     #后台运行redis
redis-climonitor                #开启redis监控
$ /elasticsearch/elasticsearch-6.4.0/bin/elasticsearch    #开启elasticsearch
[root@elasticelasticsearch-head]# grunt server          #开启elastic-head插件
logstash -f/etc/logstash/conf.d/       #启动logstash
netstat-ntlp                                  #查看端口是否监听

yum install-y yum-utils device-mapper-persistent-data lvm2

yummakecache fast && yum install docker-ce

curl -L "https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname-s)-$(uname -m)" -o /usr/local/bin/docker-compose

docker pull gliderlabs/logspout

vimlogspout.conf
input {
        tcp {
          port => 5140
        }
        udp {
          port => 5140
        }
}
output {
        stdout {
          codec => rubydebug
        }
        elasticsearch {
          hosts =>"192.168.179.134"
          index => "logspout"
        }
}
logstash -f logspout.conf

docker run --name="logspout" --volume=/var/run/docker.sock:/var/run/docker.sock --publish=192.168.179.134:8000:80

docker run -d --name=cen7 centos echo "helloworld" >>/var/log/lastlog

curl 192.168.179.134:8000/logs

docker run-d -e 'LOGSPOUT=ignore' image

docker run --name="logspout" \
    -e EXCLUDE_LABEL=logspout.exclude \
    --volume=/var/run/docker.sock:/var/run/docker.sock\
    gliderlabs/logspout
 docker run -d --label logspout.exclude=trueimage

docker run \
--volume=/var/run/docker.sock:/var/run/docker.sock\
gliderlabs/logspout\
raw://192.168.10.10:5000?filter.name=*_db

docker run \
--volume=/var/run/docker.sock:/var/run/docker.sock\
gliderlabs/logspout\
raw://192.168.10.10:5000?filter.id=3b6ba57db54a

_ "github.com/gliderlabs/logspout/transports/udp"
_"github.com/looplab/logspout-logstash"

FROM  gliderlabs/logspout:latest
COPY./modules.go /src/modules.go

ENVROUTE_URIS=logstash://host:port。
docker run --name="logspout" --volume=/var/run/docker.sock:/var/run/docker.sock -e ROUTE_URIS=logstash://192.168.179.134:5140 registry.cn-beijing.aliyuncs.com/andyyoung01/logspout-logstash