pod的日志收集
大部分内容来自：
https://blog.csdn.net/weixin_43266367/article/details/127517467
1. 介绍
日志收集分类：1来自POD的日志 2来自APP的日志输出。本次部署第1种，收集POD的日志
2. 构建logstash的镜像
内容如下:
FROM logstash:7.17.6
LABEL author="admin@163.com"
WORKDIR /usr/share/logstash
ADD logstash.yml /usr/share/logstash/config/logstash.yml
ADD logstash.conf /usr/share/logstash/pipeline/logstash.conf
USER root
RUN usermod -a -G root logstash#将logstash用户加入root组，避免因权限不足导致logstash无法读取日志文件
logstash.yml的内容：
http.host: "0.0.0.0"
Logstash.conf的内容:
input {
file {
#path => "/var/lib/docker/containers/*/*-json.log" #docker
path => "/var/log/pods/*/*/*.log"#使用containerd时，Pod的log的存放路径
start_position => "beginning"
type => "applog"#日志类型，自定义
}
file {
path => "/var/log/*.log"#操作系统日志路径
start_position => "beginning"
type => "syslog"
}
}
output {
if [type] == "applog" {#指定将applog类型的日志发送到kafka的哪个topic
kafka {
bootstrap_servers => "${KAFKA_SERVER}"
topic_id => "${TOPIC_ID}"
batch_size => 16384 #logstash每次向ES传输的数据量大小,单位为字节
codec => "${CODEC}"#日志格式
} }
if [type] == "syslog" {##指定将syslog类型的日志发送到kafka的哪个topic
kafka {
bootstrap_servers => "${KAFKA_SERVER}"
topic_id => "${TOPIC_ID}"
batch_size => 16384
codec => "${CODEC}" #系统日志不是json格式
}}
}
3. 编写yaml文件
minikube的文件内容:
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: logstash-daemonset
namespace: log
spec:
selector:
matchLabels:
app: logstash
template:
metadata:
labels:
app: logstash
spec:
containers:
- name: logstash
image: harbor-server.linux.io/n70/logstash-daemonset:7.17.6
imagePullPolicy: Always
env:
- name: KAFKA_SERVER
value: "192.168.122.33:9092,192.168.122.34:9092,192.168.122.35:9092"
- name: TOPIC_ID
value: "jsonfile-log-topic"
- name: CODEC
value: "json"
volumeMounts:
- name: varlog
mountPath: /var/log
readOnly: False
- name: varlogpods
mountPath: /var/log/pods
readOnly: False
volumes:
- name: varlog
hostPath:
path: /var/log
- name: varlogpods
hostPath:
path: /var/log/pods
