聚合组
在生产中后台服务器很多,他们会将他们产生的数据传输到一个服务器进行聚合操作,下面配置的是服务器1和服务器2分别监控自己的文件,将监听到的数据统一发送到服务器3上完成聚合操作。
a1.sources = r1 a1.sinks = k1 a1.channels = c1 a1.sources.r1.type = taildir # 文件读取位置信息路径 a1.sources.r1.positionFile = /opt/module/flume/position/position.json # 配置两个组,分别监听两个文件 a1.sources.r1.filegroups = f1 a1.sources.r1.filegroups.f1 = /home/hadoop/data3/file5 a1.sinks.k1.type = avro a1.sinks.k1.hostname=hadoop104 a1.sinks.k1.port=4141 a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 a1.sources.r1.channels = c1 a1.sinks.k1.channel = c1
a2.sources = r1 a2.sinks = k1 a2.channels = c1 a2.sources.r1.type = taildir # 文件读取位置信息路径 a2.sources.r1.positionFile = /opt/module/flume/position/position.json # 配置两个组,分别监听两个文件 a2.sources.r1.filegroups = f1 a2.sources.r1.filegroups.f1 = /home/hadoop/data3/file6 a2.sinks.k1.type = avro a2.sinks.k1.hostname=hadoop104 a2.sinks.k1.port=4141 a2.channels.c1.type = memory a2.channels.c1.capacity = 1000 a2.channels.c1.transactionCapacity = 100 a2.sources.r1.channels = c1 a2.sinks.k1.channel = c1
a3.sources = r1 a3.sinks = k1 a3.channels = c1 a3.sources.r1.type = avro a3.sources.r1.bind=hadoop104 a3.sources.r1.port=4141 a3.sinks.k1.type = logger a3.channels.c1.type = memory a3.channels.c1.capacity = 1000 a3.channels.c1.transactionCapacity = 100 a3.sources.r1.channels = c1 a3.sinks.k1.channel = c1