需求
GrayLog每天每周产生的安全告警数量较多，这些产生的告警都通过PrometheusAlert推送到钉钉webhook告警机器人 想对这些告警做一些汇总统计，报表分析
解决思路：
由于GrayLog Web界面上的告警界面无汇总统计的功能，只能寻找其他解决思路，查询prometheusalert是否有统计功能 但搜索过PrometheusAlert的相关功能，与需求不太吻合。突发奇想，看能否从PrometheusAlert.log的日志进行下手通过查看
PrometheusAlert.log的一些规律，最终确定的解决方法 包含value.go的这一行就是所需要的告警日志
2022/10/14 14:15:15.735 [D] [value.go:476] [1665728115735353131] {"event_definition_id":"62d40b5bfbbe0a2fd4faf65d","event_definition_type":"aggregation-v1","event_definition_title":"堡垒机绕过提醒","event_definition_description":"安全助手检测到绕过堡垒机直接登陆服务器行为","job_definition_id":"62d3ff2bfbbe0a2fd4fadd35","job_trigger_id":"6348fe7213d8e16f7fe61732","event":{"id":"01GFAJ3W0SZSGJNTK01QKMRQXB","event_definition_type":"aggregation-v1","event_definition_id":"62d40b5bfbbe0a2fd4faf65d","origin_context":"urn:graylog:message:es:linuxserver_23:e6490781-4b86-11ed-aea0-005056b6acae","timestamp":"2022-10-14T06:10:46.428Z","timestamp_processing":"2022-10-14T06:15:14.713Z","timerange_start":null,"timerange_end":null,"streams":[],"source_streams":["62d3eed0fbbe0a2fd4facacd"],"message":"堡垒机绕过提醒","source":"localhost","key_tuple":[],"key":"","priority":2,"alert":true,"fields":{},"group_by_fields":{}},"backlog":[{"index":"linuxserver_23","message":"Accepted password for root from 192.168.29.41 port 60382 ssh2","timestamp":"2022-10-14T06:10:46.428Z","fields":{"process_id":"3194","gl2_accounted_message_size":292,"application_name":"sshd","level":6,"gl2_remote_ip":"172.16.252.134","gl2_remote_port":64781,"facility_num":10,"Linux_server_ssh_login_ip":"192.168.29.41","gl2_message_id":"01GFAHV3ZRYPS0G895J23348A6","gl2_source_node":"d20e3549-da0a-4ae9-af4a-d352e1c3deb5","gl2_source_input":"62d3ec98fbbe0a2fd4fac816","facility":"security/authorization"},"id":"e6490781-4b86-11ed-aea0-005056b6acae","source":"ec-server-test-172-16-252-134","stream_ids":["62d3eed0fbbe0a2fd4facacd"]}]}
1、使用rsyslog服务来读取prometheusalert.log日志文件
[root@centos ~]# cd /etc/rsyslog.d/ [root@centos rsyslog.d]# vi prometheusalert_read.conf [root@centos rsyslog.d]# cat prometheusalert_read.conf module(load="imfile" PollingInterval="1") # Input for FILE1 #wildcard is allowed at file level only input( type="imfile" tag="Alertlog" ruleset="filelog" Facility="local0" Severity="info" PersistStateInterval="1" reopenOnTruncate="on" freshStartTail="on" file="/opt/PrometheusAlert/logs/prometheusalertcenter.log" ) # Define a template for file events template(name="GraylogFormatFilelog" type="string" string="%msg%\n") #Replace the Target and Port values with your GrayLogServer IP address and port. ruleset(name="filelog") { action( type="omfwd" protocol="udp" target="192.168.31.170" port="1524" template="GraylogFormatFilelog" queue.type="LinkedList" queue.filename="fileq1" queue.saveonshutdown="on" action.resumeRetryCount="-1" ) stop } [root@centos rsyslog.d]# systemctl restart rsyslog
2、创建Index,Input和Stream2、创建Index,Input和Stream
[root@centos ~]# firewall-cmd --permanent --zone=public --add-port=1524/udp success [root@centos ~]# firewall-cmd --reload success [root@centos ~]# systemctl restart rsyslog.service [root@centos ~]#
3、字段提取
在日志搜索栏中搜索告警日志后进行提取器配置，提取所需的字段
例如这里使用正则表达式进行字段的提取
以下为导出的提取器语法配置文件
{ "extractors": [ { "title": "alert_json_message", "extractor_type": "regex", "converters": [], "order": 1, "cursor_strategy": "copy", "source_field": "message", "target_field": "alert_json_message", "extractor_config": { "regex_value": "^.*\\[[0-9]{19}\\](.+)$" }, "condition_type": "string", "condition_value": "value.go" }, { "title": "json_extractor", "extractor_type": "json", "converters": [], "order": 2, "cursor_strategy": "copy", "source_field": "alert_json_message", "target_field": "", "extractor_config": { "list_separator": ", ", "kv_separator": "=", "key_prefix": "", "key_separator": "_", "replace_key_whitespace": false, "key_whitespace_replacement": "_" }, "condition_type": "none", "condition_value": "" }, { "title": "field_cut", "extractor_type": "copy_input", "converters": [ { "type": "tokenizer", "config": {} } ], "order": 3, "cursor_strategy": "copy", "source_field": "backlog", "target_field": "backlog_detail", "extractor_config": {}, "condition_type": "none", "condition_value": "" }, { "title": "level_replace", "extractor_type": "regex_replace", "converters": [], "order": 0, "cursor_strategy": "copy", "source_field": "message", "target_field": "message", "extractor_config": { "replacement": "\"alertlevel\"", "regex": "\"level\"" }, "condition_type": "none", "condition_value": "" }, { "title": "facility_num_replace", "extractor_type": "regex_replace", "converters": [], "order": 0, "cursor_strategy": "copy", "source_field": "message", "target_field": "message", "extractor_config": { "replacement": "\"alert_facility_num\"", "regex": "\"facility_num\"" }, "condition_type": "none", "condition_value": "" } ], "version": "4.2.10" }
4、字段展示和报表大屏配置
最后的效果
