小型架构实验模拟

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
 
    access_log  /var/log/nginx/access.log  main;
 
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 4096;
 
    upstream tomcat {
      server 192.168.217.77:8080;
      server 192.168.217.88:8080;
    }
    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;
 
    # Load modular configuration files from the /etc/nginx/conf.d directory.
    # See http://nginx.org/en/docs/ngx_core_module.html#include
    # for more information.
    include /etc/nginx/conf.d/*.conf;
 
    server {
        listen       80;
        listen       [::]:80;
        server_name  _;
        root         /usr/share/nginx/html;
 
 
          location ~*\.jsp$ {
        proxy_pass http://tomcat;
        }
 
        location ~* \.(jpg|jpeg|png|gif|bmp|html)$ {
        root  /usr/share/nginx/html/;
        }
}
 
  server {
        listen       80;
 
 
 
       server_name www.benet.com;
       root /data/benet;
       location / {
       proxy_pass http://192.168.217.77:8080;
       }
}
 
  server {
        listen       80;
 
       server_name www.kgc.com;
       root /data/kgc;
       location / {
       proxy_pass http://192.168.217.88:8080;
       }
 
}
        # Load configuration files for the default server block.
        include /etc/nginx/default.d/*.conf;
 
  server {
        listen       80;
        error_page 404 /404.html;
        location = /404.html {
        }
 
        error_page 500 502 503 504 /50x.html;
        location = /50x.html {
        }
 
}
 

! Configuration File for keepalived
 
global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_01
   vrrp_skip_check_adv_addr
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
 
vrrp_script check_down {
        script "/etc/keepalived/ng.sh"
        interval 1
        weight -30
        fall 3
        rise 2
        timeout 2
}
 
vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
 
    virtual_ipaddress {
        192.168.217.188
    }
 
    track_script {
        check_down
}
 
}
 
 

#=========================== Filebeat inputs =============================
 
filebeat.inputs:
 
# Each - is an input. Most options can be set at the input level, so
# you can use different inputs for various configurations.
# Below are the input specific configurations.
 
- type: log
 
  enabled: true
 
  paths:
    - /var/log/nginx/access.log
  tags: ["access"]
 
- type: log
 
  enabled: true
 
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

1. #----------------------------- Logstash output --------------------------------
2. output.logstash:
3. # The Logstash hosts
4.   hosts: ["192.168.217.66:5044"]
5. 
6. # Optional SSL. By default is off.
7. # List of root certificates for HTTPS server verifications
8. #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
9. 
10. # Certificate for SSL client authentication
11. #ssl.certificate: "/etc/pki/client/cert.pem"
12. 
13. # Client Certificate Key
14. #ssl.key: "/etc/pki/client/cert.key"

input {
    beats {
    port => "5044"
    }
}
output {
   if "access" in [tags] {
      elasticsearch {
        hosts => ["192.168.217.66:9200"]
        index => "nginx-access-%{+YYYY.MM.dd}"
        }
   }
   if "error" in [tags] {
      elasticsearch {
         hosts => ["192.168.217.66:9200"]
         index => "nginx-error-%{+YYYY.MM.dd}"
        }
   }
   stdout {
       codec => rubydebug
   }
}
 

[root@node ~]#grep -v "^#" /etc/elasticsearch/elasticsearch.yml
cluster.name: my-elk-cluster
node.name: node
path.data: /data/elk_data
path.logs: /var/log/elasticsearch/
bootstrap.memory_lock: false
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["node"]
 
http.cors.enabled: true                         #开启跨域访问支持，默认为 false
http.cors.allow-origin: "*"                     #指定跨域访问允许的域名地址为所

[root@node ~]#grep -v "^#" /etc/kibana/kibana.yml 
server.port: 5601
 
server.host: "0.0.0.0"
 
elasticsearch.url: "http://192.168.217.66:9200"
 
kibana.index: ".kibana"

[client]
port = 3306
#default-character-set=utf8
socket = /usr/local/mysql/mysql.sock
 
[mysql]
port = 3306
default-character-set=utf8
socket = /usr/local/mysql/mysql.sock
 
[mysqld]
#log-error = /var/log/mysqld/error.log
user = mysql
basedir = /usr/local/mysql
datadir = /usr/local/mysql/data
port = 3306
log_error=/usr/local/mysql/error.log
character_set_server=utf8
pid-file = /usr/local/mysql/mysqld.pid
socket = /usr/local/mysql/mysql.sock
server-id = 1
 
log_bin = master-bin
log-slave-updates = true
 
sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER,NO_AUTO_VALUE_ON_ZERO,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,PIPES_AS_CONCAT,ANSI_QUOTES
~                                                                                                   
~                                                                                                   

#=========================== Filebeat inputs =============================
 
filebeat.inputs:
 
# Each - is an input. Most options can be set at the input level, so
# you can use different inputs for various configurations.
# Below are the input specific configurations.
 
- type: log
 
  # Change to true to enable this input configuration.
  enabled: true
 
  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - /usr/local/mysql/error.log
  tags: ["error"]
 
 
 fields:           #可以使用 fields 配置选项设置一些参数字段添加到 output 中
    service_name: filebeat
    log_type: log
    service_id: 192.168.217.99
 

#----------------------------- Logstash output --------------------------------
output.logstash:
  # The Logstash hosts
  hosts: ["192.168.217.66:5044"]
 

input {
    beats {
        port => "5044"
    }
}
output {
    elasticsearch {
        hosts => ["192.168.217.66:9200"]
        index => "%{[fields][service_name]}-%{+YYYY.MM.dd}"
    }
    stdout {
        codec => rubydebug
    }
}

[root@node logstash]#/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/mysql.conf 
 
ERROR StatusLogger No log4j2 configuration file found. Using default configuration: logging only errors to the console.
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path //usr/share/logstash/config/log4j2.properties. Using default config which logs to console
07:40:53.541 [LogStash::Runner] FATAL logstash.runner - Logstash could not be started because there is already another instance using the configured data directory.  If you wish to run multiple instances, you must change the "path.data" setting.

故障修复步骤：
1．修复mysql
systemctl restart mysqld
 
2．修复主从
#在现主库服务器 Mysql2 查看二进制文件和同步点
show master status;
 
#在原主库服务器 mysql1 执行同步操作
change master to master_host='192.168.217.77',master_user='myslave',master_password='123456',master_log_file='master-bin.000002',master_log_pos=1745;
 
start slave;
 
 
3．在 manager 节点上修改配置文件app1.cnf（再把这个记录添加进去，因为它检测掉失效时候会自动消失）
vi /etc/masterha/app1.cnf
......
secondary_check_script=/usr/local/bin/masterha_secondary_check -s 192.168.10.14 -s 192.168.10.15
......
[server1]
hostname=192.168.217.77
port=3306
 
[server2]
candidate_master=1
check_repl_delay=0
hostname=192.168.217.88
port=3306
 
[server3]
hostname=192.168.217.99
port=3306
 
4．在 manager 节点上启动 MHA
nohup masterha_manager --conf=/etc/masterha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /var/log/masterha/app1/manager.log 2>&1 &

[root@node ~]#masterha_check_status --conf=/etc/masterha/app1.cnf
 
这是查看 mha  主状态
 
app1 is stopped(2:NOT_RUNNING).

[root@node ~]#masterha_check_repl -conf=/etc/masterha/app1.cnf
 
这是检测mha   主从复制情况
 
 
Mon Apr 22 01:13:23 2024 - [warning] Global configuration file /etc/masterha_default.cnf not found. 
Mon Apr 22 01:13:23 2024 - [info] Reading application default configuration from /etc/masterha/app1.
Mon Apr 22 01:13:23 2024 - [info] Reading server configuration from /etc/masterha/app1.cnf..
Mon Apr 22 01:13:23 2024 - [info] MHA::MasterMonitor version 0.57.
Mon Apr 22 01:13:24 2024 - [error][/usr/local/share/perl5/MHA/ServerManager.pm, ln781] Multi-master configurations for details. Master configurations are as below: 
Master 192.168.217.77(192.168.217.77:3306), replicating from 192.168.217.99(192.168.217.99:3306)
Master 192.168.217.99(192.168.217.99:3306), replicating from 192.168.217.77(192.168.217.77:3306)
 
Mon Apr 22 01:13:24 2024 - [error][/usr/local/share/perl5/MHA/MasterMonitor.pm, ln427] Error happene
Mon Apr 22 01:13:24 2024 - [error][/usr/local/share/perl5/MHA/MasterMonitor.pm, ln525] Error happene
Mon Apr 22 01:13:24 2024 - [info] Got exit code 1 (Not master dead).
 
MySQL Replication Health is NOT OK!
[1]+  Exit 1                  nohup masterha_manager --conf=/etc/masterha/app1.cnf --remove_dead_m

[client]
port = 3306
#default-character-set=utf8
socket = /usr/local/mysql/mysql.sock
 
[mysql]
port = 3306
default-character-set=utf8
socket = /usr/local/mysql/mysql.sock
 
[mysqld]
user = mysql
basedir = /usr/local/mysql
datadir = /usr/local/mysql/data
port = 3306
character_set_server=utf8
pid-file = /usr/local/mysql/mysqld.pid
socket = /usr/local/mysql/mysql.sock
server-id = 2
 
log_bin = master-bin
 
relay-log = relay-log-bin
 
relay-log-index = slave-relay-bin.index
 
sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER,NO_AUTO_VALUE_ON_ZERO,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,PIPES_AS_CONCAT,ANSI_QUOTES