#!/bin/bash
# 

log_info(){
   echo -e "[$(date +'%F %T')]: \033[32m $1 \033[0m"
}

log_error(){
   echo -e "[$(date +'%F %T')]: \033[31m $1 \033[0m"
}

log_file(){
   echo >&3 "[$(date +'%F %T')]: $1 "

}

log_warn(){
   echo -e "[$(date +'%F %T')]: \033[33m $1 \033[0m"
}

metrics_saving(){
   if [ -z "$3" ]; then
       echo >&4 "$1: $2"
   else
       echo >&4 "$1: $2 : $3"
   fi
}


get_container_id(){
    local ID=$(kubectl -n $ns get pods | egrep "^$1" | grep -v 'proj' | awk '{print $1}' | head -1)
    echo $ID
}

check_fio(){
    command -v fio &> /dev/null
    if [ $? -ne 0 ]; then
        rpm -ivh rpms/fio-3.2-1.el7.x86_64.rpm &> /dev/null
    fi


}


iops_unit_convert() {
    typeset -l size
    size=$1
    to_k=$(echo $size | awk '{
            if($1 ~ /k$/){
                sub(/k$/,"");
                print $1
            } else {
                print $1/1000
            }
        }')
    echo $to_k
}


get_weboffice_ver(){
    local ID=$(get_container_id edoc2)
    if [ -n "$ID" ]; then
        local vers=$(
              ${kube_exec[@]} $ID -- curl -s http://weboffice/web-apps/apps/api/documents/api.js | 
              grep '?_dc=' | awk -F'[=\"-]+' '{print $4}'
              )
        if [ -n "$vers" ]; then
           log_info "weboffice版本：$vers。"
           log_file "weboffice版本：$vers。"
        else
           log_error "未获取到weboffice版本。"
           log_file  "未获取到weboffice版本。"
           local vers=null
        fi
    fi
    metrics_saving "WEBOFFICE_VER" "$vers"

}

get_isp_ver(){
    local isp_ver=$(kubectl -n $ns get ds,deploy -o wide | 
          egrep 'isp:|udcmgt:' | awk -F'[ :]+' '{print $(NF-1)}'
          )
    if [ -n "$isp_ver" ]; then
        log_info "当前环境已部署ISP, 版本为: $isp_ver。"
        log_file "当前环境已部署ISP, 版本为: $isp_ver。"
    else
        log_info "当前环境未部署ISP。"
        log_file "当前环境未部署ISP。"
    fi
}


check_anti_virus(){
    avs=(
         symantec 
         king 
         360 
         sangfor 
         clam 
         rising 
         aliyundun 
         qaxsafe 
         sdsvrd 
         ds_agent 
         klnagent
         edr
    )
    for av in "${avs[@]}"; do
        if ps -e -o comm,args | grep -i "$av" | egrep -v 'grep|skywalking|containerd' &>/dev/null; then
            log_error "红色警告，检测到疑似杀毒软件，杀毒类型为：$av，请及时与客户确认并关闭。"
            log_file "红色警告，检测到疑似杀毒软件，杀毒类型为：$av，请及时与客户确认并关闭。"
        fi
    done

}

check_backup_software(){
    softs=(veeamagent anybackup)
    for soft in "${softs[@]}"; do
        if ps -e -o comm,args | grep -i $soft | grep -v grep &>/dev/null; then
            log_error "红色警告，检测到备份软件：$soft，请及时与客户确认并关闭。"
            log_file "红色警告，检测到备份软件：$soft，请及时与客户确认并关闭。"
        fi
    done
}


os_env_check(){
    local mem_req=64000000
    local cpu_req=20
    local total_cpu=$(lscpu | awk '/^CPU(\(s\)){0,1}:/{print $2}')
    local cpu_idle=$(top -b -d 0.1 -n 3 | grep Cpu | tail -n 1 | awk -F'[ ,]+' '{print $8}' | cut -f 1 -d ".")
    local cpu_usage=$(awk -v idle=$cpu_idle 'BEGIN{print int(100-idle)}')
    local load=$(uptime | awk -F 'load average: ' '{print $2}' | awk -F ', ' '{print int($1)}')
    local total_mem=$(free -k | sed -n '2,2p' | awk '{print $2}')
    local mem_avai=$(free -k | sed -n '2,2p' | awk '{print $NF}')
    local mem_used=$((total_mem-mem_avai))
    local mem_rate=$(awk -v t=$total_mem -v u=$mem_used 'BEGIN{print int(u/t*100)}')
    
    if [ $total_cpu -lt $cpu_req ]; then
        log_warn "黄色预警，当前cpu核心数：$total_cpu，最低cpu核心要求：$cpu_req。"
        log_file "黄色预警，当前cpu核心数：$total_cpu，最低cpu核心要求：$cpu_req。"
    else
        log_info "当前cpu核心数：$total_cpu，最低cpu核心要求：$cpu_req。"
        log_file "当前cpu核心数：$total_cpu，最低cpu核心要求：$cpu_req。"
    fi
    
    if [ $cpu_usage -ge 80 ]; then
        log_error "红色预警，当前cpu使用率：${cpu_usage}%，大于80%。"
        log_file "红色预警，当前cpu使用率：${cpu_usage}%，大于80%。"
    else
        log_info "当前cpu使用率：${cpu_usage}%，小于80%。"
        log_file "当前cpu使用率：${cpu_usage}%，小于80%。"
    fi
    
    if [ $total_mem -lt $mem_req ]; then
        log_warn "黄色预警，当前内存总数：$((total_mem/1000/1000))G，小于最低要求：$((mem_req/1000/1000))G。"
        log_file "黄色预警，当前内存总数：$((total_mem/1000/1000))G，小于最低要求：$((mem_req/1000/1000))G。"
    else
        log_info "当前内存总数：$((total_mem/1000/1000))G，满足最低要求：$((mem_req/1000/1000))G。"
        log_file "当前内存总数：$((total_mem/1000/1000))G，满足最低要求：$((mem_req/1000/1000))G。"
    fi
    
    if [ $mem_rate -ge 80 ]; then
        log_error "红色预警，当前内存使用率：${mem_rate}%，大于80%。"
        log_file "红色预警，当前内存使用率：${mem_rate}%，大于80%。"
    else
        log_info "当前内存使用率：${mem_rate}%，小于80%。"
        log_file "当前内存使用率：${mem_rate}%，小于80%。"
    fi
    
    if [ $((load/total_cpu)) -ge 1 ]; then
        log_error "红色预警，当前系统负载：$load，存在异常。"
        log_file "红色预警，当前系统负载：$load，存在异常。"
    else
        log_info "当前系统负载：$load，状态正常。"
        log_file "当前系统负载：$load，状态正常。"
    fi

    metrics_saving "CPU" "$total_cpu"
    metrics_saving "CPU_USAGE" "${cpu_usage}%"
    metrics_saving "MEMORY" "$total_mem"
    metrics_saving "MEMORY_USAGE" "${mem_rate}%"
    metrics_saving "CPU_LOAD1" "$load"

}

get_current_node(){
    declare -A HOST
    for i in $(kubectl get nodes -o wide | awk 'NR!=1{print $1":"$6}'); do
        node=$(echo $i | awk -F':' '{print $1}')
        ip=$(echo $i | awk -F':' '{print $2}')
        ip -4 a | grep $ip &> /dev/null
        if [ $? -eq 0 ]; then
            self=$node
        fi
        HOST["$node"]=$ip
    done
}


kubectl_executable_or_not(){
    kubectl get nodes &> /dev/null
    if [ $? -eq 0 ]; then
       return 0
    else
       return 1
    fi
}


check_edoc2_middleware_allinone(){
    local count=0
    for app in $(kubectl -n $ns get pods | grep edoc2 | awk '{print $1}'); do
        kubectl -n $ns get pods $app -o wide | grep $self &> /dev/null
        if [ $? -eq 0 ]; then
            metrics_saving "IS_ECM" "true"
        else
            let count++
        fi
    done

    if [ "$count" -eq "${#HOST[@]}" ]; then
        metrics_saving "IS_ECM" "false"
    fi

    local count=0
    for mid in $(kubectl -n $ns get pods | grep "mysql" | awk '{print $1}'); do
        kubectl -n $ns get pods $mid -o wide | grep $self &> /dev/null
        if [ $? -eq 0 ]; then
            metrics_saving "IS_MIDDWARE" "true"
        else
            let count++
        fi
      
   done     

   if [ "$count" -eq "${#HOST[@]}" ]; then
        metrics_saving "IS_MIDDWARE" "false"
   fi 

}


check_current_node_svc_num(){
    for svc in edoc2 transport content; do
        local count=0
        for pod in $(kubectl -n $ns get pods | grep -v 'proj' | grep $svc | awk '{print $1}'); do
            kubectl -n $ns get pods $pod -o wide | grep $self &> /dev/null
            if [ $? -eq 0 ]; then
                let count++
            fi
         done
         if [ "$count" -gt 1 ]; then
             log_error "红色预警，服务：$svc，在当前节点运行了多个，请调整为一个。"
             log_file "红色预警，服务：$svc，在当前节点运行了多个，请调整为一个。"
         fi
    done       
}


check_https_or_http(){
    local id=$(get_container_id edoc2)
    if [ -n "$id" ]; then
        local types=$(
              ${kube_exec[@]} $id -- env | 
              grep HTTP_PROTOCOL_TYPE | awk -F'=' '{print $2}'
              )
            log_info "当前环境协议类型为：${types:-http}。"
            log_file "当前环境协议类型为：${types:-http}。"
            metrics_saving "HTTP_TYPE" "${types:-http}"
    fi 
}


check_apm(){
    local id=$(get_container_id nginx)
    if [ -n "$id" ]; then
        apm_ip=$(${kube_exec[@]} $id -- dig +short +search apm)
        if [ -z "$apm_ip" ]; then
             log_error "红色预警，当前环境未安装apm，请及时安装apm并配置预警规则。"
             log_file "红色预警，当前环境未安装apm，请及时安装apm并配置预警规则。"
         else
             apm_api="http://apm/api/warningactionconfig/issettingwarningaction" 
             local result=$(${kube_exec[@]} $id -- curl -s $apm_api | grep isSuccess | awk -F'[ :]+' '{print $NF}')
             if [ "$result" == "false" ]; then
                 log_error "红色预警, apm未配置预警规则。"
                 log_file "红色预警, apm未配置预警规则。"
             else
                 log_info "当前环境已安装apm并配置预警规则。"
                 log_file "当前环境已安装apm并配置预警规则。。"
             fi
         fi
   fi

}

get_mysql_id(){
    if [ $(kubectl -n $ns get pods | grep -c mysql) -ge 3 ]; then
        local id=$(get_container_id mysql-mgr)
    else
        local id=$(get_container_id mysql)
    fi
    echo $id
}



get_mysql_conn(){
    local id=$(get_mysql_id) 
    if [ -n "$id" ]; then
        ${kube_exec[@]} $id -- mysql -urepl -pXjb0^DDy -N -e "select 1" &>/dev/null
        if [ $? -eq 0 ]; then
              mysql_conn=(mysql -urepl -pXjb0^DDy)
              return
        fi

        ${kube_exec[@]} $id -- mysql -uroot -p1qaz2WSX -N -e "select 1" &>/dev/null
        if [ $? -eq 0 ]; then
             mysql_conn=(mysql -uroot -p1qaz2WSX)
             return
        fi
    fi
    
}

check_etcd(){
    kubectl get cs 2>/dev/null | awk '/^etcd/{print $1,$2}' | while read name sts; do
        if [[ "$sts" != "Healthy" ]]; then
            log_error "红色预警: ${name}节点异常，当前状态为: ${sts}。"
            log_file "红色预警: ${name}节点异常，当前状态为: ${sts}。"
        else
            log_info "${name}节点状态: ${sts}。"
            log_file "${name}节点状态: ${sts}。"
        fi
    done
}


check_db_type(){
    local id=$(get_container_id edoc2)
    if [ -n "$id" ]; then
        local types=$(
              ${kube_exec[@]} $id -- env | grep DatabaseType | 
              awk -F'=' '{print $2}'
              )
        [ -z "$types" ] && local types=3
        case "$types" in
           1)
             local db=sqlserver;;
           2)
             local db=oracle;;
           3)
             local db=mysql;;
           4)
             local db=dm;;
           *)
             local db=unknow;;
        esac

        local db_server=$(
              ${kube_exec[@]} $id -- env | grep '^DatabaseServerName' | 
              awk -F'=' '{print $2}')
        if [ -z "$db_server" -o "$db_server" == "middle-proxy" ]; then
             local db_type=standard
        else
             db_type=external
        fi
        log_info "当前环境数据库：$db，类型为：$db_type。"
        log_file "当前环境数据库：$db，类型为：$db_type。"
        metrics_saving "DB_TYPE" "$db"
        metrics_saving "DB_MODE" "$db_type"
     fi
     
   
     local id=$(get_mysql_id)
     get_mysql_conn
     #local mysql_conn=(mysql -urepl -pXjb0^DDy)
     if [ -n "$id" -a -n "$mysql_conn" ]; then
         local binlog=$(
         ${kube_exec[@]} $id -- ${mysql_conn[@]} -N -e "
         show variables where variable_name = 'log_bin';" 2>/dev/null | awk '{print $2}'
         )
         local slowlog=$(
         ${kube_exec[@]} $id -- ${mysql_conn[@]} -N -e "
         show variables where variable_name = 'slow_query_log';" 2>/dev/null | 
         awk '{print $2}'
         )
         if [ "$binlog" == ON ]; then
             binlog_expire=$(
             ${kube_exec[@]} $id -- ${mysql_conn[@]} -N -e "
             show variables where variable_name = 'binlog_expire_logs_seconds';" 2>/dev/null | awk '{print $2}'
             )
             log_info "MySQL binlog已经开启，自动清理时长：${binlog_expire}秒。"
             log_file "MySQL binlog已经开启，自动清理时长：${binlog_expire}秒。"
             metrics_saving "MYSQL_BINLOG" "on"
             metrics_saving "MYSQL_BINLOG_EXPIRE_SECONDS" "$binlog_expire"
         else
             log_warn "黄色预警，MySQL binlog未开启。"
             log_file "黄色预警，MySQL binlog未开启。"
             metrics_saving "MYSQL_BINLOG" "off"
         fi

         if [ "$slowlog" == ON ]; then
             log_info "MySQL slowlog已经开启。"
             log_file "MySQL slowlog已经开启。"
             metrics_saving "MYSQL_SLOWLOG" "on"
         else
             log_error "红色预警，MySQL slowlog未开启。"
             log_file "红色预警，MySQL slowlog未开启。"
             metrics_saving "MYSQL_SLOWLOG" "off"
         fi
     fi

}

business_collection(){
    local id=$(get_mysql_id)
    get_mysql_conn
    if [ -n "$id" -a -n "$mysql_conn" ]; then
        #local mysql_conn=(mysql -urepl -pXjb0^DDy)
        local res=$(${kube_exec[@]} $id -- ${mysql_conn[@]} -N -e "
        SELECT IF( EXISTS( 
        select table_name from information_schema.tables where table_name = 'mig_history'), 1, 0) 
        as ecm;" 2>/dev/null
        )

        if [ "$res" == "1" ]; then
             ${kube_exec[@]} $id -- ${mysql_conn[@]} -N -e "
             select 'PRODUCT: ',
             module_id,'VERSION: ',
             version from edoc2v5.mig_history" 2>/dev/null >> ${path}/metrics.txt
        elif [ "$res" == "0" ]; then
             ${kube_exec[@]} $id -- ${mysql_conn[@]} -N -e "
             select 'PRODUCT: lower_ecm',
             'VERSION:',
             max(MigrationId) 
             from edoc2v5.__efmigrationshistory;" 2>/dev/null >> ${path}/metrics.txt
        else
             metrics_saving "PRODUCT" "null"
             metrics_saving "VERSION" "null"
        fi
        kubectl -n $ns cp scripts/analyze.sql $id:/opt/analyze.sql
        ${kube_exec[@]} $id -- ${mysql_conn[@]} -N -e "source /opt/analyze.sql" 2>/dev/null >> ${path}/metrics.txt
    fi

}


check_rabbitmq(){
   local ID=$(get_container_id rabbitmq)
   if [ -n "$ID" ]; then
      if [ $(kubectl -n $ns get pods | grep -c rabbitmq) -eq 3 ]; then
           local mode=cluster
      else
           local mode=standalone
      fi
      local queues=$(${kube_exec[@]} $ID -- rabbitmqctl list_queues -s) 
      if [ "$mode" == "cluster" ]; then
           local svc=$(kubectl -n $ns get svc |grep rabbitmq | grep 15672 | awk '{print $1}')
           local custer_status=$(
           ${kube_exec[@]} $ID -- curl -s -u admin:edoc2 $svc:15672/api/vhosts | 
           grep -oP '\"cluster_state\":.+}?,\"description' | grep -oP 'running' | wc -l
           )
           if [[ "$custer_status" -ne 3 ]]; then
                local  hc="error"
           else
                local hc="ok"
           fi
      else
           local hc="ok"
      fi

      if [ -n "$queues" ]; then
         local q_count=$(${kube_exec[@]} $ID -- rabbitmqctl list_queues -s | wc -l)
         local count=1
         ${kube_exec[@]} $ID -- rabbitmqctl list_queues -s | while read name queue; do
             if [[ "$queue" -ge 10000 ]]; then
                  log_error "红色预警，rabbitmq模式：$mode, 健康状态：$hc, 队列名：$name，消息堆积数量：$queue。" 
                  log_file "红色预警，rabbitmq模式：$mode, 健康状态：$hc, 队列名：$name，消息堆积数量：$queue。"
                  metrics_saving "MQ_BLOCK_QUEUE" "$name" "$queue"
             else
                  if [ "$count" -eq "$q_count" ]; then
                      if [ "$hc" != "ok" ]; then
                         log_error "红色预警，rabbitmq模式：$mode, 健康状态：$hc"
                         log_file "红色预警，rabbitmq模式：$mode, 健康状态：$hc"
                      else 
                         log_info "rabbitmq模式：$mode, 健康状态：$hc"
                         log_file "rabbitmq模式：$mode, 健康状态：$hc"
                      fi
                  else
                      let count++
                 fi
             fi  
         done
      else
         log_error "红色预警，rabbitmq模式：$mode, 健康状态：$hc，消息队列为空。"
         log_file "红色预警，rabbitmq模式：$mode, 健康状态：$hc，消息队列为空。"
      fi
      
      metrics_saving "MQ_TYPE" "$mode"
      metrics_saving "MQ_HC" "$hc"
   else
      log_error "未找到rabbitmq容器。"
      log_file "未找到rabbitmq容器。"
   fi
}


get_lic(){
    local id=$(get_container_id edoc2)
    [ -z "$id" ] && local id=$(get_container_id rabbitmq)
    local uri="http://edoc2/WebCore?module=SystemManager&fun=GetLicenseInfo"
    if [ -n "$id" ]; then
        local maxuser=$(
              ${kube_exec[@]} $id -- curl -s $uri | 
              grep -oP '\"MaxUsers\":\d+' | grep -oP '\d+' 
              )
        local timestamp=$(
              ${kube_exec[@]} $id -- curl -s $uri | 
              grep -oP '\"ExpiredTime\":\S+?\d+' | grep -oP '\d+'
              )
        local AccessoryProducts=$(
        ${kube_exec[@]} $id -- curl -s $uri | grep -Po 'AccessoryProducts\":\".+?\"' | 
        awk -F':' '{gsub("\"","",$2);print $2}'
        )
        local lic_timestap=$((timestamp/1000))
        local lic_times=$(date -d @$lic_timestap +%F)
        local now=$(date -d "$(date +%F -d '+30 day')" +%s)

        if [ -n "$maxuser" -a -n "$timestamp" ]; then
            if [[ "$lic_timestap" -le "$now" ]]; then
                log_warn "黄色预警，授权到期时间：$lic_times，不足30天，需要项目经理及时申请。"
                log_file "黄色预警，授权到期时间：$lic_times，不足30天，需要项目经理及时申请。"
                metrics_saving "LIC_NEAR_EXP" "true"
            else
                log_info "授权到期时间：$lic_times。"
                log_file "授权到期时间：$lic_times。"
                metrics_saving "LIC_NEAR_EXP" "false"
            fi
       
            log_info "产品系列：${AccessoryProducts:-null}"
            log_file "产品系列：${AccessoryProducts:-null}"
       
            metrics_saving "LIC_MAXUSER" "$maxuser"
            metrics_saving "LIC_EXP_TIME" "$lic_times"
        else
            log_error "无法获取lic信息。"
       fi
    else
         log_error "无法获取lic信息。"    
    fi

}

k8s_version_check(){
    local version=$(kubectl  get nodes | awk 'NR!=1{print $NF}' | head -1)
    log_info "k8s版本："$version"。"
    log_file "k8s版本："$version"。"
    metrics_saving "K8S_VER" "$version"
}

os_check(){
    local count=$(rpm -qa | egrep -i 'gnome|kde' | wc -l)
    systemctl status graphical.target | grep -q ' active '
    rest=$?

    if [[ "$count" -le 1 ]] && [[ "$rest" -ne 0 ]]; then
       log_info "当前操作系统为最小化安装。"
       log_file "当前操作系统为最小化安装。"
       metrics_saving "OS_MINIMAL" "true"
    else
       log_warn "黄色预警，当前操作系统非最小化安装。"
       log_file "黄色预警，当前操作系统非最小化安装。"
       metrics_saving "OS_MINIMAL" "false"
    fi

}

io_check(){
    local REQ_MID_DISK_IOPS=6k
    local req_mid_disk_iops="${REQ_MID_DISK_IOPS:0:-1}"
    while read types point; do
        local usage=$(df -h $point | awk -F'[ %]+' 'NR!=1{print $(NF-1)}')
        metrics_saving "DISK_SPACE" "$point" "${usage}%"
        if [ "$usage" -ge 80 ]; then
            log_error "红色预警，挂载点：$point, 存储类型：$types，使用率：${usage}%"
            metrics_saving "IOPS_REQ" "$point" "false"
            continue
        fi
        local iops=$(
        fio -filename="$point"/fio.txt -direct=1 -iodepth 6 \
        -thread -rw=write -ioengine=psync -bs=4k -size=3G \
        -numjobs=8 -runtime=30 -group_reporting -name=mytest | 
        grep IOPS | awk -F'IOPS=|,' '{print $2}' && rm -f "${point}"/fio.txt
        )
        local iops=$(iops_unit_convert "$iops")
        if [ -n "${iops}" ]; then
            if [[ $(awk 'BEGIN{print('"$iops"'<'"$req_mid_disk_iops"')?0:1}') -eq 0 ]]; then
                log_error "红色预警，挂载点：$point, 存储类型：$types 磁盘iops最低要求：${req_mid_disk_iops}k, 实际测试磁盘iops：${iops}k, 不符合运行要求"
                log_file "红色预警，挂载点：$point, 存储类型：$types 磁盘iops最低要求：${req_mid_disk_iops}k, 实际测试磁盘iops：${iops}k, 不符合运行要求"
                metrics_saving "IOPS_REQ" "$point" "false"
                metrics_saving "IOPS_TOTAL" "$iops"
            else
                log_info "挂载点：$point, 存储类型：$types 磁盘iops最低要求：${req_mid_disk_iops}k, 实际测试磁盘iops：${iops}k"
                log_file "挂载点：$point, 存储类型：$types 磁盘iops最低要求：${req_mid_disk_iops}k, 实际测试磁盘iops：${iops}k"
                metrics_saving "IOPS_REQ" "$point" "true"
            fi
        else
            log_error "测试主机磁盘 $point iops 失败。"
            log_file "测试主机磁盘 $point iops 失败。"
            metrics_saving "IOPS_REQ" "$point" "false"
        fi
    done < <(df -Th | awk '{
                      if($1~/\/dev\/.+/){
                         print $2" "$NF
                      }else if($2~/nfs|ceph|cifs/){
                         print $2" "$NF}
                      }' | grep -v 'boot'
            )
}

container_limit_check(){
    local total_mem=$(free -g | awk '/Mem:/{print $2}')
    for c_name in content transport; do
       local id=$(get_container_id $c_name)
       if [ -n "$id" ]; then
           c_mem=$(kubectl -n $ns describe deploy $c_name | 
           awk '/Limits:/,/Requests:/{if($0~/memory:/){print $NF}}' | 
           sed -r 's/([0-9]*)\.*.*/\1/'
           )
           if [ -n "$c_mem" ]; then
               if [ "$c_name" == content ]; then
                   if [[ "${c_mem}" -ge 15 ]]; then
                        log_warn "黄色预警，服务：$c_name，未做大小限制。"
                        log_file "黄色预警，服务：$c_name，未做大小限制。"
                        metrics_saving "MEM_LIMIT" "$c_name" "false"
                   else
                        log_info "服务：$c_name，内存限制大小：${c_mem}G。"
                        log_file "服务：$c_name，内存限制大小：${c_mem}G。"
                        metrics_saving "MEM_LIMIT" "$c_name" "true"
                   fi
               else
                   log_info "服务：$c_name，内存限制大小：${c_mem}G。"
                   log_file "服务：$c_name，内存限制大小：${c_mem}G。"
                   metrics_saving "MEM_LIMIT" "$c_name" "true"
               fi
           else
               log_warn "黄色预警，服务：$c_name，未做大小限制。"
               log_file "黄色预警，服务：$c_name，未做大小限制。"
               metrics_saving "MEM_LIMIT" "$c_name" "false"
           fi
       fi
    done
}

mysql_limit_check(){
    local count=$(kubectl  -n $ns get svc | grep -c -P mgr[a-c])
    if [ -n "$count" ]; then
        for i in $(kubectl  -n edoc2 get sts | grep -P mgr[a-c] | awk '{print $1}'); do
            mem_limit=$(
                kubectl -n $ns describe sts $i |
                awk '/Limits:/,/Requests:/{if($0~/memory:/){print $NF}}'
            )        
            if [ -n "$mem_limit" ]; then
                log_error "红色预警: ${i}的内存限制为${mem_limit}, 请及时取消内存限制避免宕机。"
                log_file "红色预警: ${i}的内存限制为${mem_limit}, 请及时取消内存限制避免宕机。"
            else
                log_info "数据库节点: ${i}未设置内存限制。"
                log_file "数据库节点: ${i}未设置内存限制。"
            fi
        done
    fi
}



check_redis(){
    if [ $(kubectl -n $ns get pods | grep -c redis) -eq 6 ]; then
       local mode=cluster
       if kubectl -n $ns get pods | grep -q redis-master; then
           local id=$(get_container_id redis-master)
           local vers=4
       else
           local id=$(get_container_id redis-0)
           local vers=6
       fi
    else
       local mode=standalone
       local id=$(get_container_id redis)
    fi

    if [ -n "$id" ]; then
        if [ "$mode" == standalone ]; then
            local hc=ok
            local password=$(
            ${kube_exec[@]} $id  -- grep requirepass /data/config/redis.conf | awk -F'[ "]+' '{print $2}')
            if [ -z "$password" ]; then
                command="/usr/local/bin/redis-cli"
            else
                command="/usr/local/bin/redis-cli -a $password"
            fi
            local maxcon=$(
            ${kube_exec[@]} $id -- $command config get maxclients 2>/dev/null | awk 'NR==2{print $0}')
            local connection=$(
                  ${kube_exec[@]} $id -- $command info clients 2>/dev/null | 
                  awk -F':' '/connected_clients/{printf("%d",$2)}'
                  )

            local maxcon=$(echo $maxcon)
            local connection=$(echo $connection)
            log_info "redis模式：$mode，最大链接数：$maxcon，当前连接数：$connection。"
            log_file "redis模式：$mode，最大链接数：$maxcon，当前连接数：$connection。"
        else
            if [ "$vers" -eq 4 ]; then
                local password=$(
                ${kube_exec[@]} $id -c redis-master -- grep requirepass /data/redis-master.conf | awk -F'[ "]+' '{print $2}')
            else
                local password=$(
                ${kube_exec[@]} $id -c redis -- grep requirepass /conf/redis.conf | awk -F'[ "]+' '{print $2}')
            fi
                
            if [ -z "$password" ]; then
                command="/usr/local/bin/redis-cli"
            else
                command="/usr/local/bin/redis-cli -a $password"
            fi

            if [ "$vers" -eq 4 ]; then
                local maxcon=$(
                ${kube_exec[@]} $id -c redis-master -- $command config get maxclients 2>/dev/null | 
                awk 'NR==2{print $0}'
                )
                local connection=$(
                ${kube_exec[@]} $id -c redis-master -- $command info clients 2>/dev/null | 
                awk -F':' '/connected_clients/{printf("%d",$2)}'
                )
            fi

            if [ "$vers" -eq 6 ]; then
                local maxcon=$(
                ${kube_exec[@]} $id -c redis -- $command config get maxclients 2>/dev/null | 
                awk 'NR==2{print $0}'
                )
                local connection=$(
                ${kube_exec[@]} $id -c redis -- $command info clients 2>/dev/null | 
                awk -F':' '/connected_clients/{printf("%d",$2)}'
                )
            fi
            local maxcon=$(echo $maxcon)
            local connection=$(echo $connection)

            if [ "$vers" -eq 4 ]; then
                local role=$(
                ${kube_exec[@]} $id -c redis-master -- $command -h redis-master info replication 2>/dev/null | 
                awk -F':' '/role:/{printf("%s\n",$2)}'| xargs
                )

                if [[ "$role" =~ "master" ]]; then
                    local slave_num=$(
                    ${kube_exec[@]} $id -c redis-master -- $command -h redis-master info replication 2>/dev/null | 
                    awk -F':' '/connected_slaves/{printf("%d",$2)}'
                    )
                    if [ "$slave_num" -eq 2 ]; then
                       local hc="ok"
                    else
                       local hc="error"
                    fi
                else
                    local slave1_role=$(
                    ${kube_exec[@]} $id -c redis-master -- $command -h redis-slave1 -p 36379 info replication 2>/dev/null | 
                    awk -F':' '/role:/{printf("%s\n",$2)}'| xargs
                    )
                    local slave2_role=$(
                    ${kube_exec[@]} $id -c redis-master -- $command -h redis-slave2 -p 36380 info replication 2>/dev/null | 
                    awk -F':' '/role:/{printf("%s\n",$2)}'|xargs
                    )

                    if [[ "$slave1_role" =~ "master" ]] && [[ "$slave2_role" =~ "master" ]]; then
                        local hc="error"
                    else
                        if [[ "$slave1_role" =~ "master" ]]; then
                            local slave_num=$(
                            ${kube_exec[@]} \
                            $id -c redis-master -- $command -h redis-slave1 -p 36379 info replication 2>/dev/null |
                            awk -F':' '/connected_slaves/{printf("%d",$2)}'
                            )
                       
                            if [ "$slave_num" -eq 2 ]; then
                               local hc="ok"
                            else
                               local hc="error"
                            fi
                        fi

                        if [[ "$slave2_role" =~ "master" ]]; then
                            local slave_num=$(
                            ${kube_exec[@]} \
                            $id -c redis-master -- $command -h redis-slave2 -p 36380 info replication 2>/dev/null | 
                            awk -F':' '/connected_slaves/{printf("%d",$2)}'
                            )
                            if [ "$slave_num" -eq 2 ]; then
                               local hc="ok"
                            else
                               local hc="error"
                            fi
                        fi
                    fi
                fi
            fi

            if [ "$vers" -eq 6 ]; then
                local slaves=0
                for pod in redis-0 redis-1 redis-2; do
                    if ${kube_exec[@]} $pod -c redis -- $command info &> /dev/null; then
                        local role=$(
                        ${kube_exec[@]} $pod -c redis -- $command info replication 2>/dev/null | 
                        awk -F':' '/role:/{printf("%s\n",$2)}'| xargs
                        )
                     
                        if [[ "$role" =~ "master" ]]; then
                            local slave_num=$(
                            ${kube_exec[@]} $pod -c redis -- $command info replication 2>/dev/null | 
                            awk -F':' '/connected_slaves/{printf("%d",$2)}'
                            )
                            if [ "$slave_num" -eq 2 ]; then
                                local hc="ok"
                            else
                                local hc="error"
                            fi
                        else
                            let slaves++
                        fi
                    else
                        local hc="error"
                        break
                    fi
                done
            fi
                
            if [ "$slaves" -ne 2 ]; then
                local hc="error"
            fi            

            if [ "$hc" == "ok" ]; then
                 log_info "redis模式：cluster，健康状态：$hc，最大连接数：$maxcon，当前连接数：$connection。"
                 log_file "redis模式：cluster，健康状态：$hc，最大连接数：$maxcon，当前连接数：$connection。"
            else
                 log_error "红色预警，redis模式：cluster，健康状态：$hc，最大连接数：$maxcon，当前连接数：$connection。"
                 log_file "红色预警，redis模式：cluster，健康状态：$hc，最大连接数：$maxcon，当前连接数：$connection。"
            fi
         fi

        metrics_saving "REDIS_MODE" "$mode"
        metrics_saving "REDIS_HC" "$hc"
        metrics_saving "REDIS_MAXCON" "$maxcon"
        metrics_saving "REDIS_CONS" "$connection"
    else
        log_error "没有找到redis运行容器。"
        log_file "没有找到redis运行容器。"
    fi

}


check_mysql(){
    local id=$(get_mysql_id)
    get_mysql_conn
    #local mysql_conn=(mysql -urepl -pXjb0^DDy)
    if [ -n "$id" -a -n "$mysql_conn" ]; then
        local mode=$(
              ${kube_exec[@]} $id -- env | grep 'MODE=' | awk -F'=' '{print $2}'
              )
        local max_connection=$(
              ${kube_exec[@]} $id -- ${mysql_conn[@]} -N -e "select @@max_connections;" 2>/dev/null
              )
        if [[ $(kubectl -n $ns get pods | grep -c mgr) -eq 3 ]]; then
            local connection=$(
                  ${kube_exec[@]} $id -- ${mysql_conn[@]} -hmiddle-proxy -N -e "status;" 2>/dev/null | 
                  awk '/Threads:/{print $2}'
                  )
        else
            local connection=$(
                  ${kube_exec[@]} $id -- ${mysql_conn[@]} -N -e "status;" 2>/dev/null | 
                  awk '/Threads:/{print $2}'
                  )
        fi
        local count=0
        if [ "$mode" == "cluster" ]; then
            for host in mysql-mgra mysql-mgrb mysql-mgrc; do
                if ${kube_exec[@]} $id -- ${mysql_conn[@]} -h$host -e "select 1;" &>/dev/null; then
                     local online_num=$(
                     ${kube_exec[@]} $id -- ${mysql_conn[@]} -h$host -N -e "
                     select * from performance_schema.replication_group_members;" 2>/dev/null | 
                     egrep -c 'ONLINE|RECOVERY'
                     )
                     local master_num=$(
                     ${kube_exec[@]} $id -- ${mysql_conn[@]} -h$host -N -e "
                     select * from performance_schema.replication_group_members;" 2>/dev/null | grep -c 'PRIMARY'
                     )
                     if [ "$online_num" -eq 3 -a "$master_num" -eq 1 ]; then
                          let count++ 
                     fi
                 else
                     break
                fi
            done
            if [ "$count" -ne 3 ]; then
                local hc="error"
            else
                local hc="ok"
            fi
        else
           local hc="ok"
        fi
      
        if [ -n "$max_connection" ]; then
           if [ "$hc" == "ok" ]; then
               log_info "MySQL模式：$mode，集群状态：$hc，当前链接数：$connection，最大连接数：$max_connection。"
               log_file "MySQL模式：$mode，集群状态：$hc，当前链接数：$connection，最大连接数：$max_connection。"
               ${kube_exec[@]} $id -- ${mysql_conn[@]} -hmiddle-proxy -e "
               select * from information_schema.processlist 
               where command !='sleep' 
               and user not like '%system%' 
               and state not like '%wait%' 
               and time > 3 
               order by time desc limit 10;" 2>/dev/null > ${path}/slowsql-top10.txt
               if [ $(du -b ${path}/slowsql-top10.txt | awk '{print $1}') == 0 ]; then
                   rm -f ${path}/slowsql-top10.txt
               fi
           else
               log_error "红色预警，MySQL模式：$mode，集群状态：$hc，当前链接数：$connection，最大连接数：$max_connection。"
               log_file "红色预警，MySQL模式：$mode，集群状态：$hc，当前链接数：$connection，最大连接数：$max_connection。"
           fi

           metrics_saving "MYSQL_MODE" "$mode"
           metrics_saving "MYSQL_HC" "$hc"
           metrics_saving "MYSQL_MAXCON" "$max_connection"
           metrics_saving "MYSQL_CONS" "$connection"
        else
           log_error "链接MySQL失败。"
           log_file "链接MySQL失败。"
        fi
        
    else
        log_error "未找到MySQL容器。"
        log_file "未找到MySQL容器。"
       
    fi


}


check_docker_img(){
    local mysql_tag=$(
          kubectl -n $ns get sts -o wide | grep mgr | head -1 | 
          awk -F'[:v]+' '{gsub("\\.","",$NF);print $NF}'
          )
    if [[ "${mysql_tag%%-*}" -le 801718 ]]; then
         if [[ "$mysql_tag" =~ "801718-1" ]]; then
             :
         else
             log_error "红色预警，mysql镜像版本：${mysql_tag}，存在宕机风险，需要更新mysql与haproxy镜像至ECM最新版本(参考KM)，需要区域运维处理。"
             log_file "红色预警，mysql镜像版本：${mysql_tag}，存在宕机风险，需要更新mysql与haproxy镜像至ECM最新版本(参考KM)，需要区域运维处理。"
         fi
    fi

    local haproxy_tag=$(
          kubectl -n $ns get deploy -o wide | grep '^middle-proxy' | 
          awk '{print $(NF-1)}' | awk -F'[:v]+' '{gsub("\\.","",$NF);print $NF}'
          )
    if [ -n "$haproxy_tag" ]; then
       if [[ "$haproxy_tag" -lt 24911 ]]; then
           log_error "红色预警，haproxy镜像版本：${haproxy_tag}，存在宕机风险，需要更新haproxy镜像至ECM最新版本(参考KM)，需要区域运维处理。"
           log_file "红色预警，haproxy镜像版本：${haproxy_tag}，存在宕机风险，需要更新haproxy镜像至ECM最新版本(参考KM)，需要区域运维处理。"
       fi
    fi

}


check_es(){
   local ID=$(get_container_id es)
   if [ -n "$ID" ]; then
      if ! ${kube_exec[@]} $ID -c es-container -- test -f /usr/bin/curl; then
          log_error "es容器未找到curl命令，无法获取集群状态, 请在其他容器中执行curl http://es:9200/_cat/health?v获取es集群状态。"
          log_file "es容器未找到curl命令，无法获取集群状态。请在其他容器中执行curl http://es:9200/_cat/health?v获取es集群状态。"
          return 101
      else
          es_dir=$(
          ${kube_exec[@]} $ID -c es-container -- ls /usr/local | grep "elasticsearch-" | awk -F'-' '{print $NF}'
          )
          es_ver=${es_dir%%.*}
          local svc=$(kubectl -n $ns get svc | grep '^es[0-9]*' | awk '{print $1}' | head -1)
          if [ "$es_ver" -eq 7 ]; then
              es_pass=$(
              ${kube_exec[@]} $ID -c es-container -- env | grep '^ESPASS' | awk -F'=' '{print $NF}'
              )
              local commond="curl -u elastic:${es_pass:=fY9aLwahqCG06tmYYU2E} --connect-timeout 10 -s http://$svc:9200"
          elif [[ "$es_ver" -eq 6 ]] || [[ -z "$es_ver" ]]; then
              local commond="curl --connect-timeout 10 -s http://$svc:9200"
          fi

          if [ $(kubectl -n $ns get deploy | grep -c '^es[0-9]*') -ge 3 ]; then
              local mode=cluster
              local stats=$(
                    ${kube_exec[@]} $ID -c es-container -- $commond/_cat/health | awk '{print $4}'
                    )
              if [ "$stats" == "green" ]; then
                  local hc="ok"
              else
                  local hc="error"
              fi
          else
              mode=standlone
              local stats=$(
                    ${kube_exec[@]} $ID -c es-container -- $commond/_cat/health | 
                    awk '{print $4}'
                    )
              if [ "$stats" == "yellow" ]; then
                  local hc="ok"
              else
                  local hc="error"
              fi
          fi
       fi

      if ${kube_exec[@]} $ID -c es-container -- $commond/_all/_settings?pretty | grep -q 'read_only_allow_delete'; then
            local es_readonly=Yes
      else
            local es_readonly=No
      fi
      local indiceSize=$(
         ${kube_exec[@]} $ID -c es-container -- $commond/_cat/allocation | 
         head -1 | awk '{
         if($2~/kb/){
            sub("kb"," ",$2)
            print int($2)
         }else if($2~/mb/){
            sub("mb"," ",$2)
            print int($2*1000)
         }else if($2~/gb/){
            sub("gb"," ",$2)
            print int($2*1000*1000)
         }else{
            sub("tb"," ",$2)
            print int($2*1000*1000*1000)
         }
      }')


     if [ "$hc" == "ok" -a "$es_readonly" == No ]; then
        ${kube_exec[@]} $ID -c es-container -- $commond/_cat/indices | 
        grep -P ' file_.*?| filelog_.*?| signlog_.*?' | awk '{
        if($9~/kb/){
           sub("kb"," ",$9)
           print $3,$7,int($9)
        }else if($9~/mb/){
           sub("mb"," ",$9)
           print $3,$7,int($9*1000)
        }else if($9~/gb/){
           sub("gb"," ",$9)
           print $3,$7,int($9*1000*1000)
        }else{
           sub("tb"," ",$9)
           print $3,$7,int($9*1000*1000*1000)
        }
      }' | while read index doc_count size; do
            log_info "es模式：${mode:-standalone}，集群状态：$hc，是否只读：$es_readonly，索引名：$index，文档数量：$doc_count，索引大小：$((size/1024))mb。"
            log_file "es模式：${mode:-standalone}，集群状态：$hc，是否只读：$es_readonly，索引名：$index，文档数量：$doc_count。索引大小：$((size/1024))mb。"
            metrics_saving "ES_INDEX_COUNT" "$index" "$doc_count"
            metrics_saving "ES_INDEX_SIZE_KB" "$index" "$size"
         done
     else
         log_error "红色预警，es模式：${mode:-standalone}，集群状态：$hc，是否只读：$es_readonly, 集群状态异常。"
         log_file "红色预警，es模式：${mode:-standalone}，集群状态：$hc，是否只读：$es_readonly， 集群状态异常。"

     fi

     metrics_saving "ES_MODE" "${mode:-standalone}"
     metrics_saving "ES_HC" "$hc"
     metrics_saving "ES_READONLY" "$es_readonly"
     [ -n "$indiceSize" ] && log_info "es存储空间大小：$((indiceSize/1024))mb" && \
     metrics_saving "ES_TOTAL_SIZE_KB" "$indiceSize"
  else
     log_error "未找到es容器。"
     log_file "找到es容器。"
  fi

}


edoc2_thread_check(){
    local pods=$(kubectl -n $ns get pods | grep '^edoc2-' | awk '{print $1}')
    if [ -z "$pods" ]; then return;fi
    for p in $pods; do
        local edoc2Thread=$(${kube_exec[@]} $p -- curl -s http://localhost/api/values/1 | 
              grep -oP '\"threadsCount\":\d+' | grep -oP '[0-9]+'
              )
        if [ -n "$edoc2Thread" ]; then
            if [[ "$edoc2Thread" -lt 1000 ]]; then
                 log_info "${p}线程数: $edoc2Thread，线程数正常。"
                 log_file "${p}线程数: $edoc2Thread，线程数正常。"
            else
                 log_error "红色预警, ${p}线程数: $edoc2Thread，线程数异常。"
                 log_file "红色预警, ${p}线程数: $edoc2Thread，线程数异常。"
            fi
        else
            log_error "无法获取${p}线程数。"
            log_file "无法获取${p}线程数。"
        fi
    done
}
    
service_count_check(){
    for svc in edoc2 content transport; do
        current_num=$(kubectl -n $ns get deploy $svc | awk -F'[ /]+' 'NR!=1{print $2}')
        total_num=$(kubectl -n $ns get deploy $svc | awk -F'[ /]+' 'NR!=1{print $3}')
        if [ -n "$current_num" -a -n "$total_num" ]; then
             if [ $current_num -ne $total_num ]; then
                  log_error "红色预警，${svc}服务总数量：$total_num，当前运行数量：$current_num，存在异常。"
                  log_file "红色预警，${svc}服务总数量：$total_num，当前运行数量：$current_num，存在异常。"
                  metrics_saving "SVC_COUNT_VALID" "$svc" "false"
                  metrics_saving "${svc}_counts" "$total_num"
             else
                  log_info "${svc}服务总数量：$total_num，当前运行数量：$current_num，状态正常。"
                  log_file "${svc}服务总数量：$total_num，当前运行数量：$current_num，状态正常。"
                  metrics_saving "SVC_COUNT_VALID" "$svc" "true"
                  metrics_saving "${svc}_counts" "$total_num"
             fi
         fi
    done
}


check_edoc2_restart_or_not(){
    local pods=$(kubectl -n $ns get pods | grep '^edoc2-' | awk '{print $1}')
    if [ -z "$pods" ]; then return;fi
    local local_times=$(date -d "$(date +%F -d '-7 day')" +%s)
    for p in $pods; do
        local created=$(kubectl -n $ns get pods $p -o jsonpath='{.status.startTime}' | awk -F'T' '{print $1}')
        local created_timestamp=$(date -d "$created" +%s)
        if [[ "$created_timestamp" -le "$local_times" ]]; then
            log_info "${p}七天天内未发生重启。"
            log_file "${p}七天天内未发生重启。"
        else
            log_warn "黄色预警，${p}七天内发生重启, 重启时间：$created"
            log_file "黄色预警，${p}七天内发生重启，重启时间：$created"
        fi
    done
}



#linux 7的防火墙关闭操作
firewalld(){
    systemctl status firewalld | grep "running" &>/dev/null
    is_running=$?
    systemctl is-enabled firewalld | grep "enabled" &>/dev/null
    is_enabled=$?
    if [ $is_running -eq 0 -o $is_enabled -eq 0 ]; then
            log_error "黄色预警, 防火墙处于开启状态,需要在非工作时间进行关闭操作，并重启docker。"
            log_file "黄色预警, 防火墙处于开启状态,需要在非工作时间进行关闭操作，并重启docker。"
            metrics_saving "FIREWALL_DISABLED" "false"
    else
            log_info "防火墙已经关闭。"
            log_file "防火墙已经关闭。"
            metrics_saving "FIREWALL_DISABLED" "true"
    fi
}

#selinux检查函数
selinux(){
#判断当前的状态
result=`getenforce`
    if [ "$result" != "Disabled" ]; then
            log_error "红色预警, selinux是开启状态，需要关闭"
            log_file "红色预警, selinux是开启状态，需要关闭"
            metrics_saving "SELINUX_DISABLED" "false"
    else
            log_info "selinux是关闭状态。"
            log_file "selinux是关闭状态。"
            metrics_saving "SELINUX_DISABLED" "true"
    fi
}
#检查NetworkManager是否关闭
NetworkManager(){
    systemctl status NetworkManager | grep "running" &>/dev/null
    if [ $? -eq 0 ]; then
            log_warn "黄色预警, NetworkManager处于开启状态,需要进行关闭操作！"
            log_file "黄色预警, NetworkManager处于开启状态,需要进行关闭操作！"
            metrics_saving "NET_MANAGER_DISABLED" "false"
    else
            log_info "NetworkManager已经关闭！"
            log_file "NetworkManager已经关闭！"
            metrics_saving "NET_MANAGER_DISABLED" "true"
    fi
}
#检查内存清理脚本是否配置
clean(){
    crontab -l 2>/dev/null | grep "clean-cache.sh" &>/dev/null
    if [ $? -eq 0 ]; then
            log_info "内存清理脚本已添加，无需处理。"
            log_file "内存清理脚本已添加，无需处理。"
            metrics_saving "MEM_CLEAN_CONFIG" "true"
    else
            log_warn "黄色预警，未添加内存定时清理脚本，新增定时清理脚本（参考KM），需要区域运维排查。"
            log_file "黄色预警，未添加内存定时清理脚本，新增定时清理脚本（参考KM），需要区域运维排查。"
            metrics_saving "MEM_CLEAN_CONFIG" "false"
    fi
}
#检查监控脚本是否配置
monitor(){
    crontab -l 2>/dev/null | grep "edoc2-monitor.sh" &>/dev/null
    if [ $? -eq 0 ]; then
        log_info "监控脚本已添加，无需处理。"
        log_file "监控脚本已添加，无需处理。"
        metrics_saving "MONITOR_SCRIPTS_CONFIG" "true"
    else
        log_warn "黄色预警, 监控脚本未添加，请及时添加避免不必要的问题。"
        log_file "黄色预警, 监控脚本未添加，请及时添加避免不必要的问题。"
        metrics_saving "MONITOR_SCRIPTS_CONFIG" "false"
    fi
}


#检查垃圾容器清理脚本是否配置
container(){
    crontab -l 2>/dev/null | grep "clean-container.sh" &>/dev/null
    if [ $? -eq 0 ]; then
        log_info "圾容器清理脚本已添加，无需处理。"
        log_file "圾容器清理脚本已添加，无需处理。"
        metrics_saving "CONTAINER_CLEAN_CONFIG" "true"
    else
        log_warn "黄色预警，垃圾容器清理脚本未添加，请及时添加避免不必要的问题。"
        log_file "黄色预警，垃圾容器清理脚本未添加，请及时添加避免不必要的问题。"
        metrics_saving "CONTAINER_CLEAN_CONFIG" "false"
    fi
}


#检查content清理脚本是否配置
content(){
    crontab -l 2>/dev/null | grep "delete_content.sh" &>/dev/null
    if [ $? -eq 0 ]; then
        log_info "清理临时转档脚本已添加，无需处理。"
        log_file "清理临时转档脚本已添加，无需处理。"
        metrics_saving "CONTENT_CLEAN_CONFIG" "true"
    else
        log_warn "黄色预警，清理临时转档脚本未添加，请及时添加避免不必要的问题。"
        log_file "黄色预警，清理临时转档脚本未添加，请及时添加避免不必要的问题。"
        metrics_saving "CONTENT_CLEAN_CONFIG" "false"
    fi
}

#检查清理压缩下载临时文件脚本是否配置
transport(){
    crontab -l 2>/dev/null | grep "clean-transport.sh" &>/dev/null
    if [ $? -eq 0 ]; then
        log_info "清理压缩下载临时文件脚本已添加，无需处理。"
        log_file "清理压缩下载临时文件脚本已添加，无需处理。"
        metrics_saving "TRANSPORT_CLEAN_CONFIG" "true"
    else
        log_warn "黄色预警，清理压缩下载临时文件脚本未添加，请及时添加避免不必要的问题。"
        log_file "黄色预警，清理压缩下载临时文件脚本未添加，请及时添加避免不必要的问题。"
        metrics_saving "TRANSPORT_CLEAN_CONFIG" "false"
    fi
}


function file_open() {
        is_comfig=$(sysctl -a 2>/dev/null | grep "fs.file-max" | wc -l)
        file_open_max=$(sysctl -a 2>/dev/null| grep "fs.file-max" | awk -F " " '{print $3}')
	if [ $is_comfig -eq 1 ] && [ $file_open_max -ge 655360 ] ;then
                log_info "系统文件最大打开数设置正确。"
                log_file "系统文件最大打开数设置正确。"
                metrics_saving "FILE_OPEN_VALID" "true"
	else
                log_error "红色预警，系统文件最大打开数检测不通过。"
                log_file "红色预警，系统文件最大打开数检测不通过。"
                metrics_saving "FILE_OPEN_VALID" "false"
	fi
        metrics_saving "FILE_MAX" "$file_open_max"
}

function max_user_watches(){
      if [[ $(sysctl -a 2>/dev/null | grep fs.inotify.max_user_watches | awk '{print $NF}') -lt 262144 ]]; then
          log_warn "系统参数fs.inotify.max_user_watches小于262144，请及时调整。"
          log_file "系统参数fs.inotify.max_user_watches小于262144，请及时调整。"
      else
          log_info "系统参数fs.inotify.max_user_watches大小正常。"
          log_file "系统参数fs.inotify.max_user_watches大小正常。"
      fi
}


function ulimit_check(){
    local limit=$(ulimit -n)
    if [ "$limit" -lt 65535 ]; then
        log_error "红色预警, ulimit -n设置过低，请设置为65536以上。"
        log_file "红色预警, ulimit -n设置过低，请设置为65536以上。"
    else
        log_info "ulimit -n设置正常"
        log_file "ulimit -n设置正常"

    fi
}



function check_ceph(){
       command -v ceph &> /dev/null
       if [ $? -eq 0 ]; then
          metrics_saving "IS_CEPH" "true"
          stats=$(timeout 8 ceph -s | awk '/health/{print $NF}')
          if [ "$stats" != 'HEALTH_OK' ]; then
              log_error "红色预警，ceph健康状态异常。"
              log_file "红色预警，ceph健康状态异常。"
              metrics_saving "CEPH_HC" "error"
          else
              log_info "ceph健康状态正常。"
              log_file "ceph健康状态正常。"
              metrics_saving "CEPH_HC" "ok"

              for pool in $(ceph osd pool ls); do
                  local size=$(ceph osd pool get $pool size | awk '{print $NF}')
                  if [[ "$size" -lt 3 ]]; then
                       log_error "红色预警，ceph存储池${pool}设置的副本数为: ${size}, 请调整为3副本."
                       log_file "红色预警，ceph存储池${pool}设置的副本数为: ${size}, 请调整为3副本."
                  fi
              done
          fi

          if [ -f /etc/ceph/ceph.conf ]; then
              local osd_mem=$(grep 'osd memory target' /etc/ceph/ceph.conf | grep -v '#' | awk '{print $NF}')
              if [[ "$osd_mem" -gt 2147483648 ]]; then
                  log_error "红色预警，ceph osd内存设置$((osd_mem/1024/1024/1024))GB，不合理，请联系区域运维，设置为2GB。"
                  log_file "红色预警，ceph osd内存设置$((osd_mem/1024/1024/1024))GB，不合理，请联系区域运维，设置为2GB。"
                  metrics_saving "CEPH_OSD_MEM_VALID" "false"
              else
                  metrics_saving "CEPH_OSD_MEM_VALID" "true"
              fi
          fi
        else
          metrics_saving "IS_CEPH" "false"
          log_info "当前机子未运行ceph存储。"
          log_file "当前机子未运行ceph存储。"
       fi

}


function ip_forward_check(){
       static_ip_forward=$(grep 'net.ipv4.ip_forward' /etc/sysctl.conf |grep -v '#'| awk -F'[ =]+' '{print $NF}')
       dynamic_ip_forward=$(sysctl -a 2>/dev/null | grep -w net.ipv4.ip_forward | awk '{print $NF}')
       
       if [ -n "$static_ip_forward" ]; then
             if [[ "$static_ip_forward" -eq 1 ]] && [[ "$dynamic_ip_forward" -eq 1 ]]; then
                   log_info "net.ipv4.ip_forward设置正常。"
                   log_file "net.ipv4.ip_forward设置正常。"
                   local valid=true
             else
                   log_error "红色预警，net.ipv4.ip_forward设置异常。"
                   log_file "红色预警，net.ipv4.ip_forward设置异常。"
                   local valid=false
             fi
       else
             if [ "$dynamic_ip_forward" -ne 1 ]; then
                   log_error "红色预警，net.ipv4.ip_forward设置异常。"
                   log_file "红色预警，net.ipv4.ip_forward设置异常。"
                   local valid=false
             else
                   log_info "net.ipv4.ip_forward设置正常。"
                   log_file "net.ipv4.ip_forward设置正常。"
                   local valid=true
             fi
       fi

       metrics_saving "IPV4_FORWARD_VALID" "$valid"

}

function ntp_check(){
       ntpstas=$(timedatectl | awk '/NTP enabled:/{print $NF}')
       is_sync=$(timedatectl | awk '/NTP synchronized:/{print $NF}')
       if [[ $ntpstas == "yes" ]] && [[ $is_sync == "yes" ]]; then
           log_info "ntp时间同步正常。"
           log_file "ntp时间同步正常。"
           metrics_saving "NTP_CONFIG" "true"
       else
           log_error "红色预警，ntp时间同步异常。"
           log_file "红色预警，ntp时间同步异常"
           metrics_saving "NTP_CONFIG" "false"
       fi

}

#changelog
#20231208增加fs.inotify.max_user_watches检测
#20240117增加isp检测
#20240204兼容es7与redis6
#20240325兼容redis4与redis6
#20240529增加ceph存储池副本检测
#20240627增加etcd检测、mysql内存限制检测

main(){
    VER=v3.0.20240628_k8s
    path=/tmp/system-check-$(hostname)
    #ecm running namespace
    ns=edoc2
    kube_exec=(kubectl -n $ns exec)
    if [ -f /etc/redhat-release ]; then
        if [ ! -d "$path" ] ; then
             mkdir -p $path
        else
             rm -f $path/system-check-$(date +%F).log $path/metrics.txt
        fi
        #exec 2> /dev/null
        exec 3>> $path/system-check-$(date +%F).log
        exec 4>> $path/metrics.txt
        log_info "巡检服务，版本：$VER"
        log_file "巡检服务，版本：$VER"
        log_info "主机名: $(hostname)"
        log_file "主机名: $(hostname)"
        metrics_saving "HOSTNAME" "$(hostname)"
   
        edition=`cat /etc/redhat-release|sed -r 's/.* ([0-9]+)\..*/\1/' | awk '{print $NF}'`
        ed=`cat /etc/redhat-release` 
        if [ "X$edition" == "X8" -o "X$edition" == "X7" ]; then
            check_fio
            log_info "当前服务器操作系统版本是linux ${edition}版本系统，具体版本：$ed"
            log_file "当前服务器操作系统版本是linux ${edition}版本系统，具体版本：$ed"
            os_check
            os_env_check
            firewalld
            selinux
            if [ "X$edition" == "X7" ]; then
                NetworkManager
            fi
            file_open
            ulimit_check
            max_user_watches
            ntp_check
            io_check
            ip_forward_check
            monitor
            check_ceph
            kubectl_executable_or_not
            sts=$?
            if [ $sts -eq 0 ]; then
                get_current_node
                check_anti_virus
                check_backup_software
                check_apm
                get_isp_ver
                k8s_version_check
                check_etcd
                check_https_or_http
                check_db_type
                check_edoc2_middleware_allinone
                service_count_check
                edoc2_thread_check
                check_edoc2_restart_or_not
                container_limit_check
                mysql_limit_check
                check_current_node_svc_num
                get_weboffice_ver
                get_lic
                check_es
                check_rabbitmq
                check_mysql
                #check_docker_img
                check_redis
                clean
                #container
                #content
                #transport
                business_collection
            fi
       else
           log_error "操作系统不支持。"
           exit 2
       fi
       log_warn "************开始结合指标信息，进行业务分析**************"
       log_file "************开始结合指标信息，进行业务分析**************"
       sleep 2
       /bin/bash scripts/analyze-metrics.sh ${path}/metrics.txt
       if [ -f "${path}/slowsql-top10.txt" ]; then
            log_warn "mysql慢查询记录存在至${path}/slowsql-top10.txt。"
            log_file "mysql慢查询记录存在至${path}/slowsql-top10.txt。"
       fi
       red=$(cat $path/system-check-$(date +%F).log | grep -c '红色')
       yellow=$(cat $path/system-check-$(date +%F).log | grep -c '黄色')
       log_info "巡检结束，红色预警：${red}，黄色预警：${yellow}，请及时处理相应的异常。"
       log_file "巡检结束，红色预警：${red}，黄色预警：${yellow}，请及时处理相应的异常。"
    else
       log_error "操作系统不支持。"
       exit 2
    fi
}
main

