#!/bin/bash

get_env_info() {
    find_path=`find / -name "auto_install.sh" -type f | awk -F "auto_install.sh" '{print $1}'`
    install_path="${find_path%%/}"
    echo "================================================"
    echo `"${install_path}"/scripts/remote_env.sh`
    source "${install_path}"/scripts/remote_env.sh
}

get_node_list_name() {
    MasterNodes="${K8S_MASTER01} ${K8S_MASTER02} ${K8S_MASTER03}"
    WorkNodes="${K8S_NODE01} ${K8S_NODE02} ${K8S_NODE03}"
}

get_node_ip() {
    k8s_lb_ip=$K8S_MASTER_LB_IP
    master01_ip=$K8S_MASTER_IP01
    master02_ip=$K8S_MASTER_IP02
    master03_ip=$K8S_MASTER_IP03
    node01_ip=$K8S_NODE_IP01
    node02_ip=$K8S_NODE_IP02
    node03_ip=$K8S_NODE_IP03
    k8s_service_net=$K8S_CLUSTER_NET
    k8s_service_ip=$K8S_SERVICE_IP
    k8s_cluster_dns=$K8S_CLUSTER_DNS
    #SSH_PORT=$SSH_PORT
    master_eth01=$MASTER_ETH01
    master_eth02=$MASTER_ETH02
    master_eth03=$MASTER_ETH03
    cluster_cidr=$DOCKER0_NETWORK
}

# only need exec in k8s master01
k8s_base_tar() {
    get_env_info
    get_node_ip
    cd "${install_path}"/soft/k8s1.23/
    tar -xf kubernetes-server-linux-amd64_23.10.tar.gz --strip-components=3 -C /usr/local/bin
    tar -xf kubernetes-node-linux-amd64_23.10.tar.gz --strip-components=3 -C /usr/local/bin
    tar -xf etcd-v3.5.1-linux-amd64.tar.gz --strip-components=1 -C /usr/local/bin
    tar -xf "${install_path}"/soft/helm-v3.8.2-linux-amd64.tar.gz -C "${install_path}"/soft/
    \cp ../linux-amd64/helm /usr/local/bin/
    systemctl daemon-reload 
    kubelet --version
    etcdctl version
}

rsync_k8s_bin() {
    get_env_info
    get_node_list_name
    get_node_ip
    #source /tmp/scripts/remote_env.sh
    echo "==================================="
    echo $MasterNodes
    for NODE in $MasterNodes; do
        echo "===================================="
        echo $NODE;
        echo $SSH_PORT
        #sshpass ssh -p $SSH_PORT root@${NODE} 'systemctl stop kubelet && systemctl stop kube-proxy && systemctl stop kube-scheduler && systemctl stop kube-controller-manager && systemctl stop kube-apiserver && systemctl stop etcd';
        #sleep 3;
        #sshpass ssh -p $SSH_PORT root@${NODE} 'rm -rf /opt/cni/* && rm -rf /etc/kubernetes && rm -rf /var/lib/etcd && rm -rf /var/lib/kubelet/* && rm -rf /var/lib/kubelet  && rm -rf /etc/etcd && rm -rf /etc/systemd/system/kubelet.service.d && rm -rf /var/lib/kubelet && rm -rf /var/log/kubernetes && rm -rf /etc/cni && rm -rf /root/.kube && rm -rf /etc/calico';
        sleep 1;
        sshpass scp -P $SSH_PORT -r /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} root@$NODE:/usr/local/bin/;
        sshpass scp -P $SSH_PORT -r /usr/local/bin/etcd* root@$NODE:/usr/local/bin/;
        sshpass ssh -p $SSH_PORT root@${NODE} 'mkdir -p /opt/cni/bin && mkdir -p /etc/kubernetes/pki && mkdir -p /etc/etcd/ssl';
        sshpass scp -P $SSH_PORT -r "${install_path}"/soft root@$NODE:/tmp/;
        sshpass scp -P $SSH_PORT -r "${install_path}"/scripts root@$NODE:/tmp/;
        # -o StrictHostKeyChecking=no
    done

    for NODE in $WorkNodes; do
        echo "==============================="
        echo $NODE;
        #sshpass ssh -p $SSH_PORT root@${NODE} 'systemctl stop kubelet && systemctl stop kube-proxy && systemctl stop kube-scheduler && systemctl stop kube-controller-manager && systemctl stop kube-apiserver && systemctl stop etcd';
        #sleep 3;
        #sshpass ssh -p $SSH_PORT root@${NODE} 'rm -rf /var/lib/kubelet && rm -rf /etc/systemd/system/kubelet.service.d && rm -rf /var/lib/kubelet && rm -rf /etc/calico';
        sleep 1;
        sshpass scp -P $SSH_PORT -r /usr/local/bin/kube{let,-proxy} root@$NODE:/usr/local/bin/;
        sshpass ssh -p $SSH_PORT root@${NODE} 'mkdir -p /opt/cni/bin && mkdir -p /etc/kubernetes/pki';
    done
}


create_etcd_ssl() {
    get_env_info
    get_node_ip
    #### 生成etcd CA ####
    echo "#### 创建etcd CA证书操作 ################"

    \cp "${install_path}"/soft/cfssljson_linux-amd64 /usr/local/bin/cfssljson
    \cp "${install_path}"/soft/cfssl_linux-amd64 /usr/local/bin/cfssl
    chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson
    
    cd "${install_path}"/soft/k8s1.23
    rm -rf k8s-ha-install
    tar -xf k8s-ha-install.tar.gz
    sleep 5
    sed -i "s/192.168.0.10/$K8S_CLUSTER_DNS/g" k8s-ha-install/CoreDNS/*
    sleep 3
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/pki/

    get_node_ip
    # 生成etcd CA证书和CA证书的key
    cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca
    cfssl gencert -ca=/etc/etcd/ssl/etcd-ca.pem -ca-key=/etc/etcd/ssl/etcd-ca-key.pem -config=ca-config.json -hostname=127.0.0.1,$K8S_MASTER01,$K8S_MASTER02,$K8S_MASTER03,$master01_ip,$master02_ip,$master03_ip -profile=kubernetes etcd-csr.json | cfssljson -bare /etc/etcd/ssl/etcd

    get_node_list_name
    for NODE in $MasterNodes; do
        sshpass ssh -p $SSH_PORT root@$NODE "mkdir -p /etc/etcd/ssl"
        for FILE in etcd-ca-key.pem  etcd-ca.pem  etcd-key.pem  etcd.pem; do
            sshpass scp -P $SSH_PORT -r /etc/etcd/ssl/${FILE} root@$NODE:/etc/etcd/ssl/${FILE}
        done
    done
}

# k8s组件证书
# 192.168.0.是k8s service的网段，如果说需要更改k8s service网段，那就需要更改192.168.0.1，
create_k8s_ssl() {
    echo "#### 创建所有k8s组件服务 CA证书操作 #####"
    get_env_info
    get_node_ip
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/pki/
    cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca

    cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -hostname=$k8s_service_ip,$k8s_lb_ip,127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,$master01_ip,$master02_ip,$master03_ip -profile=kubernetes apiserver-csr.json | cfssljson -bare /etc/kubernetes/pki/apiserver

    # 生成apiserver的聚合证书
    cfssl gencert -initca front-proxy-ca-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-ca

    cfssl gencert -ca=/etc/kubernetes/pki/front-proxy-ca.pem -ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem -config=ca-config.json -profile=kubernetes front-proxy-client-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-client

    # 生成controller-manage的证书
    cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -profile=kubernetes manager-csr.json | cfssljson -bare /etc/kubernetes/pki/controller-manager

    # set-cluster：设置一个集群项，
    kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://$k8s_lb_ip:8443 --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

    # 设置一个环境项，一个上下文
    kubectl config set-context system:kube-controller-manager@kubernetes --cluster=kubernetes --user=system:kube-controller-manager --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

    # set-credentials 设置一个用户项
    kubectl config set-credentials system:kube-controller-manager --client-certificate=/etc/kubernetes/pki/controller-manager.pem --client-key=/etc/kubernetes/pki/controller-manager-key.pem --embed-certs=true --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

    # 使用某个环境当做默认环境
    kubectl config use-context system:kube-controller-manager@kubernetes --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

    cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -profile=kubernetes scheduler-csr.json | cfssljson -bare /etc/kubernetes/pki/scheduler

    kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://$k8s_lb_ip:8443 --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

    kubectl config set-credentials system:kube-scheduler --client-certificate=/etc/kubernetes/pki/scheduler.pem --client-key=/etc/kubernetes/pki/scheduler-key.pem --embed-certs=true --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

    kubectl config set-context system:kube-scheduler@kubernetes --cluster=kubernetes --user=system:kube-scheduler --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

    kubectl config use-context system:kube-scheduler@kubernetes --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

    cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare /etc/kubernetes/pki/admin

    # 注意，如果不是高可用集群，10.206.16.13.236:8443改为master01的地址，8443改为apiserver的端口，默认是6443
    kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://$k8s_lb_ip:8443 --kubeconfig=/etc/kubernetes/admin.kubeconfig

    kubectl config set-credentials kubernetes-admin --client-certificate=/etc/kubernetes/pki/admin.pem --client-key=/etc/kubernetes/pki/admin-key.pem --embed-certs=true --kubeconfig=/etc/kubernetes/admin.kubeconfig

    kubectl config set-context kubernetes-admin@kubernetes --cluster=kubernetes --user=kubernetes-admin --kubeconfig=/etc/kubernetes/admin.kubeconfig

    kubectl config use-context kubernetes-admin@kubernetes --kubeconfig=/etc/kubernetes/admin.kubeconfig

    # 创建ServiceAccount Key  secret
    echo "================================================"
    echo "# 创建ServiceAccount Key secret"
    echo "================================================"
    sleep 2
    openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
    sleep 3
    openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub   
    echo "================================================"
}

rsync_k8s_ssl() {
    get_node_list_name
    for NODE in $MasterNodes; do
      echo "#### 同步k8s组件证书到"$NODE"#######################"
      for FILE in $(ls /etc/kubernetes/pki | grep -v etcd);do
          sshpass ssh -p $SSH_PORT root@$NODE "mkdir -p /etc/kubernetes/pki";
          sshpass scp -P $SSH_PORT -r /etc/kubernetes/pki/${FILE} root@$NODE:/etc/kubernetes/pki/${FILE};
          sshpass ssh -p $SSH_PORT root@$NODE "ls /etc/kubernetes/pki/";
      done;
    done

    for NODE in $MasterNodes; do
      echo "#### 同步k8s组件上下文配置到"$NODE"#################"
      for FILE in admin.kubeconfig controller-manager.kubeconfig scheduler.kubeconfig;do
          sshpass scp -P $SSH_PORT -r /etc/kubernetes/${FILE} root@$NODE:/etc/kubernetes/${FILE};
          sshpass ssh -p $SSH_PORT root@$NODE "ls /etc/kubernetes/";
      done;
    done
}

install_keepalived_haproxy() {
    get_env_info
    get_node_ip
    get_node_list_name
    echo "#### 创建并启动 keepalived haproxy服务 #######################"
    # vim /etc/haproxy/haproxy.cfg
    cat << EOF > "${install_path}"/scripts/k8s_conf_file/haproxy.cfg
global
  maxconn  2000
  ulimit-n  16384
  log  127.0.0.1 local0 err
  stats timeout 30s

defaults
  log global
  mode  http
  option  httplog
  timeout connect 5000
  timeout client  50000
  timeout server  50000
  timeout http-request 15s
  timeout http-keep-alive 15s

frontend k8s-master
  bind 0.0.0.0:8443
  bind 127.0.0.1:8443
  mode tcp
  option tcplog
  tcp-request inspect-delay 5s
  default_backend k8s-master

backend k8s-master
  mode tcp
  option tcplog
  option tcp-check
  balance roundrobin
  default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
  server $K8S_MASTER01    $master01_ip:6443  check
  server $K8S_MASTER02    $master02_ip:6443  check
  server $K8S_MASTER03    $master03_ip:6443  check
EOF

    # vim /etc/keepalived/keepalived.conf ，注意每个节点的IP和网卡（interface参数）
    cat << EOF > "${install_path}"/scripts/k8s_conf_file/keepalived01.conf
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
#注意主备参数选择
    state MASTER               #设置初始状态为"备"。
    interface $master_eth01    #设置绑定 VIP 的网卡，例如 eth0
    virtual_router_id 109       #配置集群 virtual_router_id 值
    nopreempt                  #设置非抢占模式
    preempt_delay 10           #抢占延时10分钟
    priority 101               #设置优先级，值越大优先级越高
    advert_int 2               #检查间隔，默认1秒
    authentication {           #设置认证
        auth_type PASS             #认证方式
        auth_pass K8SHA_KA_AUTH    #认证密码
    }
    unicast_src_ip $master01_ip #设置本机内网IP地址
    virtual_ipaddress {
        $k8s_lb_ip            #设置“高可用虚拟IP”为可漂移的IP
    }
    track_script {
      chk_apiserver
    }
}
EOF

cat << EOF > "${install_path}"/scripts/k8s_conf_file/keepalived02.conf
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
    state BACKUP
    interface $master_eth02
    virtual_router_id 109
    priority 91
    nopreempt
    advert_int 2
    preempt_delay 10
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    mcast_src_ip $master02_ip
    virtual_ipaddress {
        $k8s_lb_ip
    }
    track_script {
        chk_apiserver  
    }
}
EOF

cat << EOF > "${install_path}"/scripts/k8s_conf_file/keepalived03.conf
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
    state BACKUP
    interface $master_eth03
    virtual_router_id 109
    priority 81
    nopreempt
    advert_int 2
    preempt_delay 10
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    mcast_src_ip $master03_ip
    virtual_ipaddress {
        $k8s_lb_ip
    }
    track_script {
        chk_apiserver  
    }
}
EOF
    #sshpass ssh -p $SSH_PORT root@$K8S_MASTER01 'mv -f /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak';
    #sshpass ssh -p $SSH_PORT root@$K8S_MASTER02 'mv -f /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak';
    #sshpass ssh -p $SSH_PORT root@$K8S_MASTER03 'mv -f /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak';
    sshpass scp -P $SSH_PORT -r "${install_path}"/Install_Packages/*.rpm root@$K8S_MASTER01:/tmp/;
    sshpass scp -P $SSH_PORT -r "${install_path}"/Install_Packages/*.rpm root@$K8S_MASTER02:/tmp/;
    sshpass scp -P $SSH_PORT -r "${install_path}"/Install_Packages/*.rpm root@$K8S_MASTER03:/tmp/;   
    sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/keepalived01.conf root@$K8S_MASTER01:/tmp/keepalived.conf;
    sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/keepalived02.conf root@$K8S_MASTER02:/tmp/keepalived.conf;
    sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/keepalived03.conf root@$K8S_MASTER03:/tmp/keepalived.conf;
    sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/check_apiserver.sh root@$K8S_MASTER01:/tmp/;
    sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/check_apiserver.sh root@$K8S_MASTER02:/tmp/;
    sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/check_apiserver.sh root@$K8S_MASTER03:/tmp/;
    sshpass scp -P $SSH_PORT -r "${install_path}"/Install_Packages/nfscentos7 root@$K8S_MASTER01:/tmp/;
    sshpass scp -P $SSH_PORT -r "${install_path}"/Install_Packages/nfscentos7 root@$K8S_MASTER02:/tmp/;
    sshpass scp -P $SSH_PORT -r "${install_path}"/Install_Packages/nfscentos7 root@$K8S_MASTER03:/tmp/;
    for NODE in $MasterNodes; do
       echo "#### 在所有master节点上安装keepalived和haproxy rpm离线"$NODE"#################";
       #sshpass ssh -p $SSH_PORT root@$NODE 'cd /tmp/ && rpm -ivh mpfr-3.1.1-4.el7.x86_64.rpm && rpm -ivh libmpc-1.0.1-3.el7.x86_64.rpm && rpm -ivh kernel-headers-3.10.0-1160.el7.x86_64.rpm && rpm -ivh glibc-headers-2.17-317.el7.x86_64.rpm && rpm -ivh glibc-devel-2.17-317.el7.x86_64.rpm && rpm -ivh cpp-4.8.5-44.el7.x86_64.rpm && rpm -ivh gcc-4.8.5-44.el7.x86_64.rpm && rpm -ivh libstdc++-devel-4.8.5-44.el7.x86_64.rpm && rpm -ivh gcc-c++-4.8.5-44.el7.x86_64.rpm && rpm -ivh libnfnetlink-1.0.1-4.el7.x86_64.rpm && rpm -ivh libnfnetlink-devel-1.0.1-4.el7.x86_64.rpm && rpm -Uvh libnl-1.1.4-3.el7.x86_64.rpm && rpm -Uvh libnl-devel-1.1.4-3.el7.x86_64.rpm && rpm -ivh libnl3-3.2.28-4.el7.x86_64.rpm && rpm -ivh libnl3-devel-3.2.28-4.el7.x86_64.rpm && rpm -ivh --replacefiles net-snmp-libs-5.7.2-49.el7_9.2.x86_64.rpm && rpm -ivh --replacefiles net-snmp-agent-libs-5.7.2-49.el7_9.2.x86_64.rpm && rpm -e keepalived && rpm -ivh --replacefiles keepalived-1.3.5-19.el7.x86_64.rpm && rpm -ivh --replacefiles haproxy-1.5.18-9.el7_9.1.x86_64.rpm';
       #sshpass ssh -p $SSH_PORT root@$NODE 'cd /tmp/ && rpm -ivh --replacefiles net-snmp-libs-5.7.2-49.el7_9.2.x86_64.rpm && rpm -ivh --replacefiles net-snmp-agent-libs-5.7.2-49.el7_9.2.x86_64.rpm && rpm -ivh --replacefiles keepalived-1.3.5-19.el7.x86_64.rpm && rpm -ivh --replacefiles haproxy-1.5.18-9.el7_9.1.x86_64.rpm';
       echo "#### 安装 mpfr-3.1.1-4.el7.x86_64.rpm" 
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh mpfr-3.1.1-4.el7.x86_64.rpm &";
       echo "#### 安装 libmpc-1.0.1-3.el7.x86_64.rpm"
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh libmpc-1.0.1-3.el7.x86_64.rpm &";
       echo "#### 安装 kernel-headers-3.10.0-1160.el7.x86_64.rpm"
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh kernel-headers-3.10.0-1160.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh glibc-headers-2.17-317.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh glibc-devel-2.17-317.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh cpp-4.8.5-44.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh gcc-4.8.5-44.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh libstdc++-devel-4.8.5-44.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh gcc-c++-4.8.5-44.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh libnfnetlink-1.0.1-4.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh libnfnetlink-devel-1.0.1-4.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -Uvh libnl-1.1.4-3.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -Uvh libnl-devel-1.1.4-3.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh perl-5.16.3-299.el7_9.src.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh perl-devel-5.16.3-299.el7_9.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh perl-libs-5.16.3-299.el7_9.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh lm_sensors-libs-3.4.0-8.20160601gitf9185e5.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh --replacefiles net-snmp-libs-5.7.2-49.el7_9.2.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh --replacefiles net-snmp-agent-libs-5.7.2-49.el7_9.2.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh --replacefiles ipset-libs-7.1-1.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh --replacefiles ipset-7.1-1.el7.x86_64.rpm &";
       #sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh --replacefiles keepalived-1.3.5-19.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh --replacefiles keepalived-2.2.2-1.el7.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/ && rpm -ivh --replacefiles haproxy-1.5.18-9.el7_9.1.x86_64.rpm &";
       sshpass ssh -p $SSH_PORT root@$NODE "cd /tmp/nfscentos7 && rpm -ivh * --force --nodeps &";
       sshpass ssh -p $SSH_PORT root@$NODE 'systemctl restart rpcbind.service &';
       sleep 8
       sshpass ssh -p $SSH_PORT root@$NODE '\cp -f /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg_back';
       sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/haproxy.cfg root@$NODE:/etc/haproxy/haproxy.cfg;
       sshpass ssh -p $SSH_PORT root@$NODE '\cp -f /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak';
       sshpass ssh -p $SSH_PORT root@$NODE '\cp /tmp/{keepalived.conf,check_apiserver.sh} /etc/keepalived/ && chmod +x /etc/keepalived/*';
       # 所有节点均执行：
       sshpass ssh -p $SSH_PORT root@$NODE 'chmod -R 644 /etc/keepalived/keepalived.conf && systemctl daemon-reload && systemctl enable --now haproxy && systemctl enable --now keepalived && systemctl restart keepalived && systemctl restart haproxy && systemctl status keepalived && systemctl status haproxy'
    done;
    echo "#### 启动 keepalived haproxy服务 successful #######################"
}

create_etcd_service() {
    get_env_info
    get_node_ip
    get_node_list_name
    for NODE in $MasterNodes; do
        echo "#### 在所有master节点上安装并启动etcd服务 "$NODE"############################"
        sshpass ssh -p $SSH_PORT root@$NODE "mkdir -p /etc/kubernetes/pki/etcd && ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/ &";
        sleep 5;
        sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/etcd.config.yml root@$NODE:/etc/etcd/etcd.config.yml;
        sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/etcd.service root@$NODE:/usr/lib/systemd/system/etcd.service;
        sleep 5;
        master_ip=`cat /etc/hosts | grep "$NODE" | uniq | awk -F " " '{print $1}'`;
        echo "systemctl daemon-reload && systemctl enable --now etcd && systemctl restart etcd && systemctl status etcd"
        sshpass ssh -p $SSH_PORT root@$NODE "sed -i 's/masterip/$master_ip/g' /etc/etcd/etcd.config.yml && sed -i 's/mastername/$NODE/g' /etc/etcd/etcd.config.yml && sed -i 's/master01name/$K8S_MASTER01/g' /etc/etcd/etcd.config.yml && sed -i 's/master02name/$K8S_MASTER02/g' /etc/etcd/etcd.config.yml && sed -i 's/master03name/$K8S_MASTER03/g' /etc/etcd/etcd.config.yml && sed -i 's/master01ip/$master01_ip/g' /etc/etcd/etcd.config.yml && sed -i 's/master02ip/$master02_ip/g' /etc/etcd/etcd.config.yml && sed -i 's/master03ip/$master03_ip/g' /etc/etcd/etcd.config.yml";
        sleep 5
        #sshpass ssh -p $SSH_PORT root@$NODE "systemctl daemon-reload && systemctl enable --now etcd";
        #sleep 3;
        sshpass ssh -p $SSH_PORT root@$NODE "systemctl daemon-reload && systemctl restart etcd &";
        sleep 5;
        sshpass ssh -p $SSH_PORT root@$NODE "systemctl enable etcd &";
        sleep 5;
    done
    sleep 35
    systemctl status etcd
    get_env_info
    get_node_ip
    sleep 15
    export ETCDCTL_API=3
    echo "#### test etcd status ########################################################################"
    etcdctl --endpoints="$K8S_MASTER_IP01:2379,$K8S_MASTER_IP02:2379,$K8S_MASTER_IP03:2379" --cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem --cert=/etc/kubernetes/pki/etcd/etcd.pem --key=/etc/kubernetes/pki/etcd/etcd-key.pem endpoint status --write-out=table
}

create_apiserver_service() {
    get_env_info
    get_node_ip
    get_node_list_name

    cat << EOF > "${install_path}"/scripts/k8s_conf_file/kube-apiserver01.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
      --v=2 \\
      --logtostderr=true \\
      --allow-privileged=true \\
      --bind-address=0.0.0.0 \\
      --secure-port=6443 \\
      --insecure-port=0 \\
      --advertise-address=$master01_ip \\
      --service-cluster-ip-range=$k8s_service_net/24 \\
      --service-node-port-range=30000-39999 \\
      --etcd-servers=https://$master01_ip:2379,https://$master02_ip:2379,https://$master03_ip:2379 \\
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \\
      --etcd-certfile=/etc/etcd/ssl/etcd.pem \\
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \\
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \\
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\
      --service-account-key-file=/etc/kubernetes/pki/sa.pub \\
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \\
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \\
      --authorization-mode=Node,RBAC \\
      --enable-bootstrap-token-auth=true \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \\
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \\
      --requestheader-allowed-names=aggregator \\
      --requestheader-group-headers=X-Remote-Group \\
      --requestheader-extra-headers-prefix=X-Remote-Extra- \\
      --requestheader-username-headers=X-Remote-User \\
      --default-not-ready-toleration-seconds=30 \\
      --default-unreachable-toleration-seconds=30
      # --token-auth-file=/etc/kubernetes/token.csv
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF
    cat << EOF > "${install_path}"/scripts/k8s_conf_file/kube-apiserver02.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
      --v=2 \\
      --logtostderr=true \\
      --allow-privileged=true \\
      --bind-address=0.0.0.0 \\
      --secure-port=6443 \\
      --insecure-port=0 \\
      --advertise-address=$master02_ip \\
      --service-cluster-ip-range=$k8s_service_net/24 \\
      --service-node-port-range=30000-39999 \\
      --etcd-servers=https://$master01_ip:2379,https://$master02_ip:2379,https://$master03_ip:2379 \\
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \\
      --etcd-certfile=/etc/etcd/ssl/etcd.pem \\
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \\
      --client-ca-file=/etc/kubernetes/pki/ca.pem \\
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\
      --service-account-key-file=/etc/kubernetes/pki/sa.pub \\
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \\
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \\
      --authorization-mode=Node,RBAC \\
      --enable-bootstrap-token-auth=true \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \\
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \\
      --requestheader-allowed-names=aggregator \\
      --requestheader-group-headers=X-Remote-Group \\
      --requestheader-extra-headers-prefix=X-Remote-Extra- \\
      --requestheader-username-headers=X-Remote-User \\
      --default-not-ready-toleration-seconds=30 \\
      --default-unreachable-toleration-seconds=30
      # --token-auth-file=/etc/kubernetes/token.csv
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF

    cat << EOF > "${install_path}"/scripts/k8s_conf_file/kube-apiserver03.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
      --v=2 \\
      --logtostderr=true \\
      --allow-privileged=true \\
      --bind-address=0.0.0.0 \\
      --secure-port=6443 \\
      --insecure-port=0 \\
      --advertise-address=$master03_ip \\
      --service-cluster-ip-range=$k8s_service_net/24 \\
      --service-node-port-range=30000-39999 \\
      --etcd-servers=https://$master01_ip:2379,https://$master02_ip:2379,https://$master03_ip:2379 \\
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \\
      --etcd-certfile=/etc/etcd/ssl/etcd.pem \\
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \\
      --client-ca-file=/etc/kubernetes/pki/ca.pem \\
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\
      --service-account-key-file=/etc/kubernetes/pki/sa.pub \\
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \\
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \\
      --authorization-mode=Node,RBAC \\
      --enable-bootstrap-token-auth=true \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \\
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \\
      --requestheader-allowed-names=aggregator \\
      --requestheader-group-headers=X-Remote-Group \\
      --requestheader-extra-headers-prefix=X-Remote-Extra- \\
      --requestheader-username-headers=X-Remote-User \\
      --default-not-ready-toleration-seconds=30 \\
      --default-unreachable-toleration-seconds=30
      # --token-auth-file=/etc/kubernetes/token.csv
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF

    sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/kube-apiserver01.service root@$K8S_MASTER01:/usr/lib/systemd/system/kube-apiserver.service;
    sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/kube-apiserver02.service root@$K8S_MASTER02:/usr/lib/systemd/system/kube-apiserver.service;
    sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/kube-apiserver03.service root@$K8S_MASTER03:/usr/lib/systemd/system/kube-apiserver.service;
    for NODE in $MasterNodes; do
        echo "#### 在所有master节点上安装并启动apiserver服务 "$NODE"############################"
        sshpass ssh -p $SSH_PORT root@$NODE 'mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes';
        sleep 1
        sshpass ssh -p $SSH_PORT root@$NODE 'systemctl daemon-reload && systemctl enable --now kube-apiserver && systemctl restart kube-apiserver && systemctl status kube-apiserver'
    done
}

create_controller_manager_service() {
    get_env_info
    get_node_ip
    get_node_list_name
    new_cluster_cidr=`echo $cluster_cidr | awk -F "." '{print $1"."$2}'`
    cat << EOF > "${install_path}"/scripts/k8s_conf_file/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-controller-manager \\
      --v=2 \\
      --logtostderr=true \\
      --address=127.0.0.1 \\
      --root-ca-file=/etc/kubernetes/pki/ca.pem \\
      --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \\
      --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \\
      --service-account-private-key-file=/etc/kubernetes/pki/sa.key \\
      --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \\
      --leader-elect=true \\
      --use-service-account-credentials=true \\
      --node-monitor-grace-period=40s \\
      --node-monitor-period=5s \\
      --pod-eviction-timeout=2m0s \\
      --controllers=*,bootstrapsigner,tokencleaner \\
      --allocate-node-cidrs=true \\
      --cluster-cidr=$new_cluster_cidr.0.0/12 \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\
      --node-cidr-mask-size=24
Restart=always
RestartSec=10s
[Install]
WantedBy=multi-user.target
EOF

    for NODE in $MasterNodes; do
        echo "#### 在所有master节点上安装并启动controller manager服务 "$NODE"############################"
        sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/kube-controller-manager.service root@$NODE:/usr/lib/systemd/system/kube-controller-manager.service;
        sleep 1
        sshpass ssh -p $SSH_PORT root@$NODE 'systemctl daemon-reload && systemctl enable --now kube-controller-manager && systemctl status kube-controller-manager && systemctl restart kube-controller-manager'
    done
}

# 所有Master节点配置kube-scheduler service（所有master节点配置一样）
create_scheduler_service() {
    get_env_info
    get_node_ip
    get_node_list_name
    for NODE in $MasterNodes; do
        echo "#### 在所有master节点上安装并启动scheduler service服务 "$NODE"############################"
        sshpass scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/kube-scheduler.service root@$NODE:/usr/lib/systemd/system/kube-scheduler.service;
        sleep 1
        sshpass ssh -p $SSH_PORT root@$NODE 'systemctl daemon-reload && systemctl enable --now kube-scheduler && systemctl restart kube-scheduler && systemctl status kube-scheduler'
    done
}

create_bootstrap() {
    get_env_info
    get_node_ip
    get_node_list_name
    echo "#### 在master01节点上创建bootstrap kubelet kubeconfig ############################"
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/bootstrap/
    #/ecm6.2-k8s/soft/k8s1.23/k8s-ha-install/bootstrap
    kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://$k8s_lb_ip:8443 --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

    kubectl config set-credentials tls-bootstrap-token-user --token=c8ad9c.2e4d610cf3e7426e --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

    kubectl config set-context tls-bootstrap-token-user@kubernetes --cluster=kubernetes --user=tls-bootstrap-token-user --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

    kubectl config use-context tls-bootstrap-token-user@kubernetes --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
    #kubectl get cs
    #echo "#### kubectl create -f bootstrap.secret.yaml #####################"
    #echo "${install_path}/soft/k8s1.23/k8s-ha-install/bootstrap/"
    #cd "${install_path}"/soft/k8s1.23/k8s-ha-install/bootstrap/
    #kubectl apply -f bootstrap.secret.yaml
    #echo "##################################################################"
    for NODE in $MasterNodes; do
        echo "#### 在master01节点上创建/root/.kube/config "$NODE"############################"
        sshpass ssh -p $SSH_PORT root@$NODE "mkdir -p /root/.kube && \cp /etc/kubernetes/admin.kubeconfig /root/.kube/config && systemctl restart kube-scheduler && systemctl status kube-scheduler";
        sleep 5;
        kubectl get cs
    done
    sleep 5
}

# Master01节点复制证书至Node节点(yes检测脚本必须excpet做完)
rsync_k8s_node() {
    cd /etc/kubernetes/
    get_env_info
    get_node_list_name
    for NODE in $WorkNodes; do
        echo "#### 在node节点上创建pki并推送bootstrap-kubelet.kubeconfig "$NODE"#############"
        sshpass ssh -p $SSH_PORT root@$NODE 'mkdir -p /etc/kubernetes/pki /etc/etcd/ssl'
        for FILE in etcd-ca.pem etcd.pem etcd-key.pem; do
            scp -P $SSH_PORT -r /etc/etcd/ssl/$FILE root@$NODE:/etc/etcd/ssl/
        done;
        for FILE in pki/ca.pem pki/ca-key.pem pki/front-proxy-ca.pem bootstrap-kubelet.kubeconfig; do
            scp -P $SSH_PORT -r /etc/kubernetes/$FILE root@$NODE:/etc/kubernetes/${FILE}
        done;
        sshpass ssh -p $SSH_PORT root@$NODE "ls /etc/kubernetes/pki";
    done
}

create_kubelet_service() {
    get_env_info
    get_node_ip
    get_node_list_name 
    cat << EOF > "${install_path}"/scripts/k8s_conf_file/kubelet-conf.yml
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- $k8s_cluster_dns
clusterDomain: cluster.local
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
  imagefs.available: 15%
  memory.available: 100Mi
  nodefs.available: 10%
  nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
EOF
    tar xf "${install_path}"/soft/k8s1.23/cni-plugins-linux-amd64-v1.1.1.tgz -C /opt/cni/bin
    #cd /usr/local/harbor && docker-compose up -d
    sleep 5
    echo "#### kubectl create -f bootstrap.secret.yaml #####################"
    echo "${install_path}/soft/k8s1.23/k8s-ha-install/bootstrap/"
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/bootstrap/
    kubectl create -f bootstrap.secret.yaml
    echo "##################################################################"
    sleep 10
    for NODE in $WorkNodes; do
        echo "#### 在node节点上推送kubelet-conf.yml 10-kubelet.conf kubelet.service 并启动kubelet服务 "$NODE"#############"
        sshpass ssh -p $SSH_PORT root@$NODE "mkdir -p /etc/cni/net.d /opt/cni/bin && mkdir -p /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/ && docker login -u admin -p edoc2edoc2 registry.edoc2.com:5000 && docker pull registry.edoc2.com:5000/library/pause:3.6";
        sleep 10;
        scp -P $SSH_PORT -r /opt/cni/bin/* root@$NODE:/opt/cni/bin/;
        scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/kubelet.service root@$NODE:/usr/lib/systemd/system/kubelet.service;
        scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/10-kubelet.conf root@$NODE:/etc/systemd/system/kubelet.service.d/10-kubelet.conf;
        scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/kubelet-conf.yml root@$NODE:/etc/kubernetes/kubelet-conf.yml;
        sleep 5;
        sshpass ssh -p $SSH_PORT root@$NODE "rm -rf /var/lib/kubelet/pki/kubelet* && systemctl daemon-reload && systemctl enable --now kubelet && systemctl daemon-reload && systemctl restart kubelet && systemctl status kubelet";
        sleep 2
    done
    for NODE in $MasterNodes; do
        sshpass ssh -p $SSH_PORT root@$NODE "systemctl daemon-reload && sleep 1 && systemctl restart kube-apiserver && systemctl restart kube-scheduler && systemctl status kube-scheduler";
        sleep 3
    done
    sleep 15
    echo "##### kubectl get node info #####"
    kubectl get node
    echo "##### kubectl get node info #####"
    sleep 15
}

get_harbor_k8s_images() {
    get_env_info
    get_node_ip
    get_node_list_name
    for NODE in $WorkNodes; do
        echo "#### 在node节点上获取所有k8s安装需要使用到的image镜像 "$NODE"###############################################"
        sshpass ssh -p $SSH_PORT root@$NODE "docker login -u admin -p edoc2edoc2 registry.edoc2.com:5000 && docker pull registry.edoc2.com:5000/library/busybox:1.28 && docker pull registry.edoc2.com:5000/library/metrics-server:0.5.0 && docker pull registry.edoc2.com:5000/library/metrics-scraper:v1.0.7 && docker pull registry.edoc2.com:5000/library/coredns:1.8.6 && docker pull registry.edoc2.com:5000/library/dashboard:v2.4.0 && docker pull registry.edoc2.com:5000/library/node:v3.22.0 && docker pull registry.edoc2.com:5000/library/pod2daemon-flexvol:v3.22.0 && docker pull registry.edoc2.com:5000/library/cni:v3.22.0 && docker pull registry.edoc2.com:5000/library/kube-controllers:v3.22.0 && docker pull registry.edoc2.com:5000/library/typha:v3.22.0";
        sleep 3
    done
}

create_kube_proxy() {
    get_env_info
    get_node_ip
    get_node_list_name
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/
    kubectl -n kube-system create serviceaccount kube-proxy

    kubectl create clusterrolebinding system:kube-proxy --clusterrole system:node-proxier --serviceaccount kube-system:kube-proxy

    SECRET=$(kubectl -n kube-system get sa/kube-proxy --output=jsonpath='{.secrets[0].name}')

    JWT_TOKEN=$(kubectl -n kube-system get secret/$SECRET --output=jsonpath='{.data.token}' | base64 -d)

    PKI_DIR=/etc/kubernetes/pki
    K8S_DIR=/etc/kubernetes

    kubectl config set-cluster kubernetes \
    --certificate-authority=/etc/kubernetes/pki/ca.pem \
    --embed-certs=true --server=https://$k8s_lb_ip:8443 \
    --kubeconfig=${K8S_DIR}/kube-proxy.kubeconfig

    kubectl config set-credentials kubernetes \
    --token=${JWT_TOKEN} \
    --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig

    kubectl config set-context kubernetes \
    --cluster=kubernetes \
    --user=kubernetes \
    --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig

    kubectl config use-context kubernetes \
    --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig

    cat << EOF > "${install_path}"/scripts/k8s_conf_file/kube-proxy.yaml
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
  acceptContentTypes: ""
  burst: 10
  contentType: application/vnd.kubernetes.protobuf
  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
  qps: 5
clusterCIDR: $cluster_cidr
configSyncPeriod: 15m0s
conntrack:
  max: null
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: 14
  minSyncPeriod: 0s
  syncPeriod: 30s
ipvs:
  masqueradeAll: true
  minSyncPeriod: 5s
  scheduler: "rr"
  syncPeriod: 30s
kind: KubeProxyConfiguration
metricsBindAddress: 127.0.0.1:10249
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
udpIdleTimeout: 250ms
EOF
    for NODE in $WorkNodes; do
        echo "#### 在node节点上推送kube-proxy.kubeconfig kube-proxy.yaml 并启动kube proxy服务 "$NODE"#############"
        sshpass ssh -p $SSH_PORT root@$NODE "mkdir -p /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/ && docker login -u admin -p edoc2edoc2 registry.edoc2.com:5000 && docker pull registry.edoc2.com:5000/library/pause:3.6";
        scp -P $SSH_PORT -r /etc/kubernetes/kube-proxy.kubeconfig root@$NODE:/etc/kubernetes/kube-proxy.kubeconfig;
        scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/kube-proxy.yaml root@$NODE:/etc/kubernetes/kube-proxy.yaml;
        scp -P $SSH_PORT -r "${install_path}"/scripts/k8s_conf_file/kube-proxy.service root@$NODE:/usr/lib/systemd/system/kube-proxy.service;
        sleep 1
        sshpass ssh -p $SSH_PORT root@$NODE "systemctl daemon-reload && systemctl enable --now kube-proxy && systemctl restart kube-proxy && systemctl status kube-proxy";
        sleep 2
        sshpass ssh -p $SSH_PORT root@$NODE "mkdir -p /home/edoc2/macrowing/edoc2v5/data && mkdir -p /datavol/weboffice";
        sshpass ssh -p $SSH_PORT root@$NODE "mount.nfs -o noac k8s-node01:/data/nas/data /home/edoc2/macrowing/edoc2v5/data &";
        sshpass ssh -p $SSH_PORT root@$NODE "mount.nfs -o noac k8s-node01:/data/nas/weboffice /datavol/weboffice &";
    done
    sleep 10
    mkdir -p /home/edoc2/macrowing/edoc2v5/data
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/cms/UploadFile
    mkdir -p /datavol/weboffice/Data
    mkdir -p /datavol/weboffice/onlyoffice
    mkdir -p /datavol/weboffice/postgresql
    mkdir -p /datavol/weboffice/log
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/logs/config
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/logs/dms.services
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/edoc2Docs
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/edoc2/localcache
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/edoc2/watercache
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/logs/dms
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/data/edoc2Docs
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/logs/core
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/data/snapshots
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/logs/pcs
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/logs/rms.services
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/logs/rms
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/data/snapshots
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/logs/tms.services
    mkdir -p /home/edoc2/macrowing/edoc2v5/data/isov6/logs/tms
}

deploy_calico() {
    get_env_info
    get_node_ip
    get_node_list_name
    mkdir /etc/calico
    cat<<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: busybox
  namespace: default
spec:
  containers:
  - name: busybox
    image: busybox:1.28
    command:
      - sleep
      - "3600"
    imagePullPolicy: IfNotPresent
  restartPolicy: Always
EOF
    sleep 2
    #calicoctl get node --allow-version-mismatch
    #calicoctl get node -o yaml --allow-version-mismatch
    #calicoctl patch node $K8S_NODE01 --patch="{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_MASTER_IP01/24\"}}}" --allow-version-mismatch
    #calicoctl patch node $K8S_NODE01 --patch="{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_NODE_IP01/24\"}}}" --allow-version-mismatch
    #echo "#### kubectl create -f bootstrap.secret.yaml #####################"
    #echo "${install_path}/soft/k8s1.23/k8s-ha-install/bootstrap/"
    #cd "${install_path}"/soft/k8s1.23/k8s-ha-install/bootstrap/
    #kubectl create -f bootstrap.secret.yaml
    #echo "##################################################################"
    sleep 6
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install
    COREDNS_SERVICE_IP=`kubectl get svc | grep kubernetes | awk '{print $3}'`0
    sed -i "s#KUBEDNS_SERVICE_IP#${COREDNS_SERVICE_IP}#g" CoreDNS/coredns.yaml
    sleep 2
    kubectl create -f CoreDNS/coredns.yaml
    sleep 6
    kubectl get pod -A
    echo "======================================================="
    kubectl get po -n kube-system -l k8s-app=kube-dns
    echo "======================================================="
    \cp "${install_path}"/soft/calicoctl-linux-amd64 /usr/local/bin/calicoctl
    chmod +x /usr/local/bin/calicoctl
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/calico
    sed -i "s#POD_CIDR#$cluster_cidr#g" calico.yaml
    sleep 2
    nip=`echo $cluster_cidr | awk -F "." '{print $1"."$2}'`
    sed -i "s#172.17#$nip#g" calico.yaml
    sleep 2
    kubectl apply -f calico.yaml
    sleep 5
    echo "======================================================="
    kubectl get pod -n kube-system
    echo "======================================================="
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/metrics-server
    kubectl create -f .
    sleep 5
    #kubectl top node
    cd "${install_path}"/Template/cluster
    sleep 5
    echo "##### apply dashboard #####"
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/dashboard/
    kubectl apply -f .
    admin_id=`kubectl get secret -A | grep admin | awk -v FS=' ' '{print $2}'`
    add_token='token: '`kubectl describe secret -n kube-system $admin_id | grep "token:" | awk -v FS=' ' '{print $2}'`
    echo $add_token
    \cp /root/.kube/config $install_path/deploy-config/kubeconfig
    echo "$add_token" && echo "    ""$add_token" >> "${install_path}"/deploy-config/kubeconfig
    calicoctl get node --allow-version-mismatch
    calicoctl get node -o yaml --allow-version-mismatch
    #calicoctl patch node $K8S_NODE01 --patch="{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_MASTER_IP01/24\"}}}" --allow-version-mismatch
    calicoctl patch node $(hostname) --patch="{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_MASTER_IP01/24\"}}}" --allow-version-mismatch
    #calicoctl patch node $K8S_NODE02 --patch="{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_NODE_IP02/24\"}}}" --allow-version-mismatch
    #calicoctl patch node $K8S_NODE03 --patch="{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_NODE_IP03/24\"}}}" --allow-version-mismatch
    sleep 15
    kubectl get pod -A
    echo "==============================================================="
    echo "########### k8s cluster install succesful!!! ############"
    echo "请重新登录后进行部署操作： source auto_deploy_edoc2.sh start!!!"
    echo "calico执行最后注入优化信息等待5秒后确认！"
    echo "==============================================================="
    sleep 5
    if cat /etc/rc.d/rc.local | grep -q calicoctl; then
        sleep 1
        echo "###### calicoctl bgp 识别已生效！"
    else
        echo "###### 执行calicoctl bgp 识别！"
        #echo "cd /usr/local/harbor && docker-compose up -d" >> /root/.bashrc
        echo "sleep 2 && calicoctl patch node $(hostname) --patch='{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_MASTER_IP01/24\"}}}' --allow-version-mismatch" >> /etc/rc.d/rc.local
        chmod +x /etc/rc.d/rc.local
        echo "sleep 2 && cd /usr/local/harbor && docker-compose up -d" >> /etc/rc.d/rc.local
        sleep 2
        echo "calicoctl patch node $(hostname) --patch='{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_MASTER_IP01/24\"}}}' --allow-version-mismatch" >> /etc/profile
        sleep 1
        source /etc/profile
        #echo "exportfs -rv" >> /etc/profile
        echo "systemctl restart rpcbind.service" >> /etc/profile
        echo "systemctl restart nfs.service" >> /etc/profile
        #echo "systemctl restart nfs" >> /etc/profile
    fi
    cat /etc/crontab | grep calicoctl && calico_crontab_status=$?
    echo $calico_crontab_status
    if cat /etc/crontab | grep -q calicoctl; then
       sleep 1
    else
        echo "*/1 * * * * root /usr/local/bin/calicoctl patch node $(hostname) --patch='{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_MASTER_IP01/24\"}}}' --allow-version-mismatch" >> /etc/crontab
    fi
    sleep 5
    for id in `kubectl get pod -n kubernetes-dashboard | grep CrashLoopBackOff | awk -F " " '{print $1}'`;do kubectl delete pod $id -n kubernetes-dashboard;done
    sleep 5
    kubectl exec  busybox -n default -- nslookup kubernetes
    sleep 2
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/localdns/
    sed -i "s#192.168.0.10#$k8s_cluster_dns#g" nodelocaldns.yaml
    #kubectl apply -f nodelocaldns.yaml
}

apply_secret() {
    get_env_info
    get_node_ip
    get_node_list_name
    # ecm6.2-k8s/soft/k8s1.23/k8s-ha-install/bootstrap
    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/bootstrap/
    kubectl apply -f bootstrap.secret.yaml
    sleep 10
    for NODE in $WorkNodes; do
        sshpass ssh -p $SSH_PORT root@$NODE "rm -rf /var/lib/kubelet/pki/kubelet* && sleep 2 && systemctl daemon-reload && systemctl restart kubelet";
        sleep 2
    done
    sleep 5
    for NODE in $MasterNodes; do
        sshpass ssh -p $SSH_PORT root@$NODE "systemctl daemon-reload && systemctl restart kube-apiserver";
        sleep 2
    done
    source /tmp/scripts/remote_env.sh
    sleep 30
    calicoctl get node --allow-version-mismatch
    calicoctl get node -o yaml --allow-version-mismatch
    #calicoctl patch node $K8S_NODE01 --patch="{\"spec\":{\"bgp\": {\"ipv4Address\": \"$master01_ip/24\"}}}" --allow-version-mismatch
    calicoctl patch node $K8S_NODE01 --patch="{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_NODE_IP01/24\"}}}" --allow-version-mismatch
    calicoctl patch node $K8S_NODE02 --patch="{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_NODE_IP02/24\"}}}" --allow-version-mismatch
    calicoctl patch node $K8S_NODE03 --patch="{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_NODE_IP03/24\"}}}" --allow-version-mismatch
    sleep 5
    cat /etc/rc.d/rc.local | grep calicoctl
    calico_status=$?
    echo $calico_status
    if [ ${calico_status} == "1" ];then
        #echo "cd /usr/local/harbor && docker-compose up -d" >> /root/.bashrc
        echo "sleep 2 && calicoctl patch node $K8S_NODE01 --patch='{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_NODE_IP01/24\"}}}' --allow-version-mismatch" >> /etc/rc.d/rc.local
    fi
    cat /etc/crontab | grep calicoctl
    calico_crontab_status=$?
    echo $calico_crontab_status
    if [ ${calico_crontab_status} == "1" ];then
        echo "*/1 * * * * root calicoctl patch node $K8S_NODE01 --patch='{\"spec\":{\"bgp\": {\"ipv4Address\": \"$K8S_NODE_IP01/24\"}}}' --allow-version-mismatch" >> /etc/crontab
    fi
#    sleep 15
#    cd "${install_path}"/soft/k8s1.23/k8s-ha-install
#    COREDNS_SERVICE_IP=`kubectl get svc | grep kubernetes | awk '{print $3}'`0
#    sed -i "s#KUBEDNS_SERVICE_IP#${COREDNS_SERVICE_IP}#g" CoreDNS/coredns.yaml
#    kubectl delete -f CoreDNS/coredns.yaml
#    sleep 10
#    kubectl create -f CoreDNS/coredns.yaml
#    sleep 5
#    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/calico
#    sed -i "s#POD_CIDR#$cluster_cidr#g" calico.yaml
#    kubectl delete -f calico.yaml
#    sleep 5
#    kubectl apply -f calico.yaml
#    sleep 5
#    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/metrics-server
#    kubectl delete -f .
#    sleep 5
#    kubectl apply -f .
#    sleep 5
#    cd "${install_path}"/soft/k8s1.23/k8s-ha-install/dashboard/
#    kubectl create -f .
#    sleep 5
#    kubectl apply -f .
    cat<<EOF | kubectl delete -f -
apiVersion: v1
kind: Pod
metadata:
  name: busybox
  namespace: default
spec:
  containers:
  - name: busybox
    image: busybox:1.28
    command:
      - sleep
      - "3600"
    imagePullPolicy: IfNotPresent
  restartPolicy: Always
EOF
    sleep 10
    kubectl get pod -A
}

install_path=$1
k8s_base_tar
rsync_k8s_bin
create_etcd_ssl
create_k8s_ssl
rsync_k8s_ssl
install_keepalived_haproxy
create_etcd_service
create_apiserver_service
create_controller_manager_service
create_scheduler_service
create_bootstrap
rsync_k8s_node
create_kubelet_service
get_harbor_k8s_images
create_kube_proxy
deploy_calico
#sleep 10
#apply_secret
