#!/bin/bash

LOG_FILE='./ES_AUTO_BACKUP_RESTORE.log'

_color_red=$(TERM=xterm-256color tput setaf 1)
_color_normal=$(TERM=xterm-256color tput sgr0)
_color_blue=$(TERM=xterm-256color tput setaf 4)
_color_green=$(TERM=xterm-256color tput setaf 2)
_color_yellow=$(TERM=xterm-256color tput setaf 3)

function log_important()
{
    local ts=$(date -Iseconds)
    # to log file
    echo -e "$ts [IMPORTANT] $@" >> $LOG_FILE
    # to console (stderr)
    >&2 echo -e "${_color_yellow}$ts [IMPORTANT] $@${_color_normal}"
    return 0
}

function log_info()
{
    local ts=$(date -Iseconds)
    # to log file
    echo -e "$ts [INFO] $@" >> $LOG_FILE
    # to console (stderr)
    >&2 echo -e "$ts [INFO] $@"
    return 0
}

function log_error()
{
    local ts=$(date -Iseconds)
    # to log file
    echo -e "$ts [ERROR] $@" >> $LOG_FILE
    # to console (stderr)
    >&2 echo -e "${_color_red}$ts [ERROR] $@${_color_normal}"
    return 0
}

function log_debug()
{
    local ts=$(date -Iseconds)
    if [ $_DEBUG -eq 1 ]; then
        # to log file
        echo -e "$ts [DEBUG] $@" >> $LOG_FILE
        # to console (stderr)
        >&2 echo -e "$ts [DEBUG] $@"
    fi
    return 0
}

function assert_success(){
    if [ $? -ne 0 ]; then
        log_error "$@"
        return 1
    else
        log_info "$@"
        return 0
    fi
}

function check_es_status(){
    if curl -s ${1}/_cat/health | grep green &> /dev/null;then
        log_info "${1}: Elasticsearch status is green, check OK "
    elif curl -s ${1}/_cat/health | grep red &> /dev/null;then
        log_error "${1}: Elasticsearch status check is RED "
        exit 1001
    elif curl -s ${1}/_cat/health | grep yellow &> /dev/null;then
        if [ $DEPLOY_MODE == "single" ];then
            ## 单机状态ES 因有一个副本配置，不能进行分配到其他节点，故只有50% 的分片活跃，也就是主分片数量
            local result=$(curl -s ${1}/_cat/health | awk '$NF != "50.0%" {print $0}')
            if [ ! -z "$result" ];then
                log_important "${1}: Elasticsearch 单机部署模式，状态异常，请进行检查，result: $result "
                exit 2002
            fi
            log_info "${1}: Elasticsearch status yellow, check OK "
        fi
    fi
}

function create_snapshot_warehouse(){
    if [ $(curl -sIL -w "%{http_code}\n" -o /dev/null --location --request GET "${1}/_snapshot/macrowing") != 200 ];then
        log_info "${1}: Elasticsearch snapshots warehouse macrowing not exist, create it  "
        curl -L -X POST ${1}/_snapshot/macrowing -H 'Content-Type: application/json' -d '
        {
            "type": "fs",
            "settings": {
                "location": "edoc2_backup"
            }
        }
        '
    fi
}

function check_local_snapshot_status(){
    log_info "check_local_snapshot_status "
    snapshot_json=$(curl -s ${1}/_snapshot/macrowing/_all)
    len=$(echo ${snapshot_json}|jq '.snapshots|length')
    end=$(($len-2))     ## 本地保留一个
    if [ "${len}" != 0 ];then
        for index in $(seq 0 ${end})
        do
           if [ "$(echo ${snapshot_json} | jq '.snapshots[0].state' | sed 's/"//g')" == "SUCCESS" ];then
                curl --location --request DELETE ${1}/_snapshot/macrowing/$(echo ${snapshot_json} | jq ".snapshots[${index}].snapshot" | sed 's/"//g')
           else
                log_error "${1}: Elasticsearch has unfinished snapshot!!"
                exit 2000
           fi
        done
    fi
}

function create_snapshot(){
    log_info "start create_snapshot ${1} "
    create_snapshot_warehouse ${LOCAL_ES_URL}
    # curl -sL -X PUT "${LOCAL_ES_URL}/_snapshot/macrowing/${1}?wait_for_completion=true" &> /dev/null
    # if [ $(curl -sL -X GET "${LOCAL_ES_URL}/_snapshot/macrowing/${1}" | jq .snapshots[0].state | sed 's/"//g') == "SUCCESS" ];then
    #     log_info "macrowing snapshots ${1} is SUCCESS"
    # else
    #     log_error "macrowing snapshot ${1} is FAILED"
    #     exit 2001
    # fi
    if curl -sL -X PUT "${LOCAL_ES_URL}/_snapshot/macrowing/${1}" &> /dev/null;then
        log_info "wait snapshot finish 60s."
        sleep 60
        while true
        do
            if [ $(curl -sL -X GET "${LOCAL_ES_URL}/_snapshot/macrowing/${1}" | jq .snapshots[0].state|sed 's/"//g') = "SUCCESS" ];then
                log_info "macrowing snapshots ${1} is SUCCESS."
                break
            else
                log_info "macrowing snapshots ${1} isn't finished."
                sleep 30
            fi
        done
    else
        log_error "macrowing snapshot ${1} is FAILED."
        exit 2001
    fi
}

function rsync_snapshot_to_backup_server(){
    rsync -avzP --delete ${1} edoc2_backup@${2}::es/ --password-file=/etc/rsync_client.password &> /dev/null
    if [ $? -eq 0 ];then
        log_info "ES rsync backup ${1} to ${2} SUCCESS"
    else
        log_error "ES rsync backup ${1} to ${2} FAILED"
    fi
}

function get_backup_snapshot(){
    log_info "get_backup_snapshot, get the lastest one snapshot "
    local snapshot_json=$(curl -s ${1}/_snapshot/macrowing/_all)
    local len=$(echo ${snapshot_json} | jq '.snapshots|length')
    local end=$(($len-1))

    if [ "${len}" != 0 ];then
        local name=$(echo ${snapshot_json} | jq ".snapshots[${end}].snapshot" | sed 's/"//g')
        local state=$(echo ${snapshot_json} | jq '.snapshots[0].state' | sed 's/"//g')
        if [ "$state"x == "SUCCESS"x ];then
            log_info "get backup snapshot: $name "
            local _ret=$2
            eval $_ret="$name"
            return 0
        fi
    else
        log_error "${1}: not found snapshot file, 请确保备份的快照文件拷贝到 ES数据存储目录 的backup 目录下"
        exit
    fi
}

function check_recovery_status(){
    log_info "check_recovery_status: $1 "
    local indices=( $(curl -s ${1}/_cat/indices | awk '{print $3}' | xargs) )
    local finished=false
    while [[ "$finished" == "false" ]]; do
        for i in ${!indices[@]}
        do
            local recovery_status_json=
            recovery_status_json=$(curl -s ${1}/${indices[$i]}/_recovery)
            local len=$(echo ${recovery_status_json} | jq '."'${indices[$i]}'".shards|length')
            local end=$(($len-1))
            if [ ${len} -ne 0 ];then
                for shard in $(seq 0 ${end})
                do
                   if [ "$(echo ${recovery_status_json} | jq '."'${indices[$i]}.shards[${shard}]'".stage' | sed 's/"//g')" != "DONE" ];then
                        log_important "${indices[$i]} index recovery not finished "
                        finished=false
                        break 2
                   fi
                done
            fi
            log_info "${1} ${indices[$i]} index recovery is finished "
            if [ $i -eq $((${#indices[@]}-1)) ];then
                finished=true
            fi
        done
        if [ "$finished" == "false" ];then
            sleep 30
        fi
    done
}

function close_index(){
    log_info "close $REMOTE_ES_URL index $2 "
    while true; do
        local resp_code=$(curl -sIL -w "%{http_code}\n" -o /dev/null -X POST "${1}/${2}/_close")
        if [ "$resp_code"x == "200"x ];then
            log_info "close $REMOTE_ES_URL index $2 is SUCCESS "
            break
        elif [ "$resp_code"x == "403"x ];then
            curl -XPUT -H "Content-Type: application/json" "${1}/_all/_settings" -d '{"index.blocks.read_only_allow_delete": null}' &> /dev/null
            continue
        else
            log_error "close $REMOTE_ES_URL index is FAILED "
            exit 2003
        fi
    done
}

function recovery_from_snapshot(){
    log_info "recovery_from_snapshot "
    create_snapshot_warehouse ${1}
    local snapshot_name
    get_backup_snapshot $REMOTE_ES_URL snapshot_name

    close_index $REMOTE_ES_URL "_all"
    # curl -s -X POST "${1}/_snapshot/macrowing/${snapshot_name}/_restore?wait_for_completion=true"
    curl -s -X POST "${1}/_snapshot/macrowing/${snapshot_name}/_restore"
    assert_success "all_index 索引恢复检查 " || exit
}

function backup(){
	# curl -XGET -L -k -i --retry 3 ${REMOTE_ES_IP}:${REMOTE_EDOC_PORT}/api/values | grep 200
	# if [ $? -eq 0 ];then
	    check_es_status $LOCAL_ES_URL
	    create_snapshot $SNAPSHOT_NAME
	    rsync_snapshot_to_backup_server $LOCAL_BACKUP_DIR $REMOTE_BACKUP_SERVER
	    check_local_snapshot_status $LOCAL_ES_URL
	# else
	# 	log_error "Check edoc2 server FAILED,it seams not alive now"
	# 	exit 500
	# fi
}

function recovery(){
    create_snapshot_warehouse $REMOTE_ES_URL
    recovery_from_snapshot $REMOTE_ES_URL
    check_recovery_status $REMOTE_ES_URL
    check_es_status $REMOTE_ES_URL
}

function main(){
    backup
    recovery
}

# 定时脚本执行的ip
# LOCAL_IP=__LOCAL_IP__
LOCAL_ES_IP=192.168.251.41
LOCAL_ES_URL=http://${LOCAL_ES_IP}:30003
# 容器映射到 宿主机的 备份目录
LOCAL_BACKUP_DIR=/home/edoc2/macrowing/edoc2v5/data/es/backup/
# 存储快照备份文件的主机，需要配置rsync 同步到这个主机，REMOTE_ES 可以访问这个主机的快照备份目录
REMOTE_BACKUP_SERVER=192.168.251.41
SNAPSHOT_NAME=snapshot_all_indices_$(date +'%Y%m%d%H%M%S')
# ES的部署模式 single 或 cluster
# DEPLOY_MODE=__DEPLOY_MODE__
# 此项配置现在只进行 备份还原后的 状态检查
DEPLOY_MODE=single
###############################################################
# 为 cluster 时需要把宿主机的 es 数据backup目录挂载到 NFS共享存储 #
###############################################################

REMOTE_ES_IP=192.168.20.192
REMOTE_ES_URL=http://${REMOTE_ES_IP}:30003

REMOTE_EDOC_PORT=80

main


