#!/bin/bash
#
ESURL="http://192.168.251.126:30003"
LogPath=/tmp
ReportFile="es_query_result.txt"
PageSize=10000
FileTypes=(
word
ppt
excel
pdf
msg
txt
log
xml
cfg
)


function TestConn(){
curl -s -i --connect-timeout 8 $ESURL | grep -q '200 OK'
if [ "$?" -ne 0 ]; then
    echo "无法连接es地址: ${ESURL}."
    exit 10
fi
}


#获取文件id
function GetfileID(){
local FileType=$1
case "$FileType" in
    excel)
        req_body='{  
            "size": 0,
            "track_total_hits": true,
            "query": {
                "bool":{
                    "should": [
                       {
                          "match_phrase": {
                              "fileextname": ".xls"
                          }
                       },
                       {  
                           "match_phrase": {
                              "fileextname": ".xlsx"
                           }
                       }
                     ]
                }   
            }
        }'
  ;;
  word)
       req_body='{
            "size": 0,
            "track_total_hits": true,  
            "query": {
                "bool":{
                    "should": [
                      { 
                          "match_phrase": {
                              "fileextname": ".doc"
                          }
                      },
                      { 
                         "match_phrase": {
                             "fileextname": ".docx"
                         }
                      }
                    ]
                }
            }
          }'
  ;;
  ppt)
       req_body='{
            "size": 0,
            "track_total_hits": true,  
            "query": {
                "bool":{
                    "should": [
                    {   
                          "match_phrase": {
                              "fileextname": ".ppt"
                          }
                    },
                    {
                          "match_phrase": {
                              "fileextname": ".pptx"
                          }
                    }
                  ]
               }
            }
          }'
  ;;
  *)
        req_body='{
            "size": 0,
            "track_total_hits": true,
            "query": {
                "match_phrase": {
                    "fileextname": ".'$FileType'"
                }
            }
          }'
  ;;
esac

curl -s --location --request GET "$ESURL/file_1/_search" \
-H 'Content-Type: application/json' \
-d "$req_body" > fileID_tmp.txt

local Total=$(grep  -oP '"hits":.+:\d+,' fileID_tmp.txt | grep -oP '\d+')
if [[ "$Total" -le "$PageSize" ]]; then
    count=1
else
    moudle=$((Total%PageSize))
    if [[ "$moudle" -eq 0 ]]; then
        count=$((Total/PageSize))
    else
        count=$(((Total/PageSize)+1))
    fi
fi
[ -f fileID.txt ] && rm -f fileID.txt
size=0

if [ "$FileType" == "excel" ]; then
    for ((i=1;i<=count;i++)); do
        curl -s --location --request GET "$ESURL/file_1/_search" \
        -H 'Content-Type: application/json' \
        -d '{
            "from": '$size',
            "size": '$PageSize',
            "track_total_hits": true,
            "query": {
                "bool":{
                    "should": [
                       {
                          "match_phrase": {
                              "fileextname": ".xls"
                          }
                       },
                       {  
                           "match_phrase": {
                              "fileextname": ".xlsx"
                           }
                       }
                     ]
                }   
            },
            "_source": ["fileid"]
        }' > xlsid_tmp.txt
        grep -oP '\"fileid\":\d+' xlsid_tmp.txt | cut -d ':' -f 2 >> fileID.txt
        let size+=PageSize
    done
    rm -f xlsid_tmp.txt
elif [ "$FileType" == "word" ]; then
    for ((i=1;i<=count;i++)); do
        curl -s --location --request GET "$ESURL/file_1/_search" \
        -H 'Content-Type: application/json' \
        -d '{
            "from": '$size',
            "size": '$PageSize',
            "track_total_hits": true,
            "query": {
                "bool":{
                    "should": [
                       {
                          "match_phrase": {
                              "fileextname": ".doc"
                          }
                       },
                       {  
                           "match_phrase": {
                              "fileextname": ".docx"
                           }
                       }
                     ]
                }   
            },
            "_source": ["fileid"]
        }' > docid_tmp.txt
        grep -oP '\"fileid\":\d+' docid_tmp.txt | cut -d ':' -f 2 >> fileID.txt
        let size+=PageSize
    done
    rm -f docid_tmp.txt
elif [ "$FileType" == "ppt" ]; then
    for ((i=1;i<=count;i++)); do
        curl -s --location --request GET "$ESURL/file_1/_search" \
        -H 'Content-Type: application/json' \
        -d '{
            "from": '$size',
            "size": '$PageSize',
            "track_total_hits": true,
            "query": {
                "bool":{
                    "should": [
                       {
                          "match_phrase": {
                              "fileextname": ".ppt"
                          }
                       },
                       {  
                           "match_phrase": {
                              "fileextname": ".pptx"
                           }
                       }
                     ]
                }   
            },
            "_source": ["fileid"]
        }' > pptid_tmp.txt
        grep -oP '\"fileid\":\d+' pptid_tmp.txt | cut -d ':' -f 2 >> fileID.txt
        let size+=PageSize
    done
    rm -f pptid_tmp.txt
else
    for ((i=1;i<=count;i++)); do
        curl -s --location --request GET "$ESURL/file_1/_search" \
        -H 'Content-Type: application/json' \
        -d '{
             "from": '$size',
             "size": '$PageSize',
             "track_total_hits": true,
             "query": {
                  "match_phrase": {
                      "fileextname": ".'$FileType'"
             }
            },
            "_source": ["fileid"]
        }' > otherid_tmp.txt
        grep -oP '\"fileid\":\d+' otherid_tmp.txt | cut -d ':' -f 2 >> fileID.txt
        let size+=PageSize
     done
     rm -f otherid_tmp.txt
fi

}

#多线程获取文件大小
function GetFileSize(){
TOTAL_SIZE=0
THREAD_NUM=10
mkfifo /tmp/fifo
exec 6<>/tmp/fifo
rm -f /tmp/fifo

for ((i=1;i<=THREAD_NUM;i++)); do
    {
      echo 
    }
done >&6 

[ -f fileSize.txt ] && rm -f fileSize.txt

if [ -s fileID.txt ]; then
    while read ID; do
      {
        read -u 6
        {
            curl -s --request GET "$ESURL/file_1/_doc/$ID" \
            -H 'Content-Type: application/json' > fileSize_${ID}.txt
            FILE_SIZE=$(du -b fileSize_${ID}.txt | cut -f 1)
            echo "fileID: $ID Size: $FILE_SIZE" >> fileSize.txt
            rm -f fileSize_${ID}.txt
            echo "" >&6
        } &
      }
    done < fileID.txt
    wait
    TOTAL_SIZE=$(awk '{sum+=$NF}END{print sum}' fileSize.txt)
else
    TOTAL_SIZE=0
fi
echo "${1}文件总大小(Byte): ${TOTAL_SIZE}" >> $ReportFile
#echo "excel文件ID及大小信息存放路径: $LogPath/fileSize.txt"
rm -f fileSize_tmp.txt
exec 6>&-
}

#查询文件分布及文件大小
function FileDistribution(){
for File in "${FileTypes[@]}" ; do
    GetfileID $File
    GetFileSize $File 
    FileContent $File
    FileTotal=$(
        grep -oP '"hits":.+:\d+,' fileID_tmp.txt | grep -oP '\d+'
    )
    echo "${File}文件总量: $FileTotal" >> $ReportFile
done
rm -f fileID_tmp.txt

}


#查询总文件量
function QueryFileTotal(){
file_total=$(
      curl -s --location --request GET "$ESURL/file_1/_search" \
      -H 'Content-Type: application/json' \
      -d '{
            "size": 0,
            "track_total_hits": true,
            "query": {
                "match_all": {}
            }
          }' | grep -oP '"hits":.+:\d+,' | grep -oP '\d+'
)
echo "总文件量: $file_total" >> $ReportFile

} 

#开启内容索引的excel文件数量
function FileContent(){
if [ "$1" == "excel" ]; then
    curl -s --location --request GET "$ESURL/file_1/_search" \
    -H 'Content-Type: application/json' \
    -d '{
          "size": 0,
          "track_total_hits": true,
          "query": {
              "bool": {
                  "must": [
                  {
                      "bool": {
                          "should": [
                          {
                              "match_phrase": {
                                  "fileextname": ".xls"
                              }
                          },
                          { 
                              "match_phrase": {
                                  "fileextname": ".xlsx"
                              }             
                          }
                         ]
                        } 
                  },
                  {
                      "exists": {
                          "field": "filecontent"
                      }
                  }
                 ]
               }
            }         
         }' > xlscontent.txt
    local Total=$(grep -oP '"hits":.+:\d+,' xlscontent.txt | grep -oP '\d+')
    echo "${1}开启内容索引数量: $Total" >> $ReportFile
    rm -f xlscontent.txt
elif [ "$1" == "word" ]; then
    curl -s --location --request GET "$ESURL/file_1/_search" \
    -H 'Content-Type: application/json' \
    -d '{
          "size": 0,
          "track_total_hits": true,
          "query": {
              "bool": {
                  "must": [
                  {
                      "bool": {
                          "should": [
                          {
                              "match_phrase": {
                                  "fileextname": ".doc"
                              }
                          },
                          { 
                              "match_phrase": {
                                  "fileextname": ".docx"
                              }             
                          }
                         ]
                        } 
                  },
                  {
                      "exists": {
                          "field": "filecontent"
                      }
                  }
                 ]
               }
            }         
         }' > docontent.txt
    local Total=$(grep -oP '"hits":.+:\d+,' docontent.txt | grep -oP '\d+')
    echo "${1}开启内容索引数量: $Total" >> $ReportFile
    rm -f docontent.txt
elif [ "$1" == "ppt" ]; then
    curl -s --location --request GET "$ESURL/file_1/_search" \
    -H 'Content-Type: application/json' \
    -d '{
          "size": 0,
          "track_total_hits": true,
          "query": {
              "bool": {
                  "must": [
                  {
                      "bool": {
                          "should": [
                          {
                              "match_phrase": {
                                  "fileextname": ".ppt"
                              }
                          },
                          { 
                              "match_phrase": {
                                  "fileextname": ".pptx"
                              }             
                          }
                         ]
                        } 
                  },
                  {
                      "exists": {
                          "field": "filecontent"
                      }
                  }
                 ]
               }
            }         
         }' > pptcontent.txt
    local Total=$(grep -oP '"hits":.+:\d+,' pptcontent.txt | grep -oP '\d+')
    echo "${1}开启内容索引数量: $Total" >> $ReportFile
    rm -f pptcontent.txt
else
    curl -s --location --request GET "$ESURL/file_1/_search" \
    -H 'Content-Type: application/json' \
    -d '{
          "size": 0,
          "track_total_hits": true,
          "query": {
              "bool": {
                  "must": [
                  {
                     "match_phrase": {
                                  "fileextname": ".'$1'"
                      }
                  },
                  {
                      "exists": {
                          "field": "filecontent"
                      }
                  }
                 ]
               }
            }         
         }' > othercontent.txt
    local Total=$(grep -oP '"hits":.+:\d+,' othercontent.txt | grep -oP '\d+')
    echo "${1}开启内容索引数量: $Total" >> $ReportFile
    rm -f othercontent.txt
fi
    
}


main(){
cd $LogPath
TestConn
if [ -f "$ReportFile" ]; then
  rm -f $ReportFile
fi
QueryFileTotal
FileDistribution
}

main
