比较两个节点对应目录数据一致性校验

跨节点文件目录数据一致性校验-Shell组件

校验工具说明

# cksum
Print CRC checksum and byte counts of each FILE

# md5sum
Print or check MD5 (128-bit) checksums

# sha1sum
Print or check SHA1 (160-bit) checksums

# sha256sum
Print or check SHA256 (256-bit) checksums

使用效果演示

$ cksum md5_b.txt
2325705422 16392970 md5_b.txt
$ md5sum md5_b.txt 
05eb026baaeecd0cf17c5c3d991bdde4  md5_b.txt
$ sha1sum md5_b.txt 
443d40550fa3be62a9a8c614088ef53309280657  md5_b.txt
$ sha256sum md5_b.txt 
2b691daf9e0f9193448322689812839102d00c7e0e4dada2cb465157ec120f58   md5_b.txt

步骤

遍历目录中的文件并计算其哈希码，将哈希码和文件名一行行记录至一个文本中
在A、B节点分别跑步骤一，得到两个数据文本
以文件名作为遍历对象遍历A节点数据文本，每一A节点遍历内从B节点数据文本中找同名文件名，找不到则证明B节点此目录下无此文件；找到则进一步对比文件名对应的哈希码，相同则证明文件内容一致，不同则证明文件不一致

代码

data_consistency.sh

#!/bin/bash
# name:data_consistency.sh
# auth:cyrus_chen
# func:calsum_dir2txt compare_txt
# version:v1.0
# sys:centos7
# usage: bash data_consistency.sh cksum /test 127.0.0.1

set -e

sum_method=$1
sum_dir=$2
remote_ip=$3
local_txt="a.txt"
remote_txt="b.txt"

calsum_dir2txt() {
    sum_method=$1
    sum_dir=$2
    remote_ip=$3
    local_txt=$4
    remote_txt=$5
    
    if [ ! -d $sum_dir ];then
        echo "目录$sum_dir不存在"
        exit 1
    fi
    if [ $sum_method == "cksum" ];then
        # echo "这是${sum_method}"
        sudo find $sum_dir -type f|xargs cksum|awk '{print $1 "\t" $3}' > ./$local_txt
        echo "请输入$remote_ip的ssh密码"
        ssh apps@$remote_ip "bash ./cksum.sh $sum_dir ./$remote_txt"
        echo "请再次输入$remote_ip的ssh密码"
        scp $remote_ip:./$remote_txt .
        
    elif [ $sum_method == "md5sum" -o $sum_method == "sha1sum" -o $sum_method == "sha256sum" ];then
        # echo "这是${sum_method}"
        sudo find $sum_dir -type f|xargs $sum_method > ./$local_txt
        echo "请输入$remote_ip的ssh密码"
        ssh apps@$remote_ip "bash ./hashsum.sh $sum_method $sum_dir ./$remote_txt"
        echo "请再次输入$remote_ip的ssh密码"
        scp $remote_ip:./$remote_txt .
    else
        echo "目前只支持[cksum md5sum sha1sum sha256sum]"
    fi
}

compare_txt() {
    local_txt=$1
    remote_txt=$2
    if [ ! -f "./${local_txt}" ];then
        echo "./${local_txt}不存在"
        exit 1
    fi
    if [ ! -f "./${remote_txt}" ];then
        echo "./${remote_txt}不存在"
        exit 1
    fi
    for f in `awk '{print $2}' ./${local_txt}`
    do 
        if grep -qw "$f" ./${remote_txt}
        then
            md5_a=`grep -w "$f" ./${local_txt}|awk '{print 1}'`
            md5_b=`grep -w "$f" ./${remote_txt}|awk '{print 1}'`
            if [ "X${md5_a}" != "X${md5_b}" ]
            then
                echo "$f changed."
            fi
        else
            echo "$f deleted."
        fi
    done
}

main() {
    #calsum_dir2txt $sum_method $sum_dir $remote_ip $local_txt $remote_txt
    compare_txt $local_txt $remote_txt
}

main

cksum.sh(扔到remote_ip上)

#!/bin/bash
# name:cksum.sh
# auth:cyrus_chen
# version:v1.0
# sys:centos7
# usage: bash cksum.sh /test 127.0.0.1 ./b.txt

set -e

sum_dir=$1
remote_txt=$2
sudo find $sum_dir -type f|xargs cksum|awk '{print $1 "\t" $3}' > $remote_txt

hashsum.sh(扔到remote_ip上)

#!/bin/bash
# name:hashsum.sh
# auth:cyrus_chen
# version:v1.0
# sys:centos7
# usage: bash hashsum.sh md5sum /test 127.0.0.1 ./b.txt

set -e

sum_method=$1
sum_dir=$2
remote_txt=$3
sudo find $sum_dir -type f|xargs $sum_method > $remote_txt