基于bash find命令执行log日志备份和清理的一次简单实践

前言

find作为最基础的bash shell命令之一我就不多做介绍了，由于每月压缩后的日志增长量已经超过20TB，和PB相比应该还不算很大，选择的日志归档同步方案是Facebook Scribe，之后或许会基于ELK或Grafana Loki搭建日志实时分析平台，不过眼下的问题还是想办法在没有商业化集中式存储和软件定义分布式存储的支持下，用比较简单粗暴的方法苦苦支撑，本文分享了一些简单的小技巧方便回顾和二次利用。

基于bash find命令执行log日志备份和清理日志简单实践

更新历史

2019年07月29日 - 初稿

阅读原文 - https://wsgzao.github.io/post...

扩展阅读

find

find - search for files in a directory hierarchy

https://linux.die.net/man/1/find

find 命令用来在指定目录下查找文件。任何位于参数之前的字符串都将被视为欲查找的目录名。如果使用该命令时，不设置任何参数，则 find 命令将在当前目录下查找子目录与文件。并且将查找到的子目录和文件全部进行显示。

https://man.linuxde.net/find

How to use

log archive

SHELL=/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin
MAILTO=root

# For details see man 4 crontabs

# Example of job definition:
# .---------------- minute (0 - 59)
# |  .------------- hour (0 - 23)
# |  |  .---------- day of month (1 - 31)
# |  |  |  .------- month (1 - 12) OR jan,feb,mar,apr ...
# |  |  |  |  .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat
# |  |  |  |  |
# *  *  *  *  * user-name  command to be executed

#zip
#15 3 * * * root cd /opt/sa_scripts/archive_from_date && bash run.sh   /data/gop/live/primary/gop 2
15 3 * * * root bash /opt/sa_scripts/archive_from_date/one_run_2d.sh

log backup and cleanup

cd /data/scripts
./check_original_log.sh 2019-05
./move_backup.sh 2019-05
./check_backup_log.sh 2019-05
nohup sh rsync_backup_10-71-12-61.sh > log/2019-04.rsync.out 2>&1 &
nohup sh rsync_backup_10-71-14-132.sh > log/2019-05.rsync.out 2>&1 &

log archive

archive_from_date.sh

#!/bin/bash

#to compress all files like:
#auth_data-2017-01-11_00000
#auth_data-2017-01-11_00001
#auth_data-2017-01-11_00002
#auth_data-2017-01-11_00003
#auth_data-2017-01-12_00000
#auth_data-2017-01-12_00001
#
#by day till the specified date

if [ "$#" -lt 2 ]
then
    echo "Usage: $0 path archive_days [trailing_digit]"
    exit 1
fi

FILE_PATH=$1
ARCHIVE_DAYS=$2
#seems no need traling digit param here
TRAILING_DIGIT=$3
if [ -z "${TRAILING_DIGIT//}" ]
then
    TRAILING_DIGIT=6
fi

start_date="2017-01-01"

end_date=`date`
end_date=`date -d -${ARCHIVE_DAYS}days +%Y-%m-%d`
echo "Scanning from $start_date to $end_date (not inclusive)"
echo "=================================="

if [[ "$end_date" < "$start_date" ]]
then
    echo "Invalid end date: $end_date,it should be later than start date: ${start_date}. Exit..."
    exit 1
fi

RESULT_PATH=/tmp/log_archive
if [ ! -d "$RESULT_PATH" ]
then
    mkdir -p $RESULT_PATH
fi


cd $FILE_PATH

while [ "$start_date" != "$end_date" ]
do
    if ls *${start_date}*[0-9] 1>/dev/null 2>&1
    then
        echo "Compressing: $start_date"
        first_file=`ls *${start_date}*[0-9]|head -1`
        file_name=${first_file%_*}
        tar cvzf ${file_name}.tgz ${file_name}*[0-9]

#        #Case 1: compare tar size with origin, uncomment if needed
#        original_size=`ls -l *${start_date}*[0-9]|awk '{sum+=$5} END {print sum}'`
#        weighted_size=$((original_size/20))
#        tar_size=`ls -l ${file_name}.tgz|awk '{print $5}'`
#        echo $tar_size,$weighted_size
#        if (( $tar_size < $weighted_size ))
#        then
#            echo "tar size: ${tar_size}; weighted size:${weighted_size}"
#            echo "tar size too small; not deleting origin"
#            echo "`pwd`: $file_name" >>/opt/sa_scripts/archive/result
#        else
#            echo "Done.Deleting origin."
#            rm ${file_name}*[0-9]
#        fi
#        #End of Case 1

        ##############

        #Case 2, compare tar size with 0
        tar_size=`ls -l ${file_name}.tgz|awk '{print $5}'`
        if (( $tar_size > 0 ))
        then
            echo "Done.Deleting origin."
            rm ${file_name}*[0-9]
        else
            echo "tar file size is ZERO!"
            echo "`pwd`: $file_name" >>$RESULT_PATH
        fi
        #End of Case 2
    fi
    start_date=$(date -I -d "$start_date +1day")
done

archive_from_day.sh

#!/bin/bash
# bash one_day.sh /data/gop/live/primary/gop 2017-01-11

FILE_PATH=$1
ARCHIVE_DAYS=$2

start_date=$ARCHIVE_DAYS

end_date=$ARCHIVE_DAYS
echo "Scanning from $start_date to $end_date (not inclusive)"
echo "=================================="


RESULT_PATH=/tmp/log_archive
if [ ! -d "$RESULT_PATH" ]
then
    mkdir -p $RESULT_PATH
fi

cd $FILE_PATH

if ls *${start_date}*[0-9] 1>/dev/null 2>&1
then
    echo "Compressing: $start_date"
    first_file=`ls *${start_date}*[0-9]|head -1`
    file_name=${first_file%_*}
    tar cvzf ${file_name}.tgz ${file_name}*[0-9]
    #Case 2, compare tar size with 0
    tar_size=`ls -l ${file_name}.tgz|awk '{print $5}'`
    if (( $tar_size > 0 ))
    then
        echo "Done.Deleting origin."
        rm ${file_name}*[0-9]
    else
        echo "tar file size is ZERO!"
        echo "`pwd`: $file_name" >>$RESULT_PATH
    fi
    #End of Case 2
fi

run.sh

#!/bin/bash

PARENT_PATH=$1
ARCHIVE_DAY=$2

if [ "$#" -ne 2 ]
then
    echo "Usage: $0 parent_path archive_day"
    exit 1
fi

for folder in $(find $PARENT_PATH -type d -links 2)
do
    echo "processing:${folder}..."
    ./archive_from_date.sh $folder $ARCHIVE_DAY
done

one_run_2d.sh

#!/bin/bash

end_date=`date -d -2days +%Y-%m-%d`

nohup bash /opt/sa_scripts/archive_from_date/one_run.sh /data/gop/live/primary/gop $end_date > /opt/sa_scripts/archive_from_date/logs/$end_date.log 2>&1 &

one_run.sh

#!/bin/bash

PARENT_PATH=$1
ARCHIVE_DAY=$2

if [ "$#" -ne 2 ]
then
    echo "Usage: $0 parent_path archive_day"
    exit 1
fi

for folder in $(find $PARENT_PATH -type d -links 2)
do
    echo "processing:${folder}..."
    /opt/sa_scripts/archive_from_date/day.sh $folder $ARCHIVE_DAY
done

log example

total 2197444
-rw-r--r-- 1 root root 35386495 Jun  3 21:20 gop-live-point_txn_aggregator-data-2019-06-01.tgz
-rw-r--r-- 1 root root 33273034 Jun  4 20:37 gop-live-point_txn_aggregator-data-2019-06-02.tgz
-rw-r--r-- 1 root root 33390884 Jun  5 20:23 gop-live-point_txn_aggregator-data-2019-06-03.tgz
-rw-r--r-- 1 root root 35553038 Jun  6 20:18 gop-live-point_txn_aggregator-data-2019-06-04.tgz
-rw-r--r-- 1 root root 35081679 Jun  7 19:16 gop-live-point_txn_aggregator-data-2019-06-05.tgz
-rw-r--r-- 1 root root 35397705 Jun  8 19:17 gop-live-point_txn_aggregator-data-2019-06-06.tgz
-rw-r--r-- 1 root root 36039754 Jun  9 18:55 gop-live-point_txn_aggregator-data-2019-06-07.tgz
-rw-r--r-- 1 root root 38596404 Jun 10 20:22 gop-live-point_txn_aggregator-data-2019-06-08.tgz
-rw-r--r-- 1 root root 38048982 Jun 11 20:26 gop-live-point_txn_aggregator-data-2019-06-09.tgz
-rw-r--r-- 1 root root 33026017 Jun 12 19:16 gop-live-point_txn_aggregator-data-2019-06-10.tgz
-rw-r--r-- 1 root root 29198108 Jun 13 19:56 gop-live-point_txn_aggregator-data-2019-06-11.tgz
-rw-r--r-- 1 root root 25771109 Jun 14 19:19 gop-live-point_txn_aggregator-data-2019-06-12.tgz
-rw-r--r-- 1 root root 26092930 Jun 15 19:09 gop-live-point_txn_aggregator-data-2019-06-13.tgz
-rw-r--r-- 1 root root 35349519 Jun 16 19:13 gop-live-point_txn_aggregator-data-2019-06-14.tgz
-rw-r--r-- 1 root root 31780817 Jun 17 21:27 gop-live-point_txn_aggregator-data-2019-06-15.tgz
-rw-r--r-- 1 root root 36549729 Jun 18 21:57 gop-live-point_txn_aggregator-data-2019-06-16.tgz
-rw-r--r-- 1 root root 26007332 Jun 19 19:58 gop-live-point_txn_aggregator-data-2019-06-17.tgz
-rw-r--r-- 1 root root 25114440 Jun 20 19:20 gop-live-point_txn_aggregator-data-2019-06-18.tgz
-rw-r--r-- 1 root root 22494565 Jun 21 19:25 gop-live-point_txn_aggregator-data-2019-06-19.tgz
-rw-r--r-- 1 root root 25132986 Jun 22 19:19 gop-live-point_txn_aggregator-data-2019-06-20.tgz
-rw-r--r-- 1 root root 30698833 Jun 23 20:08 gop-live-point_txn_aggregator-data-2019-06-21.tgz
-rw-r--r-- 1 root root 38639032 Jun 24 21:12 gop-live-point_txn_aggregator-data-2019-06-22.tgz
-rw-r--r-- 1 root root 31422932 Jun 25 21:27 gop-live-point_txn_aggregator-data-2019-06-23.tgz
-rw-r--r-- 1 root root 25262186 Jun 26 19:57 gop-live-point_txn_aggregator-data-2019-06-24.tgz
-rw-r--r-- 1 root root 25402079 Jun 27 19:30 gop-live-point_txn_aggregator-data-2019-06-25.tgz
-rw-r--r-- 1 root root 22227072 Jun 28 17:09 gop-live-point_txn_aggregator-data-2019-06-26.tgz
-rw-r--r-- 1 root root 26174268 Jun 29 19:59 gop-live-point_txn_aggregator-data-2019-06-27.tgz
-rw-r--r-- 1 root root 27484303 Jun 30 20:43 gop-live-point_txn_aggregator-data-2019-06-28.tgz
-rw-r--r-- 1 root root 25607704 Jul  1 21:37 gop-live-point_txn_aggregator-data-2019-06-29.tgz
-rw-r--r-- 1 root root 23545276 Jul  2 21:34 gop-live-point_txn_aggregator-data-2019-06-30.tgz
-rw-r--r-- 1 root root 28956480 Jul  3 21:03 gop-live-point_txn_aggregator-data-2019-07-01.tgz
-rw-r--r-- 1 root root 27250475 Jul  4 20:21 gop-live-point_txn_aggregator-data-2019-07-02.tgz
-rw-r--r-- 1 root root 26443417 Jul  5 20:56 gop-live-point_txn_aggregator-data-2019-07-03.tgz
-rw-r--r-- 1 root root 23445274 Jul  6 20:28 gop-live-point_txn_aggregator-data-2019-07-04.tgz
-rw-r--r-- 1 root root 35914567 Jul  7 21:01 gop-live-point_txn_aggregator-data-2019-07-05.tgz
-rw-r--r-- 1 root root 42759699 Jul  8 21:46 gop-live-point_txn_aggregator-data-2019-07-06.tgz
-rw-r--r-- 1 root root 41146298 Jul  9 23:28 gop-live-point_txn_aggregator-data-2019-07-07.tgz
-rw-r--r-- 1 root root 30890861 Jul 10 21:50 gop-live-point_txn_aggregator-data-2019-07-08.tgz
-rw-r--r-- 1 root root 29505781 Jul 11 22:16 gop-live-point_txn_aggregator-data-2019-07-09.tgz
-rw-r--r-- 1 root root 25130694 Jul 12 21:04 gop-live-point_txn_aggregator-data-2019-07-10.tgz
-rw-r--r-- 1 root root 30355468 Jul 13 21:44 gop-live-point_txn_aggregator-data-2019-07-11.tgz
-rw-r--r-- 1 root root 32142222 Jul 14 22:47 gop-live-point_txn_aggregator-data-2019-07-12.tgz
-rw-r--r-- 1 root root 30846192 Jul 15 22:54 gop-live-point_txn_aggregator-data-2019-07-13.tgz
-rw-r--r-- 1 root root 32435568 Jul 17 00:21 gop-live-point_txn_aggregator-data-2019-07-14.tgz
-rw-r--r-- 1 root root 24601862 Jul 17 23:54 gop-live-point_txn_aggregator-data-2019-07-15.tgz
-rw-r--r-- 1 root root 26708856 Jul 19 00:54 gop-live-point_txn_aggregator-data-2019-07-16.tgz
-rw-r--r-- 1 root root 21915660 Jul 19 23:23 gop-live-point_txn_aggregator-data-2019-07-17.tgz
-rw-r--r-- 1 root root 19324816 Jul 21 00:01 gop-live-point_txn_aggregator-data-2019-07-18.tgz
-rw-r--r-- 1 root root 29234368 Jul 22 00:47 gop-live-point_txn_aggregator-data-2019-07-19.tgz
-rw-r--r-- 1 root root 32138171 Jul 23 02:44 gop-live-point_txn_aggregator-data-2019-07-20.tgz
-rw-r--r-- 1 root root 31359175 Jul 24 04:05 gop-live-point_txn_aggregator-data-2019-07-21.tgz
-rw-r--r-- 1 root root 26123344 Jul 24 23:36 gop-live-point_txn_aggregator-data-2019-07-22.tgz
-rw-r--r-- 1 root root 25337107 Jul 26 00:10 gop-live-point_txn_aggregator-data-2019-07-23.tgz
-rw-r--r-- 1 root root 23113614 Jul 26 23:52 gop-live-point_txn_aggregator-data-2019-07-24.tgz
-rw-r--r-- 1 root root 22578291 Jul 28 00:37 gop-live-point_txn_aggregator-data-2019-07-25.tgz
-rw-r--r-- 1 root root 28572628 Jul 29 01:21 gop-live-point_txn_aggregator-data-2019-07-26.tgz
-rw-r--r-- 1 root root 29820440 Jul 30 01:18 gop-live-point_txn_aggregator-data-2019-07-27.tgz
-rw-r--r-- 1 root root 25214160 Jul 31 01:26 gop-live-point_txn_aggregator-data-2019-07-28.tgz
-rw-r--r-- 1 root root 20978337 Aug  1 00:27 gop-live-point_txn_aggregator-data-2019-07-29.tgz
-rw-r--r-- 1 root root 22192620 Aug  1 23:12 gop-live-point_txn_aggregator-data-2019-07-30.tgz
-rw-r--r-- 1 root root  6329205 Jul 31 00:59 gop-live-point_txn_aggregator-data-2019-07-31_00000
-rw-r--r-- 1 root root  5786176 Jul 31 02:00 gop-live-point_txn_aggregator-data-2019-07-31_00001
-rw-r--r-- 1 root root  5451181 Jul 31 03:00 gop-live-point_txn_aggregator-data-2019-07-31_00002
-rw-r--r-- 1 root root  5621760 Jul 31 04:00 gop-live-point_txn_aggregator-data-2019-07-31_00003
-rw-r--r-- 1 root root  5820921 Jul 31 05:00 gop-live-point_txn_aggregator-data-2019-07-31_00004
-rw-r--r-- 1 root root  6088472 Jul 31 05:59 gop-live-point_txn_aggregator-data-2019-07-31_00005
-rw-r--r-- 1 root root  6048663 Jul 31 06:59 gop-live-point_txn_aggregator-data-2019-07-31_00006
-rw-r--r-- 1 root root  6434107 Jul 31 08:00 gop-live-point_txn_aggregator-data-2019-07-31_00007
-rw-r--r-- 1 root root  6937321 Jul 31 09:00 gop-live-point_txn_aggregator-data-2019-07-31_00008
-rw-r--r-- 1 root root  6717315 Jul 31 09:59 gop-live-point_txn_aggregator-data-2019-07-31_00009
-rw-r--r-- 1 root root  6667836 Jul 31 10:59 gop-live-point_txn_aggregator-data-2019-07-31_00010
-rw-r--r-- 1 root root  6559363 Jul 31 12:00 gop-live-point_txn_aggregator-data-2019-07-31_00011
-rw-r--r-- 1 root root  6107832 Jul 31 12:59 gop-live-point_txn_aggregator-data-2019-07-31_00012
-rw-r--r-- 1 root root  6323896 Jul 31 13:59 gop-live-point_txn_aggregator-data-2019-07-31_00013
-rw-r--r-- 1 root root  6389052 Jul 31 15:00 gop-live-point_txn_aggregator-data-2019-07-31_00014
-rw-r--r-- 1 root root  6504740 Jul 31 16:00 gop-live-point_txn_aggregator-data-2019-07-31_00015
-rw-r--r-- 1 root root  6796378 Jul 31 17:00 gop-live-point_txn_aggregator-data-2019-07-31_00016
-rw-r--r-- 1 root root  7878303 Jul 31 18:00 gop-live-point_txn_aggregator-data-2019-07-31_00017
-rw-r--r-- 1 root root  8900771 Jul 31 19:00 gop-live-point_txn_aggregator-data-2019-07-31_00018
-rw-r--r-- 1 root root  9379681 Jul 31 20:00 gop-live-point_txn_aggregator-data-2019-07-31_00019
-rw-r--r-- 1 root root 10435828 Jul 31 21:00 gop-live-point_txn_aggregator-data-2019-07-31_00020
-rw-r--r-- 1 root root 10127803 Jul 31 22:00 gop-live-point_txn_aggregator-data-2019-07-31_00021
-rw-r--r-- 1 root root  8788741 Jul 31 22:59 gop-live-point_txn_aggregator-data-2019-07-31_00022
-rw-r--r-- 1 root root  7747234 Jul 31 23:59 gop-live-point_txn_aggregator-data-2019-07-31_00023
-rw-r--r-- 1 root root  6983387 Aug  1 00:59 gop-live-point_txn_aggregator-data-2019-08-01_00000
-rw-r--r-- 1 root root  6572435 Aug  1 02:00 gop-live-point_txn_aggregator-data-2019-08-01_00001
-rw-r--r-- 1 root root  5959884 Aug  1 03:00 gop-live-point_txn_aggregator-data-2019-08-01_00002
-rw-r--r-- 1 root root  5834298 Aug  1 04:00 gop-live-point_txn_aggregator-data-2019-08-01_00003
-rw-r--r-- 1 root root  6008950 Aug  1 04:59 gop-live-point_txn_aggregator-data-2019-08-01_00004
-rw-r--r-- 1 root root  6614511 Aug  1 06:00 gop-live-point_txn_aggregator-data-2019-08-01_00005
-rw-r--r-- 1 root root  6934464 Aug  1 07:00 gop-live-point_txn_aggregator-data-2019-08-01_00006
-rw-r--r-- 1 root root  7501841 Aug  1 08:00 gop-live-point_txn_aggregator-data-2019-08-01_00007
-rw-r--r-- 1 root root  8018389 Aug  1 09:00 gop-live-point_txn_aggregator-data-2019-08-01_00008
-rw-r--r-- 1 root root  8113113 Aug  1 10:00 gop-live-point_txn_aggregator-data-2019-08-01_00009
-rw-r--r-- 1 root root  8247397 Aug  1 11:00 gop-live-point_txn_aggregator-data-2019-08-01_00010
-rw-r--r-- 1 root root  9334051 Aug  1 12:00 gop-live-point_txn_aggregator-data-2019-08-01_00011
-rw-r--r-- 1 root root  8284898 Aug  1 12:59 gop-live-point_txn_aggregator-data-2019-08-01_00012
-rw-r--r-- 1 root root  9218796 Aug  1 14:00 gop-live-point_txn_aggregator-data-2019-08-01_00013
-rw-r--r-- 1 root root  7875013 Aug  1 15:00 gop-live-point_txn_aggregator-data-2019-08-01_00014
-rw-r--r-- 1 root root  4420827 Aug  1 16:00 gop-live-point_txn_aggregator-data-2019-08-01_00015
-rw-r--r-- 1 root root  7999875 Aug  1 16:59 gop-live-point_txn_aggregator-data-2019-08-01_00016
-rw-r--r-- 1 root root  9480688 Aug  1 18:00 gop-live-point_txn_aggregator-data-2019-08-01_00017
-rw-r--r-- 1 root root 10069991 Aug  1 19:00 gop-live-point_txn_aggregator-data-2019-08-01_00018
-rw-r--r-- 1 root root 11397317 Aug  1 20:00 gop-live-point_txn_aggregator-data-2019-08-01_00019
-rw-r--r-- 1 root root 12597957 Aug  1 21:00 gop-live-point_txn_aggregator-data-2019-08-01_00020
-rw-r--r-- 1 root root 11541451 Aug  1 22:00 gop-live-point_txn_aggregator-data-2019-08-01_00021
-rw-r--r-- 1 root root 10813577 Aug  1 23:00 gop-live-point_txn_aggregator-data-2019-08-01_00022
-rw-r--r-- 1 root root  9507499 Aug  2 00:00 gop-live-point_txn_aggregator-data-2019-08-01_00023
-rw-r--r-- 1 root root  8110763 Aug  2 01:00 gop-live-point_txn_aggregator-data-2019-08-02_00000
-rw-r--r-- 1 root root  7238339 Aug  2 02:00 gop-live-point_txn_aggregator-data-2019-08-02_00001
-rw-r--r-- 1 root root  6577030 Aug  2 02:59 gop-live-point_txn_aggregator-data-2019-08-02_00002
-rw-r--r-- 1 root root  6586196 Aug  2 04:00 gop-live-point_txn_aggregator-data-2019-08-02_00003
-rw-r--r-- 1 root root  6883540 Aug  2 05:00 gop-live-point_txn_aggregator-data-2019-08-02_00004
-rw-r--r-- 1 root root  7172237 Aug  2 06:00 gop-live-point_txn_aggregator-data-2019-08-02_00005
-rw-r--r-- 1 root root  7293336 Aug  2 07:00 gop-live-point_txn_aggregator-data-2019-08-02_00006
-rw-r--r-- 1 root root  7728992 Aug  2 08:00 gop-live-point_txn_aggregator-data-2019-08-02_00007
-rw-r--r-- 1 root root  8154425 Aug  2 09:00 gop-live-point_txn_aggregator-data-2019-08-02_00008
-rw-r--r-- 1 root root  8036346 Aug  2 09:59 gop-live-point_txn_aggregator-data-2019-08-02_00009
-rw-r--r-- 1 root root  7940669 Aug  2 11:00 gop-live-point_txn_aggregator-data-2019-08-02_00010
-rw-r--r-- 1 root root  7901900 Aug  2 12:00 gop-live-point_txn_aggregator-data-2019-08-02_00011
-rw-r--r-- 1 root root  8501400 Aug  2 13:00 gop-live-point_txn_aggregator-data-2019-08-02_00012
-rw-r--r-- 1 root root  8024730 Aug  2 13:59 gop-live-point_txn_aggregator-data-2019-08-02_00013
-rw-r--r-- 1 root root  8413981 Aug  2 14:59 gop-live-point_txn_aggregator-data-2019-08-02_00014
-rw-r--r-- 1 root root  5899646 Aug  2 15:50 gop-live-point_txn_aggregator-data-2019-08-02_00015
lrwxrwxrwx 1 root root       51 Aug  2 15:00 gop-live-point_txn_aggregator-data_current -> gop-live-point_txn_aggregator-data-2019-08-02_00015

backup and rsync scripts

check_original_log.sh

#!/bin/bash
var_date=$1
var_src="/data/gop/live/primary/gop"
find ${var_src} | grep -i ${var_date}

check_backup_log.sh


#!/bin/bash
var_date=$1
var_src="/data/backup/"
find ${var_src} | grep -i ${var_date}

move_backup.sh

#!/bin/bash
var_date=\$1
var_path="/data/gop/live/primary/gop"
var_str="/data/backup"

# analyze log

find ${var_path} | grep -i ${var_date} > log/${var_date}.ori.out
cp log/${var_date}.ori.out log/${var_date}.mod.out
sed -i "s:/data/gop/live/primary/gop:${var_str}:g" log/\${var_date}.mod.out

# merge move action

paste -d "|" log/${var_date}.ori.out log/${var_date}.mod.out > log/\${var_date}.out

# move files

for i in `cat log/${var_date}.out`
do
var_a=`echo ${i} | cut -f1 -d"|"`
var_b=`echo ${i} | cut -f2 -d"|"`
mv ${var_a} ${var_b}
done

rsync_backup.sh


#!/bin/bash
var_src="/data/backup/"
var_des="rsync://10.71.12.61:873/backup/"
rsync -aP ${var_src} ${var_des}

clean_backup_log.sh


#!/bin/bash
var_date=$1
var_src="/data/backup/"
find ${var_src} | grep -i ${var_date} | xargs rm -f

log out samples

# 2019-05.ori.out
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-25.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-27.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-23.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-30.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-31.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-28.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-24.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-26.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-29.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-31.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-29.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-23.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-30.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-27.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-28.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-24.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-25.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-26.tgz

# 2019-05.mod.out
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-25.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-27.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-23.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-30.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-31.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-28.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-24.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-26.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-29.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-31.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-29.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-23.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-30.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-27.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-28.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-24.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-25.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-26.tgz

# 2019-05.out
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-25.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-25.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-27.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-27.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-23.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-23.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-30.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-30.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-31.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-31.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-28.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-28.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-24.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-24.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-26.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-26.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-29.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-29.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-31.tgz|/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-31.tgz

# 2019-05.rsync.out
nohup: ignoring input
sending incremental file list
./
gop-live-account.garena.com-access/
gop-live-account.garena.com-access/gop-live-account.garena.com-access-2019-05-01.tgz
^M         32,768   0%    0.00kB/s    0:00:00  ^M     75,551,185 100%  188.53MB/s    0:00:00 (xfr#1, ir-chk=1053/1441)
gop-live-account.garena.com-access/gop-live-account.garena.com-access-2019-05-02.tgz
^M         32,768   0%   81.01kB/s    0:15:24  ^M     74,896,153 100%   82.01MB/s    0:00:00 (xfr#2, ir-chk=1052/1441)

log cleanup exclude list

/data/nc_backup

gop-live-tcp_server
gop-live-sso_website
gop-live-app_point
gop-live-gop_vk_data_server
gop-live-gop_huawei_data_server
gop-live-gop_line_data_server
gop-live-gop_google_data_server
gop-live-gop_fb_data_server
gop-live-gop_data_server
gop-live-general-api_server
gop-live-payment_center
gop-live-msdk_api
gop-live-cron_autofix
gop-live-shell_api_server
gop-live-api_server
gop-staging-payment_center
gop-staging-msdk_api

create python script to combine multiple lines

data=open("file").readlines()
for n,line in enumerate(data):
    if line.startswith("line"):
       data[n] = "\n"+line.rstrip()
    else:
       data[n]=line.rstrip()
print ('|'.join(data))

# simple example
Englist
中文

Englist|中文

# can change line.startwith
line1
text1
text2
text3
line2
something1
something2

line1|text1|text2|text3|
line2|something1|something2

use find to filter the log

# use find command filter directory
find /data/nc_backup -type d > log_source
find /data/nc_backup -type d | egrep -v "gop-live-tcp_server|gop-live-sso_website|gop-live-app_point|gop-live-gop_vk_data_server|gop-live-gop_huawei_data_server|gop-live-gop_line_data_server|gop-live-gop_google_data_server|gop-live-gop_fb_data_server|gop-live-gop_data_server|gop-live-general-api_server|gop-live-payment_center|gop-live-msdk_api|gop-live-cron_autofix|gop-live-shell_api_server|gop-live-api_server|gop-staging-payment_center|gop-staging-msdk_api" > log_target
# remove the fist line
sed -i '1d' log_source
sed -i '1d' log_target

基于bash find命令执行log日志备份和清理的一次简单实践

前言

更新历史

find

How to use

log archive

backup and rsync scripts

log cleanup exclude list

王奥OX

引用和评论

字节跳动和TikTok内推