Linux7上安装Oracle RAC 19C一键配置脚本（后续补充其他版本）

Linux7上安装Oracle RAC 19C一键配置脚本（后续补充其他版本）

#!/bin/bash
## Linux7上安装Oracle数据库一键配置脚本
# 安装介质所在目录
# You must put the installation package in the /tmp/soft directory, Like this:
# /tmp/soft/
# ├── 11
# ├── 12
# ├── 19
# │   ├── LINUX.X64_193000_db_home.zip
# │   ├── LINUX.X64_193000_grid_home.zip
# │   ├── p34773504_190000_Linux-x86-64_OJVM+GI_RU_19.18.0.0.230117.zip
# │   ├── p6880880_190000_Linux-x86-64-OPatch_12.2.0.1.36.zip
# │   └── patch
# │       ├── 34762026
# │       ├── 34786990
# │       ├── PatchSearch.xml
# │       └── README.html
# └── compat-libstdc++-33-3.2.3-72.el7.x86_64.rpm
# 定义变量
# db_type | si/SI --单实例；rac/RAC --RAC(两节点)
export db_type=RAC
# 定义 Public IP、Virtual IP、SCAN IP 等业务网段 IP 地址信息
export PubIP1=192.168.11.31
export hostname1=Jeff-Test-RAC01
export PubIP2=192.168.11.32
export hostname2=Jeff-Test-RAC02
export VIP1=192.168.11.33
export VIP1NAME=Jeff-Test-VIP1
export VIP2=192.168.11.34
export VIP2NAME=Jeff-Test-VIP2
export SCANIP=192.168.11.35
export SCANNAME=Jeff-Test-SCAN
# Pub_Mask 业务网络子网掩码，数字 23 或者 255.255.254.0 均支持
export Pub_Mask=23
export scanPort=1525
export cluster_name=Jeff-Test
export ORA_SID_PREFIX=Jeff0
export DB_NAME=Jeff
export DB_UNIQUE_NAME=JeffTest
export ORA_SID1=Jeff01
export ORA_SID2=Jeff02
# 定义心跳网络 Private IP 信息
export PrivIP1=11.11.11.31
export Priv1NAME=Jeff-Test-Priv1
export PrivIP2=11.11.11.32
export Priv2NAME=Jeff-Test-Priv2
# Priv_Mask 心跳网络子网掩码，数字 23 或者 255.255.254.0 均支持
export Priv_Mask=24
# ntpserver 用于配置 chronyd 服务
export ntpserver=11.11.11.1
# oraver:安装的数据库版本:11、12、18、19、23
export oraver=19
# 指定root密码、oracle&grid密码、DB管理密码
export rootpw="Test01"
export ORAPWD="Oracle_19C"       # oracle && grid 设置相同密码
export db_admin_pwd="Oracle_19C" # 预设数据库管理使用的统一密码
# needcdb 指定是否Y(y)需要 CDB 容器数据库，needpdb 是否Y(y)需要创建 PDB（限制创建 1 个）,PDB_SID 指定 PDB 名字
export needcdb=Y
export needpdb=y
export PDB_SID=Jeff_PDB
# 指定内存、FRA参数设置，如下是很小的参数了，再小会导致安装失败
export sgasize=1800MB
export sga_max_size=2000MB
export pgasize=600MB
export recovery_size=18000MB
export processes=5000
export redoLogFileSize=2048
export useOMF=true
# 软件安装目录设置
export rootdir=/u01
export ORA_BASE=$rootdir/app/oracle
export ORA_HOME=$ORA_BASE/product/19.3.0/db_1
export ORA_INV=$rootdir/app/oraInventory
export GI_BASE=$rootdir/app/grid
export GI_HOME=$rootdir/app/19.3.0/grid
export logfile=/tmp/install_$(date +%F).log
# ASM 磁盘组分配
# mapper_dir 指定提供给ASM管理的磁盘识别路径,或者 /dev/mapper 等
export mapper_dir=/dev
# sd : 提供给ASM管理的磁盘前缀
export mapper_prefix=sd
# asm_ocr_disks: OCR\Voting 磁盘组,多组使用空格\,\;等字符隔开，建议命令行使用,，不容易混乱；不写前缀
export asm_ocr_disks="b c d"
export asm_ocr_dir=OCR
# asm_fra_disks:归档日志、闪回区 磁盘组
export asm_fra_disks="e f g"
export asm_fra_dir=FRA
# asm_data_disks:数据文件 磁盘组；默认 redo 存放 $asm_data_dir 和 $asm_fra_dir，需要另外配置则调整脚本
export asm_data_disks="h"
export asm_data_dir=DATA
# 指定是否需要格式化磁盘 Y(y)格式化，其他不执行
export need_format=y
# 安装介质指定
export softroot=/tmp/soft
export softdir=$softroot/$oraver
export db_zip_file=LINUX.X64_193000_db_home.zip
export grid_zip_file=LINUX.X64_193000_grid_home.zip
export opatch_zip_file=p6880880_190000_Linux-x86-64-OPatch_12.2.0.1.36.zip
export ru=34762026

############################################ 以上是主要参数定义，以下部分可不设置 ############################################
# 以下部分没有参数设置
split1() {
    echo -e "\033[1;40;33m\n\n  ===================================== $(date +'%F %T') =====================================\n$1\033[0m"
}
result_err() {
    echo -e "\033[5;31m\n      (*>﹏<*)(*>﹏<*)(*>﹏<*)  $1  (*>﹏<*)(*>﹏<*)(*>﹏<*)\n\033[0m"
}
result_ok() {
    echo -e "\n\033[32m      (*^_^*)(*^_^*)(*^_^*)   $1   (*^_^*)(*^_^*)(*^_^*)\033[0m\n"
}
# db_zip_file 默认值 LINUX.X64_193000_db_home.zip
if [ -z ${db_zip_file} ];then
    db_zip_file=LINUX.X64_193000_db_home.zip
fi
# grid_zip_file 默认值 LINUX.X64_193000_grid_home.zip
if [ -z ${grid_zip_file} ];then
    grid_zip_file=LINUX.X64_193000_grid_home.zip
fi
# opatch_zip_file 默认值 p6880880_190000_Linux-x86-64.zip，每次下载最新文件
if [ -z ${opatch_zip_file} ];then
    opatch_zip_file=p6880880_190000_Linux-x86-64.zip
fi
# 需要apply的RU，不 apply 则置空
export ru_zip_file=$softdir/p${ru}*.zip
# 识别 RU 对应的 zip 文件
export RUs=$softdir/patch/$ru
# 安装时需要同时安装的 RU 绝对路径
if [ -d $softroot ] && [ ! -d $softdir/patch ];then
    mkdir -p $softdir/patch
fi

echo "################    Begin : $(date +'%F %T')    ################">$logfile
chmod -R 777 $softroot
export db_install_file=$softdir/$db_zip_file
export grid_install_file=$softdir/$grid_zip_file
export opatch_install_file=$softdir/$opatch_zip_file
export upper_db_type=$(echo $db_type | tr [:lower:] [:upper:])
if [ ! -f ${db_install_file} ];then
    result_err "Error: DB install file not found!" >>$logfile
    exit 1
    elif [ ${upper_db_type} == "RAC" ] && [ ! -f ${grid_install_file} ];then
        result_err "You select Db type as RAC, But can not found GRID install file!">>$logfile
        exit 1
        elif [ -n ${ru} ] && [ ! -f ${ru_zip_file} ];then
            result_err "You Want to apply an RU, But not found zip file!">>$logfile
            exit 1
            if [ ! -f ${opatch_install_file} ];then
                result_err "You choose to apply RU and must update OPatch">>$logfile
                exit 1
            fi
fi
split1 "# 按照输入参数解析各文件压缩包名如下，The input installation media is analyzed as follows:
DB install file: $db_install_file
GRID install file: $grid_install_file
OPatch install file: $opatch_install_file
RU zip file: $ru_zip_file">>$logfile
split1 "# 按照输入参数解析各文件压缩包名如下，The input installation media is analyzed as follows:
DB install file: $db_install_file
GRID install file: $grid_install_file
OPatch install file: $opatch_install_file
RU zip file: $ru_zip_file"
sleep 5

# 如下实现掩码数字计算:255.255.254.0 转换 23
export tmp1=$(echo $Pub_Mask | grep -o '\.' | wc -l)
if [ $tmp1 -eq 3 ]; then
    export Pub_Mask=$(echo $Pub_Mask | sed 's/\./ + /g;s/255/8/g;s/254/7/g;s/252/6/g;s/248/5/g;s/240/4/g;s/224/3/g;s/192/2/g;s/128/1/g' | bc)
elif [ $tmp1 -gt 0 ]; then
    result_err "# 请输入正确的 Public 网卡子网掩码($Pub_Mask)">>$logfile
    exit 1
elif [ $Pub_Mask -lt 32 ] && [ $Pub_Mask -gt 1 ] && [ $tmp1 -eq 0 ]; then
    echo -e
else
    result_err "# 请输入正确的 Public($Pub_Mask) 网卡掩码位数字 2-31">>$logfile
    exit 1
fi
echo "# SCAN_IP:  $SCANIP/$Pub_Mask" >>$logfile

export tmp2=$(echo $Priv_Mask | grep -o '\.' | wc -l)
if [ $tmp2 -eq 3 ]; then
    export Priv_Mask=$(echo $Priv_Mask | sed 's/\./ + /g;s/255/8/g;s/254/7/g;s/252/6/g;s/248/5/g;s/240/4/g;s/224/3/g;s/192/2/g;s/128/1/g' | bc)
elif [ $tmp2 -gt 0 ]; then
    result_err "# 请输入正确的 Private 网卡子网掩码($Priv_Mask)">>$logfile
    exit 1
elif [ $Priv_Mask -lt 32 ] && [ $Priv_Mask -gt 1 ] && [ $tmp2 -eq 0 ]; then
    echo -e
else
    result_err "# 请输入正确的 Private($Priv_Mask) 网卡掩码位数字 2-31">>$logfile
    exit 1
fi
echo "# PrivIP1:  $PrivIP1/$Priv_Mask" >>$logfile

# 获取子网函数
get_ipgate() {
    #ip地址转换整数
    ipgate_int=$(echo $1 | awk -F'.' '{print $1*(256^3) + $2*(256^2) + $3*256 + $4}')
    #主机位:32 - $2
    #按位运算，右移，ip地址移除主机位，保留网络位
    #按位运算，左移，ip地址以0补全主机位
    ((ipgate = ((ipgate_int >> ((32 - $2)))) << ((32 - $2))))
    #获取子网:整数转换ip地址,整数右移取低8位
    echo $((ipgate >> 24 & 0xFF)).$((ipgate >> 16 & 0xFF)).$((ipgate >> 8 & 0xFF)).$((ipgate & 0xFF))
}
export scan_subnet=$(get_ipgate $SCANIP $Pub_Mask)
export priv_subnet=$(get_ipgate $PrivIP1 $Priv_Mask)
echo "# pub-subnet: $scan_subnet ; Priv-subnet: $priv_subnet">>$logfile
netcard_name() {
    findIP=$1
    # 按照网卡名行，开头为数字编号，进行过滤选择
    for devEnum in $(ip link show | grep ^[0-9]\\+: | awk -F ' ' {'print $2'}); do
        # 去掉后缀冒号
        devName=${devEnum%:}
        # 去掉可能的子网卡接口后缀
        devName=${devName%@*}
        if [ $(ip addr show $devName | grep -c $findIP) -gt 0 ]; then
            echo $devName
            break
        fi
    done
}
if [ "$(hostname)" == "$hostname1" ]; then
    pub_netcard=$(netcard_name $PubIP1)
    priv_netcard=$(netcard_name $PrivIP1)
    echo "# pub_netcard: $pub_netcard ; priv_netcard: $priv_netcard" >>$logfile
fi

# 提取 ASM 磁盘信息，转换后提供 for 循环使用(-->21 asm 配置)
export asm_ocr_disks=$(echo $asm_ocr_disks | sed 's/,/ /g' | sed 's/\./ /g' | sed 's/;/ /g' | sed 's/\// /g' | sed 's/\\/ /g')
export asm_fra_disks=$(echo $asm_fra_disks | sed 's/,/ /g' | sed 's/\./ /g' | sed 's/;/ /g' | sed 's/\// /g' | sed 's/\\/ /g')
export asm_data_disks=$(echo $asm_data_disks | sed 's/,/ /g' | sed 's/\./ /g' | sed 's/;/ /g' | sed 's/\// /g' | sed 's/\\/ /g')

# 确认临时安装文件存储目录 /tmp 存在，且可用空间大于 10GB
echo "# $(hostname) 检查 /tmp 目录存在，且可用空间大于 10GB:" >>$logfile
if [ ! -d /tmp ]; then
    echo "# /tmp Filesystem does not exist,Create it." >>$logfile
    mkdir /tmp
fi
export tmpsizemb=$(cd /tmp && df -PmT . | grep -v Filesystem | awk '{print $5}')
if [ $tmpsizemb -lt 10240 ]; then
    cd /tmp && df -PmT .
    result_err "# /tmp Filesystem has size $tmpsizemb MB, less than 10G, exit this scripts."
    result_err "# /tmp Filesystem has size $tmpsizemb MB, less than 10G, exit this scripts." >>$logfile
    exit 1
fi

# 0. yum 检查及安装依赖包
split1 "# 0. 检查 yum 安装源是否可用,并安装依赖包($(hostname)):"
if [ $(hostname) == "$hostname1" ]; then
    split1 "# 0. 检查 yum 安装源是否可用,并安装依赖包($(hostname)):" >>$logfile
    echo '#!/bin/bash' >/tmp/yumlist.sh
    cat >>/tmp/yumlist.sh <<yumlist
export yumlist=\$(yum list 2>/dev/null | wc -l)
if [ \$yumlist -gt 2000 ]; then
    export yumusable=1
    echo "# yum avaliable on \$(hostname)"
else
    export yumusable=0
    echo "# YUM Unavailable on \$(hostname), Try to mount /dev/cdrom make it available"
    if [ -h /dev/cdrom ];then
        echo "# /dev/cdrom exist, mount it to /mnt for use"
        mount /dev/cdrom /mnt/
        if [ ! -d /etc/yum.repos.d/bak ]; then
            mkdir /etc/yum.repos.d/bak
        fi
        mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/bak/
        echo "[Local_mnt]
name=Local_mnt
baseurl=file:///mnt
enabled=1
gpgcheck=0" >/etc/yum.repos.d/Local_mnt.repo
        yum -y clean all
        yum makecache
    else
        echo "# /dev/cdrom does not exist, do nothing."
    fi
    if [ \$(yum list 2>/dev/null | wc -l) -lt 2000 ];then
        echo "# Yum is not available on \$(hostname), try Failed"
        exit 1
    fi
fi
# Install need rpms
echo "# Installing need rpms on \$(hostname) ... ..."
sleep 5
yum install -y bc binutils compat-libcap1 compat-libstdc++-33 elfutils-libelf elfutils-libelf-devel fontconfig-devel glibc glibc-devel ksh libaio libaio-devel libXrender libXrender-devel libX11 libXau libXi libXtst libgcc libstdc++ libstdc++-devel libxcb make policycoreutils policycoreutils-python smartmontools sysstat net-tools nfs-utils python python-configshell python-rtslib python-six targetcli xorg-x11-xauth xorg-x11-fonts-* xorg-x11-font-utils xorg-x11-fonts-Type1 xorg-x11-apps xclock ncurses-devel redhat-lsb-core readline-devel vim-enhanced bzip2 chrony iotop xterm lsof tree expect gcc-c++ zip unzip 2>/dev/null
yumlist

    chmod 777 /tmp/yumlist.sh
    sh /tmp/yumlist.sh >>$logfile
    if [ $? -ne 0 ]; then
        result_err "# $PubIP1 执行 yumlist.sh, Failed"
        result_err "# $PubIP1 执行 yumlist.sh, Failed" >>$logfile
        cat /tmp/yumlist.sh >>$logfile
        exit 1
    else
        result_ok "# $PubIP1 执行 yumlist.sh, successful"
        result_ok "# $PubIP1 执行 yumlist.sh, successful" >>$logfile
    fi

    expect >>$logfile <<EOF
set timeout 100
spawn ssh -o StrictHostKeyChecking=no $PubIP2
expect {
        "(yes/no)" {send "yes\r"; exp_continue}
        "password:" {send "$rootpw\r"}
}
expect "root@*"  {send "if \[ ! -d /tmp \]; then mkdir /tmp; fi\r"}
expect "root@*"  {send "exit\r"}
expect eof
EOF

    # 远程 $hostname2 做相同脚本
    # 拷贝脚本到对端
    echo "# 远程 $hostname2 执行相同 yumlist 脚本"
    echo "# 远程 $hostname2 执行相同 yumlist 脚本" >>$logfile
    if [ $(grep $PubIP2 ~/.ssh/authorized_keys 2>/dev/null | wc -l) -lt 1 ];then
        expect >>$logfile <<EOF
set timeout 100
spawn scp /tmp/yumlist.sh $PubIP2:/tmp/
expect {
        "password:" {send "$rootpw\r"}
}
expect eof
EOF
    else
        scp /tmp/yumlist.sh $PubIP2:/tmp/
    fi

    if [ $? -ne 0 ]; then
        result_err "# scp yumlist.sh 到 $PubIP2, Failed"
        result_err "# scp yumlist.sh 到 $PubIP2, Failed" >>$logfile
        exit 1
    else
        result_ok "# scp yumlist.sh 到 $PubIP2, successful"
        result_ok "# scp yumlist.sh 到 $PubIP2, successful" >>$logfile
    fi
    # 执行脚本
    expect >>$logfile <<EOF
set timeout 100
spawn ssh $PubIP2
expect {
        "password:" {send "$rootpw\r"}
}
expect "root@*"  {send "sh /tmp/yumlist.sh\r"}
expect "root@*"  {send "exit\r"}
expect eof
EOF
    if [ $? -ne 0 ]; then
        result_err "# $PubIP2 执行 yumlist.sh, Failed"
        result_err "# $PubIP2 执行 yumlist.sh, Failed" >>$logfile
        exit 1
    else
        result_ok "# $PubIP2 执行 yumlist.sh, successful"
        result_ok "# $PubIP2 执行 yumlist.sh, successful" >>$logfile
    fi

else
    result_err "# Please run this script on $hostname1 node!"
    result_err "# Please run this script on $hostname1 node!" >>$logfile
    exit 1
fi

# 1. 配置hosts
split1 "# 1. 配置 $(hostname) /etc/hosts 文件:"
split1 "# 1. 配置 $(hostname) /etc/hosts 文件:" >>$logfile
hostnamectl set-hostname $hostname1 --static
cp /etc/hosts /etc/hosts_$(date +"%Y%m%d_%H%M%S")
sed -i "/IP\|^$\|$PubIP1\|$PubIP2\|$PrivIP1\|$PrivIP2\|$VIP1\|$VIP2\|$SCANIP/d" /etc/hosts
echo "# Public IP
$PubIP1 $hostname1
$PubIP2 $hostname2

# Private IP
$PrivIP1 $Priv1NAME
$PrivIP2 $Priv2NAME

# Virtual IP (VIP)
$VIP1 $VIP1NAME
$VIP2 $VIP2NAME

# SCAN IP
$SCANIP $SCANNAME" >>/etc/hosts
cat /etc/hosts >>$logfile

# 拷贝脚本到对端
echo "# 拷贝 /etc/hosts 文件到 $hostname2 :" 
echo "# 拷贝 /etc/hosts 文件到 $hostname2 :" >>$logfile
expect >>$logfile<< EOF
set timeout 100
spawn scp /etc/hosts $PubIP2:/etc/
expect {
        "password:" {send "$rootpw\r"}
}
expect eof
EOF
# 执行脚本
expect >>$logfile << EOF
set timeout 100
spawn ssh $PubIP2
expect {
        "password:" {send "$rootpw\r"}
}
expect "root@*"  {send "hostnamectl set-hostname $hostname2 --static\r"}
expect "root@*"  {send "exit\r"}
expect eof
EOF

# 2. 配置 OS 组及用户
# https://docs.oracle.com/en/database/oracle/oracle-database/19/cwsol/example-of-creating-minimal-groups-users-paths.html#GUID-103186A1-74E0-42A8-AC3D-15AF833DCB40
split1 "# 2. 配置 OS 组及用户 on $(hostname) :"
split1 "# 2. 配置 OS 组及用户 on $(hostname) :" >>$logfile
echo '#!/bin/bash' >/tmp/usergrp.sh
cat >>/tmp/usergrp.sh <<userg
echo "# \$(hostname) 配置 OS 组及用户:"
groupadd -g 54321 oinstall
groupadd -g 54322 dba
groupadd -g 54323 oper
groupadd -g 54324 backupdba
groupadd -g 54325 dgdba
groupadd -g 54326 kmdba
groupadd -g 54327 asmdba
groupadd -g 54328 asmoper
groupadd -g 54329 asmadmin
groupadd -g 54330 racdba
u1=\$(grep oracle /etc/passwd | wc -l)
u2=\$(grep oracle /etc/passwd | wc -l)
if [ \$u1 -ge 1 ]; then
    userdel oracle
fi
if [ \$u2 -ge 1 ]; then
    userdel grid
fi
useradd -g oinstall -G dba,oper,backupdba,dgdba,kmdba,asmdba,asmoper,asmadmin,racdba -u 1000 -m oracle
useradd -g oinstall -G dba,asmdba,asmoper,asmadmin,racdba -u 1001 -m grid
echo "$ORAPWD" | passwd --stdin oracle
echo "$ORAPWD" | passwd --stdin grid
mkdir -p $GI_HOME
mkdir -p $GI_BASE
mkdir -p $ORA_BASE
mkdir -p $ORA_HOME
mkdir -p $ORA_INV
chown -R grid:oinstall $rootdir
chown -R oracle:oinstall $ORA_BASE
chmod -R 775 $rootdir
userg
chmod +x /tmp/usergrp.sh; sh /tmp/usergrp.sh >>$logfile

# 拷贝脚本到 $hostname2 并执行
echo "# 拷贝脚本 /tmp/usergrp.sh 到 $hostname2 并执行:"
echo "# 拷贝脚本 /tmp/usergrp.sh 到 $hostname2 并执行:" >>$logfile
expect >>$logfile << EOF
set timeout 100
spawn scp /tmp/usergrp.sh $PubIP2:/tmp/
expect {
        "(yes/no)" {send "yes\r"; exp_continue}
        "password:" {send "$rootpw\r"}
}
expect eof
EOF
# 执行脚本
expect >>$logfile << EOF
set timeout 100
spawn ssh $PubIP2
expect {
        "password:" {send "$rootpw\r"}
}
expect "root@*"  {send "sh /tmp/usergrp.sh\r"}
expect "root@*"  {send "exit\r"}
expect eof
EOF

# 3. 设置环境变量配置文件
split1 "# 3. 设置环境变量配置文件 .bash_profile:"
split1 "# 3. 设置环境变量配置文件 .bash_profile:" >>$logfile
if [ "$(hostname)" == "$hostname1" ]; then
    # For root on node01
    sed -i '/ORACLE_HOME/d' /root/.bash_profile
    echo "export ORACLE_HOME=$GI_HOME
export GRID_HOME=\$ORACLE_HOME
export PATH=\$PATH:\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch" >>/root/.bash_profile
    echo -e "# root: \n$(cat /root/.bash_profile)\n" >>$logfile
    
    # For root on node02
    expect >>$logfile << EOF
set timeout 100
spawn scp /root/.bash_profile $PubIP2:/root/
expect {
        "password:" {send "$rootpw\r"}
}
expect eof
EOF

    # # For oracle on node01
    sed -i '/export\|alias\|umask\|^$/d' /home/oracle/.bash_profile
    cat >>/home/oracle/.bash_profile <<EOF
umask 022
export TMP=/tmp
export TMPDIR=\$TMP
export ORACLE_HOSTNAME=$hostname1
export ORACLE_BASE=$ORA_BASE
export ORACLE_HOME=$ORA_HOME
export ORACLE_SID=$ORA_SID1
export ORACLE_INVENTORY=$ORA_INV
export INVENTORY_LOCATION=\$ORACLE_INVENTORY
# export ORACLE_PDB_SID=$PDB_SID
export NLS_LANG="AMERICAN_AMERICA.AL32UTF8"
export NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"
export TNS_ADMIN=\$ORACLE_HOME/network/admin
export LD_LIBRARY_PATH=\$ORACLE_HOME/lib:/lib:/usr/lib
export PATH=.:\$PATH:\$HOME/bin:\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch
export THREADS_FLAG=native
# alias sqlplus='rlwrap sqlplus'
# alias rman='rlwrap rman'
alias ss='sqlplus / as sysdba'
EOF
    echo -e "# oracle: \n$(cat /home/oracle/.bash_profile)\n" >>$logfile
    
    # For oracle on node02
    expect >>$logfile << EOF
set timeout 100
spawn scp /home/oracle/.bash_profile oracle@$PubIP2:/home/oracle/
expect {
        "password:" {send "$ORAPWD\r"}
}
expect eof
EOF
# 执行脚本
expect >>$logfile << EOF
set timeout 100
spawn ssh oracle@$PubIP2
expect {
        "password:" {send "$ORAPWD\r"}
}
expect "oracle@*"  {send "sed -i 's/$ORA_SID1/$ORA_SID2/g' /home/oracle/.bash_profile\r"}
expect "oracle@*"  {send "sed -i 's/$hostname1/$hostname2/g' /home/oracle/.bash_profile\r"}
expect "oracle@*"  {send "exit\r"}
expect eof
EOF

    # # For grid on node01
    sed -i '/export\|alias\|umask\|^$/d' /home/grid/.bash_profile
    cat >>/home/grid/.bash_profile <<EOF
umask 022
export TMP=/tmp
export TMPDIR=\$TMP
export ORACLE_HOSTNAME=$hostname1
export ORACLE_SID=+ASM1
export ORACLE_BASE=$GI_BASE
export GRID_BASE=\$ORACLE_BASE
export ORACLE_HOME=$GI_HOME
export GRID_HOME=\$ORACLE_HOME
export ORACLE_INVENTORY=$ORA_INV
export INVENTORY_LOCATION=\$ORACLE_INVENTORY
export NLS_LANG="AMERICAN_AMERICA.AL32UTF8"
export NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"
export TNS_ADMIN=\$ORACLE_HOME/network/admin
export LD_LIBRARY_PATH=\$ORACLE_HOME/lib:/lib:/usr/lib
export PATH=.:\$PATH:\$HOME/bin:\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch
export THREADS_FLAG=native
# alias sqlplus='rlwrap sqlplus'
# alias rman='rlwrap rman'
alias ss='sqlplus / as sysasm'
EOF
    echo -e "# grid: \n$(cat /home/grid/.bash_profile)\n" >>$logfile
    # For grid on node02
    expect >>$logfile << EOF
set timeout 100
spawn scp /home/grid/.bash_profile grid@$PubIP2:/home/grid/
expect {
        "(yes/no)" {send "yes\r"; exp_continue}
        "password:" {send "$ORAPWD\r"}
}
expect eof
EOF
# 执行脚本
expect >>$logfile << EOF
set timeout 100
spawn ssh grid@$PubIP2
expect {
        "(yes/no)" {send "yes\r"; exp_continue}
        "password:" {send "$ORAPWD\r"}
}
expect "grid@*"  {send "sed -i 's/ASM1/ASM2/g' /home/grid/.bash_profile\r"}
expect "grid@*"  {send "sed -i 's/$hostname1/$hostname2/g' /home/grid/.bash_profile\r"}
expect "grid@*"  {send "exit\r"}
expect eof
EOF

else
    result_err "# Please execute this script on $hostname1,while this machine hostname is: $(hostname)"
    result_err "# Please execute this script on $hostname1,while this machine hostname is: $(hostname)" >>$logfile
    exit 1
fi
if [ $? -ne 0 ]; then
    result_err "# 设置环境变量, Failed"
    result_err "# 设置环境变量, Failed" >>$logfile
    exit 1
else
    result_ok "# 设置环境变量, successful"
    result_ok "# 设置环境变量, successful" >>$logfile
fi

# 4. Check OS and Configuration 检查系统配置
# $ISVirtual ≥ 1, is Virtual Machine
# export routeip=$(route -n | egrep -iv "Gateway|routing" | awk '{print $2}' | grep -v "0\.0\.0\.0")
# export routeprefix=$(echo $routeip | awk -F"." '{print $1"."$2}')
split1 "# 4. Check OS and Configuration 检查系统配置:"
split1 "# 4. Check OS and Configuration 检查系统配置:" >>$logfile
echo '#!/bin/bash'>/tmp/osinfo.sh
cat >>/tmp/osinfo.sh <<osinfo
export routeprefix=\$(route -n | egrep -iv "Gateway|routing" | awk '{print \$2}' | grep -v "0\.0\.0\.0" | awk -F"." '{print \$1"."\$2}')
export client_ip=\$(ip a | grep -w inet | grep -Eiv "secondary|virbr|docker|:|127.0.0.1|192.168.1\." | awk -F " +|/" '{ print \$3}' | grep \$routeprefix)
export ISVirtual=\$(dmidecode -s system-product-name | grep -i virtual | wc -l)
export MachineName=\$(dmidecode -s system-product-name)
if [ -f /etc/yum/pluginconf.d/subscription-manager.conf ]; then
    sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/subscription-manager.conf
    yum remove subscription-manager -y
else
    echo "subscription-manager.conf file not exist, Do nothing"
fi
if [ \$ISVirtual -eq 0 ]; then
    machine_type="物理机"
    export RAM_GB=\$(dmidecode | grep -P -A5 "Memory\s+Device" | grep GB | awk '{sum+=\$2} END {print sum}')
else
    machine_type="虚拟机"
    export RAM_GB=\$(free -g | grep Mem | awk '{print \$2}')
fi
cpus=\$(lscpu | grep ^'CPU(s):' | awk '{print \$2}' | grep -v ^\$)
sockets=\$(lscpu | grep ^'Socket(s):' | awk -F":" '{print \$2}' | grep -v ^\$ | sed 's/^[ ]*//g')
cores_per_sock=\$(lscpu | grep "Core(s) per socket:" | awk -F":" '{print \$2}' | grep -v ^\$ | sed 's/^[ ]*//g')
cpu_name=\$(lscpu | grep "Model name:" | awk -F":" '{print \$2}' | grep -v ^\$ | sed 's/^[ ]*//g')
Cores=\$((\${sockets} * \${cores_per_sock}))
echo "# \$(hostname) 硬件及OS Version信息采集:
    服务器IP:\$client_ip
    服务器类型:\$machine_type
    机器型号/平台:\$MachineName
    CPU name:\$cpu_name
    CPU(s):\$cpus
    Core(s):\$Cores
    Socket(s):\$sockets
    Core(s) per socket:\$cores_per_sock"
osinfo

chmod +x /tmp/osinfo.sh; sh /tmp/osinfo.sh >>$logfile
if [ $? -ne 0 ]; then
    result_err "# Check OS and Configuration 检查系统配置, Failed"
    result_err "# Check OS and Configuration 检查系统配置, Failed" >>$logfile
    exit 1
else
    result_ok "# Check OS and Configuration 检查系统配置, successful"
    result_ok "# Check OS and Configuration 检查系统配置, successful" >>$logfile
fi

# 拷贝到 $hostname2 并执行脚本
expect >>$logfile << EOF
set timeout 100
spawn scp /tmp/osinfo.sh $PubIP2:/tmp/
expect {
        "password:" {send "$rootpw\r"}
}
expect eof
EOF
# 执行脚本
expect >>$logfile << EOF
set timeout 100
spawn ssh $PubIP2
expect {
        "password:" {send "$rootpw\r"}
}
expect "root@*"  {send "sh /tmp/osinfo.sh\r"}
expect "root@*"  {send "exit\r"}
expect eof
EOF
if [ $? -ne 0 ]; then
    result_err "# $hostname2 检查系统配置, Failed"
    result_err "# $hostname2 检查系统配置, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 检查系统配置, successful"
    result_ok "# $hostname2 检查系统配置, successful" >>$logfile
fi

# 5. 解压软件，如果存在，仅节点1执行
split1 "# 5. 解压软件，如果存在，仅节点1执行:"
split1 "# 5. 解压软件，如果存在，仅节点1执行:" >>$logfile
if [ -f $grid_install_file ] && [ "$(hostname)" == "$hostname1" ]; then
    echo "# unziping grid install file to $GI_HOME..."
    echo "# unziping grid install file to $GI_HOME..." >>$logfile
    su - grid -c "unzip -qo $grid_install_file -d $GI_HOME/; mv $GI_HOME/OPatch $GI_HOME/OPatch.$(date +%Y%m%d%H%M); unzip -qo $opatch_install_file -d $GI_HOME/" >>$logfile
    echo -e "\n# unzip grid install file End!\n$(ls $GI_HOME/)\n" >>$logfile
fi
if [ -f $db_install_file ] && [ "$(hostname)" == "$hostname1" ]; then
    echo "# unziping DB install file to $ORA_HOME..."
    echo "# unziping DB install file to $ORA_HOME..." >>$logfile
    su - oracle -c "unzip -qo $db_install_file -d $ORA_HOME/; mv $ORA_HOME/OPatch $ORA_HOME/OPatch.$(date +%Y%m%d%H%M); unzip -qo $opatch_install_file -d $ORA_HOME/" >>$logfile
    echo -e "\n# unzip DB install file End!\n$(ls $ORA_HOME/)\n" >>$logfile
fi
if [ -f ${ru_zip_file} ] && [ "$(hostname)" == "$hostname1" ];then
    echo "# unziping RU zip file to $softdir/patch/..."
    echo "# unziping RU zip file to $softdir/patch/..." >>$logfile
    unzip -qo ${ru_zip_file} -d $softdir/patch/; chmod -R 777 $softdir/patch/
    echo -e "\n# unzip Ru zip file End!\n$(ls -l $softdir/patch/)\n" >>$logfile
fi
chown -R grid:oinstall $rootdir
chown -R oracle:oinstall $ORA_BASE
chmod -R 775 $rootdir

# 测试环境空间不太够用，$rootdir 和 /tmp 都在/ 下，判断目录不足时清理zip文件
tmpmnt=$(df -m /tmp/ | grep -v Filesystem | awk '{print $6}' | sed -e '/^$/d')
rootdirmnt=$(df -m $rootdir | grep -v Filesystem | awk '{print $6}' | sed -e '/^$/d')
rootdirleft=$(df -m $rootdir | grep -v Filesystem | awk '{print $4}' | sed -e '/^$/d')
if [ "$tmpmnt" == "$rootdirmnt" ] && [ $rootdirleft -le 20480 ];then
    echo "# $rootdirmnt 剩余空间太少，且同 zip 文件相同文件系统，删除 zip 文件"
    echo "# $rootdirmnt 剩余空间太少，且同 zip 文件相同文件系统，删除 zip 文件" >>$logfile
    rm -rf $softdir/*.zip >>$logfile
elif [ $rootdirleft -le 20480 ]; then
    echo "# $rootdirmnt 剩余空间太少, 请注意"
    echo "# $rootdirmnt 剩余空间太少, 请注意" >>$logfile
fi

# 6. 配置 ssh 互信
split1 "# 6. 配置 ssh 互信(oracle grid root),仅在解压文件的节点1执行:"
split1 "# 6. 配置 ssh 互信(oracle grid root),仅在解压文件的节点1执行:" >>$logfile
if [ -f $GI_HOME/oui/prov/resources/scripts/sshUserSetup.sh ]; then
    for i in oracle grid root; do
        if [ "$i" == "root" ]; then
            export pass=$rootpw
        else
            export pass=$ORAPWD
        fi
        expect >>$logfile << EOF
spawn $GI_HOME/oui/prov/resources/scripts/sshUserSetup.sh -user $i -hosts "$hostname1 $hostname2" -advanced -noPromptPassphrase
expect {
    "(yes/no)" {send "yes\r"; exp_continue}
    "*assword:" {send "$pass\r";exp_continue}
}
expect eof
EOF

        expect >>$logfile << EOF
spawn $GI_HOME/oui/prov/resources/scripts/sshUserSetup.sh -user $i -hosts "$Priv1NAME $Priv2NAME" -advanced -noPromptPassphrase
expect {
    "(yes/no)" {send "yes\r"; exp_continue}
    "*assword:" {send "$pass\r";exp_continue}
}
expect eof
EOF

        for g in $hostname1 $hostname2 $Priv1NAME $Priv2NAME; do
            echo "# Testing $g for $i" >>$logfile
            if [ "$i" == "root" ]; then
                ssh -l $i -o StrictHostKeyChecking=no $g date >>$logfile
            else
                su - $i -c "ssh -l $i -o StrictHostKeyChecking=no $g date" >>$logfile
            fi
            if [ $? -ne 0 ]; then
                result_err "# $i Test $g sshUserSetup Maybe Failed"
                result_err "# $i Test $g sshUserSetup Maybe Failed" >>$logfile
                exit 1
            else
                result_ok "# $i 配置 $g ssh 互信, successful"
                result_ok "# $i 配置 $g ssh 互信, successful" >>$logfile
            fi
        done
    done
fi

# 7. 检查 RAM\Swap\shm
# swap 不需要设置过大，最大32GB，Oracle建议是16GB(内存>=16GB)；eq 内存大小(小于16GB)
# shm 默认是内存的1/2，配置Oracle会不够用，调整为等同可用内存大小(本人未取总内存)
split1 "# 7. 检查 RAM\Swap\shm"
split1 "# 7. 检查 RAM\Swap\shm" >>$logfile
echo '#!/bin/bash'>/tmp/shm.sh
cat >>/tmp/shm.sh<<shm
echo "# 设置 /dev/shm:\$(hostname)"
shmsizegb=\$(echo "scale=0;\$(df -PmT /dev/shm | grep shm | awk '{print \$3}')/1024" | bc)
swapgb=\$(echo "scale=0;\$(grep SwapTotal /proc/meminfo | awk '{print \$2}')/1024/1024" | bc)
memgb=\$(echo "scale=0;\$(grep MemTotal /proc/meminfo | awk '{print \$2}')/1024/1024" | bc)
if [ \$shmsizegb -lt \$memgb ]; then
    echo "resize /dev/shm to \${memgb}G"
    sed -i '/\/dev\/shm/d' /etc/fstab
    echo "tmpfs /dev/shm tmpfs defaults,size=\${memgb}G,noatime,nodiratime 0 0" >>/etc/fstab
    mount -o remount /dev/shm
else
    echo "/dev/shm equal to MemTotal, Now \$(df -h | grep /dev/shm | awk '{print $2}')"
fi
shm
chmod +x /tmp/shm.sh;sh /tmp/shm.sh >>$logfile
if [ $? -ne 0 ]; then
    result_err "# 检查 RAM\Swap\shm, Failed"
    result_err "# 检查 RAM\Swap\shm, Failed" >>$logfile
    exit 1
else
    result_ok "# 检查 RAM\Swap\shm, successful"
    result_ok "# 检查 RAM\Swap\shm, successful" >>$logfile
fi
scp /tmp/shm.sh $hostname2:/tmp/ >>$logfile
echo "sh /tmp/shm.sh;exit" | ssh $hostname2 >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 检查 RAM\Swap\shm, Failed"
    result_err "# $hostname2 检查 RAM\Swap\shm, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 检查 RAM\Swap\shm, successful"
    result_ok "# $hostname2 检查 RAM\Swap\shm, successful" >>$logfile
fi

# 8. 禁用防火墙firewalld
split1 "# 8. $(hostname) 禁用防火墙 firewalld:" 
split1 "# 8. $(hostname) 禁用防火墙 firewalld:" >>$logfile
systemctl stop firewalld
systemctl disable firewalld
echo "# $hostname2 禁用防火墙 firewalld" >>$logfile
echo "systemctl stop firewalld;systemctl disable firewalld;exit" | ssh $hostname2 >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 禁用防火墙 firewalld, Failed"
    result_err "# $hostname2 禁用防火墙 firewalld, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 禁用防火墙 firewalld, successful"
    result_ok "# $hostname2 禁用防火墙 firewalld, successful" >>$logfile
fi

# 9. 禁用 SElinux
split1 "# 9. 禁用 SElinux:"
split1 "# 9. 禁用 SElinux:" >>$logfile
echo '#!/bin/bash' >/tmp/sel.sh
cat >>/tmp/sel.sh <<sel
echo "# 9 禁用 SElinux: \$(hostname)"
if [ \$(getenforce) != "Disabled" ]; then
    export SELINUX=\$(grep ^SELINUX= /etc/selinux/config)
    if [ \$SELINUX != "SELINUX=disabled" ]; then
        cp /etc/selinux/config /etc/selinux/config_\$(date +"%Y%m%d_%H%M%S") && sed -i 's/SELINUX\=enforcing/SELINUX\=disabled/g' /etc/selinux/config
        echo "Changed SElinux configuration, reboot later..."
    else
        echo "The SELINUX configuration file has been modified before."
    fi
    setenforce 0
else
    echo "SELINUX is already disabled, Do nothing."
fi
sel
chmod 777 /tmp/sel.sh;sh /tmp/sel.sh >>$logfile
if [ $? -ne 0 ]; then
    result_err "# 禁用 SElinux, Failed"
    result_err "# 禁用 SElinux, Failed" >>$logfile
    exit 1
else
    result_ok "# 禁用 SElinux, successful"
    result_ok "# 禁用 SElinux, successful" >>$logfile
fi
scp /tmp/sel.sh $hostname2:/tmp/ >>$logfile
echo "sh /tmp/sel.sh;exit" | ssh $hostname2 >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 禁用 SElinux, Failed"
    result_err "# $hostname2 禁用 SElinux, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 禁用 SElinux, successful"
    result_ok "# $hostname2 禁用 SElinux, successful" >>$logfile
fi

# 10. 禁用虚拟网卡
split1 "# 10. 禁用虚拟网卡 libvirtd on $hostname1:"
split1 "# 10. 禁用虚拟网卡 libvirtd on $hostname1:" >>$logfile
systemctl stop libvirtd
systemctl disable libvirtd
yum remove libvirt-libs
echo "# 10 禁用虚拟网卡，for $hostname2"
echo "# 10 禁用虚拟网卡，for $hostname2" >>$logfile
echo "systemctl stop libvirtd;systemctl disable libvirtd;yum remove libvirt-libs;exit"|ssh $hostname2 >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 禁用虚拟网卡, Failed"
    result_err "# $hostname2 禁用虚拟网卡, Failed" >>$logfile
else
    result_ok "# $hostname2 禁用虚拟网卡, successful"
    result_ok "# $hostname2 禁用虚拟网卡, successful" >>$logfile
fi

# 11. 禁用 zeroconf 路由
# Zero configuration networking(zeroconf)零配置网络服务规范，是一种用于自动生成可用IP地址的网络技术，不需要额外的手动配置和专属的配置服务器。Avahi 是Zeroconf规范的开源实现
split1 "# 11. 禁用 zeroconf 路由:"
split1 "# 11. 禁用 zeroconf 路由:" >>$logfile
# 配置 NOZEROCONF
sed -i '/NOZEROCONF/d' /etc/sysconfig/network
echo "NOZEROCONF=yes" >>/etc/sysconfig/network
# 禁用 Avahi
systemctl stop avahi-daemon.service
systemctl stop avahi-daemon.socket
systemctl disable avahi-daemon.service
systemctl disable avahi-daemon.socket
echo "# 禁用 zeroconf 路由 for $hostname2:"
echo "# 禁用 zeroconf 路由 for $hostname2:" >>$logfile
scp /etc/sysconfig/network $hostname2:/etc/sysconfig/ >>$logfile
echo "systemctl stop avahi-daemon.service;systemctl stop avahi-daemon.socket;systemctl disable avahi-daemon.service;systemctl disable avahi-daemon.socket;exit"|ssh $hostname2 >>$logfile

# 12. 禁用透明大页 THP 和 NUMA
split1 "# 12. 禁用透明大页 THP 和 NUMA:"
split1 "# 12. 禁用透明大页 THP 和 NUMA:" >>$logfile
echo '#!/bin/bash'>/tmp/thp.sh
cat >>/tmp/thp.sh<<thp
echo "# 12 \$(hostname) 禁用透明大页 THP 和 NUMA:"
export AnonHP=\$(grep AnonHugePages /proc/meminfo | awk '{print \$2}')
export TPHstatus=\$(cat /sys/kernel/mm/transparent_hugepage/enabled | awk -F"[][]" '{print \$2}')
if [ \$AnonHP -gt 0 ] || [ \$TPHstatus != "never" ]; then
    echo "Checked TPH is not completely disabled!"
    cp /etc/default/grub /etc/default/grub_\$(date +"%Y%m%d_%H%M%S") && sed -i 's/quiet"/quiet numa=off transparent_hugepage=never"/' /etc/default/grub
    if [ -d /sys/firmware/efi ]; then
        # On UEFI:
        echo "OS running use UEFI, Turn off TPH By grub /boot/efi/EFI/redhat/grub.cfg"
        cp /boot/efi/EFI/redhat/grub.cfg /boot/efi/EFI/redhat/grub.cfg_\$(date +"%Y%m%d_%H%M%S") && grub2-mkconfig -o /boot/efi/EFI/redhat/grub.cfg
    else
        # On BIOS:
        echo "OS running use BIOS, Turn off TPH By grub /boot/grub2/grub.cfg"
        cp /boot/grub2/grub.cfg /boot/grub2/grub.cfg_\$(date +"%Y%m%d_%H%M%S") && grub2-mkconfig -o /boot/grub2/grub.cfg
    fi
else
    echo "TPH already disabled."
fi
sed -i '/transparent_hugepage/d' /etc/rc.local
sed -i "/^fi\b/Id" /etc/rc.local
echo "if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
  echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
  echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi" >>/etc/rc.local
thp
chmod +x /tmp/thp.sh;sh /tmp/thp.sh >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname1 禁用透明大页 THP 和 NUMA, Failed"
    result_err "# $hostname1 禁用透明大页 THP 和 NUMA, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname1 禁用透明大页 THP 和 NUMA, successful"
    result_ok "# $hostname1 禁用透明大页 THP 和 NUMA, successful" >>$logfile
fi
scp /tmp/thp.sh $hostname2:/tmp/
echo "sh /tmp/thp.sh;exit" | ssh $hostname2 >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 禁用透明大页 THP 和 NUMA, Failed"
    result_err "# $hostname2 禁用透明大页 THP 和 NUMA, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 禁用透明大页 THP 和 NUMA, successful"
    result_ok "# $hostname2 禁用透明大页 THP 和 NUMA, successful" >>$logfile
fi

# 13. 配置时钟同步机制 chronyd
split1 "# 13. 配置 chronyd 时钟同步机制 chronyd:"
split1 "# 13. 配置 chronyd 时钟同步机制 chronyd:" >>$logfile
systemctl stop ntpd
systemctl disable ntpd
sed -i "/$ntpserver\|logchange\|syslog/Id"
sed -i "/server 0/i server $ntpserver iburst" /etc/chrony.conf
sed -i "/pool.ntp.org/d" /etc/chrony.conf
sed -i "/makestep/d" /etc/chrony.conf
echo "# Send message to syslog when clock adjustment is larger than 0.1 seconds.
logchange 0.1" >>/etc/chrony.conf
systemctl restart chronyd
systemctl enable chronyd
scp /etc/chrony.conf $hostname2:/etc/ >>$logfile
echo "systemctl stop ntpd;systemctl disable ntpd;systemctl restart chronyd;systemctl enable chronyd;exit"|ssh $hostname2 >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 配置 chronyd 时钟同步机制, Failed"
    result_err "# $hostname2 配置 chronyd 时钟同步机制, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 配置 chronyd 时钟同步机制, successful"
    result_ok "# $hostname2 配置 chronyd 时钟同步机制, successful" >>$logfile
fi

# 14. 修改登录验证
split1 "# 14. 修改 pam 登录验证:"
split1 "# 14. 修改 pam 登录验证:" >>$logfile
sed -i '/pam_limits.so/d' /etc/pam.d/login
echo "session required pam_limits.so
session required /lib64/security/pam_limits.so" >>/etc/pam.d/login
echo "# 14 修改 pam 登录验证:for $hostname2" >>$logfile
scp /etc/pam.d/login $hostname2:/etc/pam.d/ >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 修改 pam 登录验证, Failed"
    result_err "# $hostname2 修改 pam 登录验证, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 修改 pam 登录验证, successful"
    result_ok "# $hostname2 修改 pam 登录验证, successful" >>$logfile
fi

# 15. 修改 limit 资源限制
split1 "# 15. 修改 /etc/security/limits.conf limit 资源限制:"
split1 "# 15. 修改 /etc/security/limits.conf limit 资源限制:" >>$logfile
sed -i "/oracle\|grid\|oinstall/d" /etc/security/limits.conf
echo "# Add oinstall group limits
@oinstall     soft   nofile   16384
@oinstall     hard   nofile   65536
@oinstall     soft   nproc    16384
@oinstall     hard   nproc    65536
@oinstall     soft   stack    16384
@oinstall     hard   stack    32768
@oinstall     soft   memlock  unlimited
@oinstall     hard   memlock  unlimited" >>/etc/security/limits.conf
echo "# 修改 /etc/security/limits.conf limit 资源限制: for $hostname2"
echo "# 修改 /etc/security/limits.conf limit 资源限制: for $hostname2" >>$logfile
scp /etc/security/limits.conf $hostname2:/etc/security/ >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 修改 /etc/security/limits.conf limit 资源限制, Failed"
    result_err "# $hostname2 修改 /etc/security/limits.conf limit 资源限制, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 修改 /etc/security/limits.conf limit 资源限制, successful"
    result_ok "# $hostname2 修改 /etc/security/limits.conf limit 资源限制, successful" >>$logfile
fi

# 16. 修改 kernel 配置
split1 "# 16. 修改 /etc/sysctl.conf kernel 配置:"
split1 "# 16. 修改 /etc/sysctl.conf kernel 配置:" >>$logfile
sed -i '/[^fs.|^vm.|^kernel.|^net.|^$]/Id' /etc/sysctl.conf
memTotal=$(grep MemTotal /proc/meminfo | awk '{print $2}')
MEM=$(expr $(grep MemTotal /proc/meminfo | awk '{print $2}') \* 1024)
SHMALL=$(($MEM * 9 / 10 / $(getconf PAGE_SIZE)))
SHMMAX=$(($MEM * 9 / 10)) # 这里配置为 90% RAM大小
# shmmax(bytes) = shmmni(page size, default 4k) * shmall (page的个数)
if [ $SHMALL -lt 2097152 ]; then ## 2097152*4k/1024/1024=8G
    SHMALL=2097152
fi
SHMMAX=$((memTotal * 1024 - 1))
if [ "$SHMMAX" -lt 4294967295 ]; then ## 4294967295k/1024/1024=4095MB
    SHMMAX=4294967295
fi
echo "fs.file-max = 6815744
fs.aio-max-nr = 1048576
kernel.sem = 250 32000 100 128
kernel.shmmni = 4096
kernel.shmall = $SHMALL
kernel.shmmax = $SHMMAX
kernel.panic_on_oops = 1
net.core.rmem_default = 262144
net.core.rmem_max = 41943044
net.core.wmem_default = 262144
net.core.wmem_max = 4194304
net.ipv4.ip_local_port_range = 10000 65500
net.ipv4.conf.all.rp_filter = 2
net.ipv4.conf.default.rp_filter = 2
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv4.tcp_max_tw_buckets = 10000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_keepalive_time = 30
net.ipv4.tcp_keepalive_intvl = 10
net.ipv4.tcp_retries2 = 12
net.ipv4.ip_local_reserved_ports = 15400-15407,20050-20057
net.ipv4.tcp_rmem = 8192 250000 16777216
net.ipv4.tcp_wmem = 8192 250000 16777216
# vm.dirty_background_ratio = 5
# vm.dirty_ratio = 40
# vm.dirty_expire_centisecs = 500
# vm.dirty_writeback_centisecs = 100
# vm.min_free_kbytes= 1048576
vm.swappiness = 1" >>/etc/sysctl.conf
sed -i '/^$/Id' /etc/sysctl.conf
sysctl -p
sysctl --system >>$logfile
scp /etc/sysctl.conf $hostname2:/etc/ >>$logfile
echo "sysctl -p;sysctl --system;exit"|ssh $hostname2 >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 修改 /etc/sysctl.conf kernel 配置, Failed"
    result_err "# $hostname2 修改 /etc/sysctl.conf kernel 配置, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 修改 /etc/sysctl.conf kernel 配置, successful"
    result_ok "# $hostname2 修改 /etc/sysctl.conf kernel 配置, successful" >>$logfile
fi

# 17. 修改启动 runlevel
split1 "# 17. 修改启动 runlevel=3:"
split1 "# 17. 修改启动 runlevel=3:" >>$logfile
systemctl set-default multi-user.target
# systemctl get-default  #查看默认启动方式是什么，如果显示multi-user.target， 说明是默认命令行启动
# systemctl set-default graphical.target   #设置开机默认图形桌面启动
# systemctl set-default multi-user.target  #设置开机默认命令行启动
echo "systemctl set-default multi-user.target;exit"|ssh $hostname2 >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 修改启动 runlevel=3, Failed"
    result_err "# $hostname2 修改启动 runlevel=3, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 修改启动 runlevel=3, successful"
    result_ok "# $hostname2 修改启动 runlevel=3, successful" >>$logfile
fi

# 18. 设置ssh，防止登录过慢,启用X11 Forwing 转发(默认访问接收端6000端口
split1 "# 18. 设置 /etc/ssh/sshd_config，防止登录过慢,启用X11 Forwing 转发(默认访问接收端6000端口)"
split1 "# 18. 设置 /etc/ssh/sshd_config，防止登录过慢,启用X11 Forwing 转发(默认访问接收端6000端口)" >>$logfile
cp /etc/ssh/sshd_config /etc/ssh/sshd_config_$(date +"%Y%m%d_%H%M%S") && sed -i '/UseDNS no\|^AllowTcpForwarding yes\|X11UseLocalhost no/d' /etc/ssh/sshd_config
sed -i '/#UseDNS yes/a\UseDNS no' /etc/ssh/sshd_config
sed -i '/#AllowTcpForwarding yes/a\AllowTcpForwarding yes' /etc/ssh/sshd_config
sed -i '/#X11UseLocalhost yes/a\X11UseLocalhost no' /etc/ssh/sshd_config
egrep "UseDNS|Forwarding|X11" /etc/ssh/sshd_config >>$logfile
systemctl restart sshd
scp /etc/ssh/sshd_config $hostname2:/etc/ssh/ >>$logfile
echo "systemctl restart sshd;exit"|ssh $hostname2 >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 设置ssh，防止登录过慢,启用X11 Forwing 转发, Failed"
    result_err "# $hostname2 设置ssh，防止登录过慢,启用X11 Forwing 转发, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 设置ssh，防止登录过慢,启用X11 Forwing 转发, successful"
    result_ok "# $hostname2 设置ssh，防止登录过慢,启用X11 Forwing 转发, successful" >>$logfile
fi

# 19. 设置sudoers，授予oinstall组sudo ALL 权限
split1 "# 19. 设置 /etc/sudoers，授予oinstall组sudo ALL 权限:"
split1 "# 19. 设置 /etc/sudoers，授予oinstall组sudo ALL 权限:" >>$logfile
cp /etc/sudoers /etc/sudoers_$(date +"%Y%m%d_%H%M%S") && sed -i '/oinstall/d' /etc/sudoers && sed -i '/^%wheel/a\%oinstall  ALL=(ALL)       ALL' /etc/sudoers && grep oinstall /etc/sudoers
grep oinstall /etc/sudoers >>$logfile
scp /etc/sudoers $hostname2:/etc/ >>$logfile
if [ $? -ne 0 ]; then
    result_err "# $hostname2 设置sudoers，授予oinstall组sudo ALL 权限, Failed"
    result_err "# $hostname2 设置sudoers，授予oinstall组sudo ALL 权限, Failed" >>$logfile
    exit 1
else
    result_ok "# $hostname2 设置sudoers，授予oinstall组sudo ALL 权限, successful"
    result_ok "# $hostname2 设置sudoers，授予oinstall组sudo ALL 权限, successful" >>$logfile
fi

# 20. cvu 安装
split1 "# 20. cvu 1节点安装，并传送2节点安装:"
split1 "# 20. cvu 1节点安装，并传送2节点安装:" >>$logfile
if [ -f $GI_HOME/cv/rpm/cvuqdisk*.rpm ]; then
    rpm -ivh $GI_HOME/cv/rpm/cvuqdisk*.rpm >>$logfile
    echo "# 2节点安装" >>$logfile
    scp $GI_HOME/cv/rpm/cvuqdisk*.rpm $hostname2:/tmp/ >>$logfile
    ssh $hostname2 rpm -ivh /tmp/cvuqdisk*.rpm >>$logfile
else
    result_err "# not found package cvuqdisk*.rpm"
    result_err "# not found package cvuqdisk*.rpm" >>$logfile
    exit 1
fi
if [ $? -ne 0 ]; then
    result_err "# cvu 1节点安装，并传送2节点安装, Failed"
    result_err "# cvu 1节点安装，并传送2节点安装, Failed" >>$logfile
    exit 1
else
    result_ok "# cvu 1节点安装，并传送2节点安装, successful"
    result_ok "# cvu 1节点安装，并传送2节点安装, successful" >>$logfile
fi

# 21. asm 配置
# ASM 磁盘组分配
split1 "# 21. UDEV 配置磁盘,$hostname1:"
split1 "# 21. UDEV 配置磁盘,$hostname1:" >>$logfile
>/etc/udev/rules.d/99-oracle-asmdevices.rules
for i in $asm_ocr_disks; do
    echo "KERNEL==\"${mapper_prefix}?\",SUBSYSTEM==\"block\", PROGRAM==\"/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=$mapper_dir/\$name\",RESULT==\"$(/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=$mapper_dir/${mapper_prefix}$i)\", SYMLINK+=\"asm_ocr_disk$i\",OWNER=\"grid\",ACTION==\"add|change\", GROUP=\"asmadmin\",MODE=\"0660\"" >>/etc/udev/rules.d/99-oracle-asmdevices.rules
    ocr_tmp=$(echo "$ocr_tmp""/dev/asm_ocr_disk$i,")
done
export ocr_fgs=$(echo $ocr_tmp | sed 's/,\//,,\//g')
export ocr_diskgs=$(echo $ocr_tmp | sed 's/,$//')
for i in $asm_fra_disks; do
    echo "KERNEL==\"${mapper_prefix}?\",SUBSYSTEM==\"block\", PROGRAM==\"/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=$mapper_dir/\$name\",RESULT==\"$(/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=$mapper_dir/${mapper_prefix}$i)\", SYMLINK+=\"asm_fra_disk$i\",OWNER=\"grid\",ACTION==\"add|change\", GROUP=\"asmadmin\",MODE=\"0660\"" >>/etc/udev/rules.d/99-oracle-asmdevices.rules
    fra_tmp=$(echo "$fra_tmp""/dev/asm_fra_disk$i,")
done
export fra_fgs=$(echo $fra_tmp | sed 's/,\//,,\//g')
export fra_diskgs=$(echo $fra_tmp | sed 's/,$//')
for i in $asm_data_disks; do
    echo "KERNEL==\"${mapper_prefix}?\",SUBSYSTEM==\"block\", PROGRAM==\"/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=$mapper_dir/\$name\",RESULT==\"$(/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=$mapper_dir/${mapper_prefix}$i)\", SYMLINK+=\"asm_data_disk$i\",OWNER=\"grid\",ACTION==\"add|change\", GROUP=\"asmadmin\",MODE=\"0660\"" >>/etc/udev/rules.d/99-oracle-asmdevices.rules
    data_tmp=$(echo "$data_tmp""/dev/asm_data_disk$i,")
done
export data_fgs=$(echo $data_tmp | sed 's/,\//,,\//g')
export data_diskgs=$(echo $data_tmp | sed 's/,$//')
systemctl restart systemd-udev-trigger.service >>$logfile
udevadm control --reload-rules >>$logfile
udevadm trigger --type=devices >>$logfile
udevadm trigger --type=devices --action=change >>$logfile
sleep 10
ls -l /dev/ | grep asm >>$logfile
if [ $(ssh $hostname1 ls /dev/asm* | wc -l) -lt 5 ]; then
    result_err "# $hostname1 UDEV 配置磁盘, Failed"
    result_err "# $hostname1 UDEV 配置磁盘, Failed" >>$logfile
    echo "$(ssh $hostname1 ls /dev/asm*)"
    echo "$(ssh $hostname1 ls /dev/asm*)" >>$logfile
    exit 1
else
    result_ok "# $hostname1 UDEV 配置磁盘, successful"
    result_ok "# $hostname1 UDEV 配置磁盘, successful" >>$logfile
    echo "$(ssh $hostname1 ls /dev/asm*)"
    echo "$(ssh $hostname1 ls /dev/asm*)" >>$logfile
fi
echo "# $hostname2 UDEV 配置磁盘"
echo "# $hostname2 UDEV 配置磁盘" >>$logfile
scp /etc/udev/rules.d/99-oracle-asmdevices.rules $hostname2:/etc/udev/rules.d/ >>$logfile
echo "systemctl restart systemd-udev-trigger.service;udevadm control --reload-rules;udevadm trigger --type=devices;udevadm trigger --type=devices --action=change;sleep 10;exit"|ssh $hostname2 >>$logfile
if [ $(ssh $hostname2 ls /dev/asm* | wc -l) -lt 5 ]; then
    result_err "# $hostname2 UDEV 配置磁盘, Failed"
    echo "$(ssh $hostname2 ls /dev/asm*)"
    result_err "# $hostname2 UDEV 配置磁盘, Failed" >>$logfile
    echo "$(ssh $hostname2 ls /dev/asm*)" >>$logfile
    exit 1
else
    result_ok "# $hostname2 UDEV 配置磁盘, successful"
    echo "$(ssh $hostname2 ls /dev/asm*)"
    result_ok "# $hostname2 UDEV 配置磁盘, successful" >>$logfile
    echo "$(ssh $hostname2 ls /dev/asm*)" >>$logfile
fi

# 格式化磁盘
if [ $need_format == "y" ] || [ $need_format == "Y" ];then
    for i in $(ls -l /dev | grep grid | awk '{print $NF}'); do
        cnt=$(lsblk -b | grep -w $i | awk '{print $4/1024/1024}')
        echo -e "\n# 开始格式化 /dev/$i, 大小：${cnt}MB ..."
        echo "dd if=/dev/zero of=/dev/$i bs=1M count=$cnt"
        dd if=/dev/zero of=/dev/$i bs=1M count=$cnt
        echo "# 格式化 /dev/$i 完成"
    done
    if [ -z $i ];then
        result_err "# You need to format the disk, but did not execute it. Please check it."
        result_err "# You need to format the disk, but did not execute it. Please check it." >>$logfile
        exit 1
    fi
fi

# 22. 预检查
split1 "# 22. grid 安装预检查(仅节点1运行):"
split1 "# 22. grid 安装预检查(仅节点1运行):" >>$logfile
if [ "$(hostname)" == "$hostname1" ]; then
    echo '#!/usr/bin/expect -f' >/tmp/precheck.sh
    echo "spawn $GI_HOME/runcluvfy.sh stage -pre crsinst -n $hostname1,$hostname2 -fixup -verbose -method root
expect {
    \"*assword:\" {send \"$rootpw\r\";exp_continue}
    \"*again:\" {send \"$rootpw\r\"}
}
expect eof" >>/tmp/precheck.sh
    chmod 777 /tmp/precheck.sh
    su - grid -c "expect /tmp/precheck.sh" >>$logfile
    if [ $? -ne 0 ]; then
        result_err "# grid 安装预检查(仅节点1运行), Failed"
        result_err "# grid 安装预检查(仅节点1运行), Failed" >>$logfile
        exit 1
    else
        result_ok "# grid 安装预检查(仅节点1运行), successful"
        result_ok "# grid 安装预检查(仅节点1运行), successful" >>$logfile
    fi
fi


# 23. grid 软件安装
split1 "# 23. grid 软件安装(仅节点1运行):"
split1 "# 23. grid 软件安装(仅节点1运行):" >>$logfile

# $ sed -e 's/[\t ]\+$//' gridsetup.rsp | egrep -v "^$|^#|=$"
# oracle.install.responseFileVersion=/oracle/install/rspfmt_crsinstall_response_schema_v19.0.0
# INVENTORY_LOCATION=/u01/app/oraInventory
# oracle.install.option=CRS_CONFIG
# ORACLE_BASE=/u01/app/grid
# oracle.install.asm.OSDBA=asmdba
# oracle.install.asm.OSOPER=asmoper
# oracle.install.asm.OSASM=asmadmin
# oracle.install.crs.config.scanType=LOCAL_SCAN
# oracle.install.crs.config.gpnp.scanName=Jeff-Test-SCAN
# oracle.install.crs.config.gpnp.scanPort=1525
# oracle.install.crs.config.ClusterConfiguration=STANDALONE
# oracle.install.crs.config.configureAsExtendedCluster=false
# oracle.install.crs.config.clusterName=Jeff-Test
# oracle.install.crs.config.gpnp.configureGNS=false
# oracle.install.crs.config.autoConfigureClusterNodeVIP=false
# oracle.install.crs.config.clusterNodes=Ora-Jeff-Test-Prod-RAC1:Jeff-Test-VIP1:HUB,Ora-Jeff-Test-Prod-RAC2:Jeff-Test-VIP2:HUB
# oracle.install.crs.config.networkInterfaceList=Bond0:10.10.164.0:1,Bond1:11.11.11.0:5
# oracle.install.crs.configureGIMR=false
# oracle.install.asm.configureGIMRDataDG=false
# oracle.install.crs.config.storageOption=FLEX_ASM_STORAGE
# oracle.install.asm.SYSASMPassword=Oracle_19C
# oracle.install.crs.config.useIPMI=false
# oracle.install.asm.storageOption=ASM
# oracle.install.asm.diskGroup.name=OCR
# oracle.install.asm.diskGroup.redundancy=NORMAL
# oracle.install.asm.diskGroup.AUSize=4
# oracle.install.asm.diskGroup.disksWithFailureGroupNames=/dev/asm_ocr_diska,,/dev/asm_ocr_diskb,,/dev/asm_ocr_diskc,
# oracle.install.asm.diskGroup.disks=/dev/asm_ocr_diska,/dev/asm_ocr_diskb,/dev/asm_ocr_diskc
# oracle.install.asm.diskGroup.diskDiscoveryString=/dev/asm*
# oracle.install.asm.monitorPassword=Oracle_19C
# oracle.install.asm.gimrDG.AUSize=1
# oracle.install.asm.configureAFD=false
# oracle.install.crs.configureRHPS=false
# oracle.install.crs.config.ignoreDownNodes=false
# oracle.install.config.managementOption=NONE
# oracle.install.config.omsPort=0
# oracle.install.crs.rootconfig.executeRootScript=false

if [ -f ${GI_HOME}/gridSetup.sh ] && [ "$(hostname)" == "$hostname1" ]; then
    cat >/tmp/gridsetup.rsp <<gridsetuprsp
oracle.install.responseFileVersion=/oracle/install/rspfmt_crsinstall_response_schema_v19.0.0
INVENTORY_LOCATION=$ORA_INV
oracle.install.option=CRS_CONFIG
ORACLE_BASE=$GI_BASE
oracle.install.asm.OSDBA=asmdba
oracle.install.asm.OSOPER=asmoper
oracle.install.asm.OSASM=asmadmin
oracle.install.crs.config.scanType=LOCAL_SCAN
oracle.install.crs.config.gpnp.scanName=$SCANNAME
oracle.install.crs.config.gpnp.scanPort=$scanPort
oracle.install.crs.config.ClusterConfiguration=STANDALONE
oracle.install.crs.config.configureAsExtendedCluster=false
oracle.install.crs.config.clusterName=$cluster_name
oracle.install.crs.config.gpnp.configureGNS=false
oracle.install.crs.config.autoConfigureClusterNodeVIP=false
oracle.install.crs.config.clusterNodes=$hostname1:$VIP1NAME:HUB,$hostname2:$VIP2NAME:HUB
oracle.install.crs.config.networkInterfaceList=$pub_netcard:$scan_subnet:1,$priv_netcard:$priv_subnet:5
oracle.install.crs.configureGIMR=false
oracle.install.asm.configureGIMRDataDG=false
oracle.install.crs.config.storageOption=FLEX_ASM_STORAGE
oracle.install.asm.SYSASMPassword=$db_admin_pwd
oracle.install.crs.config.sharedFileSystemStorage.ocrLocations=
oracle.install.crs.config.useIPMI=false
oracle.install.asm.storageOption=ASM
oracle.install.asm.diskGroup.name=${asm_ocr_dir}
oracle.install.asm.diskGroup.redundancy=NORMAL
oracle.install.asm.diskGroup.AUSize=4
oracle.install.asm.diskGroup.disksWithFailureGroupNames=$ocr_fgs
oracle.install.asm.diskGroup.disks=$ocr_diskgs
oracle.install.asm.diskGroup.diskDiscoveryString=/dev/asm*
oracle.install.asm.monitorPassword=$db_admin_pwd
oracle.install.asm.gimrDG.AUSize=1
oracle.install.asm.configureAFD=false
oracle.install.crs.configureRHPS=false
oracle.install.crs.config.ignoreDownNodes=false
oracle.install.config.managementOption=NONE
oracle.install.config.omsPort=0
oracle.install.crs.rootconfig.executeRootScript=false
oracle.install.crs.rootconfig.configMethod=
gridsetuprsp
    chmod 777 /tmp/gridsetup.rsp
#    echo '#!/usr/bin/expect -f' >/tmp/grid_install.sh
#    echo "spawn su - grid -c \"${GI_HOME}/gridSetup.sh -silent -ignorePrereqFailure -responseFile /tmp/gridsetup.rsp -applyRU $RUs\"
#expect {
#    \"*assword:\" {send \"$rootpw\r\"}
#}
#expect eof" >>/tmp/grid_install.sh
#    chmod 777 /tmp/grid_install.sh
    echo -e "# 开始使用rsp响应文件安装grid软件\n命令: ${GI_HOME}/gridSetup.sh -silent -ignorePrereqFailure -waitforcompletion -responseFile /tmp/gridsetup.rsp -applyRU $RUs\n响应文件内容:\n$(cat /tmp/gridsetup.rsp)"
    echo -e "# 开始使用rsp响应文件安装grid软件\n命令: ${GI_HOME}/gridSetup.sh -silent -ignorePrereqFailure -waitforcompletion -responseFile /tmp/gridsetup.rsp -applyRU $RUs\n响应文件内容:\n$(cat /tmp/gridsetup.rsp)" >>$logfile
#    expect /tmp/grid_install.sh >>$logfile
    su - grid -c "${GI_HOME}/gridSetup.sh -silent -ignorePrereqFailure -waitforcompletion -responseFile /tmp/gridsetup.rsp -applyRU $RUs" >>$logfile
    sleep 10
    if [ $(tail -20 $logfile | grep orainstRoot | wc -l) -lt 1 ];then
        result_err "# ${GI_HOME}/gridSetup.sh -silent Failed"
        result_err "# ${GI_HOME}/gridSetup.sh -silent Failed" >>$logfile
        exit 1
    else
        result_ok "# Grid 软件安装 ${GI_HOME}/gridSetup.sh -silent successful"
        result_ok "# Grid 软件安装 ${GI_HOME}/gridSetup.sh -silent successful" >>$logfile
    fi
    if [ -f $ORA_INV/orainstRoot.sh ] && [ -f $GI_HOME/root.sh ];then
        echo "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 执行(节点1运行) " >>$logfile
        sh $ORA_INV/orainstRoot.sh >>$logfile
        sh $GI_HOME/root.sh >>$logfile
        if [ $? -ne 0 ]; then
            result_err "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 执行(节点1运行) Failed"
            result_err "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 执行(节点1运行) Failed" >>$logfile
            exit 1
        else
            result_ok "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 执行(节点1运行) successful"
            result_ok "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 执行(节点1运行) successful" >>$logfile
            if [ "$db_type" == "RAC" ] && [ -n $hostname2 ];then
                echo "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 开始执行(ssh节点2运行)"
                echo "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 开始执行(ssh节点2运行)" >>$logfile
                ssh $hostname2 $ORA_INV/orainstRoot.sh >>$logfile
                ssh $hostname2 $GI_HOME/root.sh >>$logfile
                if [ $? -ne 0 ]; then
                    result_err "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 执行(节点2运行) Failed"
                    result_err "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 执行(节点2运行) Failed" >>$logfile
                    exit 1
                else
                    result_ok "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 执行(节点2运行) successful"
                    result_ok "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 执行(节点2运行) successful" >>$logfile
                fi
            else
                result_err "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 节点2未运行"
                result_err "# $ORA_INV/orainstRoot.sh;$GI_HOME/root.sh 节点2未运行" >>$logfile
                exit
            fi
        fi
    else
        result_err "# $ORA_INV/orainstRoot.sh or $GI_HOME/root.sh file can not found"
        result_err "# $ORA_INV/orainstRoot.sh or $GI_HOME/root.sh file can not found" >>$logfile
        exit 1
    fi
    if [ $? -ne 0 ]; then
        result_err "# grid 软件安装 Failed"
        result_err "# grid 软件安装 Failed" >>$logfile
        exit 1
    else
        result_ok "# grid 软件安装 successful"
        result_ok "# grid 软件安装 successful" >>$logfile
    fi
else
    result_err "# ${GI_HOME}/gridSetup.sh not found" 
    result_err "# ${GI_HOME}/gridSetup.sh not found" >>$logfile
    exit 1
fi
# As a root user, execute the following script(s):
#         1. /u01/app/oraInventory/orainstRoot.sh       --> $ORA_INV/orainstRoot.sh
#         2. /u01/app/19.3.0/grid/root.sh               --> $GI_HOME/root.sh
# 
# Execute /u01/app/oraInventory/orainstRoot.sh on the following nodes:
# [Jeff-Test-RAC01, Jeff-Test-RAC02]
# Execute /u01/app/19.3.0/grid/root.sh on the following nodes:
# [Jeff-Test-RAC01, Jeff-Test-RAC02]
# 
# Run the script on the local node first. After successful completion, you can start the script in parallel on all other nodes.
# 
# Successfully Setup Software with warning(s).
# As install user, execute the following command to complete the configuration.
#         /u01/app/19.3.0/grid/gridSetup.sh -executeConfigTools -responseFile /tmp/gridsetup.rsp [-silent]

# 24. ASM 磁盘组添加
split1 "# 24. ASM 磁盘组添加 ${asm_fra_dir} \ ${asm_data_dir}(仅节点1运行):"
split1 "# 24. ASM 磁盘组添加 ${asm_fra_dir} \ ${asm_data_dir}(仅节点1运行):" >>$logfile
if [ -n $fra_diskgs ]; then
    echo -e "# 添加 ${asm_fra_dir} 执行命令:\nsu - grid -c \"asmca -silent -sysAsmPassword $db_admin_pwd -createDiskGroup -diskString '/dev/asm*' -diskGroupName ${asm_fra_dir} -diskList '$fra_diskgs' -redundancy EXTERNAL -au_size 4\""
    echo -e "# 添加 ${asm_fra_dir} 执行命令:\nsu - grid -c \"asmca -silent -sysAsmPassword $db_admin_pwd -createDiskGroup -diskString '/dev/asm*' -diskGroupName ${asm_fra_dir} -diskList '$fra_diskgs' -redundancy EXTERNAL -au_size 4\"" >>$logfile
    su - grid -c "asmca -silent -sysAsmPassword $db_admin_pwd -createDiskGroup -diskString '/dev/asm*' -diskGroupName ${asm_fra_dir} -diskList '$fra_diskgs' -redundancy EXTERNAL -au_size 4" >>$logfile
    if [ $? -ne 0 ]; then
        result_err "# ASM 磁盘组添加 ${asm_fra_dir}, Failed"
        result_err "# ASM 磁盘组添加 ${asm_fra_dir}, Failed" >>$logfile
        exit 1
    else
        result_ok "# ASM 磁盘组添加 ${asm_fra_dir}, successful"
        result_ok "# ASM 磁盘组添加 ${asm_fra_dir}, successful" >>$logfile
    fi
else
    echo "# No ${asm_fra_dir} Disk input, \$$fra_diskgs:$$fra_diskgs"
fi
if [ -n $data_diskgs ]; then
    echo -e "\n# 添加 ${asm_data_dir} 执行命令:\nsu - grid -c \"asmca -silent -sysAsmPassword $db_admin_pwd -createDiskGroup -diskString '/dev/asm*' -diskGroupName ${asm_data_dir} -diskList '$data_diskgs' -redundancy EXTERNAL -au_size 4\""
    echo -e "\n# 添加 ${asm_data_dir} 执行命令:\nsu - grid -c \"asmca -silent -sysAsmPassword $db_admin_pwd -createDiskGroup -diskString '/dev/asm*' -diskGroupName ${asm_data_dir} -diskList '$data_diskgs' -redundancy EXTERNAL -au_size 4\"" >>$logfile
    su - grid -c "asmca -silent -sysAsmPassword $db_admin_pwd -createDiskGroup -diskString '/dev/asm*' -diskGroupName ${asm_data_dir} -diskList '$data_diskgs' -redundancy EXTERNAL -au_size 4" >>$logfile
    if [ $? -ne 0 ]; then
        result_err "# ASM 磁盘组添加 ${asm_data_dir}, Failed"
        result_err "# ASM 磁盘组添加 ${asm_data_dir}, Failed" >>$logfile
        exit 1
    else
        result_ok "# ASM 磁盘组添加 ${asm_data_dir}, successful"
        result_ok "# ASM 磁盘组添加 ${asm_data_dir}, successful" >>$logfile
    fi
else
    echo "# No ${asm_data_dir} Disk input, \$data_diskgs:$data_diskgs"
fi


# 25. DB 软件安装
split1 "# 25. DB 软件安装(仅节点1运行):"
split1 "# 25. DB 软件安装(仅节点1运行):" >>$logfile
# $ sed -e 's/[\t ]\+$//' db_install_manual.rsp | egrep -v "^$|^#|=$"
# oracle.install.responseFileVersion=/oracle/install/rspfmt_dbinstall_response_schema_v19.0.0
# oracle.install.option=INSTALL_DB_SWONLY
# UNIX_GROUP_NAME=oinstall
# INVENTORY_LOCATION=/u01/app/oraInventory
# ORACLE_BASE=/u01/app/oracle
# oracle.install.db.InstallEdition=EE
# oracle.install.db.OSDBA_GROUP=dba
# oracle.install.db.OSOPER_GROUP=oper
# oracle.install.db.OSBACKUPDBA_GROUP=backupdba
# oracle.install.db.OSDGDBA_GROUP=dgdba
# oracle.install.db.OSKMDBA_GROUP=kmdba
# oracle.install.db.OSRACDBA_GROUP=racdba
# oracle.install.db.rootconfig.executeRootScript=false
# oracle.install.db.CLUSTER_NODES=ora-Jeff-Test-prod-rac1,ora-Jeff-Test-prod-rac2
# oracle.install.db.config.starterdb.type=GENERAL_PURPOSE
# oracle.install.db.ConfigureAsContainerDB=false
# oracle.install.db.config.starterdb.memoryOption=false
# oracle.install.db.config.starterdb.installExampleSchemas=false
# oracle.install.db.config.starterdb.managementOption=DEFAULT
# oracle.install.db.config.starterdb.omsPort=0
# oracle.install.db.config.starterdb.enableRecovery=false
cat >/tmp/db_install.rsp<<dbinst
oracle.install.responseFileVersion=/oracle/install/rspfmt_dbinstall_response_schema_v19.0.0
oracle.install.option=INSTALL_DB_SWONLY
UNIX_GROUP_NAME=oinstall
INVENTORY_LOCATION=$ORA_INV
ORACLE_HOME=$ORA_HOME
ORACLE_BASE=$ORA_BASE
oracle.install.db.InstallEdition=EE
oracle.install.db.OSDBA_GROUP=dba
oracle.install.db.OSOPER_GROUP=oper
oracle.install.db.OSBACKUPDBA_GROUP=backupdba
oracle.install.db.OSDGDBA_GROUP=dgdba
oracle.install.db.OSKMDBA_GROUP=kmdba
oracle.install.db.OSRACDBA_GROUP=racdba
oracle.install.db.CLUSTER_NODES=$hostname1,$hostname2
oracle.install.db.config.starterdb.type=GENERAL_PURPOSE
oracle.install.db.isRACOneInstall=false
oracle.install.db.rac.serverpoolCardinality=0
oracle.install.db.ConfigureAsContainerDB=false
# oracle.install.db.config.starterdb.memoryOption=false
# oracle.install.db.config.starterdb.installExampleSchemas=false
# oracle.install.db.config.starterdb.managementOption=DEFAULT
# oracle.install.db.config.starterdb.omsPort=0
# oracle.install.db.config.starterdb.enableRecovery=false
oracle.install.db.rootconfig.executeRootScript=false
SECURITY_UPDATES_VIA_MYORACLESUPPORT=false
DECLINE_SECURITY_UPDATES=true
dbinst
chmod 777 /tmp/db_install.rsp
if [ -f $ORA_HOME/runInstaller ]; then
    echo -e "# 执行 DB 软件安装命令:\nsu - oracle -c \"$ORA_HOME/runInstaller -silent -force -noconfig -ignorePrereq -responseFile /tmp/db_install.rsp -applyRU $RUs\""
    echo -e "# 执行 DB 软件安装命令:\nsu - oracle -c \"$ORA_HOME/runInstaller -silent -force -noconfig -ignorePrereq -responseFile /tmp/db_install.rsp -applyRU $RUs\"" >>$logfile
    su - oracle -c "$ORA_HOME/runInstaller -silent -force -noconfig -ignorePrereq -responseFile /tmp/db_install.rsp -applyRU $RUs" >>$logfile
    sleep 10
    if [ $(tail -20 $logfile | grep root.sh | wc -l) -lt 1 ];then
        result_err "# $ORA_HOME/runInstaller -silent Failed"
        result_err "# $ORA_HOME/runInstaller -silent Failed" >>$logfile
        exit 1
    else
        result_ok "# $ORA_HOME/runInstaller -silent successful"
        result_ok "# $ORA_HOME/runInstaller -silent successful" >>$logfile
    fi
    echo "# root 执行脚本 $ORA_HOME/root.sh (节点1运行)"
    echo "# root 执行脚本 $ORA_HOME/root.sh (节点1运行)" >>$logfile
    $ORA_HOME/root.sh >>$logfile
    if [ $? -ne 0 ]; then
        result_err "# root 执行脚本 $ORA_HOME/root.sh (节点1运行) Failed"
        result_err "# root 执行脚本 $ORA_HOME/root.sh (节点1运行) Failed" >>$logfile
        exit 1
    else
        result_ok "# root 执行脚本 $ORA_HOME/root.sh (节点1运行) successful"
        result_ok "# root 执行脚本 $ORA_HOME/root.sh (节点1运行) successful" >>$logfile
        if [ "$db_type" == "RAC" ] && [ -n $hostname2 ]; then
            echo -e "\n# root 执行脚本 $ORA_HOME/root.sh (节点2运行)"
            echo -e "\n# root 执行脚本 $ORA_HOME/root.sh (节点2运行)" >>$logfile
            $ORA_HOME/root.sh >>$logfile
        fi
    fi
    if [ $? -ne 0 ]; then
        result_err "# DB 软件安装 Failed"
        result_err "# DB 软件安装 Failed" >>$logfile
        exit 1
    else
        result_ok "# DB 软件安装 successful"
        result_ok "# DB 软件安装 successful" >>$logfile
    fi
else
    result_err "# $ORA_HOME/runInstaller file not found"
    result_err "# $ORA_HOME/runInstaller file not found" >>$logfile
    exit 1
fi


# 26. DBCA 创建数据库
split1 "# 26 DBCA 创建数据库(仅节点1运行):"
split1 "# 26 DBCA 创建数据库(仅节点1运行):" >>$logfile
# $ sed -e 's/[\t ]\+$//' dbca_MANUAL.rsp | egrep -v "^$|^#|=$"
# responseFileVersion=/oracle/assistants/rspfmt_dbca_response_schema_v12.2.0
# gdbName=JeffTest
# sid=Jeff0
# databaseConfigType=RAC
# policyManaged=false
# createServerPool=false
# force=false
# createAsContainerDatabase=true
# numberOfPDBs=1
# pdbName=Jeff_pdb
# useLocalUndoForPDBs=true
# nodelist=jeff-test-rac01,jeff-test-rac02
# templateName=/u01/app/oracle/product/19.3.0/db_1/assistants/dbca/templates/New_Database.dbt
# emExpressPort=5500
# runCVUChecks=TRUE
# omsPort=0
# dvConfiguration=false
# olsConfiguration=false
# datafileJarLocation={ORACLE_HOME}/assistants/dbca/templates/
# datafileDestination=+DATA/{DB_UNIQUE_NAME}/
# recoveryAreaDestination=+FRA
# storageType=ASM
# diskGroupName=+DATA/{DB_UNIQUE_NAME}/
# recoveryGroupName=+FRA
# characterSet=AL32UTF8
# nationalCharacterSet=UTF8
# registerWithDirService=false
# skipListenerRegistration=true
# variables=ORACLE_BASE_HOME=/u01/app/oracle/product/19.3.0/db_1,DB_UNIQUE_NAME=JeffTest,ORACLE_BASE=/u01/app/oracle,PDB_NAME=,DB_NAME=Jeff,ORACLE_HOME=/u01/app/oracle/product/19.3.0/db_1,SID=Jeff0
# initParams=Jeff01.undo_tablespace=UNDOTBS1,Jeff02.undo_tablespace=UNDOTBS2,enable_pluggable_database=true,sga_target=1824MB,db_block_size=8192BYTES,cluster_database=true,standby_file_management=AUTO,family:dw_helper.instance_mode=read-only,log_archive_dest_1='LOCATION=+FRA',nls_language=AMERICAN,filesystemio_options=SETALL,dispatchers=(PROTOCOL=TCP) (SERVICE=Jeff0XDB),diagnostic_dest={ORACLE_BASE},remote_login_passwordfile=exclusive,db_create_file_dest=+DATA/{DB_UNIQUE_NAME}/,db_create_online_log_dest_2=+FRA,db_create_online_log_dest_1=+DATA,nls_date_format="YYYY/MM/DD HH24:MI:SS",parallel_force_local=TRUE,audit_file_dest={ORACLE_BASE}/admin/{DB_UNIQUE_NAME}/adump,processes=5000,pga_aggregate_target=609MB,Jeff01.thread=1,Jeff02.thread=2,nls_territory=AMERICA,undo_retention=10800,session_cached_cursors=200,local_listener=-oraagent-dummy-,db_recovery_file_dest_size=18000MB,db_unique_name=JeffTest,optimizer_adaptive_plans=FALSE,open_cursors=1000,control_file_record_keep_time=31,log_archive_format=arch_%t_%s_%r.arc,compatible=19.0.0,sga_max_size=1000MB,db_name=Jeff,archive_lag_target=1800,result_cache_max_size=0MB,db_files=2000,enable_ddl_logging=TRUE,Jeff01.instance_number=1,Jeff02.instance_number=2,db_recovery_file_dest=+FRA,audit_trail=db
# sampleSchema=false
# memoryPercentage=40
# databaseType=MULTIPURPOSE
# automaticMemoryManagement=false
# totalMemory=0
cat >/tmp/dbca.rsp<<dbcarsp
responseFileVersion=/oracle/assistants/rspfmt_dbca_response_schema_v12.2.0
gdbName=$DB_UNIQUE_NAME
sid=$ORA_SID_PREFIX
databaseConfigType=RAC
RACOneNodeServiceName=
policyManaged=false
createServerPool=false
force=false
createAsContainerDatabase=true
numberOfPDBs=1
pdbName=$PDB_SID
useLocalUndoForPDBs=true
pdbAdminPassword=$ORAPWD
nodelist=$hostname1,$hostname2
templateName=$ORA_HOME/assistants/dbca/templates/New_Database.dbt
sysPassword=$ORAPWD
systemPassword=$ORAPWD
serviceUserPassword=$ORAPWD
emConfiguration=
emExpressPort=5500
runCVUChecks=TRUE
dbsnmpPassword=$ORAPWD
omsPort=0
dvConfiguration=false
olsConfiguration=false
datafileJarLocation={ORACLE_HOME}/assistants/dbca/templates/
datafileDestination=+${asm_data_dir}/{DB_UNIQUE_NAME}/
recoveryAreaDestination=+${asm_fra_dir}
storageType=ASM
diskGroupName=+${asm_data_dir}/{DB_UNIQUE_NAME}/
asmsnmpPassword=$ORAPWD
recoveryGroupName=+${asm_fra_dir}
characterSet=AL32UTF8
nationalCharacterSet=UTF8
registerWithDirService=false
skipListenerRegistration=true
variables=ORACLE_BASE_HOME=$ORA_HOME,DB_UNIQUE_NAME=$DB_UNIQUE_NAME,ORACLE_BASE=$ORA_BASE,PDB_NAME=$PDB_SID,DB_NAME=$DB_NAME,ORACLE_HOME=$ORA_HOME,SID=$ORA_SID_PREFIX
initParams=$ORA_SID1.undo_tablespace=UNDOTBS1,$ORA_SID2.undo_tablespace=UNDOTBS2,enable_pluggable_database=true,sga_target=$sgasize,db_block_size=8192BYTES,cluster_database=true,standby_file_management=AUTO,family:dw_helper.instance_mode=read-only,log_archive_dest_1='LOCATION=+${asm_fra_dir}',nls_language=AMERICAN,filesystemio_options=SETALL,dispatchers=(PROTOCOL=TCP) (SERVICE=${ORA_SID_PREFIX}XDB),diagnostic_dest={ORACLE_BASE},remote_login_passwordfile=exclusive,db_create_file_dest=+${asm_data_dir}/{DB_UNIQUE_NAME}/,db_create_online_log_dest_2=+${asm_fra_dir},db_create_online_log_dest_1=+${asm_data_dir},nls_date_format="YYYY/MM/DD HH24:MI:SS",parallel_force_local=TRUE,audit_file_dest={ORACLE_BASE}/admin/{DB_UNIQUE_NAME}/adump,processes=$processes,pga_aggregate_target=$pgasize,$ORA_SID1.thread=1,$ORA_SID2.thread=2,nls_territory=AMERICA,undo_retention=10800,session_cached_cursors=200,local_listener=-oraagent-dummy-,db_recovery_file_dest_size=$recovery_size,db_unique_name=$DB_UNIQUE_NAME,optimizer_adaptive_plans=FALSE,open_cursors=1000,max_dump_file_size='1024m',control_file_record_keep_time=31,log_archive_format=arch_%t_%s_%r.arc,compatible=19.0.0,sga_max_size=$sga_max_size,db_name=$DB_NAME,archive_lag_target=1800,result_cache_max_size=0MB,db_files=2000,enable_ddl_logging=TRUE,$ORA_SID1.instance_number=1,$ORA_SID2.instance_number=2,db_recovery_file_dest=+${asm_fra_dir},audit_trail=db
sampleSchema=false
memoryPercentage=40
databaseType=MULTIPURPOSE
automaticMemoryManagement=false
totalMemory=0
dbcarsp

chmod 777 /tmp/dbca.rsp
if [ $needcdb != "y" ] && [ $needcdb != "Y" ];then
    echo "# Do not need CDB"
    sed -i 's/createAsContainerDatabase=true/createAsContainerDatabase=false/g' /tmp/dbca.rsp
elif [ $needpdb != "y" ] && [ $needpdb != "Y" ];then
    sed -i "s/numberOfPDBs=1/numberOfPDBs=0/g" /tmp/dbca.rsp
    sed -i "s/PDB_NAME=$PDB_SID,//g" /tmp/dbca.rsp
    sed -i "/PDB_NAME=$PDB_SID/d" /tmp/dbca.rsp
fi
cat /tmp/dbca.rsp >>$logfile
echo "# 开始 DBCA 建库"
echo "# 开始 DBCA 建库" >>$logfile
su - oracle -c "dbca -silent -createDatabase -ignorePrereqFailure -responseFile /tmp/dbca.rsp -redoLogFileSize $redoLogFileSize -useOMF $useOMF -enableArchive true -emConfiguration NONE"
if [ $? -ne 0 ];then
    result_err "# DBCA 建库失败,Failed"
    result_err "# DBCA 建库失败,Failed" >>$logfile
    exit 1
else
    result_ok "# DBCA 建库成功,successful"
    result_ok "# DBCA 建库成功,successful" >>$logfile
fi