使用 Docker 搭建 Hadoop 集群

创建容器

# 创建自定义网络
docker network create --subnet=172.18.0.0/16 mynetwork

# 启动容器
# cluster-master
docker run -itd --privileged  --name cluster-master -h cluster-master -p 18088:18088 -p 9870:9870 --net mynetwork --ip 172.18.0.2 centos:latest /usr/sbin/init
# cluster-slaves
docker run -itd --privileged --name cluster-slave1 -h cluster-slave1 --net mynetwork --ip 172.18.0.3 centos:latest /usr/sbin/init
docker run -itd --privileged --name cluster-slave2 -h cluster-slave2 --net mynetwork --ip 172.18.0.4 centos:latest /usr/sbin/init
docker run -itd --privileged --name cluster-slave3 -h cluster-slave3 --net mynetwork --ip 172.18.0.5 centos:latest /usr/sbin/init
# docker run -itd --privileged -v /System/Volumes/Data/data/personal/docker/hadoop/sys/fs/cgroup:/sys/fs/cgroup --name cluster-slave4 -h cluster-slave4 --net mynetwork --ip 172.18.0.6 centos:latest /usr/sbin/init

安装 OpenSSH 免密登录

master 安装 openSSH

# 进入master容器中
docker exec -it cluster-master /bin/bash
# 在容器内部执行
yum -y install openssh openssh-server openssh-clients vim

# 去掉文件 Port 22  PermitRootLogin(开启远程登录) yes  PasswordAuthentication(使用密码) yes
vim /etc/ssh/sshd_config

# 启动 openSSH
systemctl start sshd

# 生成ssh key
ssh-keygen -t rsa # 一路回车就好

cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys

slave 安装 openSSH（slave1 slave2 slave3 同理安装）

# 进入master容器中
docker exec -it cluster-slave1 /bin/bash
# 在容器内部执行
yum -y install openssh openssh-server openssh-clients vim

# 去掉文件 Port 22  PermitRootLogin(开启远程登录) yes  PasswordAuthentication(使用密码) yes
vim /etc/ssh/sshd_config

# 启动 openSSH
systemctl start sshd

# 使用 master 的 ~/.ssh/*替换 slave1的~/.ssh/*

systemctl start sshd

Ansible安装

yum -y install epel-release
yum -y install ansible

# 修改
vi /etc/ansible/hosts
`
[cluster]
cluster-master
cluster-slave1
cluster-slave2
cluster-slave3

[master]
cluster-master

[slaves]
cluster-slave1
cluster-slave2
cluster-slave3
`

修改 hosts

:>/etc/hosts
cat >>/etc/hosts<<EOF
127.0.0.1   localhost
172.18.0.2  cluster-master
172.18.0.3  cluster-slave1
172.18.0.4  cluster-slave2
172.18.0.5  cluster-slave3
EOF

source ~/.bashrc

# 分发至slave
ansible cluster -m copy -a "src=~/.bashrc dest=~/"

安装 jdk和hodoop （四台机器都要安装）

# 将宿主机 jdk文件和hadoop文件移动至docker容器
docker cp ~/Download/hadoop-3.3.0.tar.gz  容器ID或容器名称:/opt
docker cp ~/Download/jdk-8u261-linux-x64.tar.gz 容器ID或容器名称:/opt

# 将文件移动至 slave
scp ./* root@cluster-slave1:/opt/
scp ./* root@cluster-slave2:/opt/
scp ./* root@cluster-slave3:/opt/

# 解压
tar -zxvf hadoop-3.3.0.tar.gz
tar -zxvf jdk-8u261-linux-x64.tar.gz

# 配置环境变量 
vim ~/.bashrc
`
# hadoop
export HADOOP_HOME=/opt/hadoop-3.3.0
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

#java
export JAVA_HOME=/opt/jdk1.8.0_261
export PATH=$JAVA_HOME/bin:$PATH
`
source ~/.bashrc

配置hadoop运行所需配置文件