使用路由泄漏实现VPC-PEERING

实验拓扑

拓扑说明

实验环境是一台16G内存的主机。上面使用vmware运行了三个虚拟机，运行的系统为ubuntu-19.04。三个虚拟机采用host-only模式连接。

• spine，leaf1，leaf2三个设备均为ubuntu-19.04.上面运行了FRR程序。
• host1，host2，host3，host4为网络命名空间。
• underlay网络采用的是二层模式(局限于实验条件)

实验中host1和host3在一个vrf，使用l3vni 100 进行互通，host2和host4在另外一个vrf，使用l3vni 200进行互通。要想让evpn-vrf和evpn-vrf1中的主机相互通信，需要使用bgp的路由泄漏功能，完成vpc-peering功能。

spine配置

bgp evpn配置

router bgp 7677
 bgp router-id 192.168.59.130
 bgp bestpath as-path multipath-relax
 neighbor fabric peer-group
 neighbor fabric remote-as external
 neighbor 192.168.59.128 peer-group fabric
 neighbor 192.168.59.129 peer-group fabric
 !
 address-family l2vpn evpn
  neighbor fabric activate
 exit-address-family
!

leaf1配置

接口配置

ubuntu@ubuntu:~$ cat work/frr-frr-7.1/vpc-peering.sh    
#!/bin/bash

#开启转发
sudo sysctl -w net.ipv4.ip_forward=1  
sudo sysctl -p

#添加host1
sudo ip netns add host1
sudo ip link add veth1 type veth peer name eth0 netns host1
sudo ip netns exec host1 ip link set lo up
sudo ip netns exec host1 ip link set eth0 up
sudo ip netns exec host1 ip addr add 1.1.1.1/24 dev eth0
sudo ip netns exec host1 ip route add default via 1.1.1.254 dev eth0

sudo ip link add br10 type bridge
sudo ip link set br10 up
sudo ip link set veth1 up
sudo ip link set veth1 master br10
sudo ip addr add 1.1.1.254/24 dev br10

#添加host2
sudo ip netns add host2
sudo ip link add veth2 type veth peer name eth0 netns host2
sudo ip netns exec host2 ip link set lo up
sudo ip netns exec host2 ip link set eth0 up
sudo ip netns exec host2 ip addr add 2.2.2.2/24 dev eth0
sudo ip netns exec host2 ip route add default via 2.2.2.254 dev eth0

sudo ip link add br20 type bridge
sudo ip link set br20 up
sudo ip link set veth2 up
sudo ip link set veth2 master br20
sudo ip addr add 2.2.2.254/24 dev br20

#添加vni 100，作为l3vni
sudo ip link add br100 type bridge
sudo ip link add vxlan100 type vxlan id 100 local 192.168.59.128 dstport 4789 nolearning
sudo ip link set br100 up
sudo ip link set vxlan100 up
sudo ip link set vxlan100 master br100  
#sudo ip addr add 5.5.5.254/24 dev br100 切记，作为l3vni的svi接口不能配置IP，否则收到type-5路由不会安装。
sudo ip link set dev br100 address 00:00:01:02:03:04 #这个是路由mac

#添加vni 200，作为evpn-vrf1 l3vni
sudo ip link add br200 type bridge
sudo ip link add vxlan200 type vxlan id 200 local 192.168.59.128 dstport 4789 nolearning
sudo ip link set br200 up
sudo ip link set vxlan200 up
sudo ip link set vxlan200 master br200  
#sudo ip addr add 5.5.5.254/24 dev br100 切记，作为l3vni的svi接口不能配置IP，否则收到type-5路由不会安装。
sudo ip link set dev br200 address 00:00:01:02:03:05 #这个是路由mac

#添加vrf
sudo ip link add evpn-vrf type vrf table 100
sudo ip link set evpn-vrf up
sudo ip link set br100 master evpn-vrf  
sudo ip link set br10 master evpn-vrf 

sudo ip link add evpn-vrf1 type vrf table 200
sudo ip link set evpn-vrf1 up
sudo ip link set br200 master evpn-vrf1  
sudo ip link set br20 master evpn-vrf1 

#close rp filter
sudo sysctl -w net.ipv4.conf.all.rp_filter=0
sudo sysctl -w net.ipv4.conf.default.rp_filter=0
sudo sysctl -w net.ipv4.conf.br100.rp_filter=0
sudo sysctl -w net.ipv4.conf.br200.rp_filter=0

#startup bgp
sudo zebra/zebra -d
sudo staticd/staticd -d
sudo bgpd/bgpd -d
sudo vtysh/vtysh

bgp evpn配置

vrf evpn-vrf
 vni 100
 exit-vrf
!
vrf evpn-vrf1
 vni 200
 exit-vrf
!
router bgp 7675
 bgp router-id 192.168.59.128
 bgp bestpath as-path multipath-relax
 neighbor fabric peer-group
 neighbor fabric remote-as external
 neighbor 192.168.59.130 peer-group fabric
 !
 address-family l2vpn evpn
  neighbor fabric activate
  advertise-all-vni
 exit-address-family
!
router bgp 7675 vrf evpn-vrf1
 !
 address-family ipv4 unicast
  network 2.2.2.0/24
  import vrf evpn-vrf
 exit-address-family
 !
 address-family l2vpn evpn
  advertise ipv4 unicast
 exit-address-family
!
router bgp 7675 vrf evpn-vrf
 !
 address-family ipv4 unicast
  network 1.1.1.0/24
  network 5.1.1.0/24
  import vrf evpn-vrf1
 exit-address-family
 !
 address-family l2vpn evpn
  advertise ipv4 unicast
 exit-address-family
!
line vty
!
end
ubuntu# 

注：

import vrf evpn-vrf1

这段指令表示从evpn-vrf1中引入路由。

sudo sysctl -w net.ipv4.conf.all.rp_filter=0
sudo sysctl -w net.ipv4.conf.default.rp_filter=0
sudo sysctl -w net.ipv4.conf.br100.rp_filter=0
sudo sysctl -w net.ipv4.conf.br200.rp_filter=0

这段指令用于关闭linux内核的反向路径检查，由于br100和br200没有配置ip，属于unnumber interface，需要禁止掉反向路径检查。

leaf2配置

接口配置

#!/bin/bash
#开启转发
sudo sysctl -w net.ipv4.ip_forward=1  
sudo sysctl -p

#添加host3
sudo ip netns add host3
sudo ip link add veth3 type veth peer name eth0 netns host3
sudo ip netns exec host3 ip link set lo up
sudo ip netns exec host3 ip link set eth0 up
sudo ip netns exec host3 ip addr add 3.3.3.3/24 dev eth0
sudo ip netns exec host3 ip route add default via 3.3.3.254 dev eth0 

# 添加网桥，将veth3加入网桥
sudo ip link add br30 type bridge
sudo ip link set br30 up
sudo ip link set veth3 up
sudo ip link set veth3 master br30
sudo ip addr add 3.3.3.254/24 dev br30

#添加host4
sudo ip netns add host4
sudo ip link add veth4 type veth peer name eth0 netns host4
sudo ip netns exec host4 ip link set lo up
sudo ip netns exec host4 ip link set eth0 up
sudo ip netns exec host4 ip addr add 4.4.4.4/24 dev eth0
sudo ip netns exec host4 ip route add default via 4.4.4.254 dev eth0

sudo ip link add br40 type bridge
sudo ip link set br40 up
sudo ip link set veth4 up
sudo ip link set veth4 master br40
sudo ip addr add 4.4.4.254/24 dev br40

#添加vni 100，作为ievpn-vrf l3vni
sudo ip link add br100 type bridge
sudo ip link add vxlan100 type vxlan id 100 local 192.168.59.129 dstport 4789 nolearning
sudo ip link set br100 up
sudo ip link set vxlan100 up
sudo ip link set vxlan100 master br100  
#sudo ip addr add 5.5.5.253/24 dev br100 切记一定不能添加IP地址，否则type5路由不能正确下内核
sudo ip link set dev br100 address 00:00:01:02:03:06  #这个是rmac，即路由mac

sudo ip link add br200 type bridge
sudo ip link add vxlan200 type vxlan id 200 local 192.168.59.129 dstport 4789 nolearning
sudo ip link set br200 up
sudo ip link set vxlan200 up
sudo ip link set vxlan200 master br200  
#sudo ip addr add 5.5.5.253/24 dev br100 切记一定不能添加IP地址，否则type5路由不能正确下内核
sudo ip link set dev br200 address 00:00:01:02:03:07  #这个是rmac，即路由mac

#添加vrf
sudo ip link add evpn-vrf type vrf table 100
sudo ip link set evpn-vrf up
sudo ip link set br100 master evpn-vrf  
sudo ip link set br30 master evpn-vrf 

sudo ip link add evpn-vrf1 type vrf table 200
sudo ip link set evpn-vrf1 up
sudo ip link set br200 master evpn-vrf1  
sudo ip link set br40 master evpn-vrf1 

#添加跨vrf静态路由，打通host3和host4
sudo ip route add 4.4.4.4 dev br40 vrf evpn-vrf
sudo ip route add 3.3.3.3 dev br40 vrf evpn-vrf1

#访问外网

#添加连接evpn-vrf到默认vrf的vtep接口
sudo ip link add ext1 type veth peer name ext
sudo ip link set ext1 up
sudo ip link set ext up

#添加连接evpn-vrf1到默认vrf的vtep接口
sudo ip link add ext2 type veth peer name ext3
sudo ip link set ext2 up
sudo ip link set ext3 up

#其中ext1在evpn-vrf，ext在default
sudo ip link set ext1 master evpn-vrf

#使用网段5.5.5.0/24作为relay网段
sudo ip addr add 5.5.5.253/24 dev ext1
sudo ip addr add 5.5.5.254/24 dev ext

sudo ip addr add 5.5.6.253/24 dev ext2
sudo ip addr add 5.5.6.254/24 dev ext3

#在evpn中加入默认路由，默认让流量访问公网
sudo ip route add default via 5.5.5.254 dev ext1 vrf evpn-vrf
sudo ip route add default via 5.5.6.254 dev ext2 vrf evpn-vrf1

#配置snat，让私网流量改smac后范文公网
sudo nft add table nat
sudo nft add chain nat prerouting { type nat hook prerouting priority 0 \; }
sudo nft add chain nat postrouting { type nat hook postrouting priority 100 \; }
sudo nft add rule nat postrouting oifname ext1  counter masquerade
sudo nft add rule nat postrouting oifname ens33  counter masquerade
sudo nft add rule nat postrouting oifname ext2  counter masquerade

sudo sysctl -w net.ipv4.conf.all.rp_filter=0
sudo sysctl -w net.ipv4.conf.default.rp_filter=0
sudo sysctl -w net.ipv4.conf.br100.rp_filter=0
sudo sysctl -w net.ipv4.conf.br200.rp_filter=0

sudo chmod 777 /var/run/
sudo zebra -d
sudo staticd -d
sudo bgpd -d
sudo vtysh 

bgp evpn配置

vrf evpn-vrf
 vni 100
 exit-vrf
!
vrf evpn-vrf1
 vni 200
 exit-vrf
!
router bgp 7676
 bgp router-id 192.168.59.129
 bgp bestpath as-path multipath-relax
 neighbor fabric peer-group
 neighbor fabric remote-as external
 neighbor 192.168.59.130 peer-group fabric
 !
 address-family l2vpn evpn
  neighbor fabric activate
  advertise-all-vni
 exit-address-family
!
router bgp 7676 vrf evpn-vrf1
 !
 address-family ipv4 unicast
  network 4.4.4.0/24
  import vrf evpn-vrf
 exit-address-family
 !
 address-family l2vpn evpn
  advertise ipv4 unicast
 exit-address-family
!
router bgp 7676 vrf evpn-vrf
 !
 address-family ipv4 unicast
  network 0.0.0.0/0
  network 3.3.3.0/24
  import vrf evpn-vrf1
 exit-address-family
 !
 address-family l2vpn evpn
  advertise ipv4 unicast
 exit-address-family
!
line vty
!
end

查看bgp信息

leaf1

• 查看路由信息

ubuntu# show bgp l2vpn evpn 
BGP table version is 6, local router ID is 192.168.59.128
Status codes: s suppressed, d damped, h history, * valid, > best, i - internal
Origin codes: i - IGP, e - EGP, ? - incomplete

   Network          Next Hop            Metric LocPrf Weight Path
Route Distinguisher: ip 1.1.1.254:3

*> [5]:[0]:[24]:[1.1.1.0]
                    192.168.59.128           0         32768 i
*> [5]:[0]:[24]:[4.4.4.0]
                    192.168.59.128                         0 7677 7676 i
*> [5]:[0]:[24]:[5.1.1.0]
                    192.168.59.128           0         32768 i
Route Distinguisher: ip 2.2.2.254:2

*> [5]:[0]:[0]:[0.0.0.0]
                    192.168.59.128                         0 7677 7676 i
*> [5]:[0]:[24]:[2.2.2.0]
                    192.168.59.128           0         32768 i
*> [5]:[0]:[24]:[3.3.3.0]
                    192.168.59.128                         0 7677 7676 i
Route Distinguisher: ip 5.5.5.253:3

*> [5]:[0]:[0]:[0.0.0.0]
                    192.168.59.129                         0 7677 7676 i
*> [5]:[0]:[24]:[3.3.3.0]
                    192.168.59.129                         0 7677 7676 i
Route Distinguisher: ip 5.5.6.253:2

*> [5]:[0]:[24]:[4.4.4.0]
                    192.168.59.129                         0 7677 7676 i

Displayed 9 out of 9 total prefixes
ubuntu# 

leaf2

• 查看路由信息

ubuntu# show bgp l2vpn evpn 
BGP table version is 6, local router ID is 192.168.59.129
Status codes: s suppressed, d damped, h history, * valid, > best, i - internal
Origin codes: i - IGP, e - EGP, ? - incomplete

   Network          Next Hop            Metric LocPrf Weight Path
Route Distinguisher: ip 1.1.1.254:3

*> [5]:[0]:[24]:[1.1.1.0]
                    192.168.59.128                         0 7677 7675 i
*> [5]:[0]:[24]:[5.1.1.0]
                    192.168.59.128                         0 7677 7675 i
Route Distinguisher: ip 2.2.2.254:2

*> [5]:[0]:[24]:[2.2.2.0]
                    192.168.59.128                         0 7677 7675 i
Route Distinguisher: ip 5.5.5.253:3

*> [5]:[0]:[0]:[0.0.0.0]
                    192.168.59.129           0         32768 i
*> [5]:[0]:[24]:[2.2.2.0]
                    192.168.59.129                         0 7677 7675 i
*> [5]:[0]:[24]:[3.3.3.0]
                    192.168.59.129           0         32768 i
Route Distinguisher: ip 5.5.6.253:2

*> [5]:[0]:[24]:[1.1.1.0]
                    192.168.59.129                         0 7677 7675 i
*> [5]:[0]:[24]:[4.4.4.0]
                    192.168.59.129           0         32768 i
*> [5]:[0]:[24]:[5.1.1.0]
                    192.168.59.129                         0 7677 7675 i

Displayed 9 out of 9 total prefixes
ubuntu# 

总结

这个实验的关键点：

• bgp route leak。
• linux内核路由支持下一跳接口不在同一个vrf中。
• 没有配置ip的接口转发报文时需要关闭反向路径检查。