ovn实现容器外网访问，fip

实验拓扑

物理拓扑

逻辑拓扑

其中172.24.4.8为pod 100.60.0.31的fip

步骤

准备

创建逻辑路由器 ovn-cluster

ovn-nbctl lr-add ovn-cluster
ovn-nbctl lrp-add ovn-cluster ovn-cluster-fip-ns1 00:00:00:65:77:09 100.69.0.1/16

创建逻辑交换机 fip-ns1，连接ovn-cluster

ovn-nbctl ls-add fip-ns1
ovn-nbctl lsp-add fip-ns1 fip-ns1-ovn-cluster 
ovn-nbctl lsp-set-type fip-ns1-ovn-cluster router
ovn-nbctl lsp-set-addresses fip-ns1-ovn-cluster 00:00:00:65:77:09
ovn-nbctl lsp-set-options fip-ns1-ovn-cluster router-port=ovn-cluster-fip-ns1

在node3上创建容器，连接到br-int （ovn-nbctl都是在centorl节点node1上执行）

# 在 fip-ns1上创建port
ovn-nbctl lsp-add fip-ns1 app1.fip-ns1
ovn-nbctl lsp-set-addresses app1.fip-ns1 "02:ac:10:ff:01:30 100.69.0.31"
# 启动容器
docker run -itd --name app1 --net=none halfcrazy/toolbox entrypoint.sh
ovs-docker add-port br-int eth0 app1 --ipaddress=100.69.0.31/24
# 关联
ovs-vsctl set Interface app1 external_ids:iface-id=app1.fip-ns1

查看逻辑网络

[root@node1 ovn]#  ovn-nbctl show
switch 8dc28655-dbd7-4018-9495-f5fc6cca672e (fip-ns1)
    port app1.fip-ns1
        addresses: ["02:ac:10:ff:01:30 100.69.0.31"]
    port fip-ns1-ovn-cluster
        type: router
        addresses: ["00:00:00:65:77:09"]
        router-port: ovn-cluster-fip-ns1
router 84923ba1-cb82-424c-93f3-042349311c60 (ovn-cluster)
    port ovn-cluster-fip-ns1
        mac: "00:00:00:65:77:09"
        networks: ["100.69.0.1/16"]

[root@node3 /]# ovs-vsctl show
bdb72edf-98e7-4854-aac6-cde2883c3da9
    Bridge br-int
        fail_mode: secure
        Port br-int
            Interface br-int
                type: internal
        Port "a1268ee29b43_l"
            Interface "a1268ee29b43_l"
        Port "ovn-5b4d77-0"
            Interface "ovn-5b4d77-0"
                type: geneve
                options: {csum="true", key=flow, remote_ip="172.29.101.161"}
        Port "ovn-7ef11f-0"
            Interface "ovn-7ef11f-0"
                type: geneve
                options: {csum="true", key=flow, remote_ip="172.29.101.164"}
    ovs_version: "2.11.2"

创建网桥

在node3上，创建网桥br-ex，添加网络口ens7

ovs-vsctl add-br br-ex
# ens7是机器上的网口
ovs-vsctl add-port br-ex ens7
ip addr add 172.24.4.1/24 dev br-ex
ip link set br-ex up

创建逻辑交换机public，连接br-ex和ovn-cluster

# ovn-cluster 添加端口lrp-0000001 
ovn-nbctl lrp-add ovn-cluster lrp-0000001  00:00:00:4C:3F:15 172.24.4.9/24
ovn-nbctl lrp-set-gateway-chassis lrp-0000001 a0b25a91-20f8-4466-bf63-368c66b8203f

# public 添加端口ae9b52 
ovn-nbctl ls-add public
ovn-nbctl lsp-add public ae9b52  -- set logical_switch_port ae9b52   type=router -- set logical_switch_port ae9b52  options:router-port=lrp-0000001
ovn-nbctl lsp-set-addresses ae9b52 00:00:00:4C:3F:15

# public 添加端口provnet-d1ac28
ovn-nbctl lsp-add public provnet-d1ac28 -- set logical_switch_port provnet-d1ac28  type=localnet
ovn-nbctl lsp-set-addresses   provnet-d1ac28 unknown
ovn-nbctl lsp-set-options provnet-d1ac28 network-name="fip-test"

#public provnet-d1ac28和br-ex映射
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=fip-test:br-ex

创建nat，实现fip

ovn-nbctl lr-nat-add ovn-cluster dnat_and_snat 172.24.4.8 100.69.0.31
ovn-nbctl lr-nat-add ovn-cluster snat 172.24.4.9 100.69.0.0/16 

查看逻辑网络

# ovn-nbctl show
switch 93b1256d-2e3d-430a-9ef3-b67c4f508624 (public)
    port ae9b52
        type: router
        addresses: ["00:00:00:4C:3F:15"]
        router-port: lrp-0000001
    port provnet-d1ac28
        type: localnet
        addresses: ["unknown"]
switch 8dc28655-dbd7-4018-9495-f5fc6cca672e (fip-ns1)
    port app1-6d65577797-qq49p.fip-ns1
        addresses: ["dynamic 100.69.0.31"]
    port fip-ns1-ovn-cluster
        type: router
        addresses: ["00:00:00:65:77:09"]
        router-port: ovn-cluster-fip-ns1
router 84923ba1-cb82-424c-93f3-042349311c60 (ovn-cluster)
    port lrp-0000001
        mac: "00:00:00:4C:3F:15"
        networks: ["172.24.4.9/24"]
        gateway chassis: [1c8f9fa3-ea79-46f7-b844-b516c4aec5d5]
    port ovn-cluster-fip-ns1
        mac: "00:00:00:65:77:09"
        networks: ["100.69.0.1/16"]
    nat 289844f5-9135-421b-b2f0-aacffdb25379
        external ip: "172.24.4.8"
        logical ip: "100.69.0.31"
        type: "dnat_and_snat"
    nat 4f298e67-9d99-4140-86c6-d3fca11dbc99
        external ip: "172.24.4.9"
        logical ip: "100.69.0.0/16"
        type: "snat"

[root@node1 ovn]#  ovn-sbctl  show
Chassis "7ef11fe6-2251-4323-ae81-80d39886d934"
    hostname: "node4"
    Encap geneve
        ip: "172.29.101.164"
        options: {csum="true"}
    Port_Binding "node-node4"
Chassis "1c8f9fa3-ea79-46f7-b844-b516c4aec5d5"
    hostname: "node3"
    Encap geneve
        ip: "172.29.101.163"
        options: {csum="true"}
    Port_Binding "node-node3"
    Port_Binding "app1.fip-ns1"
    Port_Binding "cr-lrp-0000001"
Chassis "5b4d7788-751c-4b03-a9a5-ea1e600e7142"
    hostname: "node1"
    Encap geneve
        ip: "172.29.101.161"
        options: {csum="true"}
    Port_Binding "node-node1"

[root@node3 /]# ovs-vsctl show
bdb72edf-98e7-4854-aac6-cde2883c3da9
    Bridge br-int
        fail_mode: secure
        Port br-int
            Interface br-int
                type: internal
        Port "a1268ee29b43_h"
            Interface "a1268ee29b43_h"
        Port "ovn-5b4d77-0"
            Interface "ovn-5b4d77-0"
                type: geneve
                options: {csum="true", key=flow, remote_ip="172.29.101.161"}
        Port "patch-br-int-to-provnet-d1ac28"
            Interface "patch-br-int-to-provnet-d1ac28"
                type: patch
                options: {peer="patch-provnet-d1ac28-to-br-int"}
        Port "ovn-7ef11f-0"
            Interface "ovn-7ef11f-0"
                type: geneve
                options: {csum="true", key=flow, remote_ip="172.29.101.164"}
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "ens7"
            Interface "ens7"
        Port "patch-provnet-d1ac28-to-br-int"
            Interface "patch-provnet-d1ac28-to-br-int"
                type: patch
                options: {peer="patch-br-int-to-provnet-d1ac28"}
    ovs_version: "2.11.2"

node3上查看物理网络

[root@node3 kube-ovn]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 52:54:00:b3:1c:0e brd ff:ff:ff:ff:ff:ff
    inet 172.29.101.163/24 brd 172.29.101.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::5054:ff:feb3:1c0e/64 scope link 
       valid_lft forever preferred_lft forever
7: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 3e:15:b4:82:87:ac brd ff:ff:ff:ff:ff:ff
8: br-int: <BROADCAST,MULTICAST> mtu 1442 qdisc noop state DOWN group default qlen 1000
    link/ether e6:33:68:1c:5a:4e brd ff:ff:ff:ff:ff:ff
9: genev_sys_6081: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 65000 qdisc noqueue master ovs-system state UNKNOWN group default qlen 1000
    link/ether da:db:66:4c:51:d0 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::d8db:66ff:fe4c:51d0/64 scope link 
       valid_lft forever preferred_lft forever
10: ovn0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1442 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether 0a:00:00:40:00:03 brd ff:ff:ff:ff:ff:ff
    inet 100.64.0.2/16 brd 100.64.255.255 scope global ovn0
       valid_lft forever preferred_lft forever
    inet6 fe80::800:ff:fe40:3/64 scope link 
       valid_lft forever preferred_lft forever
11: br-ex: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether 0a:09:c5:7e:c0:4c brd ff:ff:ff:ff:ff:ff
    inet 172.24.4.1/24 scope global br-ex
       valid_lft forever preferred_lft forever
    inet6 fe80::809:c5ff:fe7e:c04c/64 scope link 
       valid_lft forever preferred_lft forever
12: ens7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master ovs-system state UP group default qlen 1000
    link/ether 52:54:00:9e:90:ae brd ff:ff:ff:ff:ff:ff
    inet6 fe80::5054:ff:fe9e:90ae/64 scope link 
       valid_lft forever preferred_lft forever
14: a1268ee29b43_h@if13: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1442 qdisc noqueue master ovs-system state UP group default 
    link/ether 0a:00:00:45:00:20 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet6 fe80::800:ff:fe45:20/64 scope link 
       valid_lft forever preferred_lft forever

验证

在容器内部

[root@node3 pods]# docker exec -ti app1 bash
bash-4.4# 
bash-4.4# curl 172.24.4.8
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>
bash-4.4# 

在node3上

[root@node3 /]# curl 172.24.4.8
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>
[root@msxu3 /]#