DVR介绍
dvr是distributed virtual router的缩写，目的是为了解决openstack neutron部署的扩展问题，分发网络节点的流量负载到计算节点上。如果实例绑定了floating ip，外网流量直接从计算节点出去，不经过网络节点。对于那些没绑定floating ip的实例外网流量还是走snat，要经过网络节点。
DVR配置
这里说明一下，我的controller node和network node在同一台机器上
controller节点
[root@controller-162 ~(keystone_admin)]# vim /etc/neutron/neutron.conf # 添加如下配置选项
router_distributed = True
[root@controller-162 ~(keystone_admin)]# vim /etc/neutron/plugins/ml2/ml2_conf.ini # 网络采用ml2+vxlan,配置如下
[ml2]
type_drivers = flat,vxlan,vlan,gre
tenant_network_types = vxlan
mechanism_drivers = openvswitch,l2population
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges = 1:1000
vxlan_group = 239.1.1.1
[agent]
l2_population = True
tunnel_types = vxlan
enable_distributed_routing = True
[ovs]
enable_tunneling = True
tunnel_type = vxlan
integration_bridge = br-int
local_ip = 10.0.0.162
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
enable_security_group = True
enable_ipset = True
[root@controller-162 ~(keystone_admin)]# vim /etc/neutron/l3_agent.ini
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
ovs_use_veth = True
use_namespaces = True
external_network_bridge = br-ex
router_delete_namespaces = True
agent_mode = dvr_snat # 虚拟机snat上外网走network node 的L3
compute节点
[root@compute-2 ~]# vim /etc/neutron/plugins/ml2/ml2_conf.ini # 网络采用ml2+vxlan,配置如下
[ml2]
type_drivers = flat,vxlan,vlan,gre
tenant_network_types = vxlan
mechanism_drivers = openvswitch,l2population
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges = 1:1000
vxlan_group = 239.1.1.1
[agent]
l2_population = True
tunnel_types = vxlan
enable_distributed_routing = True
[ovs]
enable_tunneling = True
tunnel_type = vxlan
integration_bridge = br-int
local_ip=10.0.0.2
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
enable_security_group = True
enable_ipset = True
[root@compute-2 ~]# vim /etc/neutron/l3_agent.ini # compute node也要起l3-agent,记住还要add-br br-ex,add-port br-ex eth2这些
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
ovs_use_veth = True
use_namespaces = True
external_network_bridge = br-ex
router_delete_namespaces = True
agent_mode = dvr # floating ip直接走compute node的l3
[root@compute-2 ~]# vim /etc/neutron/metadata_agent.ini # compute node也起了l3，所以会有找metadata的问题
[DEFAULT]
auth_url = http://controller-162:35357/v2.0
auth_region = regionOne
admin_tenant_name = service
admin_user = neutron
admin_password = neutron
nova_metadata_ip = controller-162 # controller node的metadata地址(nova-api接管metadata服务)
metadata_proxy_shared_secret = meta_pass
DVR功能
先来看下controller node的namespace
[root@controller-162 ~(keystone_admin)]# ip netns
qdhcp-f8876645-352e-48d2-b96c-304cb8de805f
snat-ac4a4d9b-27c7-492b-824a-ae384710ab2a # 虚拟机上外网专门的snat，单独出来的namespace
qrouter-ac4a4d9b-27c7-492b-824a-ae384710ab2a
[root@compute-3 ~]# ip netns
qrouter-ac4a4d9b-27c7-492b-824a-ae384710ab2a # qrouter-xxxxxxx 跟上面一样的哦，floating ip的dnat规则在这里
# 我在dashboard上面绑定floating ip的时候找不到port，后台命令行可以绑定，这个bug openstack官方已经修复。
# 详情见这里 https://review.openstack.org/#/c/132383/3/openstack_dashboard/api/neutron.py
# https://ask.openstack.org/en/question/51634/juno-dvr-associate-floating-ip-reported-no-ports-available/
[root@controller-162 ~(keystone_admin)]# nova floating-ip-associate test-5 172.16.101.2 # 绑定floating ip，出现了下面这个东东
[root@compute-3 ~]# ip netns
fip-53f6877e-2f46-43e3-93b7-7e22786cacb2 # 绑定floating ip后，多出来这个namespace
qrouter-ac4a4d9b-27c7-492b-824a-ae384710ab2a中rfp接口和fip-53f6877e-2f46-43e3-93b7-7e22786cacb2中的fpr接口是一对veth，打通两个namespace
[root@compute-3 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.16.101.254 0.0.0.0 UG 0 0 0 fg-d7d8b46c-14 # 外网的网关在这里
169.254.30.174 0.0.0.0 255.255.255.254 U 0 0 0 fpr-02fa05dc-3
169.254.31.184 0.0.0.0 255.255.255.254 U 0 0 0 fpr-30e06cb3-7
169.254.31.224 0.0.0.0 255.255.255.254 U 0 0 0 fpr-1eaac5a6-2
172.16.101.0 0.0.0.0 255.255.255.0 U 0 0 0 fg-f57ba4c0-e1
172.16.101.0 0.0.0.0 255.255.255.0 U 0 0 0 fg-d7d8b46c-14
172.16.101.24 169.254.31.184 255.255.255.255 UGH 0 0 0 fpr-30e06cb3-7
172.16.101.70 169.254.31.224 255.255.255.255 UGH 0 0 0 fpr-1eaac5a6-2
172.16.101.71 169.254.31.224 255.255.255.255 UGH 0 0 0 fpr-1eaac5a6-2
172.16.101.72 169.254.31.224 255.255.255.255 UGH 0 0 0 fpr-1eaac5a6-2
Trouble Shooting
问题：
ERROR neutron.agent.linux.utils [req-adef3886-1fdd-4904-a442-0f5d43d70d1b None]
Command: ['sudo', 'neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ip', 'netns', 'exec', 'qrouter-02fa05dc-355f-42fb-8608-6b5d920ece6a', 'ip', '-4', 'neigh', 'del', '10.0.0.38', 'lladdr', 'fa:16:3e:b8:6d:15', 'dev', 'qr-55a5ec64-31']
Exit code: 2
Stdout: ''
Stderr: 'RTNETLINK answers: No such file or directory\n'
# neutron/agent/l3_agent.py中没有判断这个device是否存在，这个bug官方已修。
解决方法：
vim /usr/lib/python2.7/site-packages/neutron/agent/l3_agent.py # 编辑l3_agent.py文件
int_dev = ip_wrapper.add_veth(rtr_2_fip_name, # 查找这些代码所在位置
fip_2_rtr_name, fip_ns_name)
self.internal_ns_interface_added(str(rtr_2_fip),
rtr_2_fip_name, ri.ns_name)
self.internal_ns_interface_added(str(fip_2_rtr),
fip_2_rtr_name, fip_ns_name)
int_dev[0].link.set_up()
int_dev[1].link.set_up()
# 在这些代码前面，加个判断
if not ip_lib.device_exists(rtr_2_fip_name,self.root_helper,
namespace=ri.ns_name):
int_dev = ip_wrapper.add_veth(rtr_2_fip_name,
fip_2_rtr_name, fip_ns_name)
self.internal_ns_interface_added(str(rtr_2_fip),
rtr_2_fip_name, ri.ns_name)
self.internal_ns_interface_added(str(fip_2_rtr),
fip_2_rtr_name, fip_ns_name)
int_dev[0].link.set_up()
int_dev[1].link.set_up()
参考链接
https://wiki.openstack.org/wiki/Neutron/DVR/HowTo
https://wiki.openstack.org/wiki/Neutron/DVR#Juno_and_Distributed_Routing
http://www.openstack.cn/p2510.html
DistributedVirtualRouter-East-WestNorthSouthwithServices.pdf
