Broadcom 高端交换芯片 配置动态流量负载分担。
纪念一位已经离开了我们的同事Leo Wu.
|
Products: |
|
|
|
Domains: |
|
Summary
Trident/Trident2/Triumph3 and later devicessupport a load balancing mode call Dynamic Load Balancing (DLB), which takes into account the existing traffic load of aggregate member when assigning new flows to achieve a more balanced utilization of aggregate members. This article demonstrates how to configure DLB using BCM API.
Description
The following figure shows the components used in DLB (see article HT1119 for DLB hardware tables/registers) :
RTAG7 hashing is still used for flow classification. Once a flow is identified, the agregate member assignment depends on the DLB mode:
- Spray Mode - Traffic is equally distributed per packet and in a round robin manner among the HG trunk members
- Inactivity-based Eligibility Mode - a new member is picked only after the inactivity duration elapses.
- Fixed Assignment Mode - Pick the fixed member as configured in the flow set member table
In the following example configuration, I set up a HG trunk group which consists of four HG ports {27, 28, 29, 30}:
port xe26-xe29 enc=higig2port hg speed=10000 lb=phy
The following function configures RTAG7 hashing:
int dlb_trunk_init(int unit){ int rv, arg; /* L2 - This section to setup RTAG 7 with L2 header parameters only*/ BCM_IF_ERROR_RETURN(bcm_switch_control_set(unit, bcmSwitchHashL2Field0, BCM_HASH_FIELD_SRCMOD | BCM_HASH_FIELD_SRCPORT | BCM_HASH_FIELD_ETHER_TYPE | BCM_HASH_FIELD_MACDA_LO | BCM_HASH_FIELD_MACDA_MI | BCM_HASH_FIELD_MACDA_HI | BCM_HASH_FIELD_MACSA_LO | BCM_HASH_FIELD_MACSA_MI | BCM_HASH_FIELD_MACSA_HI | BCM_HASH_FIELD_VLAN)); BCM_IF_ERROR_RETURN(bcm_switch_control_set(unit, bcmSwitchHashL2Field1, BCM_HASH_FIELD_SRCMOD | BCM_HASH_FIELD_SRCPORT | BCM_HASH_FIELD_ETHER_TYPE | BCM_HASH_FIELD_MACDA_LO | BCM_HASH_FIELD_MACDA_MI | BCM_HASH_FIELD_MACDA_HI | BCM_HASH_FIELD_MACSA_LO | BCM_HASH_FIELD_MACSA_MI | BCM_HASH_FIELD_MACSA_HI | BCM_HASH_FIELD_VLAN)); /* Configure A0 and B0 HASH Selection */ BCM_IF_ERROR_RETURN(bcm_switch_control_set(unit, bcmSwitchHashField0Config, BCM_HASH_FIELD_CONFIG_CRC16XOR8)); BCM_IF_ERROR_RETURN(bcm_switch_control_set(unit, bcmSwitchHashField1Config, BCM_HASH_FIELD_CONFIG_CRC16)); /* Configure A0 and B0 HASH Seed */ BCM_IF_ERROR_RETURN(bcm_switch_control_set(unit, bcmSwitchHashSeed0, 0x33333333)); BCM_IF_ERROR_RETURN(bcm_switch_control_set(unit, bcmSwitchHashSeed1, 0x55555555)); /* Enable RTAG7 HASH on Non-unicast traffic */ BCM_IF_ERROR_RETURN(bcm_switch_control_get(unit, bcmSwitchHashControl, &arg)); arg |= BCM_HASH_CONTROL_TRUNK_NUC_ENHANCE; BCM_IF_ERROR_RETURN(bcm_switch_control_set(unit, bcmSwitchHashControl, arg)); /* Disable RTAG7 hash selection on IP4 and IP6 field*/ BCM_IF_ERROR_RETURN(bcm_switch_control_get(unit,bcmSwitchHashSelectControl, &arg)); arg |=(BCM_HASH_FIELD0_DISABLE_IP4| BCM_HASH_FIELD1_DISABLE_IP4| BCM_HASH_FIELD0_DISABLE_IP6| BCM_HASH_FIELD1_DISABLE_IP6); BCM_IF_ERROR_RETURN( bcm_switch_control_set(unit, bcmSwitchHashSelectControl, arg)); return 0;}
The DLB configuration is done when the HG trunk is created, you provide the DLB mode via the bcm_trunk_info_t structure.
int dlb_trunk_create(int unit, int *gport, int nport, int dlb_mode, bcm_trunk_t *tid)
{
int i, rv, my_modid, flags=0;
bcm_trunk_info_t trunk_info;
bcm_trunk_member_t *p_trunk_member;
/* Create trunk group, allocate trunk group id */
*tid = 1025;
rv = bcm_trunk_create(unit, BCM_TRUNK_FLAG_WITH_ID, tid);
if (BCM_FAILURE(rv)) {
printf ("failed, error code: %s\n", bcm_errmsg(BCM_FAILURE(rv)));
return rv;
}
bcm_trunk_info_t_init(&trunk_info);
trunk_info.dlf_index = BCM_TRUNK_UNSPEC_INDEX;
trunk_info.mc_index = BCM_TRUNK_UNSPEC_INDEX;
trunk_info.ipmc_index = BCM_TRUNK_UNSPEC_INDEX;
trunk_info.psc = dlb_mode;
trunk_info.dynamic_age = 16; /* inactivity duration in usec */
trunk_info.dynamic_size = 512; /* Flow Set size */
/* sizeof(bcm_trunk_member_t) is 16 */
p_trunk_member = sal_alloc (16 * nport, "Trunk Members");
if (!p_trunk_member) {
printf("Failed to allocate memory. \n");
return BCM_E_MEMORY;
}
for (i = 0; i < nport; i++) {
bcm_trunk_member_t_init(&p_trunk_member[i]);
p_trunk_member[i].gport = gport[i];
p_trunk_member[i].dynamic_load_weight = 100;
p_trunk_member[i].dynamic_scaling_factor = 1;
}
printf("Configure DLB information for trunk %d ", *tid);
rv = bcm_trunk_set(unit, *tid, &trunk_info, nport, p_trunk_member);
if (BCM_FAILURE(rv)) {
printf ("failed, error code: %s\n", bcm_errmsg(BCM_FAILURE(rv)));
return rv;
}
for (i = 0; i < nport; i++) {
rv = bcm_trunk_member_status_set(unit, p_trunk_member[i].gport, BCM_TRUNK_DYNAMIC_MEMBER_FORCE_UP);
if (BCM_FAILURE(rv)) {
printf("DLB Trunk member force up set failue! rv: %d, unit: %d, gport: %d\n", rv, unit, gport[i]);
return rv;
}
}
sal_free (p_trunk_member);
return 0;
}
Now, we can create the HG trunk:
int unit = 0;int mymodid=1;int remote_modid=11;int nport = 4;int stackingport[nport] = {27, 28, 29, 30};int stackinggport[nport];int tid;int i;print bcm_stk_modid_set(unit, mymodid);for (i=0; i<nport; i++) { printf ("Stacking port %d: Partner Modid: %d \n", stackingport[i], remote_modid); print bcm_stk_modport_set(unit, remote_modid, stackingport[i]);}/* Create the HG trunk, using DLB mode BCM_TRUNK_PSC_DYNAMIC_OPTIMAL, BCM_TRUNK_PSC_DYNAMIC */for (i=0; i<nport; i++) { print bcm_port_gport_get(unit, stackingport[i], &stackinggport[i]);}print dlb_trunk_init(unit);print dlb_trunk_create(unit, stackinggport, nport, BCM_TRUNK_PSC_DYNAMIC_OPTIMAL, &tid);
To test DLB, we add the following L2 entry, note that the port is a remote port, which means the packet with the DA and VLAN matching this entry will be switched to the HG trunk group.
bcm_l2_addr_t l2_addr;bcm_mac_t mac = "00:00:00:00:00:0a";bcm_vlan_t vid = 1;print bcm_l2_addr_t_init(&l2_addr, mac, vid);l2_addr.port = 1;l2_addr.modid = remote_modid;print bcm_l2_addr_add(unit, &l2_addr);
If we send traffic with DA == "00:00:00:00:00:0a", and VLAN Id 1, we can then check the counters:
BCM.0> show c TPOKTPOK.hg0 : 9,903 +9,903TPOK.hg1 : 9,785 +9,785TPOK.hg2 : 9,706 +9,706TPOK.hg3 : 10,606 +10,606
As you can see, the packet are evenly distributed among the HG trunk members.
Environment
This was tested on BCM56850_A2 with SDK-6.3.11
