K8S ipvs之ipset.go源码解读

//go:build linux
// +build linux

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ipvs

import (
	"k8s.io/apimachinery/pkg/util/sets"
	utilversion "k8s.io/apimachinery/pkg/util/version"
	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"

	"fmt"
	"strings"
)

const (
	// MinIPSetCheckVersion is the min ipset version we need.  IPv6 is supported in ipset 6.x
	MinIPSetCheckVersion = "6.0"

	kubeLoopBackIPSetComment = "Kubernetes endpoints dst ip:port, source ip for solving hairpin purpose"
	kubeLoopBackIPSet        = "KUBE-LOOP-BACK"

	kubeClusterIPSetComment = "Kubernetes service cluster ip + port for masquerade purpose"
	kubeClusterIPSet        = "KUBE-CLUSTER-IP"

	kubeExternalIPSetComment = "Kubernetes service external ip + port for masquerade and filter purpose"
	kubeExternalIPSet        = "KUBE-EXTERNAL-IP"

	kubeExternalIPLocalSetComment = "Kubernetes service external ip + port with externalTrafficPolicy=local"
	kubeExternalIPLocalSet        = "KUBE-EXTERNAL-IP-LOCAL"

	kubeLoadBalancerSetComment = "Kubernetes service lb portal"
	kubeLoadBalancerSet        = "KUBE-LOAD-BALANCER"

	kubeLoadBalancerLocalSetComment = "Kubernetes service load balancer ip + port with externalTrafficPolicy=local"
	kubeLoadBalancerLocalSet        = "KUBE-LOAD-BALANCER-LOCAL"

	kubeLoadBalancerFWSetComment = "Kubernetes service load balancer ip + port for load balancer with sourceRange"
	kubeLoadBalancerFWSet        = "KUBE-LOAD-BALANCER-FW"

	kubeLoadBalancerSourceIPSetComment = "Kubernetes service load balancer ip + port + source IP for packet filter purpose"
	kubeLoadBalancerSourceIPSet        = "KUBE-LOAD-BALANCER-SOURCE-IP"

	kubeLoadBalancerSourceCIDRSetComment = "Kubernetes service load balancer ip + port + source cidr for packet filter purpose"
	kubeLoadBalancerSourceCIDRSet        = "KUBE-LOAD-BALANCER-SOURCE-CIDR"

	kubeNodePortSetTCPComment = "Kubernetes nodeport TCP port for masquerade purpose"
	kubeNodePortSetTCP        = "KUBE-NODE-PORT-TCP"

	kubeNodePortLocalSetTCPComment = "Kubernetes nodeport TCP port with externalTrafficPolicy=local"
	kubeNodePortLocalSetTCP        = "KUBE-NODE-PORT-LOCAL-TCP"

	kubeNodePortSetUDPComment = "Kubernetes nodeport UDP port for masquerade purpose"
	kubeNodePortSetUDP        = "KUBE-NODE-PORT-UDP"

	kubeNodePortLocalSetUDPComment = "Kubernetes nodeport UDP port with externalTrafficPolicy=local"
	kubeNodePortLocalSetUDP        = "KUBE-NODE-PORT-LOCAL-UDP"

	kubeNodePortSetSCTPComment = "Kubernetes nodeport SCTP port for masquerade purpose with type 'hash ip:port'"
	kubeNodePortSetSCTP        = "KUBE-NODE-PORT-SCTP-HASH"

	kubeNodePortLocalSetSCTPComment = "Kubernetes nodeport SCTP port with externalTrafficPolicy=local with type 'hash ip:port'"
	kubeNodePortLocalSetSCTP        = "KUBE-NODE-PORT-LOCAL-SCTP-HASH"

	kubeHealthCheckNodePortSetComment = "Kubernetes health check node port"
	kubeHealthCheckNodePortSet        = "KUBE-HEALTH-CHECK-NODE-PORT"

	kubeIPVSSetComment = "Addresses on the ipvs interface"
	kubeIPVSSet        = "KUBE-IPVS-IPS"
)

//这段代码定义了一系列常量，用于在IPSet中标识不同的集合名称和注释。
//这些集合用于标识Kubernetes中的不同网络实体，例如LoopBack、ClusterIP、ExternalIP、LoadBalancer等。
//每个集合都有一个相应的注释，描述了它的用途。
//此外，还定义了一个最小IPSet检查版本常量，表示我们所需的最小IPSet版本。

// IPSetVersioner can query the current ipset version.
type IPSetVersioner interface {
	// returns "X.Y"
	GetVersion() (string, error)
}

// IPSet wraps util/ipset which is used by IPVS proxier.
type IPSet struct {
	utilipset.IPSet
	// activeEntries is the current active entries of the ipset.
	activeEntries sets.Set[string]
	// handle is the util ipset interface handle.
	handle utilipset.Interface
}

//该代码定义了一个IPSetVersioner接口和一个IPSet结构体。
//IPSetVersioner接口有一个方法GetVersion，用于获取当前ipset的版本号，返回一个字符串和一个错误。
//IPSet结构体包含了一个utilipset.IPSet类型的字段，一个sets.Set[string]类型的activeEntries字段和一个utilipset.Interface类型的handle字段。
//其中，utilipset.IPSet是用于IPVS proxier的ipset工具包的封装，activeEntries存储了当前活跃的ipset条目，handle是util ipset的接口句柄。

// NewIPSet initialize a new IPSet struct
func NewIPSet(handle utilipset.Interface, name string, setType utilipset.Type, isIPv6 bool, comment string) *IPSet {
	hashFamily := utilipset.ProtocolFamilyIPV4
	if isIPv6 {
		hashFamily = utilipset.ProtocolFamilyIPV6
		// In dual-stack both ipv4 and ipv6 ipset's can co-exist. To
		// ensure unique names the prefix for ipv6 is changed from
		// "KUBE-" to "KUBE-6-". The "KUBE-" prefix is kept for
		// backward compatibility. The maximum name length of an ipset
		// is 31 characters which must be taken into account.  The
		// ipv4 names are not altered to minimize the risk for
		// problems on upgrades.
		if strings.HasPrefix(name, "KUBE-") {
			name = strings.Replace(name, "KUBE-", "KUBE-6-", 1)
			if len(name) > 31 {
				klog.InfoS("Ipset name truncated", "ipSetName", name, "truncatedName", name[:31])
				name = name[:31]
			}
		}
	}
	set := &IPSet{
		IPSet: utilipset.IPSet{
			Name:       name,
			SetType:    setType,
			HashFamily: hashFamily,
			Comment:    comment,
		},
		activeEntries: sets.New[string](),
		handle:        handle,
	}
	return set
}

//该函数用于初始化一个新的IPSet结构体。
//- 参数handle是一个utilipset.Interface接口，用于操作IPSet。
//- 参数name是IPSet的名称。
//- 参数setType是IPSet的类型。
//- 参数isIPv6是一个布尔值，表示IPSet是IPv4还是IPv6。
//- 参数comment是IPSet的注释。
//函数根据isIPv6的值选择IP协议版本，并根据名称前缀判断是否需要修改名称。
//如果name以"KUBE-"开头且isIPv6为true，则将名称中的"KUBE-"替换为"KUBE-6-"，并确保名称长度不超过31个字符。
//最后，根据输入参数创建并返回一个新的IPSet结构体实例。

func (set *IPSet) validateEntry(entry *utilipset.Entry) bool {
	return entry.Validate(&set.IPSet)
}

func (set *IPSet) isEmpty() bool {
	return set.activeEntries.Len() == 0
}

func (set *IPSet) getComment() string {
	return fmt.Sprintf("\"%s\"", set.Comment)
}

func (set *IPSet) resetEntries() {
	set.activeEntries = sets.New[string]()
}

func (set *IPSet) syncIPSetEntries() {
	appliedEntries, err := set.handle.ListEntries(set.Name)
	if err != nil {
		klog.ErrorS(err, "Failed to list ip set entries")
		return
	}

	// currentIPSetEntries represents Endpoints watched from API Server.
	currentIPSetEntries := sets.New[string]()
	for _, appliedEntry := range appliedEntries {
		currentIPSetEntries.Insert(appliedEntry)
	}

	if !set.activeEntries.Equal(currentIPSetEntries) {
		// Clean legacy entries
		for _, entry := range currentIPSetEntries.Difference(set.activeEntries).UnsortedList() {
			if err := set.handle.DelEntry(entry, set.Name); err != nil {
				if !utilipset.IsNotFoundError(err) {
					klog.ErrorS(err, "Failed to delete ip set entry from ip set", "ipSetEntry", entry, "ipSet", set.Name)
				}
			} else {
				klog.V(3).InfoS("Successfully deleted legacy ip set entry from ip set", "ipSetEntry", entry, "ipSet", set.Name)
			}
		}
		// Create active entries
		for _, entry := range set.activeEntries.Difference(currentIPSetEntries).UnsortedList() {
			if err := set.handle.AddEntry(entry, &set.IPSet, true); err != nil {
				klog.ErrorS(err, "Failed to add ip set entry to ip set", "ipSetEntry", entry, "ipSet", set.Name)
			} else {
				klog.V(3).InfoS("Successfully added ip set entry to ip set", "ipSetEntry", entry, "ipSet", set.Name)
			}
		}
	}
}

//该函数用于同步IPSet的条目。
//它首先列出当前IPSet中的所有条目，并将其与活动条目进行比较。
//如果两者不相等，则删除IPSet中活动条目中不存在的条目，并将活动条目中不存在的条目添加到IPSet中。

func ensureIPSet(set *IPSet) error {
	if err := set.handle.CreateSet(&set.IPSet, true); err != nil {
		klog.ErrorS(err, "Failed to make sure existence of ip set", "ipSet", set)
		return err
	}
	return nil
}

//该函数用于确保IPSet已经创建，如果未创建则尝试创建。
//1. 首先，它通过调用set.handle.CreateSet方法来尝试创建IPSet。
//2. 如果创建成功，则返回nil，表示没有错误发生。
//3. 如果创建失败，则会使用klog.ErrorS记录错误日志，并返回错误。

// checkMinVersion checks if ipset current version satisfies required min version
func checkMinVersion(vstring string) bool {
	version, err := utilversion.ParseGeneric(vstring)
	if err != nil {
		klog.ErrorS(err, "Got invalid version string", "versionString", vstring)
		return false
	}

	minVersion, err := utilversion.ParseGeneric(MinIPSetCheckVersion)
	if err != nil {
		klog.ErrorS(err, "Got invalid version string", "versionString", MinIPSetCheckVersion)
		return false
	}
	return !version.LessThan(minVersion)
}

//该函数用于检查给定的IPSet版本是否满足最小版本要求。
//它首先解析传入的版本字符串，然后解析最小版本字符串。
//如果任何一个解析出现错误，函数将返回false。
//最后，它会比较解析后的版本和最小版本，如果传入的版本小于最小版本，则返回false，否则返回true。