什么是CNI

CNI是Kubernetes提供的网络接口。要使用CNI，需要在kubelet上加参数–network-plugin=cni，需要需要也可以设置–cni-conf-dir和–cni-bin-dir参数。对于Kubernetes来说，network plugin就是一个二进制文件。本次分析将介绍CNI的调用过程。

CNI

之前的分析中我们提到过，Kubernetes通过调用SetUpPod()来设置pod的网络，在/pkg/kubelet/dockertools/docker_manager.go中：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
func (dm *DockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubecontainer.PodStatus, pullSecrets []api.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) {
	......
	// If we should create infra container then we do it first.
	//***Fankang***//
	//***起pause容器***//
	podInfraContainerID := containerChanges.InfraContainerId
	if containerChanges.StartInfraContainer && (len(containerChanges.ContainersToStart) > 0) {
		......
		//***Fankang***//
		//***设置pause容器网络***//
		setupNetworkResult := kubecontainer.NewSyncResult(kubecontainer.SetupNetwork, kubecontainer.GetPodFullName(pod))
		result.AddSyncResult(setupNetworkResult)
		if !kubecontainer.IsHostNetworkPod(pod) {
			glog.V(3).Infof("Calling network plugin %s to setup pod for %s", dm.networkPlugin.Name(), format.Pod(pod))
			//***Fankang***//
			//***默认情况networkPlugin为kubernetes.io/no-op***//
			err = dm.networkPlugin.SetUpPod(pod.Namespace, pod.Name, podInfraContainerID.ContainerID())
			......
}

SetUpPod()定义在/pkg/kubelet/network/cni/cni.go中：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID) error {
	if err := plugin.checkInitialized(); err != nil {
		return err
	}
	netnsPath, err := plugin.host.GetNetNS(id.ID)
	if err != nil {
		return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
	}
	_, err = plugin.loNetwork.addToNetwork(name, namespace, id, netnsPath)
	if err != nil {
		glog.Errorf("Error while adding to cni lo network: %s", err)
		return err
	}
	_, err = plugin.getDefaultNetwork().addToNetwork(name, namespace, id, netnsPath)
	if err != nil {
		glog.Errorf("Error while adding to cni network: %s", err)
		return err
	}
	return err
}

可以看到，在设置网络时，会设置lo和一个由plugin提供的网络。
所以，在设置网络时，调用的是addToNetwork()：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
func (network *cniNetwork) addToNetwork(podName string, podNamespace string, podInfraContainerID kubecontainer.ContainerID, podNetnsPath string) (*cnitypes.Result, error) {
	rt, err := buildCNIRuntimeConf(podName, podNamespace, podInfraContainerID, podNetnsPath)
	if err != nil {
		glog.Errorf("Error adding network: %v", err)
		return nil, err
	}
	netconf, cninet := network.NetworkConfig, network.CNIConfig
	glog.V(4).Infof("About to run with conf.Network.Type=%v", netconf.Network.Type)
	res, err := cninet.AddNetwork(netconf, rt)
	if err != nil {
		glog.Errorf("Error adding network: %v", err)
		return nil, err
	}
	return res, nil
}

addToNetwork()最后调用的是AddNetwork()。

类似的，删除网络调用的是TearDownPod()，定义在/pkg/kubelet/network/cni/cni.go中：

1
2
3
4
5
6
7
8
9
10
11
func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id kubecontainer.ContainerID) error {
	if err := plugin.checkInitialized(); err != nil {
		return err
	}
	netnsPath, err := plugin.host.GetNetNS(id.ID)
	if err != nil {
		return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
	}
	return plugin.getDefaultNetwork().deleteFromNetwork(name, namespace, id, netnsPath)
}

TearDownPod()最后调用的是deleteFromNetwork():

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
func (network *cniNetwork) deleteFromNetwork(podName string, podNamespace string, podInfraContainerID kubecontainer.ContainerID, podNetnsPath string) error {
	rt, err := buildCNIRuntimeConf(podName, podNamespace, podInfraContainerID, podNetnsPath)
	if err != nil {
		glog.Errorf("Error deleting network: %v", err)
		return err
	}
	netconf, cninet := network.NetworkConfig, network.CNIConfig
	glog.V(4).Infof("About to run with conf.Network.Type=%v", netconf.Network.Type)
	err = cninet.DelNetwork(netconf, rt)
	if err != nil {
		glog.Errorf("Error deleting network: %v", err)
		return err
	}
	return nil
}

所以，在删除网络时，最后调用的是DelNetwork()。

libcni包

来看下AddNetwork()和DelNetwork()的实现，定义在/containernetworking/cni/libcni/api.go中：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
func (c *CNIConfig) AddNetwork(net *NetworkConfig, rt *RuntimeConf) (*types.Result, error) {
	pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path)
	if err != nil {
		return nil, err
	}
	return invoke.ExecPluginWithResult(pluginPath, net.Bytes, c.args("ADD", rt))
}
func (c *CNIConfig) DelNetwork(net *NetworkConfig, rt *RuntimeConf) error {
	pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path)
	if err != nil {
		return err
	}
	return invoke.ExecPluginWithoutResult(pluginPath, net.Bytes, c.args("DEL", rt))
}

可以看到，AddNetwork()和DelNetwork()仅通过调用ExecPluginWithResult()或ExecPluginWithoutResult()完成对plugin二进制文件的调用。

ExecPluginWithResult()或ExecPluginWithoutResult()定义在/containernetworking/cni/pkg/invoke/exec.go中，实现也相当简单：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
//***Fankang***//
//***调用execPlugin()***//
func ExecPluginWithResult(pluginPath string, netconf []byte, args CNIArgs) (*types.Result, error) {
	stdoutBytes, err := execPlugin(pluginPath, netconf, args)
	if err != nil {
		return nil, err
	}
	res := &types.Result{}
	err = json.Unmarshal(stdoutBytes, res)
	return res, err
}
//***Fankang***//
//***调用execPlugin()，并忽略返回值***//
func ExecPluginWithoutResult(pluginPath string, netconf []byte, args CNIArgs) error {
	_, err := execPlugin(pluginPath, netconf, args)
	return err
}
//***Fankang***//
//***调用plugin的方式是直接执行plugin的二进制文件***//
func execPlugin(pluginPath string, netconf []byte, args CNIArgs) ([]byte, error) {
	stdout := &bytes.Buffer{}
	c := exec.Cmd{
		Env:    args.AsEnv(),
		Path:   pluginPath,
		Args:   []string{pluginPath},
		Stdin:  bytes.NewBuffer(netconf),
		Stdout: stdout,
		Stderr: os.Stderr,
	}
	if err := c.Run(); err != nil {
		return nil, pluginErr(err, stdout.Bytes())
	}
	return stdout.Bytes(), nil
}

plugin

所以，plugin二进制文件有处理”ADD”和”DEL”两个子命令的功能。可以参考下flannel plugin的实现，在/cni/plugins/metaflannel/flannel.go中：

1
2
3
func main() {
	skel.PluginMain(cmdAdd, cmdDel)
}

flannel.go中实现了cmdAdd()和cmdDel()两个函数，然后通过skel.PluginMain()管理这两个方法。
再来看PluginMain()，定义在/pkg/skel/skel.go中：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
// PluginMain is the "main" for a plugin. It accepts
// two callback functions for add and del commands.
func PluginMain(cmdAdd, cmdDel func(_ *CmdArgs) error) {
	var cmd, contID, netns, ifName, args, path string
	vars := []struct {
		name string
		val  *string
		req  bool
	}{
		{"CNI_COMMAND", &cmd, true},
		{"CNI_CONTAINERID", &contID, false},
		{"CNI_NETNS", &netns, true},
		{"CNI_IFNAME", &ifName, true},
		{"CNI_ARGS", &args, false},
		{"CNI_PATH", &path, true},
	}
	argsMissing := false
	for _, v := range vars {
		*v.val = os.Getenv(v.name)
		if v.req && *v.val == "" {
			log.Printf("%v env variable missing", v.name)
			argsMissing = true
		}
	}
	if argsMissing {
		dieMsg("required env variables missing")
	}
	stdinData, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		dieMsg("error reading from stdin: %v", err)
	}
	cmdArgs := &CmdArgs{
		ContainerID: contID,
		Netns:       netns,
		IfName:      ifName,
		Args:        args,
		Path:        path,
		StdinData:   stdinData,
	}
	switch cmd {
	case "ADD":
		err = cmdAdd(cmdArgs)
	case "DEL":
		err = cmdDel(cmdArgs)
	default:
		dieMsg("unknown CNI_COMMAND: %v", cmd)
	}
	if err != nil {
		if e, ok := err.(*types.Error); ok {
			// don't wrap Error in Error
			dieErr(e)
		}
		dieMsg(err.Error())
	}
}

PluginMain()会从env中读取信息，组装成cmdArgs后调用cmdAdd()或cmdDel()完成网络的创建或销毁。

总结

目前只分析了调用流程。具体plugin的实现在以后分析。