在controller.go中的New()函数中，会调用c.startExternalKeyListener()启动Unix sock监听。本次分析就介绍controller是如何完成外部net namespace导入到libnetork中的。

(五) 监听unix socket

libnetwork-setkey

我们先来看libnetwork-setkey子命令的执行流程。libnetwork-setkey子命令用于runc向libnetwork报告创建好的net namespace。

在libnetwork包中的sandbox_externalkey.go中，有：

1
2
3
4
func init() {
	//***调用docker的reexec，注册libnetwork-setkey***//
	reexec.Register("libnetwork-setkey", processSetKeyReexec)
}

该init()函数向docker reexec注册”libnetwork-setkey”子命令，其处理的函数是processSetKeyReexec。
processSetKeyReexec定义在/libnetwork/sandbox_externalkey_unix.go中：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
// processSetKeyReexec is a private function that must be called only on an reexec path
// It expects 3 args { [0] = "libnetwork-setkey", [1] = <container-id>, [2] = <controller-id> }
// It also expects configs.HookState as a json string in <stdin>
// Refer to https://github.com/opencontainers/runc/pull/160/ for more information
func processSetKeyReexec() {
	var err error
	// Return a failure to the calling process via ExitCode
	defer func() {
		if err != nil {
			logrus.Fatalf("%v", err)
		}
	}()
	// expecting 3 args {[0]="libnetwork-setkey", [1]=<container-id>, [2]=<controller-id> }
	if len(os.Args) < 3 {
		err = fmt.Errorf("Re-exec expects 3 args, received : %d", len(os.Args))
		return
	}
	//***从参数获取contaienrID***//
	containerID := os.Args[1]
	// We expect configs.HookState as a json string in <stdin>
	//***获取state信息，state中包含容器进程pid***//
	//***configs.HookState会赋给command的Stdin***//
	stateBuf, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return
	}
	var state configs.HookState
	if err = json.Unmarshal(stateBuf, &state); err != nil {
		return
	}
	//***从参数获取controllerID***//
	controllerID := os.Args[2]
	//***生成key，并调用SetExternalKey()***//
	err = SetExternalKey(controllerID, containerID, fmt.Sprintf("/proc/%d/ns/net", state.Pid))
	return
}

processSetKeyReexec()最后调用的是SetExternalKey():

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
// SetExternalKey provides a convenient way to set an External key to a sandbox
func SetExternalKey(controllerID string, containerID string, key string) error {
	//***生成keyData***//
	keyData := setKeyData{
		ContainerID: containerID,
		Key:         key}
	//***往unix sock中写入keyData***//
	c, err := net.Dial("unix", udsBase+controllerID+".sock")
	if err != nil {
		return err
	}
	defer c.Close()
	if err = sendKey(c, keyData); err != nil {
		return fmt.Errorf("sendKey failed with : %v", err)
	}
	return processReturn(c)
}
func sendKey(c net.Conn, data setKeyData) error {
	var err error
	defer func() {
		if err != nil {
			c.Close()
		}
	}()
	var b []byte
	if b, err = json.Marshal(data); err != nil {
		return err
	}
	_, err = c.Write(b)
	return err
}

SetExternalKey()先生成keyData，然后写入unix sock。

unix sock

先来看startExternalKeyListener()，定义在/libnetwork/sandbox_externalkey_unix.go中：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
//***启动unix sock监听***//
func (c *controller) startExternalKeyListener() error {
	if err := os.MkdirAll(udsBase, 0600); err != nil {
		return err
	}
	uds := udsBase + c.id + ".sock"
	l, err := net.Listen("unix", uds)
	if err != nil {
		return err
	}
	if err := os.Chmod(uds, 0600); err != nil {
		l.Close()
		return err
	}
	c.Lock()
	c.extKeyListener = l
	c.Unlock()
	go c.acceptClientConnections(uds, l)
	return nil
}

startExternalKeyListener()会启动一个goroutine执行acceptClientConnections()：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
func (c *controller) acceptClientConnections(sock string, l net.Listener) {
	for {
		conn, err := l.Accept()
		if err != nil {
			if _, err1 := os.Stat(sock); os.IsNotExist(err1) {
				logrus.Debugf("Unix socket %s doesn't exist. cannot accept client connections", sock)
				return
			}
			logrus.Errorf("Error accepting connection %v", err)
			continue
		}
		go func() {
			defer conn.Close()
			err := c.processExternalKey(conn)
			ret := success
			if err != nil {
				ret = err.Error()
			}
			_, err = conn.Write([]byte(ret))
			if err != nil {
				logrus.Errorf("Error returning to the client %v", err)
			}
		}()
	}
}

acceptClientConnections()会接收来自unix sock的数据，并启动goroutine处理。处理主要调用的是processExternalKey():

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
func (c *controller) processExternalKey(conn net.Conn) error {
	buf := make([]byte, 1280)
	nr, err := conn.Read(buf)
	if err != nil {
		return err
	}
	var s setKeyData
	if err = json.Unmarshal(buf[0:nr], &s); err != nil {
		return err
	}
	var sandbox Sandbox
	//***依据ContainerID获取sandbox***//
	search := SandboxContainerWalker(&sandbox, s.ContainerID)
	c.WalkSandboxes(search)
	if sandbox == nil {
		return types.BadRequestErrorf("no sandbox present for %s", s.ContainerID)
	}
	//***调用sandbox的SetKey()方法***//
	return sandbox.SetKey(s.Key)
}

processExternalKey()先从数据中提取ContainerID，然后依据ContainerID找到对应的sandbox，最后调用sandbox的SetKey()方法完成处理。
SetKey()定义在/libnetwork/sandbox.go中，可以认为SetKey()更新了原来sandbox中的net namespace：

sandbox::SetKey()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
//***external key使用***//
func (sb *sandbox) SetKey(basePath string) error {
	start := time.Now()
	defer func() {
		log.Debugf("sandbox set key processing took %s for container %s", time.Now().Sub(start), sb.ContainerID())
	}()
	if basePath == "" {
		return types.BadRequestErrorf("invalid sandbox key")
	}
	sb.Lock()
	oldosSbox := sb.osSbox
	sb.Unlock()
	if oldosSbox != nil {
		// If we already have an OS sandbox, release the network resources from that
		// and destroy the OS snab. We are moving into a new home further down. Note that none
		// of the network resources gets destroyed during the move.
		sb.releaseOSSbox()
	}
	//***调用osl.GetSandboxForExternalKey()***//
	osSbox, err := osl.GetSandboxForExternalKey(basePath, sb.Key())
	if err != nil {
		return err
	}
	sb.Lock()
	sb.osSbox = osSbox
	sb.Unlock()
	defer func() {
		if err != nil {
			sb.Lock()
			sb.osSbox = nil
			sb.Unlock()
		}
	}()
	// If the resolver was setup before stop it and set it up in the
	// new osl sandbox.
	if oldosSbox != nil && sb.resolver != nil {
		sb.resolver.Stop()
		if err := sb.osSbox.InvokeFunc(sb.resolver.SetupFunc()); err == nil {
			if err := sb.resolver.Start(); err != nil {
				log.Errorf("Resolver Start failed for container %s, %q", sb.ContainerID(), err)
			}
		} else {
			log.Errorf("Resolver Setup Function failed for container %s, %q", sb.ContainerID(), err)
		}
	}
	for _, ep := range sb.getConnectedEndpoints() {
		//***调用populateNetworkResources()***//
		if err = sb.populateNetworkResources(ep); err != nil {
			return err
		}
	}
	return nil
}

其中populateNetworkResources()会重新设置更新后sandbox中endpoint的信息。
GetSandboxForExternalKey()定义在/libnetwork/osl/namespace_linux.go中：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
//***docker run -d --network bridge nginx:v1会执行到***//
func GetSandboxForExternalKey(basePath string, key string) (Sandbox, error) {
	//***basePath:  /proc/20022/ns/net***//
	//***key:  /var/run/docker/netns/b05bfc66c1bb***//
	if err := createNamespaceFile(key); err != nil {
		return nil, err
	}
	//***Fankang***//
	//***把basePath挂载到key上***//
	if err := mountNetworkNamespace(basePath, key); err != nil {
		return nil, err
	}
	n := &networkNamespace{path: key}
	sboxNs, err := netns.GetFromPath(n.path)
	if err != nil {
		return nil, fmt.Errorf("failed get network namespace %q: %v", n.path, err)
	}
	defer sboxNs.Close()
	n.nlHandle, err = netlink.NewHandleAt(sboxNs, syscall.NETLINK_ROUTE)
	if err != nil {
		return nil, fmt.Errorf("failed to create a netlink handle: %v", err)
	}
	if err = n.loopbackUp(); err != nil {
		n.nlHandle.Delete()
		return nil, err
	}
	return n, nil
}

GetSandboxForExternalKey()会把进程所对应的net namespace，即/proc//ns/net挂载到/var/run/docker/netns/上。从而完成外边的netnamespace的导入。完成更新net namespace后，populateNetworkResources()会重新设置sandbox中endpoint的信息。这些细节还有等深入分析。