apiserver分析(一)-authenticator-v1.5.2

什么是Authenticator

Authenticator负责Kubernetes中对请求进行认证，只有通过认证的请求才会被执行。在Kubernetes中，有BasicAuth, Keystone, X509, Token, ServiceAccount, OIDCIssuer, WebhookToken, AnyToken等认证器。本次分析将介绍Kubernetes是如何管理认证器的，及如何对请求进行认证。

认证器的生成

认证器的生成在/cmd/kube-apiserver/app/server.go的Run()函数中：

//***认证器***//
apiAuthenticator, securityDefinitions, err := authenticator.New(authenticator.AuthenticatorConfig{
    Anonymous:                   s.GenericServerRunOptions.AnonymousAuth,
    AnyToken:                    s.GenericServerRunOptions.EnableAnyToken,
    BasicAuthFile:               s.GenericServerRunOptions.BasicAuthFile,
    ClientCAFile:                s.GenericServerRunOptions.ClientCAFile,
    TokenAuthFile:               s.GenericServerRunOptions.TokenAuthFile,
    OIDCIssuerURL:               s.GenericServerRunOptions.OIDCIssuerURL,
    OIDCClientID:                s.GenericServerRunOptions.OIDCClientID,
    OIDCCAFile:                  s.GenericServerRunOptions.OIDCCAFile,
    OIDCUsernameClaim:           s.GenericServerRunOptions.OIDCUsernameClaim,
    OIDCGroupsClaim:             s.GenericServerRunOptions.OIDCGroupsClaim,
    ServiceAccountKeyFiles:      s.ServiceAccountKeyFiles,
    ServiceAccountLookup:        s.ServiceAccountLookup,
    ServiceAccountTokenGetter:   serviceAccountGetter,
    KeystoneURL:                 s.GenericServerRunOptions.KeystoneURL,
    KeystoneCAFile:              s.GenericServerRunOptions.KeystoneCAFile,
    WebhookTokenAuthnConfigFile: s.WebhookTokenAuthnConfigFile,
    WebhookTokenAuthnCacheTTL:   s.WebhookTokenAuthnCacheTTL,
    RequestHeaderConfig:         s.GenericServerRunOptions.AuthenticationRequestHeaderConfig(),
})

所以，接下来看authenticator.New()，定义在/pkg/apiserver/authenticator/authn.go中：

func New(config AuthenticatorConfig) (authenticator.Request, *spec.SecurityDefinitions, error) {
    var authenticators []authenticator.Request
    securityDefinitions := spec.SecurityDefinitions{}
    hasBasicAuth := false
    hasTokenAuth := false
    // front-proxy, BasicAuth methods, local first, then remote
    // Add the front proxy authenticator if requested
    if config.RequestHeaderConfig != nil {
        requestHeaderAuthenticator, err := headerrequest.NewSecure(
            config.RequestHeaderConfig.ClientCA,
            config.RequestHeaderConfig.AllowedClientNames,
            config.RequestHeaderConfig.UsernameHeaders,
        )
        if err != nil {
            return nil, nil, err
        }
        authenticators = append(authenticators, requestHeaderAuthenticator)
    }
    if len(config.BasicAuthFile) > 0 {
        basicAuth, err := newAuthenticatorFromBasicAuthFile(config.BasicAuthFile)
        if err != nil {
            return nil, nil, err
        }
        authenticators = append(authenticators, basicAuth)
        hasBasicAuth = true
    }
    if len(config.KeystoneURL) > 0 {
        keystoneAuth, err := newAuthenticatorFromKeystoneURL(config.KeystoneURL, config.KeystoneCAFile)
        if err != nil {
            return nil, nil, err
        }
        authenticators = append(authenticators, keystoneAuth)
        hasBasicAuth = true
    }
    // X509 methods
    if len(config.ClientCAFile) > 0 {
        certAuth, err := newAuthenticatorFromClientCAFile(config.ClientCAFile)
        if err != nil {
            return nil, nil, err
        }
        authenticators = append(authenticators, certAuth)
    }
    // Bearer token methods, local first, then remote
    if len(config.TokenAuthFile) > 0 {
        tokenAuth, err := newAuthenticatorFromTokenFile(config.TokenAuthFile)
        if err != nil {
            return nil, nil, err
        }
        authenticators = append(authenticators, tokenAuth)
        hasTokenAuth = true
    }
    if len(config.ServiceAccountKeyFiles) > 0 {
        serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.ServiceAccountTokenGetter)
        if err != nil {
            return nil, nil, err
        }
        authenticators = append(authenticators, serviceAccountAuth)
        hasTokenAuth = true
    }
    // NOTE(ericchiang): Keep the OpenID Connect after Service Accounts.
    //
    // Because both plugins verify JWTs whichever comes first in the union experiences
    // cache misses for all requests using the other. While the service account plugin
    // simply returns an error, the OpenID Connect plugin may query the provider to
    // update the keys, causing performance hits.
    if len(config.OIDCIssuerURL) > 0 && len(config.OIDCClientID) > 0 {
        oidcAuth, err := newAuthenticatorFromOIDCIssuerURL(config.OIDCIssuerURL, config.OIDCClientID, config.OIDCCAFile, config.OIDCUsernameClaim, config.OIDCGroupsClaim)
        if err != nil {
            return nil, nil, err
        }
        authenticators = append(authenticators, oidcAuth)
        hasTokenAuth = true
    }
    if len(config.WebhookTokenAuthnConfigFile) > 0 {
        webhookTokenAuth, err := newWebhookTokenAuthenticator(config.WebhookTokenAuthnConfigFile, config.WebhookTokenAuthnCacheTTL)
        if err != nil {
            return nil, nil, err
        }
        authenticators = append(authenticators, webhookTokenAuth)
        hasTokenAuth = true
    }
    // always add anytoken last, so that every other token authenticator gets to try first
    if config.AnyToken {
        authenticators = append(authenticators, bearertoken.New(anytoken.AnyTokenAuthenticator{}))
        hasTokenAuth = true
    }
    if hasBasicAuth {
        securityDefinitions["HTTPBasic"] = &spec.SecurityScheme{
            SecuritySchemeProps: spec.SecuritySchemeProps{
                Type:        "basic",
                Description: "HTTP Basic authentication",
            },
        }
    }
    if hasTokenAuth {
        securityDefinitions["BearerToken"] = &spec.SecurityScheme{
            SecuritySchemeProps: spec.SecuritySchemeProps{
                Type:        "apiKey",
                Name:        "authorization",
                In:          "header",
                Description: "Bearer Token authentication",
            },
        }
    }
    if len(authenticators) == 0 {
        if config.Anonymous {
            return anonymous.NewAuthenticator(), &securityDefinitions, nil
        }
    }
    switch len(authenticators) {
    case 0:
        return nil, &securityDefinitions, nil
    }
    authenticator := union.New(authenticators...)
    //***GroupAdder定义在/pkg/auth/group***//
    //***所以，antenticator调用的是GroupAdder的AuthenticateRequest()***//
    authenticator = group.NewGroupAdder(authenticator, []string{user.AllAuthenticated})
    if config.Anonymous {
        // If the authenticator chain returns an error, return an error (don't consider a bad bearer token anonymous).
        authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator())
    }
    return authenticator, &securityDefinitions, nil
}

New()会根据kube-apiserver的参数来生成各个认证器，并把认证器放在authenticators变量中。比如，如果指定了–experimental-keystone-url，则就会生成keystoneAuthenticator。这里要强调的是unionAuthenticator可以封装多个认证器，关于unionAuthenticator，稍后分析。

这里还要涉及到一个概念，GroupAdder，封装了authenticator，定义在/pkg/auth/group/group_adder.go中：

// GroupAdder adds groups to an authenticated user.Info
type GroupAdder struct {
    // Authenticator is delegated to make the authentication decision
    Authenticator authenticator.Request
    // Groups are additional groups to add to the user.Info from a successful authentication
    Groups []string
}
// NewGroupAdder wraps a request authenticator, and adds the specified groups to the returned user when authentication succeeds
func NewGroupAdder(auth authenticator.Request, groups []string) authenticator.Request {
    return &GroupAdder{auth, groups}
}

New()会根据kube-apiserver的参数来生成各个认证器，并把认证器放在authenticators变量中。比如，如果指定了–experimental-keystone-url，则就会生成keystoneAuthenticator。这里要强调的是unionAuthenticator可以封装多个认证器，关于unionAuthenticator，稍后分析。

这里还要涉及到一个概念，GroupAdder，封装了authenticator，定义在/pkg/auth/group/group_adder.go中：

// GroupAdder adds groups to an authenticated user.Info
type GroupAdder struct {
    // Authenticator is delegated to make the authentication decision
    Authenticator authenticator.Request
    // Groups are additional groups to add to the user.Info from a successful authentication
    Groups []string
}
// NewGroupAdder wraps a request authenticator, and adds the specified groups to the returned user when authentication succeeds
func NewGroupAdder(auth authenticator.Request, groups []string) authenticator.Request {
    return &GroupAdder{auth, groups}
}

所以，kube-apiserver的认证器就一个GroupAdder，GroupAdder定义有认证的入口：

//***实现AuthenticateRequest()***//
//***返回user.DefaultInfo***//
func (g *GroupAdder) AuthenticateRequest(req *http.Request) (user.Info, bool, error) {
    u, ok, err := g.Authenticator.AuthenticateRequest(req)
    if err != nil || !ok {
        return nil, ok, err
    }
    return &user.DefaultInfo{
        Name:   u.GetName(),
        UID:    u.GetUID(),
        Groups: append(u.GetGroups(), g.Groups...),
        Extra:  u.GetExtra(),
    }, true, nil
}

GroupAdder的AuthenticateRequest()会调用unionAuthenticator的AuthenticateRequest()对请求进行认证，得到user信息之后返回。

关于Group，定义在/pkg/auth/user/user.go中，authenticator.New()传入的是AllAuthenticated，估计是为了把该用户标记为已经通过认证。但现在还不知道user为什么要这么分类。

// well-known user and group names
const (
    SystemPrivilegedGroup = "system:masters"
    NodesGroup            = "system:nodes"
    AllUnauthenticated    = "system:unauthenticated"
    AllAuthenticated      = "system:authenticated"
    Anonymous     = "system:anonymous"
    APIServerUser = "system:apiserver"
)

unionAuthenticator

unionAuthenticator定义在/plugin/pkg/auth/authenticator/request/union/union.go:

// unionAuthRequestHandler authenticates requests using a chain of authenticator.Requests
type unionAuthRequestHandler struct {
    // Handlers is a chain of request authenticators to delegate to
    Handlers []authenticator.Request
    // FailOnError determines whether an error returns short-circuits the chain
    FailOnError bool
}
//***Go语言中参数三个点表示可变参数***//
func New(authRequestHandlers ...authenticator.Request) authenticator.Request {
    if len(authRequestHandlers) == 1 {
        return authRequestHandlers[0]
    }
    return &unionAuthRequestHandler{Handlers: authRequestHandlers, FailOnError: false}
}

unionAuthenticator的AuthenticateRequest()方法定义如下：

//***逐个调用authRequestHandler，如果有一个成功，则返回***//
func (authHandler *unionAuthRequestHandler) AuthenticateRequest(req *http.Request) (user.Info, bool, error) {
    var errlist []error
    for _, currAuthRequestHandler := range authHandler.Handlers {
        info, ok, err := currAuthRequestHandler.AuthenticateRequest(req)
        if err != nil {
            if authHandler.FailOnError {
                return info, ok, err
            }
            errlist = append(errlist, err)
            continue
        }
        if ok {
            return info, ok, err
        }
    }
    return nil, false, utilerrors.NewAggregate(errlist)
}

AuthenticateRequest()会轮询每一个authenticator，如果有一个authenticator认证成功，则直接返回认证成功。

keystoneAuthenticator

接下来回到/pkg/apiserver/authenticator/authn.go的New()中。New()调用newAuthenticatorFromKeystoneURL()生成keystoneAuthenticator。newAuthenticatorFromKeystoneURL()定义如下：

// newAuthenticatorFromTokenFile returns an authenticator.Request or an error
func newAuthenticatorFromKeystoneURL(keystoneURL string, keystoneCAFile string) (authenticator.Request, error) {
	keystoneAuthenticator, err := keystone.NewKeystoneAuthenticator(keystoneURL, keystoneCAFile)
	if err != nil {
		return nil, err
	}

	return basicauth.New(keystoneAuthenticator), nil
}

其中，basicAuthenticator封装了keystoneAuthenticator的AuthenticatePassword()方法，定义在/plugin/pkg/auth/authenticator/request/basicauth/basicauth.go中：

// Authenticator authenticates requests using basic auth
type Authenticator struct {
	auth authenticator.Password
}

// New returns a request authenticator that validates credentials using the provided password authenticator
func New(auth authenticator.Password) *Authenticator {
	return &Authenticator{auth}
}

// AuthenticateRequest authenticates the request using the "Authorization: Basic" header in the request
func (a *Authenticator) AuthenticateRequest(req *http.Request) (user.Info, bool, error) {
	username, password, found := req.BasicAuth()
	if !found {
		return nil, false, nil
	}
	return a.auth.AuthenticatePassword(username, password)
}

可以看出，basicAuthenticator实现了AuthenticateRequest()，该方法会从请求中获取用户名和密码，然后调用所封装的认证器的AuthenticatePassword()方法进行认证。

回到正题来看keystoneAuthenticator。keystoneAuthenticator实现了authenticatePassword()方法。keystoneAuthenticator定义在/pkg/auth/authenticator/password/keystone/keystone.go中：

// KeystoneAuthenticator contacts openstack keystone to validate user's credentials passed in the request.
// The keystone endpoint is passed during apiserver startup
type KeystoneAuthenticator struct {
	authURL   string
	transport http.RoundTripper
}

// NewKeystoneAuthenticator returns a password authenticator that validates credentials using openstack keystone
func NewKeystoneAuthenticator(authURL string, caFile string) (*KeystoneAuthenticator, error) {
	if !strings.HasPrefix(authURL, "https") {
		return nil, errors.New("Auth URL should be secure and start with https")
	}
	if authURL == "" {
		return nil, errors.New("Auth URL is empty")
	}
	if caFile != "" {
		roots, err := certutil.NewPool(caFile)
		if err != nil {
			return nil, err
		}
		config := &tls.Config{}
		config.RootCAs = roots
		transport := netutil.SetOldTransportDefaults(&http.Transport{TLSClientConfig: config})
		return &KeystoneAuthenticator{authURL, transport}, nil
	}

	return &KeystoneAuthenticator{authURL: authURL}, nil
}

再来看下authenticatePassword()方法：

// AuthenticatePassword checks the username, password via keystone call
func (keystoneAuthenticator *KeystoneAuthenticator) AuthenticatePassword(username string, password string) (user.Info, bool, error) {
	opts := gophercloud.AuthOptions{
		IdentityEndpoint: keystoneAuthenticator.authURL,
		Username:         username,
		Password:         password,
	}

	_, err := keystoneAuthenticator.AuthenticatedClient(opts)
	if err != nil {
		glog.Info("Failed: Starting openstack authenticate client:" + err.Error())
		return nil, false, errors.New("Failed to authenticate")
	}

	return &user.DefaultInfo{Name: username}, true, nil
}

// AuthenticatedClient logs in to an OpenStack cloud found at the identity endpoint specified by options, acquires a
// token, and returns a Client instance that's ready to operate.
func (keystoneAuthenticator *KeystoneAuthenticator) AuthenticatedClient(options gophercloud.AuthOptions) (*gophercloud.ProviderClient, error) {
	client, err := openstack.NewClient(options.IdentityEndpoint)
	if err != nil {
		return nil, err
	}

	if keystoneAuthenticator.transport != nil {
		client.HTTPClient.Transport = keystoneAuthenticator.transport
	}

	err = openstack.Authenticate(client, options)
	return client, err
}

实现也很简单，生成openstack client，然后去认证用户名和密码，最后返回的是user。

对请求进行认证

现在来看下如何对请求进行处理。
在GenericAPIServer结构体(定义在/pkg/genericapiserver/genericapiserver.go中)中，有HandlerContainer和Handler两个字段。其中HandlerContainer封装了所有的API处理函数，具体如何封装以后分析；而Handler在HandlerContainer的基础上还封装了授权，认证等处理函数。详见/pkg/genericapiserver/config.go:

//***通过调用WithAuthentication(), WithAuthorization()等构建handler处理链***//
func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) (secure, insecure http.Handler) {
	attributeGetter := apiserverfilters.NewRequestAttributeGetter(c.RequestContextMapper)

	generic := func(handler http.Handler) http.Handler {
		handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")
		handler = genericfilters.WithPanicRecovery(handler, c.RequestContextMapper)
		handler = apiserverfilters.WithRequestInfo(handler, NewRequestInfoResolver(c), c.RequestContextMapper)
		handler = api.WithRequestContext(handler, c.RequestContextMapper)
		handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc)
		handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.LongRunningFunc)
		return handler
	}
	audit := func(handler http.Handler) http.Handler {
		return apiserverfilters.WithAudit(handler, attributeGetter, c.AuditWriter)
	}
	protect := func(handler http.Handler) http.Handler {
		//***封装在/pkg/apiserver/filters/authorization.go中***//
		handler = apiserverfilters.WithAuthorization(handler, attributeGetter, c.Authorizer)
		handler = apiserverfilters.WithImpersonation(handler, c.RequestContextMapper, c.Authorizer)
		handler = audit(handler) // before impersonation to read original user
		handler = authhandlers.WithAuthentication(handler, c.RequestContextMapper, c.Authenticator, authhandlers.Unauthorized(c.SupportsBasicAuth))
		return handler
	}

	return generic(protect(apiHandler)), generic(audit(apiHandler))
}

以WithAuthentication()为例来看下是如何对handler进行层层封装的。WithAuthentication()定义在/pkg/auth/handlers/handlers.go中：

// WithAuthentication creates an http handler that tries to authenticate the given request as a user, and then
// stores any such user found onto the provided context for the request. If authentication fails or returns an error
// the failed handler is used. On success, "Authorization" header is removed from the request and handler
// is invoked to serve the request.
func WithAuthentication(handler http.Handler, mapper api.RequestContextMapper, auth authenticator.Request, failed http.Handler) http.Handler {
	if auth == nil {
		glog.Warningf("Authentication is disabled")
		return handler
	}
	return api.WithRequestContext(
		http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			//***handler中的认证***//
			user, ok, err := auth.AuthenticateRequest(req)
			if err != nil || !ok {
				if err != nil {
					glog.Errorf("Unable to authenticate the request due to an error: %v", err)
				}
				failed.ServeHTTP(w, req)
				return
			}

			// authorization header is not required anymore in case of a successful authentication.
			req.Header.Del("Authorization")

			if ctx, ok := mapper.Get(req); ok {
				mapper.Update(req, api.WithUser(ctx, user))
			}

			authenticatedUserCounter.WithLabelValues(compressUsername(user.GetName())).Inc()

			handler.ServeHTTP(w, req)
		}),
		mapper,
	)
}

其中，http.HandlerFunc本身就是一个handler。所以WithAuthentication()先调用auth.AuthenticateRequest()对请求进行验证，然后调用前handler的ServeHTTP()对请求接着作处理，以达到在处理前进行认证的目的。

回到DefaultBuildHandlerChain()，调用DefaultBuildHandlerChain()函数的地方在/pkg/genericapiserver/config.go中的New()方法中：

//***调用BuildHandlerChainsFunc()生成Handler***//
//***把HandlerContainer进一步封装，以进行认证，授权等动作***//
s.Handler, s.InsecureHandler = c.BuildHandlerChainsFunc(s.HandlerContainer.ServeMux, c.Config)

其中，BuildHandlerChainsFunc就是DefaultBuildHandlerChain。可见，Handler是在handlerContainer的基础上封装了protect和generic；InsecureHandler是在handlerContainer的基础上封装了audit和generic。

那么，是怎样关联server和handler呢？

来看/pkg/genericapiserver/genericapiserver.go中preparedGenericAPIServer的Run()方法：

// Run spawns the http servers (secure and insecure). It only returns if stopCh is closed
// or one of the ports cannot be listened on initially.
func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) {
	//***监听安全端口***//
	if s.SecureServingInfo != nil && s.Handler != nil {
		if err := s.serveSecurely(stopCh); err != nil {
			glog.Fatal(err)
		}
	}

	//***监听非安全端口***//
	if s.InsecureServingInfo != nil && s.InsecureHandler != nil {
		if err := s.serveInsecurely(stopCh); err != nil {
			glog.Fatal(err)
		}
	}

	s.RunPostStartHooks()

	// err == systemd.SdNotifyNoSocket when not running on a systemd system
	if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
		glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
	}

	//***监听stpCh channel，如果有信号，则退出***//
	<-stopCh
}

在Run()方法中，有serveSecurely()和serveInsecuurely()两个函数调用，这两个函数定义在/pkg/genericapiserver/serve.go中：

//***监听安全端口***//
func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
	namedCerts, err := getNamedCertificateMap(s.SecureServingInfo.SNICerts)
	if err != nil {
		return fmt.Errorf("unable to load SNI certificates: %v", err)
	}

	secureServer := &http.Server{
		Addr:           s.SecureServingInfo.BindAddress,
		Handler:        s.Handler,
		MaxHeaderBytes: 1 << 20,
		TLSConfig: &tls.Config{
			NameToCertificate: namedCerts,
			// Can't use SSLv3 because of POODLE and BEAST
			// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
			// Can't use TLSv1.1 because of RC4 cipher usage
			MinVersion: tls.VersionTLS12,
			// enable HTTP2 for go's 1.7 HTTP Server
			NextProtos: []string{"h2", "http/1.1"},
		},
	}

	if len(s.SecureServingInfo.ServerCert.CertFile) != 0 || len(s.SecureServingInfo.ServerCert.KeyFile) != 0 {
		secureServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
		secureServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile)
		if err != nil {
			return fmt.Errorf("unable to load server certificate: %v", err)
		}
	}

	// append all named certs. Otherwise, the go tls stack will think no SNI processing
	// is necessary because there is only one cert anyway.
	// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
	// cert will become the default cert. That's what we expect anyway.
	for _, c := range namedCerts {
		secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c)
	}

	if len(s.SecureServingInfo.ClientCA) > 0 {
		clientCAs, err := certutil.NewPool(s.SecureServingInfo.ClientCA)
		if err != nil {
			return fmt.Errorf("unable to load client CA file: %v", err)
		}
		// Populate PeerCertificates in requests, but don't reject connections without certificates
		// This allows certificates to be validated by authenticators, while still allowing other auth types
		secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
		// Specify allowed CAs for client certificates
		secureServer.TLSConfig.ClientCAs = clientCAs
	}

	glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
	s.effectiveSecurePort, err = runServer(secureServer, s.SecureServingInfo.BindNetwork, stopCh)
	return err
}

//***监听非安全端口***//
func (s *GenericAPIServer) serveInsecurely(stopCh <-chan struct{}) error {
	insecureServer := &http.Server{
		Addr:           s.InsecureServingInfo.BindAddress,
		Handler:        s.InsecureHandler,
		MaxHeaderBytes: 1 << 20,
	}
	glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress)
	var err error
	s.effectiveInsecurePort, err = runServer(insecureServer, s.InsecureServingInfo.BindNetwork, stopCh)
	return err
}

可以看出，这两个函数中的server的Handler分别为s.Handler和s.InsecureHandler。在来看下runServer()：

// runServer listens on the given port, then spawns a go-routine continuously serving
// until the stopCh is closed. The port is returned. This function does not block.
func runServer(server *http.Server, network string, stopCh <-chan struct{}) (int, error) {
	if len(server.Addr) == 0 {
		return 0, errors.New("address cannot be empty")
	}

	if len(network) == 0 {
		network = "tcp"
	}

	// first listen is synchronous (fail early!)
	ln, err := net.Listen(network, server.Addr)
	if err != nil {
		return 0, fmt.Errorf("failed to listen on %v: %v", server.Addr, err)
	}

	// get port
	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
	if !ok {
		ln.Close()
		return 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
	}

	lock := sync.Mutex{} // to avoid we close an old listener during a listen retry
	go func() {
		<-stopCh
		lock.Lock()
		defer lock.Unlock()
		ln.Close()
	}()

	go func() {
		defer utilruntime.HandleCrash()

		for {
			var listener net.Listener
			listener = tcpKeepAliveListener{ln.(*net.TCPListener)}
			if server.TLSConfig != nil {
				listener = tls.NewListener(listener, server.TLSConfig)
			}

			//***server开始监听***//
			err := server.Serve(listener)
			glog.Errorf("Error serving %v (%v); will try again.", server.Addr, err)

			// listen again
			func() {
				lock.Lock()
				defer lock.Unlock()
				for {
					time.Sleep(15 * time.Second)

					ln, err = net.Listen("tcp", server.Addr)
					if err == nil {
						return
					}
					select {
					case <-stopCh:
						return
					default:
					}
					glog.Errorf("Error listening on %v (%v); will try again.", server.Addr, err)
				}
			}()

			select {
			case <-stopCh:
				return
			default:
			}
		}
	}()

	return tcpAddr.Port, nil
}

runServer()中会开始监听端口，并把请求交给Handler或InsecureHandler进行处理。