什么是evaluator

大家都知道，Kubernetes中使用resourcequota对配额进行管理。配额的管理涉及两个步骤：1、计算请求所需要的资源；2、比较并更新配额。所以解读resourcequota将分为两次进行。
evaluator就是用来计算请求所需要的资源的。

GenericEvaluator

GenericEvaluator实现了evaluator，是一个基础的evaluator。
我们先来看下GenericEvaluator的定义，在/pkg/quota/generic/evaluator.go中：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// GenericEvaluator provides an implementation for quota.Evaluator
type GenericEvaluator struct {
	// Name used for logging
	Name string
	// The GroupKind that this evaluator tracks
	InternalGroupKind unversioned.GroupKind
	// The set of resources that are pertinent to the mapped operation
	InternalOperationResources map[admission.Operation][]api.ResourceName
	// The set of resource names this evaluator matches
	MatchedResourceNames []api.ResourceName
	// A function that knows how to evaluate a matches scope request
	MatchesScopeFunc MatchesScopeFunc
	// A function that knows how to return usage for an object
	UsageFunc UsageFunc
	// A function that knows how to list resources by namespace
	ListFuncByNamespace ListFuncByNamespace
	// A function that knows how to get resource in a namespace
	// This function must be specified if the evaluator needs to handle UPDATE
	GetFuncByNamespace GetFuncByNamespace
	// A function that checks required constraints are satisfied
	ConstraintsFunc ConstraintsFunc
}

其中：

  • Name: 表示该Evaluator的名称；
  • InternalGroupKind: 表明该Evaluator所处理资源的内部的类型；
  • InternalOperationResources: 表明该Evaluator所支持的请求的类型，如Create, Update等及这些操作所支持的资源；
  • MatchedResourceNames: 表明该Evaluator所对应的资源名称，如ResourceCPU, ResourcePods等；
  • MatchesScopeFunc: resourcequota的scope判断函数。resourcequota只处理满足scope判断函数的请求(即只统计部分对象的配额)，目前有Terminating, NotTerminating, BestEffort, NotBestEffort这些Scope；
  • UsageFunc: 用来计算对象所占资源；
  • ListFuncByNamespace: 对象List函数；
  • GetFuncByNamespace: 对象获取函数；
  • ConstraintsFunc: 对对象申请的资源进行合理性检查，如requests<limits。

Matches()

Matches()方法判断该Evaluator及resourceQuota是否需要处理该请求。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
// Matches returns true if the evaluator matches the specified quota with the provided input item
func (g *GenericEvaluator) Matches(resourceQuota *api.ResourceQuota, item runtime.Object) bool {
	if resourceQuota == nil {
		return false
	}
	// verify the quota matches on resource, by default its false
	matchResource := false
	//***如果resourceQuota中的项有该evaluator处理时所需要的项，则更新matchResource为true***//
	for resourceName := range resourceQuota.Status.Hard {
		if g.MatchesResource(resourceName) {
			matchResource = true
			break
		}
	}
	// by default, no scopes matches all
	matchScope := true
	for _, scope := range resourceQuota.Spec.Scopes {
		matchScope = matchScope && g.MatchesScope(scope, item)
	}
	return matchResource && matchScope
}
// MatchesResource returns true if this evaluator can match on the specified resource
func (g *GenericEvaluator) MatchesResource(resourceName api.ResourceName) bool {
	for _, matchedResourceName := range g.MatchedResourceNames {
		if resourceName == matchedResourceName {
			return true
		}
	}
	return false
}
// MatchesScope returns true if the input object matches the specified scope
func (g *GenericEvaluator) MatchesScope(scope api.ResourceQuotaScope, object runtime.Object) bool {
	return g.MatchesScopeFunc(scope, object)
}

Usage()

Usage()方法可以计算object所需要的资源。

1
2
3
4
//***计算资源使用量***//
func (g *GenericEvaluator) Usage(object runtime.Object) api.ResourceList {
	return g.UsageFunc(object)
}

UsageStats()

UsageStats()可以计算出某命名空间下某类对象的资源使用情况。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
func (g *GenericEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
	// default each tracked resource to zero
	result := quota.UsageStats{Used: api.ResourceList{}}
	for _, resourceName := range g.MatchedResourceNames {
		result.Used[resourceName] = resource.MustParse("0")
	}
	//***获取资源***//
	items, err := g.ListFuncByNamespace(options.Namespace, api.ListOptions{
		LabelSelector: labels.Everything(),
	})
	if err != nil {
		return result, fmt.Errorf("%s: Failed to list %v: %v", g.Name, g.GroupKind(), err)
	}
	for _, item := range items {
		// need to verify that the item matches the set of scopes
		matchesScopes := true
		for _, scope := range options.Scopes {
			if !g.MatchesScope(scope, item) {
				matchesScopes = false
			}
		}
		// only count usage if there was a match
		//***计算并累加资源使用量***//
		if matchesScopes {
			result.Used = quota.Add(result.Used, g.Usage(item))
		}
	}
	return result, nil
}

PodEvaluator

上小节介绍了Evaluator，在这小节将以PodEvaluator。PodEvaluator可以计算Pod的所需资源量。
PodEvaluator定义在/pkg/quota/evaluator/core/pods.go中，其本身就是一个Evaluator:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
//***pod资源统计器***//
func NewPodEvaluator(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Evaluator {
	computeResources := []api.ResourceName{
		api.ResourceCPU,
		api.ResourceMemory,
		api.ResourceRequestsCPU,
		api.ResourceRequestsMemory,
		api.ResourceLimitsCPU,
		api.ResourceLimitsMemory,
	}
	//***与pod相关的所有资源***//
	allResources := append(computeResources, api.ResourcePods)
	//***用来获取具体namespace下的pods***//
	listFuncByNamespace := listPodsByNamespaceFuncUsingClient(kubeClient)
	if f != nil {
		listFuncByNamespace = generic.ListResourceUsingInformerFunc(f, unversioned.GroupResource{Resource: "pods"})
	}
	return &generic.GenericEvaluator{
		Name:              "Evaluator.Pod",
		InternalGroupKind: api.Kind("Pod"),
		//***支持的操作有Create，需要更新allResourcces***//
		InternalOperationResources: map[admission.Operation][]api.ResourceName{
			admission.Create: allResources,
			// TODO: the quota system can only charge for deltas on compute resources when pods support updates.
			// admission.Update: computeResources,
		},
		GetFuncByNamespace: func(namespace, name string) (runtime.Object, error) {
			return kubeClient.Core().Pods(namespace).Get(name)
		},
		ConstraintsFunc:      PodConstraintsFunc,
		MatchedResourceNames: allResources,
		MatchesScopeFunc:     PodMatchesScopeFunc,
		UsageFunc:            PodUsageFunc,
		ListFuncByNamespace:  listFuncByNamespace,
	}
}

这里要着重说下listFuncByNamespace，有listPodsByNamespaceFuncUsingClient和ListResourceUsingInformerFunc两种，而且ListResourceUsingInformerFunc优先级更高，具体由NewPodEvaluator()的参数来控制。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
//***生成pod list获取函数***//
func listPodsByNamespaceFuncUsingClient(kubeClient clientset.Interface) generic.ListFuncByNamespace {
	// TODO: ideally, we could pass dynamic client pool down into this code, and have one way of doing this.
	// unfortunately, dynamic client works with Unstructured objects, and when we calculate Usage, we require
	// structured objects.
	//***可以获取某namespace下的pods***//
	return func(namespace string, options api.ListOptions) ([]runtime.Object, error) {
		itemList, err := kubeClient.Core().Pods(namespace).List(options)
		if err != nil {
			return nil, err
		}
		results := make([]runtime.Object, 0, len(itemList.Items))
		for i := range itemList.Items {
			results = append(results, &itemList.Items[i])
		}
		return results, nil
	}
}

1
2
3
4
5
6
7
8
9
10
// ListResourceUsingInformerFunc returns a listing function based on the shared informer factory for the specified resource.
func ListResourceUsingInformerFunc(f informers.SharedInformerFactory, groupResource unversioned.GroupResource) ListFuncByNamespace {
	return func(namespace string, options api.ListOptions) ([]runtime.Object, error) {
		informer, err := f.ForResource(groupResource)
		if err != nil {
			return nil, err
		}
		return informer.Lister().ByNamespace(namespace).List(options.LabelSelector)
	}
}

PodUsageFunc()

PodUsageFunc()函数用来计算Pod的所需资源。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
//***计算pod的资源使用量***//
func PodUsageFunc(object runtime.Object) api.ResourceList {
	pod, ok := object.(*api.Pod)
	if !ok {
		return api.ResourceList{}
	}
	// by convention, we do not quota pods that have reached an end-of-life state
	if !QuotaPod(pod) {
		return api.ResourceList{}
	}
	// TODO: fix this when we have pod level cgroups
	// when we have pod level cgroups, we can just read pod level requests/limits
	requests := api.ResourceList{}
	limits := api.ResourceList{}
	//***统计requests和limits***//
	for i := range pod.Spec.Containers {
		requests = quota.Add(requests, pod.Spec.Containers[i].Resources.Requests)
		limits = quota.Add(limits, pod.Spec.Containers[i].Resources.Limits)
	}
	// InitContainers are run sequentially before other containers start, so the highest
	// init container resource is compared against the sum of app containers to determine
	// the effective usage for both requests and limits.
	for i := range pod.Spec.InitContainers {
		requests = quota.Max(requests, pod.Spec.InitContainers[i].Resources.Requests)
		limits = quota.Max(limits, pod.Spec.InitContainers[i].Resources.Limits)
	}
	return podUsageHelper(requests, limits)
}
//***根据收集到的requests和limits生成result***//
func podUsageHelper(requests api.ResourceList, limits api.ResourceList) api.ResourceList {
	result := api.ResourceList{}
	//***占用1个pod数量配额***//
	result[api.ResourcePods] = resource.MustParse("1")
	if request, found := requests[api.ResourceCPU]; found {
		result[api.ResourceCPU] = request
		result[api.ResourceRequestsCPU] = request
	}
	if limit, found := limits[api.ResourceCPU]; found {
		result[api.ResourceLimitsCPU] = limit
	}
	if request, found := requests[api.ResourceMemory]; found {
		result[api.ResourceMemory] = request
		result[api.ResourceRequestsMemory] = request
	}
	if limit, found := limits[api.ResourceMemory]; found {
		result[api.ResourceLimitsMemory] = limit
	}
	return result
}

GenericRegistry

如PodEvaluator这样的Evaluator有非常多个，所以需要有一个地方来管理这些Evaluator，这个管理Evaluator的就是GenericRegistry。
GenericRegistry定义在/pkg/quota/generic/registry.go中：

1
2
3
4
5
6
7
8
9
10
11
12
13
// GenericRegistry implements Registry
type GenericRegistry struct {
	// internal evaluators by group kind
	InternalEvaluators map[unversioned.GroupKind]quota.Evaluator
}
``` 
可以看出，GenericRegistry中有字段InternalEvaluators，里面记录了GK和对应Evaluator的映射关系。可以通过Evaluators()方法获取InternalEvaluators。
``` Go
// Evaluators returns the map of evaluators by groupKind
func (r *GenericRegistry) Evaluators() map[unversioned.GroupKind]quota.Evaluator {
	return r.InternalEvaluators
}

GenericRegistry的生成函数定义在/pkg/quota/evaluator/core/registry.go中：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// NewRegistry returns a registry that knows how to deal with core kubernetes resources
// If an informer factory is provided, evaluators will use them.
func NewRegistry(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Registry {
	pod := NewPodEvaluator(kubeClient, f)
	service := NewServiceEvaluator(kubeClient)
	replicationController := NewReplicationControllerEvaluator(kubeClient)
	resourceQuota := NewResourceQuotaEvaluator(kubeClient)
	secret := NewSecretEvaluator(kubeClient)
	configMap := NewConfigMapEvaluator(kubeClient)
	persistentVolumeClaim := NewPersistentVolumeClaimEvaluator(kubeClient, f)
	return &generic.GenericRegistry{
		InternalEvaluators: map[unversioned.GroupKind]quota.Evaluator{
			pod.GroupKind():                   pod,
			service.GroupKind():               service,
			replicationController.GroupKind(): replicationController,
			secret.GroupKind():                secret,
			configMap.GroupKind():             configMap,
			resourceQuota.GroupKind():         resourceQuota,
			persistentVolumeClaim.GroupKind(): persistentVolumeClaim,
		},
	}
}

可以看出，NewRegistry()会把PodEvaluator, ReplicationControllerEvaluator, ResourceQuotaEvaluator, SecretEvaluator, ConfigMapEvaluator, PersistentVolumeClaimEvaluator注册到GenericRegistry。

入口

整个Evaluator的入口定义在/pkg/quota/install/registry.go中：

1
2
3
4
5
6
// NewRegistry returns a registry of quota evaluators.
// If a shared informer factory is provided, it is used by evaluators rather than performing direct queries.
func NewRegistry(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Registry {
	// TODO: when quota supports resources in other api groups, we will need to merge
	return core.NewRegistry(kubeClient, f)
}

这里的core.NewRegistry()就是上面的NewRegistry()，返回GenericRegistry。得到GenericRegistry后，通过调用Evaluators()方法，即可获取全部的Evaluator。