是直接通过pod的ip暴露的，我们可以直接通过get pod 获取容器的ip，在node上直接curl访问到
[root@k8s-master01 ink8s-pod-metrics]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
ink8s-pod-metrics-deployment-85d9795d6-95lsp 1/1 Running 0 13h 10.100.85.207 k8s-node01 <none><none>[root@k8s-master01 ink8s-pod-metrics]# curl -s 10.100.85.207:8080/metrics |grep ink8s # HELP ink8s_pod_metrics_get_node_detail k8s node detail each# TYPE ink8s_pod_metrics_get_node_detail gauge
ink8s_pod_metrics_get_node_detail{containerRuntimeVersion="containerd://1.4.4",hostname="k8s-master01",ip="172.20.70.205",kubeletVersion="v1.20.1"}1
ink8s_pod_metrics_get_node_detail{containerRuntimeVersion="containerd://1.4.4",hostname="k8s-node01",ip="172.20.70.215",kubeletVersion="v1.20.1"}1# HELP ink8s_pod_metrics_get_node_last_duration_seconds get node last duration seconds# TYPE ink8s_pod_metrics_get_node_last_duration_seconds gauge
ink8s_pod_metrics_get_node_last_duration_seconds 0.008506914# HELP ink8s_pod_metrics_get_pod_control_plane_pod_detail k8s pod detail of control plane# TYPE ink8s_pod_metrics_get_pod_control_plane_pod_detail gauge
ink8s_pod_metrics_get_pod_control_plane_pod_detail{component="etcd",ip="172.20.70.205",pod_name="etcd-k8s-master01"}1
ink8s_pod_metrics_get_pod_control_plane_pod_detail{component="kube-apiserver",ip="172.20.70.205",pod_name="kube-apiserver-k8s-master01"}1
ink8s_pod_metrics_get_pod_control_plane_pod_detail{component="kube-controller-manager",ip="172.20.70.205",pod_name="kube-controller-manager-k8s-master01"}1
ink8s_pod_metrics_get_pod_control_plane_pod_detail{component="kube-scheduler",ip="172.20.70.205",pod_name="kube-scheduler-k8s-master01"}1# HELP ink8s_pod_metrics_get_pod_last_duration_seconds get pod last duration seconds# TYPE ink8s_pod_metrics_get_pod_last_duration_seconds gauge
ink8s_pod_metrics_get_pod_last_duration_seconds 0.012481561
target页面举例图片
容器基础资源指标
kubelet 内置cadvisor metrics接口暴露的
我们可以先获取token，再使用token作为header访问各个节点的cadvisor指标
TOKEN=$(kubectl -n kube-system get secret $(kubectl -n kube-system get serviceaccount prometheus -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}'| base64 --decode )curl -s https://172.20.70.215:10250/metrics/cadvisor --header "Authorization: Bearer $TOKEN" --insecure |head# HELP cadvisor_version_info A metric with a constant '1' value labeled by kernel version, OS version, docker version, cadvisor version & cadvisor revision.# TYPE cadvisor_version_info gauge
cadvisor_version_info{cadvisorRevision="",cadvisorVersion="",dockerVersion="1.13.1",kernelVersion="3.10.0-957.1.3.el7.x86_64",osVersion="CentOS Linux 7 (Core)"}1# HELP container_cpu_cfs_periods_total Number of elapsed enforcement period intervals.# TYPE container_cpu_cfs_periods_total counter
container_cpu_cfs_periods_total{container="",id="/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod6ab97c68_b0ac_48ce_ba39_6ffa72a2f4c8.slice",image="",name="",namespace="default",pod="ink8s-pod-metrics-deployment-85d9795d6-95lsp"}466641629771810858
container_cpu_cfs_periods_total{container="",id="/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podbf3f353a_92fa_4436_a8ca_6cb632d48ada.slice",image="",name="",namespace="kube-admin",pod="k8s-mon-daemonset-z6sfw"}7629651629771819606
container_cpu_cfs_periods_total{container="",id="/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd9a95d67_a843_4369_8d5c_34a5333f1480.slice",image="",name="",namespace="kube-admin",pod="k8s-mon-deployment-6d7d58bdc8-rxj42"}4588221629771809776
container_cpu_cfs_periods_total{container="",id="/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pode27c9fe7_9d82_4228_86fb_b9c920611c15.slice",image="",name="",namespace="kube-system",pod="prometheus-0"}9413741629771809770
container_cpu_cfs_periods_total{container="ink8s-pod-metrics",id="/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod6ab97c68_b0ac_48ce_ba39_6ffa72a2f4c8.slice/cri-containerd-2f85fd45a67cc4bb775b99d4676200b412ea18ef7ae4976fc93a8a7cff1c5f34.scope",image="docker.io/library/ink8s-pod-metrics:v1",name="2f85fd45a67cc4bb775b99d4676200b412ea18ef7ae4976fc93a8a7cff1c5f34",namespace="default",pod="ink8s-pod-metrics-deployment-85d9795d6-95lsp"}466671629771818053
target页面举例图片
k8s对象资源指标
这是ksm直接暴露指标，prometheus通过dns解析到域名然后访问的
我们可以通过dig获取ksm 的service_ip，然后访问 service_ip:8080
dig +short kube-state-metrics.kube-system.svc.cluster.local @10.96.0.10
10.100.85.200
curl -s 10.100.85.200:8080/metrics |head# HELP kube_certificatesigningrequest_labels Kubernetes labels converted to Prometheus labels.# TYPE kube_certificatesigningrequest_labels gauge# HELP kube_certificatesigningrequest_created Unix creation timestamp# TYPE kube_certificatesigningrequest_created gauge# HELP kube_certificatesigningrequest_condition The number of each certificatesigningrequest condition# TYPE kube_certificatesigningrequest_condition gauge# HELP kube_certificatesigningrequest_cert_length Length of the issued cert# TYPE kube_certificatesigningrequest_cert_length gauge# HELP kube_configmap_info Information about configmap.# TYPE kube_configmap_info gauge
target页面举例图片
k8s服务组件指标
是由服务组件自身直接暴露的，我们也可以通过带token直接访问
TOKEN=$(kubectl -n kube-system get secret $(kubectl -n kube-system get serviceaccount prometheus -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}'| base64 --decode )curl -s https://localhost:6443/metrics --header "Authorization: Bearer $TOKEN" --insecure |head# HELP aggregator_openapi_v2_regeneration_count [ALPHA] Counter of OpenAPI v2 spec regeneration count broken down by causing APIService name and reason.# TYPE aggregator_openapi_v2_regeneration_count counter
aggregator_openapi_v2_regeneration_count{apiservice="*",reason="startup"}0
aggregator_openapi_v2_regeneration_count{apiservice="k8s_internal_local_delegation_chain_0000000002",reason="update"}0# HELP aggregator_openapi_v2_regeneration_duration [ALPHA] Gauge of OpenAPI v2 spec regeneration duration in seconds.# TYPE aggregator_openapi_v2_regeneration_duration gauge
aggregator_openapi_v2_regeneration_duration{reason="startup"}0.812717406
aggregator_openapi_v2_regeneration_duration{reason="update"}0.848521427# HELP aggregator_unavailable_apiservice [ALPHA] Gauge of APIServices which are marked as unavailable broken down by APIService name.# TYPE aggregator_unavailable_apiservice gauge[root@k8s-master01 ink8s-pod-metrics]#
