Consolidate YAML files [part-4] (#9241)
This PR consolidates YAML files used in the `tasks/debug-application-cluster` subdirectory. Depends-On: #9236
This commit is contained in:
@@ -1,10 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: counter
|
||||
spec:
|
||||
containers:
|
||||
- name: count
|
||||
image: busybox
|
||||
args: [/bin/sh, -c,
|
||||
'i=0; while true; do echo "$i: $(date)"; i=$((i+1)); sleep 1; done']
|
||||
@@ -28,15 +28,15 @@ the description of how logs are stored and handled on the node to be useful.
|
||||
|
||||
In this section, you can see an example of basic logging in Kubernetes that
|
||||
outputs data to the standard output stream. This demonstration uses
|
||||
a [pod specification](/docs/concepts/cluster-administration/counter-pod.yaml) with
|
||||
a [pod specification](/examples/debug/counter-pod.yaml) with
|
||||
a container that writes some text to standard output once per second.
|
||||
|
||||
{{< code file="counter-pod.yaml" >}}
|
||||
{{< codenew file="debug/counter-pod.yaml" >}}
|
||||
|
||||
To run this pod, use the following command:
|
||||
|
||||
```shell
|
||||
$ kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/counter-pod.yaml
|
||||
$ kubectl create -f https://k8s.io/examples/debug/counter-pod.yaml
|
||||
pod "counter" created
|
||||
```
|
||||
|
||||
@@ -251,4 +251,4 @@ You can implement cluster-level logging by exposing or pushing logs directly fro
|
||||
every application; however, the implementation for such a logging mechanism
|
||||
is outside the scope of Kubernetes.
|
||||
|
||||
{{% /capture %}}
|
||||
{{% /capture %}}
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
apiVersion: audit.k8s.io/v1beta1 # This is required.
|
||||
kind: Policy
|
||||
# Don't generate audit events for all requests in RequestReceived stage.
|
||||
omitStages:
|
||||
- "RequestReceived"
|
||||
rules:
|
||||
# Log pod changes at RequestResponse level
|
||||
- level: RequestResponse
|
||||
resources:
|
||||
- group: ""
|
||||
# Resource "pods" doesn't match requests to any subresource of pods,
|
||||
# which is consistent with the RBAC policy.
|
||||
resources: ["pods"]
|
||||
# Log "pods/log", "pods/status" at Metadata level
|
||||
- level: Metadata
|
||||
resources:
|
||||
- group: ""
|
||||
resources: ["pods/log", "pods/status"]
|
||||
|
||||
# Don't log requests to a configmap called "controller-leader"
|
||||
- level: None
|
||||
resources:
|
||||
- group: ""
|
||||
resources: ["configmaps"]
|
||||
resourceNames: ["controller-leader"]
|
||||
|
||||
# Don't log watch requests by the "system:kube-proxy" on endpoints or services
|
||||
- level: None
|
||||
users: ["system:kube-proxy"]
|
||||
verbs: ["watch"]
|
||||
resources:
|
||||
- group: "" # core API group
|
||||
resources: ["endpoints", "services"]
|
||||
|
||||
# Don't log authenticated requests to certain non-resource URL paths.
|
||||
- level: None
|
||||
userGroups: ["system:authenticated"]
|
||||
nonResourceURLs:
|
||||
- "/api*" # Wildcard matching.
|
||||
- "/version"
|
||||
|
||||
# Log the request body of configmap changes in kube-system.
|
||||
- level: Request
|
||||
resources:
|
||||
- group: "" # core API group
|
||||
resources: ["configmaps"]
|
||||
# This rule only applies to resources in the "kube-system" namespace.
|
||||
# The empty string "" can be used to select non-namespaced resources.
|
||||
namespaces: ["kube-system"]
|
||||
|
||||
# Log configmap and secret changes in all other namespaces at the Metadata level.
|
||||
- level: Metadata
|
||||
resources:
|
||||
- group: "" # core API group
|
||||
resources: ["secrets", "configmaps"]
|
||||
|
||||
# Log all other resources in core and extensions at the Request level.
|
||||
- level: Request
|
||||
resources:
|
||||
- group: "" # core API group
|
||||
- group: "extensions" # Version of group should NOT be included.
|
||||
|
||||
# A catch-all rule to log all other requests at the Metadata level.
|
||||
- level: Metadata
|
||||
# Long-running requests like watches that fall under this rule will not
|
||||
# generate an audit event in RequestReceived.
|
||||
omitStages:
|
||||
- "RequestReceived"
|
||||
@@ -77,7 +77,7 @@ A policy with no (0) rules is treated as illegal.
|
||||
|
||||
Below is an example audit policy file:
|
||||
|
||||
{{< code file="audit-policy.yaml" >}}
|
||||
{{< codenew file="audit/audit-policy.yaml" >}}
|
||||
|
||||
You can use a minimal audit policy file to log all requests at the `Metadata` level:
|
||||
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: counter
|
||||
spec:
|
||||
containers:
|
||||
- name: count
|
||||
image: busybox
|
||||
args: [/bin/sh, -c,
|
||||
'i=0; while true; do echo "$i: $(date)"; i=$((i+1)); sleep 1; done']
|
||||
@@ -22,12 +22,12 @@ your pods. But there are a number of ways to get even more information about you
|
||||
|
||||
For this example we'll use a Deployment to create two pods, similar to the earlier example.
|
||||
|
||||
{{< code file="nginx-dep.yaml" >}}
|
||||
{{< codenew file="application/nginx-with-request.yaml" >}}
|
||||
|
||||
Create deployment by running following command:
|
||||
|
||||
```shell
|
||||
$ kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/nginx-dep.yaml
|
||||
$ kubectl create -f https://k8s.io/examples/application/nginx-with-request.yaml
|
||||
deployment "nginx-deployment" created
|
||||
```
|
||||
|
||||
|
||||
@@ -34,11 +34,11 @@ In this exercise, you create a Pod that runs one container.
|
||||
The configuration file specifies a command that runs when
|
||||
the container starts.
|
||||
|
||||
{{< code file="termination.yaml" >}}
|
||||
{{< codenew file="debug/termination.yaml" >}}
|
||||
|
||||
1. Create a Pod based on the YAML configuration file:
|
||||
|
||||
kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/termination.yaml
|
||||
kubectl create -f https://k8s.io/examples/debug/termination.yaml
|
||||
|
||||
In the YAML file, in the `cmd` and `args` fields, you can see that the
|
||||
container sleeps for 10 seconds and then writes "Sleep expired" to
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: event-exporter-sa
|
||||
namespace: default
|
||||
labels:
|
||||
app: event-exporter
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: event-exporter-rb
|
||||
labels:
|
||||
app: event-exporter
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: view
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: event-exporter-sa
|
||||
namespace: default
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: event-exporter-v0.1.0
|
||||
namespace: default
|
||||
labels:
|
||||
app: event-exporter
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: event-exporter
|
||||
replicas: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: event-exporter
|
||||
spec:
|
||||
serviceAccountName: event-exporter-sa
|
||||
containers:
|
||||
- name: event-exporter
|
||||
image: k8s.gcr.io/event-exporter:v0.1.0
|
||||
command:
|
||||
- '/event-exporter'
|
||||
terminationGracePeriodSeconds: 30
|
||||
@@ -60,7 +60,7 @@ average, approximately 100Mb RAM and 100m CPU is needed.
|
||||
Deploy event exporter to your cluster using the following command:
|
||||
|
||||
```shell
|
||||
kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/event-exporter-deploy.yaml
|
||||
kubectl create -f https://k8s.io/examples/debug/event-exporter.yaml
|
||||
```
|
||||
|
||||
Since event exporter accesses the Kubernetes API, it requires permissions to
|
||||
@@ -70,7 +70,7 @@ to allow event exporter to read events. To make sure that event exporter
|
||||
pod will not be evicted from the node, you can additionally set up resource
|
||||
requests. As mentioned earlier, 100Mb RAM and 100m CPU should be enough.
|
||||
|
||||
{{< code file="event-exporter-deploy.yaml" >}}
|
||||
{{< codenew file="debug/event-exporter.yaml" >}}
|
||||
|
||||
## User Guide
|
||||
|
||||
@@ -94,4 +94,4 @@ jsonPayload.involvedObject.name:"nginx-deployment"
|
||||
|
||||
{{< figure src="/images/docs/stackdriver-event-exporter-filter.png" alt="Filtered events in the Stackdriver Logging interface" width="500" >}}
|
||||
|
||||
{{% /capture %}}
|
||||
{{% /capture %}}
|
||||
|
||||
@@ -1,390 +0,0 @@
|
||||
kind: ConfigMap
|
||||
apiVersion: v1
|
||||
data:
|
||||
containers.input.conf: |-
|
||||
# This configuration file for Fluentd is used
|
||||
# to watch changes to Docker log files that live in the
|
||||
# directory /var/lib/docker/containers/ and are symbolically
|
||||
# linked to from the /var/log/containers directory using names that capture the
|
||||
# pod name and container name. These logs are then submitted to
|
||||
# Google Cloud Logging which assumes the installation of the cloud-logging plug-in.
|
||||
#
|
||||
# Example
|
||||
# =======
|
||||
# A line in the Docker log file might look like this JSON:
|
||||
#
|
||||
# {"log":"2014/09/25 21:15:03 Got request with path wombat\\n",
|
||||
# "stream":"stderr",
|
||||
# "time":"2014-09-25T21:15:03.499185026Z"}
|
||||
#
|
||||
# The record reformer is used to write the tag to focus on the pod name
|
||||
# and the Kubernetes container name. For example a Docker container's logs
|
||||
# might be in the directory:
|
||||
# /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b
|
||||
# and in the file:
|
||||
# 997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
|
||||
# where 997599971ee6... is the Docker ID of the running container.
|
||||
# The Kubernetes kubelet makes a symbolic link to this file on the host machine
|
||||
# in the /var/log/containers directory which includes the pod name and the Kubernetes
|
||||
# container name:
|
||||
# synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
||||
# ->
|
||||
# /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
|
||||
# The /var/log directory on the host is mapped to the /var/log directory in the container
|
||||
# running this instance of Fluentd and we end up collecting the file:
|
||||
# /var/log/containers/synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
||||
# This results in the tag:
|
||||
# var.log.containers.synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
||||
# The record reformer is used is discard the var.log.containers prefix and
|
||||
# the Docker container ID suffix and "kubernetes." is pre-pended giving the tag:
|
||||
# kubernetes.synthetic-logger-0.25lps-pod_default-synth-lgr
|
||||
# Tag is then parsed by google_cloud plugin and translated to the metadata,
|
||||
# visible in the log viewer
|
||||
|
||||
# Example:
|
||||
# {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
|
||||
<source>
|
||||
type tail
|
||||
format json
|
||||
time_key time
|
||||
path /var/log/containers/*.log
|
||||
pos_file /var/log/gcp-containers.log.pos
|
||||
time_format %Y-%m-%dT%H:%M:%S.%N%Z
|
||||
tag reform.*
|
||||
read_from_head true
|
||||
</source>
|
||||
|
||||
<filter reform.**>
|
||||
type parser
|
||||
format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<log>.*)/
|
||||
reserve_data true
|
||||
suppress_parse_error_log true
|
||||
key_name log
|
||||
</filter>
|
||||
|
||||
<match reform.**>
|
||||
type record_reformer
|
||||
enable_ruby true
|
||||
tag raw.kubernetes.${tag_suffix[4].split('-')[0..-2].join('-')}
|
||||
</match>
|
||||
|
||||
# Detect exceptions in the log output and forward them as one log entry.
|
||||
<match raw.kubernetes.**>
|
||||
@type copy
|
||||
|
||||
<store>
|
||||
@type prometheus
|
||||
|
||||
<metric>
|
||||
type counter
|
||||
name logging_line_count
|
||||
desc Total number of lines generated by application containers
|
||||
<labels>
|
||||
tag ${tag}
|
||||
</labels>
|
||||
</metric>
|
||||
</store>
|
||||
<store>
|
||||
@type detect_exceptions
|
||||
|
||||
remove_tag_prefix raw
|
||||
message log
|
||||
stream stream
|
||||
multiline_flush_interval 5
|
||||
max_bytes 500000
|
||||
max_lines 1000
|
||||
</store>
|
||||
</match>
|
||||
system.input.conf: |-
|
||||
# Example:
|
||||
# 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081
|
||||
<source>
|
||||
type tail
|
||||
format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
|
||||
time_format %Y-%m-%d %H:%M:%S
|
||||
path /var/log/salt/minion
|
||||
pos_file /var/log/gcp-salt.pos
|
||||
tag salt
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script
|
||||
<source>
|
||||
type tail
|
||||
format syslog
|
||||
path /var/log/startupscript.log
|
||||
pos_file /var/log/gcp-startupscript.log.pos
|
||||
tag startupscript
|
||||
</source>
|
||||
|
||||
# Examples:
|
||||
# time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
|
||||
# time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
|
||||
<source>
|
||||
type tail
|
||||
format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
|
||||
path /var/log/docker.log
|
||||
pos_file /var/log/gcp-docker.log.pos
|
||||
tag docker
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# 2016/02/04 06:52:38 filePurge: successfully removed file /var/etcd/data/member/wal/00000000000006d0-00000000010a23d1.wal
|
||||
<source>
|
||||
type tail
|
||||
# Not parsing this, because it doesn't have anything particularly useful to
|
||||
# parse out of it (like severities).
|
||||
format none
|
||||
path /var/log/etcd.log
|
||||
pos_file /var/log/gcp-etcd.log.pos
|
||||
tag etcd
|
||||
</source>
|
||||
|
||||
# Multi-line parsing is required for all the kube logs because very large log
|
||||
# statements, such as those that include entire object bodies, get split into
|
||||
# multiple lines by glog.
|
||||
|
||||
# Example:
|
||||
# I0204 07:32:30.020537 3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
|
||||
<source>
|
||||
type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /var/log/kubelet.log
|
||||
pos_file /var/log/gcp-kubelet.log.pos
|
||||
tag kubelet
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# I1118 21:26:53.975789 6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
|
||||
<source>
|
||||
type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /var/log/kube-proxy.log
|
||||
pos_file /var/log/gcp-kube-proxy.log.pos
|
||||
tag kube-proxy
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# I0204 07:00:19.604280 5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
|
||||
<source>
|
||||
type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /var/log/kube-apiserver.log
|
||||
pos_file /var/log/gcp-kube-apiserver.log.pos
|
||||
tag kube-apiserver
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
|
||||
# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
|
||||
<source>
|
||||
type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\S+\s+AUDIT:/
|
||||
# Fields must be explicitly captured by name to be parsed into the record.
|
||||
# Fields may not always be present, and order may change, so this just looks
|
||||
# for a list of key="\"quoted\" value" pairs separated by spaces.
|
||||
# Unknown fields are ignored.
|
||||
# Note: We can't separate query/response lines as format1/format2 because
|
||||
# they don't always come one after the other for a given query.
|
||||
# TODO: Maybe add a JSON output mode to audit log so we can get rid of this?
|
||||
format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
|
||||
time_format %FT%T.%L%Z
|
||||
path /var/log/kube-apiserver-audit.log
|
||||
pos_file /var/log/gcp-kube-apiserver-audit.log.pos
|
||||
tag kube-apiserver-audit
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# I0204 06:55:31.872680 5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kubernetes-dashboard
|
||||
<source>
|
||||
type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /var/log/kube-controller-manager.log
|
||||
pos_file /var/log/gcp-kube-controller-manager.log.pos
|
||||
tag kube-controller-manager
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# W0204 06:49:18.239674 7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
|
||||
<source>
|
||||
type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /var/log/kube-scheduler.log
|
||||
pos_file /var/log/gcp-kube-scheduler.log.pos
|
||||
tag kube-scheduler
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# I1104 10:36:20.242766 5 rescheduler.go:73] Running Rescheduler
|
||||
<source>
|
||||
type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /var/log/rescheduler.log
|
||||
pos_file /var/log/gcp-rescheduler.log.pos
|
||||
tag rescheduler
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# I0603 15:31:05.793605 6 cluster_manager.go:230] Reading config from path /etc/gce.conf
|
||||
<source>
|
||||
type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /var/log/glbc.log
|
||||
pos_file /var/log/gcp-glbc.log.pos
|
||||
tag glbc
|
||||
</source>
|
||||
|
||||
# Example:
|
||||
# I0603 15:31:05.793605 6 cluster_manager.go:230] Reading config from path /etc/gce.conf
|
||||
<source>
|
||||
type tail
|
||||
format multiline
|
||||
multiline_flush_interval 5s
|
||||
format_firstline /^\w\d{4}/
|
||||
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
||||
time_format %m%d %H:%M:%S.%N
|
||||
path /var/log/cluster-autoscaler.log
|
||||
pos_file /var/log/gcp-cluster-autoscaler.log.pos
|
||||
tag cluster-autoscaler
|
||||
</source>
|
||||
|
||||
# Logs from systemd-journal for interesting services.
|
||||
<source>
|
||||
type systemd
|
||||
filters [{ "_SYSTEMD_UNIT": "docker.service" }]
|
||||
pos_file /var/log/gcp-journald-docker.pos
|
||||
read_from_head true
|
||||
tag docker
|
||||
</source>
|
||||
|
||||
<source>
|
||||
type systemd
|
||||
filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
|
||||
pos_file /var/log/gcp-journald-kubelet.pos
|
||||
read_from_head true
|
||||
tag kubelet
|
||||
</source>
|
||||
monitoring.conf: |-
|
||||
# Prometheus monitoring
|
||||
<source>
|
||||
@type prometheus
|
||||
port 80
|
||||
</source>
|
||||
|
||||
<source>
|
||||
@type prometheus_monitor
|
||||
</source>
|
||||
output.conf: |-
|
||||
# We use 2 output stanzas - one to handle the container logs and one to handle
|
||||
# the node daemon logs, the latter of which explicitly sends its logs to the
|
||||
# compute.googleapis.com service rather than container.googleapis.com to keep
|
||||
# them separate since most users don't care about the node logs.
|
||||
<match kubernetes.**>
|
||||
@type copy
|
||||
|
||||
<store>
|
||||
@type google_cloud
|
||||
|
||||
# Set the buffer type to file to improve the reliability and reduce the memory consumption
|
||||
buffer_type file
|
||||
buffer_path /var/log/fluentd-buffers/kubernetes.containers.buffer
|
||||
# Set queue_full action to block because we want to pause gracefully
|
||||
# in case of the off-the-limits load instead of throwing an exception
|
||||
buffer_queue_full_action block
|
||||
# Set the chunk limit conservatively to avoid exceeding the GCL limit
|
||||
# of 10MiB per write request.
|
||||
buffer_chunk_limit 2M
|
||||
# Cap the combined memory usage of this buffer and the one below to
|
||||
# 2MiB/chunk * (6 + 2) chunks = 16 MiB
|
||||
buffer_queue_limit 6
|
||||
# Never wait more than 5 seconds before flushing logs in the non-error case.
|
||||
flush_interval 5s
|
||||
# Never wait longer than 30 seconds between retries.
|
||||
max_retry_wait 30
|
||||
# Disable the limit on the number of retries (retry forever).
|
||||
disable_retry_limit
|
||||
# Use multiple threads for processing.
|
||||
num_threads 2
|
||||
</store>
|
||||
<store>
|
||||
@type prometheus
|
||||
|
||||
<metric>
|
||||
type counter
|
||||
name logging_entry_count
|
||||
desc Total number of log entries generated by either an application container or a system component
|
||||
<labels>
|
||||
tag ${tag}
|
||||
component container
|
||||
</labels>
|
||||
</metric>
|
||||
</store>
|
||||
</match>
|
||||
|
||||
# Keep a smaller buffer here since these logs are less important than the user's
|
||||
# container logs.
|
||||
<match **>
|
||||
@type copy
|
||||
|
||||
<store>
|
||||
@type google_cloud
|
||||
|
||||
detect_subservice false
|
||||
buffer_type file
|
||||
buffer_path /var/log/fluentd-buffers/kubernetes.system.buffer
|
||||
buffer_queue_full_action block
|
||||
buffer_chunk_limit 2M
|
||||
buffer_queue_limit 2
|
||||
flush_interval 5s
|
||||
max_retry_wait 30
|
||||
disable_retry_limit
|
||||
num_threads 2
|
||||
</store>
|
||||
<store>
|
||||
@type prometheus
|
||||
|
||||
<metric>
|
||||
type counter
|
||||
name logging_entry_count
|
||||
desc Total number of log entries generated by either an application container or a system component
|
||||
<labels>
|
||||
tag ${tag}
|
||||
component system
|
||||
</labels>
|
||||
</metric>
|
||||
</store>
|
||||
</match>
|
||||
metadata:
|
||||
name: fluentd-gcp-config
|
||||
labels:
|
||||
addonmanager.kubernetes.io/mode: Reconcile
|
||||
@@ -1,113 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: fluentd-gcp-v2.0
|
||||
labels:
|
||||
k8s-app: fluentd-gcp
|
||||
kubernetes.io/cluster-service: "true"
|
||||
addonmanager.kubernetes.io/mode: Reconcile
|
||||
version: v2.0
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
k8s-app: fluentd-gcp
|
||||
kubernetes.io/cluster-service: "true"
|
||||
version: v2.0
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: fluentd-gcp
|
||||
kubernetes.io/cluster-service: "true"
|
||||
version: v2.0
|
||||
# This annotation ensures that fluentd does not get evicted if the node
|
||||
# supports critical pod annotation based priority scheme.
|
||||
# Note that this does not guarantee admission on the nodes (#40573).
|
||||
annotations:
|
||||
scheduler.alpha.kubernetes.io/critical-pod: ''
|
||||
spec:
|
||||
dnsPolicy: Default
|
||||
containers:
|
||||
- name: fluentd-gcp
|
||||
image: k8s.gcr.io/fluentd-gcp:2.0.2
|
||||
# If fluentd consumes its own logs, the following situation may happen:
|
||||
# fluentd fails to send a chunk to the server => writes it to the log =>
|
||||
# tries to send this message to the server => fails to send a chunk and so on.
|
||||
# Writing to a file, which is not exported to the back-end prevents it.
|
||||
# It also allows to increase the fluentd verbosity by default.
|
||||
command:
|
||||
- '/bin/sh'
|
||||
- '-c'
|
||||
- '/run.sh $FLUENTD_ARGS 2>&1 >>/var/log/fluentd.log'
|
||||
env:
|
||||
- name: FLUENTD_ARGS
|
||||
value: --no-supervisor
|
||||
resources:
|
||||
limits:
|
||||
memory: 300Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 200Mi
|
||||
volumeMounts:
|
||||
- name: varlog
|
||||
mountPath: /var/log
|
||||
- name: varlibdockercontainers
|
||||
mountPath: /var/lib/docker/containers
|
||||
readOnly: true
|
||||
- name: libsystemddir
|
||||
mountPath: /host/lib
|
||||
readOnly: true
|
||||
- name: config-volume
|
||||
mountPath: /etc/fluent/config.d
|
||||
# Liveness probe is aimed to help in situations where fluentd
|
||||
# silently hangs for no apparent reasons until manual restart.
|
||||
# The idea of this probe is that if fluentd is not queueing or
|
||||
# flushing chunks for 5 minutes, something is not right. If
|
||||
# you want to change the fluentd configuration, reducing amount of
|
||||
# logs fluentd collects, consider changing the threshold or turning
|
||||
# liveness probe off completely.
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 600
|
||||
periodSeconds: 60
|
||||
exec:
|
||||
command:
|
||||
- '/bin/sh'
|
||||
- '-c'
|
||||
- >
|
||||
LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300};
|
||||
STUCK_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-900};
|
||||
if [ ! -e /var/log/fluentd-buffers ];
|
||||
then
|
||||
exit 1;
|
||||
fi;
|
||||
LAST_MODIFIED_DATE=`stat /var/log/fluentd-buffers | grep Modify | sed -r "s/Modify: (.*)/\1/"`;
|
||||
LAST_MODIFIED_TIMESTAMP=`date -d "$LAST_MODIFIED_DATE" +%s`;
|
||||
if [ `date +%s` -gt `expr $LAST_MODIFIED_TIMESTAMP + $STUCK_THRESHOLD_SECONDS` ];
|
||||
then
|
||||
rm -rf /var/log/fluentd-buffers;
|
||||
exit 1;
|
||||
fi;
|
||||
if [ `date +%s` -gt `expr $LAST_MODIFIED_TIMESTAMP + $LIVENESS_THRESHOLD_SECONDS` ];
|
||||
then
|
||||
exit 1;
|
||||
fi;
|
||||
nodeSelector:
|
||||
beta.kubernetes.io/fluentd-ds-ready: "true"
|
||||
tolerations:
|
||||
- key: "node.alpha.kubernetes.io/ismaster"
|
||||
effect: "NoSchedule"
|
||||
terminationGracePeriodSeconds: 30
|
||||
volumes:
|
||||
- name: varlog
|
||||
hostPath:
|
||||
path: /var/log
|
||||
- name: varlibdockercontainers
|
||||
hostPath:
|
||||
path: /var/lib/docker/containers
|
||||
- name: libsystemddir
|
||||
hostPath:
|
||||
path: /usr/lib64
|
||||
- name: config-volume
|
||||
configMap:
|
||||
name: fluentd-gcp-config
|
||||
@@ -28,12 +28,12 @@ running Container.
|
||||
In this exercise, you create a Pod that has one Container. The Container
|
||||
runs the nginx image. Here is the configuration file for the Pod:
|
||||
|
||||
{{< code file="shell-demo.yaml" >}}
|
||||
{{< codenew file="application/shell-demo.yaml" >}}
|
||||
|
||||
Create the Pod:
|
||||
|
||||
```shell
|
||||
kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/shell-demo.yaml
|
||||
kubectl create -f https://k8s.io/examples/application/shell-demo.yaml
|
||||
```
|
||||
|
||||
Verify that the Container is running:
|
||||
|
||||
@@ -96,7 +96,7 @@ than Google Kubernetes Engine. Proceed at your own risk.
|
||||
1. Deploy a `ConfigMap` with the logging agent configuration by running the following command:
|
||||
|
||||
```
|
||||
kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/fluentd-gcp-configmap.yaml
|
||||
kubectl create -f https://k8s.io/examples/debug/fluentd-gcp-configmap.yaml
|
||||
```
|
||||
|
||||
The command creates the `ConfigMap` in the `default` namespace. You can download the file
|
||||
@@ -105,7 +105,7 @@ than Google Kubernetes Engine. Proceed at your own risk.
|
||||
1. Deploy the logging agent `DaemonSet` by running the following command:
|
||||
|
||||
```
|
||||
kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/fluentd-gcp-ds.yaml
|
||||
kubectl create -f https://k8s.io/examples/debug/fluentd-gcp-ds.yaml
|
||||
```
|
||||
|
||||
You can download and edit this file before using it as well.
|
||||
@@ -129,16 +129,16 @@ default fluentd-gcp-v2.0 3 3 3 beta.kubernetes.i
|
||||
```
|
||||
|
||||
To understand how logging with Stackdriver works, consider the following
|
||||
synthetic log generator pod specification [counter-pod.yaml](/docs/tasks/debug-application-cluster/counter-pod.yaml):
|
||||
synthetic log generator pod specification [counter-pod.yaml](/examples/debug/counter-pod.yaml):
|
||||
|
||||
{{< code file="counter-pod.yaml" >}}
|
||||
{{< codenew file="debug/counter-pod.yaml" >}}
|
||||
|
||||
This pod specification has one container that runs a bash script
|
||||
that writes out the value of a counter and the date once per
|
||||
second, and runs indefinitely. Let's create this pod in the default namespace.
|
||||
|
||||
```shell
|
||||
kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/counter-pod.yaml
|
||||
kubectl create -f https://k8s.io/examples/debug/counter-pod.yaml
|
||||
```
|
||||
|
||||
You can observe the running pod:
|
||||
@@ -175,7 +175,7 @@ pod "counter" deleted
|
||||
and then recreating it:
|
||||
|
||||
```shell
|
||||
$ kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/counter-pod.yaml
|
||||
$ kubectl create -f https://k8s.io/examples/debug/counter-pod.yaml
|
||||
pod "counter" created
|
||||
```
|
||||
|
||||
|
||||
@@ -64,7 +64,7 @@ customized node problems.
|
||||
|
||||
* **Step 1:** `node-problem-detector.yaml`:
|
||||
|
||||
{{< code file="node-problem-detector.yaml" >}}
|
||||
{{< codenew file="debug/node-problem-detector.yaml" >}}
|
||||
|
||||
|
||||
***Notice that you should make sure the system log directory is right for your
|
||||
@@ -73,7 +73,7 @@ OS distro.***
|
||||
* **Step 2:** Start node problem detector with `kubectl`:
|
||||
|
||||
```shell
|
||||
kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/node-problem-detector.yaml
|
||||
kubectl create -f https://k8s.io/examples/debug/node-problem-detector.yaml
|
||||
```
|
||||
|
||||
### Addon Pod
|
||||
@@ -98,14 +98,14 @@ following the steps:
|
||||
node-problem-detector-config --from-file=config/`.
|
||||
* **Step 3:** Change the `node-problem-detector.yaml` to use the ConfigMap:
|
||||
|
||||
{{< code file="node-problem-detector-configmap.yaml" >}}
|
||||
{{< codenew file="debug/node-problem-detector-configmap.yaml" >}}
|
||||
|
||||
|
||||
* **Step 4:** Re-create the node problem detector with the new yaml file:
|
||||
|
||||
```shell
|
||||
kubectl delete -f https://k8s.io/docs/tasks/debug-application-cluster/node-problem-detector.yaml # If you have a node-problem-detector running
|
||||
kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/node-problem-detector-configmap.yaml
|
||||
kubectl delete -f https://k8s.io/examples/debug/node-problem-detector.yaml # If you have a node-problem-detector running
|
||||
kubectl create -f https://k8s.io/examples/debug/node-problem-detector-configmap.yaml
|
||||
```
|
||||
|
||||
***Notice that this approach only applies to node problem detector started with `kubectl`.***
|
||||
@@ -177,4 +177,4 @@ resource overhead on each node. Usually this is fine, because:
|
||||
* Even under high load, the resource usage is acceptable.
|
||||
(see [benchmark result](https://github.com/kubernetes/node-problem-detector/issues/2#issuecomment-220255629))
|
||||
|
||||
{{% /capture %}}
|
||||
{{% /capture %}}
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: nginx-deployment
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: nginx
|
||||
replicas: 2
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx
|
||||
spec:
|
||||
containers:
|
||||
- name: nginx
|
||||
image: nginx
|
||||
resources:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
ports:
|
||||
- containerPort: 80
|
||||
@@ -1,49 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: node-problem-detector-v0.1
|
||||
namespace: kube-system
|
||||
labels:
|
||||
k8s-app: node-problem-detector
|
||||
version: v0.1
|
||||
kubernetes.io/cluster-service: "true"
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
k8s-app: node-problem-detector
|
||||
version: v0.1
|
||||
kubernetes.io/cluster-service: "true"
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: node-problem-detector
|
||||
version: v0.1
|
||||
kubernetes.io/cluster-service: "true"
|
||||
spec:
|
||||
hostNetwork: true
|
||||
containers:
|
||||
- name: node-problem-detector
|
||||
image: k8s.gcr.io/node-problem-detector:v0.1
|
||||
securityContext:
|
||||
privileged: true
|
||||
resources:
|
||||
limits:
|
||||
cpu: "200m"
|
||||
memory: "100Mi"
|
||||
requests:
|
||||
cpu: "20m"
|
||||
memory: "20Mi"
|
||||
volumeMounts:
|
||||
- name: log
|
||||
mountPath: /log
|
||||
readOnly: true
|
||||
- name: config # Overwrite the config/ directory with ConfigMap volume
|
||||
mountPath: /config
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: log
|
||||
hostPath:
|
||||
path: /var/log/
|
||||
- name: config # Define ConfigMap volume
|
||||
configMap:
|
||||
name: node-problem-detector-config
|
||||
@@ -1,43 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: node-problem-detector-v0.1
|
||||
namespace: kube-system
|
||||
labels:
|
||||
k8s-app: node-problem-detector
|
||||
version: v0.1
|
||||
kubernetes.io/cluster-service: "true"
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
k8s-app: node-problem-detector
|
||||
version: v0.1
|
||||
kubernetes.io/cluster-service: "true"
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: node-problem-detector
|
||||
version: v0.1
|
||||
kubernetes.io/cluster-service: "true"
|
||||
spec:
|
||||
hostNetwork: true
|
||||
containers:
|
||||
- name: node-problem-detector
|
||||
image: k8s.gcr.io/node-problem-detector:v0.1
|
||||
securityContext:
|
||||
privileged: true
|
||||
resources:
|
||||
limits:
|
||||
cpu: "200m"
|
||||
memory: "100Mi"
|
||||
requests:
|
||||
cpu: "20m"
|
||||
memory: "20Mi"
|
||||
volumeMounts:
|
||||
- name: log
|
||||
mountPath: /log
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: log
|
||||
hostPath:
|
||||
path: /var/log/
|
||||
@@ -1,14 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: shell-demo
|
||||
spec:
|
||||
volumes:
|
||||
- name: shared-data
|
||||
emptyDir: {}
|
||||
containers:
|
||||
- name: nginx
|
||||
image: nginx
|
||||
volumeMounts:
|
||||
- name: shared-data
|
||||
mountPath: /usr/share/nginx/html
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: termination-demo
|
||||
spec:
|
||||
containers:
|
||||
- name: termination-demo-container
|
||||
image: debian
|
||||
command: ["/bin/sh"]
|
||||
args: ["-c", "sleep 10 && echo Sleep expired > /dev/termination-log"]
|
||||
Reference in New Issue
Block a user