diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..df4d6c8710 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,22 @@ +language: ruby +rvm: +- 2.1 +branches: + only: + - release-1.3 +script: +- cd $HOME +- git config --global user.email ${GIT_EMAIL} +- git config --global user.name "${GIT_NAME}" +- git clone https://${GIT_USERNAME}:${GH_TOKEN}@github.com/kubernetes-v1-3/kubernetes-v1-3.github.io.git +- cd kubernetes-v1-3.github.io +- git remote add --fetch --track release-1.3 homebase "https://${GIT_USERNAME}:${GH_TOKEN}@github.com/kubernetes/kubernetes.github.io.git" +- git merge -s recursive -X theirs homebase/release-1.3 -m "Sync from homebase" +- git push +env: + global: + - secure: Fd6wlE2mjPb1fAACxklQcJumpJWycYkaJQBfKRcjGCFlmw1XWVFGhpUC7Ni/MOyzTolqOvtb2rXnYpaujMlJP1UXqVFJ+zPbwur2lc8unQF8PcqPezl8DbPsr6HdceOjRdut/dN8zTw6+hZRDzw/mG4Rf8IVaozlYycAOnWZdAZsLXdbBpAvBp0WYHP9+8wn9xiet1L/QpSiawa/Q35Q9UM1tciPmfFBL4Fkq6Mm8/w6ECaxHyA2EX+eH5ea9EtykrzB5cMA/odJLptjnmfzsPeGS5F1MxuTqrH2Z09emZcqjfXW+fNNzioiZdD4BQe+rqOA1Ktpw645szJVJJenIMzM4P8dDiW+9cJyddGdTC9A7APgLlLVdG/tuoYcjiY1F068SWx8sUWW/xc7YenCzGj2nXIeHjUuWsjqjMorpBKe6Y1QSGfi7HsLq1DpJDR1xQZbPgM/FOaovVIPIozgAIBpS1ukdNNwadmrCw55tUKfr4s2SNv3SUmeprL37QEtIxtBpPDjL1z+qTLjc6JzRr7J57guPBdNbj6Ukg/uxW+z1CwDYb3uAIIa5e6eDxfxGQsUEj8NmkIHBLLGTnKb1WAyPXAIf9ZvZHUwQQzHvOWXGZRd32enRh00uvuwHEy3OuGEqTAOIs/31q5WAamZ1uqzW7KBTMU/nq4QA9W2yfM= + - secure: Lvj1MT/I2zaJp9mQi46us64O4JfeT9cz1m+nqKY/nuC828ULfqMYLlpw08jvrc25JhAaqHNawAwdtaxHTQqHztsmq1ixHFYA6G/1oll+YPhym29toFT9oIkXThf/L77FTJfM1gVFMyiHFbz11Ob8R15tRB1WF1+Uu1RBmtudJD7HSy0u6Uc6lPxpqtycWVRCyPdvZqF3e0KIZaTDRkRzJpcVMHa/5ZJDDcMtuJyjJYXZqS1WR3QHC1z44LlnqyB5ZM2PU4H3LyWnY8wHF2mutF0QtDDVdEuBqILBHiFuKHMxpLY92UgHm2n51RR63MxFjjEzE+iu5f9ComEm5JC0N/cc8sunIiol+d8SRC30/00Vs0tvmeAjRX9IMCExiP3mv7Tz6mqEVk6PyrVlg675hxRg0eVdaNNv92gWzCSIecZ5TuCaRG4JaWO0P8lQC4NreONt6gbnwMD350hMQZLpUUE0QoSEdafpudaD+agl4ZzTFVTOOcSOz2Sa/+RT96Msazq2YlddYXaEKZeyYzqHVkk60PiQzQcAuwCMFrAagqMe3bNI2aCFEWbc9CoR13K/wwRDwAeSzBq/UkylZ8AayJLnIpewr/iYBOQasrXLrorW969Rfr8d/nwhDN5VpRgItU0arDzngaJRoZEAtHx4zaCvZF/H/nvbORcD2gsFP9Q= + - secure: WGmgvSnT1OeHPYtsm/JzcRDnZYp+SeelgbSVNbtwOXGImkeqDo/sDIRH80HxVC3zkUu5wvRO/NkMA7N3LXN2BqENacWBFQ+j+Hf3lnDw9yETBaBpLQPO/MElKGd48GoSiukCipfaITUfwtjAQ3wx0W/z06dYC9BdUYvUM73OtsGtlrY1MEQnBfff/neDGZbzfvTdgDwg1n6d69WP+WSUJt5Aysz0ZoAbYO2rzBrKNwcL7KEhoI/ketXXQ4xEW0nnxU/qfxqhVfJbck1HpSz+HMsVGXF4tT3zdRXmU+P47KGbfYn0GmlviGrRSWCu9/elKTlX9fIqRnR1/UvdTkflwLsk4i3hWnEUflKpIT7soJrKjNQseyd06KI8qvVRXRcll92vCtsYDuOO1AlemxIqp2dIFduCf+5FWH06PbS8oHU8NBe5aVYaoBY8nK14EB3A98saJ8Un7ziB8moRaWZX9bMjfJWfxDF526NdHIusqLQrxvUthtBjf5aRULE2PeIyqYVY8/IKmBEfsy9IUENmlM5KItvgZZt4xdk8o//kceLrKabudHGVC1M1RLZo9ejkdLySz2i8nlJyJ03Hn30H9DfDjW/OMQTn/b1HU4CYSbd/yGR+6+5+tdPeKy/Z6gI6+EjHRYaaQrBLo0RXm2620cb7uDTBIsKVs2TetyhTq54= + - secure: lqboLQKFpfa6F+UkSCs9+NPv027EnSWZA4/cmAXj0hZvxc4W+qMFIlPKytcq+bbZrp7kx3cZgRd5NrgmOTqlJ9merVK1cgMsDegMv/YImHjhGVBH+3ATvqMDb7I7dLg4x/DjhKt/ogyZpoh8vsS77T9MQXaIJM8JMM3ISxYI6MMloCh0z/hbT2qv3IrPMFjpHohHdF1yorxKvSefNgPx6Y4wC+t1t+u8wS9RBhvrmOkVynlU2NgbSiMPyz3syJmnszHDTXXJlqciGT5Lc8lzpzt2OGOnZmAupEYVGok7Jx4aAxU9O8/bnoDwOcGGVrd/pGHTI5+rvT4TPy+WG1Aec55dgva4XZfrJQ88snHLvq0VrvdiI/fHJEPF18QnrBwKSni1Vd1jOzsuPIF6HL0Wwv8tklO8OX3D5wqVsXpeJd4Cj4HPPAtzxOVsV6Fz3On9vuLdV7StC/ZafQL40koaytqkNO0gX8zeUiaaVQtMCf/2MVTbX9x5m51gkGpwT0JBvqQpSMOlIM5S1fiN4X9DLBLqmsARYeZw1Jeiq7Zm39Sy1QeCgfUXS8+6t62BjWx70iftYGIkoljXtD7x/3pjKymdpwjcyUcS8KQ5W6vbMXFxmZe2phtgGTCxOIDuASPspD8zwm/ckIGArHj4qkrg94/mUpgDDIbVsaUpWXjYvSc= + - secure: bPZMNH79Lx0Wb2SCxTwyZek28w/keKxEAlfad8RDMwJrct6Bi2B0o4KjkrwFS5DyCUU7Ndk192145XnUKOWX2YVs6cQ8ge+5LvtgYhWLgX8g5Ycro+JyzOBskn+o1gQjvi8+3b42X31efcUmTEhfRdKVrrUpONEIcjG1NpLGk/mQJ6AM6hGWO2xdNfAezeWq5ISnpK0b6VUWZyTEDg3NivrTfCL2juWWCnjYm1BWSHUblXwRQ/Rl7Tcldl6cMMUVsalUQ1iG0h8YZDrxNz0cm3XTZZZJKuSHYeTCLd57RBeHD5/iMxjCmfzfq6ETNLONWLmtvA8yhWMQZ0DPFtLZzbqFIfOR6P1feBZFqP7/X5KZeFKBufN15JbcXIqHE8homLY9mS0LhyNffOs5G/P/x8ChE1DJaYiZIWCZ60umvpqibScZB3z5uFPTxLk9rJSOtT7hCWjcmg3EdJ+R4ExOiBDd62ZS5jH72WU4uysPXORRofUCL+zHycJoJxsFWQSW49GAGrohllrW45jnTgpalErxvjJFSKy8JW9w634eetz9ct2fObep7m8bfVMl8U2H3ITIoXHm+f8ooHUiNzHFLSl4wtcoAOtrCAvGtJPNfv8T2eTkznzj7Tk+XzTvFg03u+J99TmnC58Bs516Oc7E40NarmsZceOD2sN4BA3X9tE= diff --git a/README.md b/README.md index d2ad5c1682..31824d6f36 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ ## Instructions for Contributing to the Docs/Website +**Note: This branch is for pre-release documentation for 1.3 of Kubernetes. Changes to this branch are automatically served as a website at [http://kubernetes-v1-3.github.io/](http://kubernetes-v1-3.github.io/). See the "master" branch to edit the current stable release, which is served at http://kubernetes.io.** + Welcome! We are very pleased you want to contribute to the documentation and/or website for Kubernetes. You can click the "Fork" button in the upper-right area of the screen to create a copy of our site on your GitHub account called a "fork." Make any changes you want in your fork, and when you are ready to send those changes to us, go to the index page for your fork and click "New Pull Request" to let us know about it. diff --git a/_config.yml b/_config.yml index c3eaba8b58..5094499bfe 100644 --- a/_config.yml +++ b/_config.yml @@ -16,9 +16,9 @@ defaults: scope: path: "" values: - version: "v1.2" - githubbranch: "release-1.2" - docsbranch: "master" + version: "v1.3" + githubbranch: "master" + docsbranch: "release-1.3" - scope: path: "docs" diff --git a/_data/guides.yml b/_data/guides.yml index cc1d2b8a9a..06c07fee19 100644 --- a/_data/guides.yml +++ b/_data/guides.yml @@ -58,6 +58,8 @@ toc: path: /docs/user-guide/config-best-practices/ - title: Using kubectl to Manage Resources path: /docs/user-guide/working-with-resources/ + - title: Garbage collection + path: /docs/user-guide/garbage-collector/ - title: Batch Jobs section: @@ -113,6 +115,8 @@ toc: path: /docs/user-guide/downward-api/volume - title: Persistent Volumes Walkthrough path: /docs/user-guide/persistent-volumes/walkthrough/ + - title: Bootstrapping Pet Sets + path: /docs/user-guide/petset/bootstrapping/ - title: Monitoring, Logging, and Debugging Containers section: @@ -205,9 +209,9 @@ toc: path: /docs/getting-started-guides/fedora/fedora-calico/ - title: rkt section: - - title: Running Kubernetes on rkt + - title: Running Kubernetes with rkt path: /docs/getting-started-guides/rkt/ - - title: Notes on Different UX with rkt Container Runtime + - title: Known Issues when Using rkt path: /docs/getting-started-guides/rkt/notes/ - title: Kubernetes on Mesos path: /docs/getting-started-guides/mesos/ @@ -258,6 +262,8 @@ toc: path: /docs/admin/cluster-components/ - title: Configuring Kubernetes Use of etcd path: /docs/admin/etcd/ + - title: Federating Clusters + path: /docs/admin/federation/ - title: Using Multiple Clusters path: /docs/admin/multi-cluster/ - title: Changing Cluster Size @@ -276,7 +282,11 @@ toc: path: /docs/admin/network-plugins/ - title: Static Pods path: /docs/admin/static-pods/ - - title: Configuring Garbage Collection + - title: Configuring kubelet Garbage Collection path: /docs/admin/garbage-collection/ + - title: Configuring Out Of Resource Handling + path: /docs/admin/out-of-resource/ - title: Configuring Kubernetes with Salt path: /docs/admin/salt/ + - title: Monitoring Node Health + path: /docs/admin/node-problem/ diff --git a/_data/reference.yml b/_data/reference.yml index 3b9b864cc7..a11214565a 100644 --- a/_data/reference.yml +++ b/_data/reference.yml @@ -224,6 +224,10 @@ toc: path: /docs/admin/resource-quota/ - title: Replica Sets path: /docs/user-guide/replicasets/ + - title: Pet Sets + path: /docs/user-guide/petset/ + - title: Network Policies + path: /docs/user-guide/networkpolicies/ - title: Kubernetes Design Docs section: diff --git a/docs/admin/authorization.md b/docs/admin/authorization.md index ed8652fe47..efd93fb497 100644 --- a/docs/admin/authorization.md +++ b/docs/admin/authorization.md @@ -19,9 +19,20 @@ need authorization. - `--authorization-mode=ABAC`allows for user-configured authorization policy. ABAC stands for Attribute-Based Access Control. + - `--authorization-mode=RBAC` is an experimental implementation which allows +for authorization to be driven by the Kubernetes API. +RBAC stands for Roles-Based Access Control. - `--authorization-mode=Webhook` allows for authorization to be driven by a remote service using REST. +If multiple modes are provided the set is unioned, and only a single authorizer is required to admit the action. This means the flag: + +``` +--authorization-mode=AlwaysDeny,AlwaysAllow +``` + +will always allow. + ## ABAC Mode ### Request Attributes @@ -152,6 +163,154 @@ file: The apiserver will need to be restarted to pickup the new policy lines. +## RBAC Mode + +When specified "RBAC" (Role-Based Access Control) uses the +"rbac.authorization.k8s.io" API group to drive authorization decisions, +allowing admins to dynamically configure permission policies through the +Kubernetes API. + +As of 1.3 RBAC mode is in alpha and considered experimental. + +### Roles, RolesBindings, ClusterRoles, and ClusterRoleBindings + +The RBAC API Group declares four top level types which will be covered in this +section. Users can interact with these resources as they would with any other +API resource. Through `kubectl`, direct calls to the API, etc. For instance, +`kubectl create -f (resource).yml` can be used with any of these examples, +though readers who wish to follow along should review the following section on +bootstrapping first. + +In the RBAC API Group, roles hold a logical grouping of permissions. These +permissions map very closely to ABAC policies, but only contain information +about requests being made. Permission are purely additive, rules may only omit +permissions they do not wish to grant. + +Here's an example of a role which grants read access to pods within the +"default" namespace. + +```yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + namespace: default + name: pod-reader +rules: + - apiGroups: [""] # The API group "" indicates the default API Group. + resources: ["pods"] + verbs: ["get", "watch", "list"] + nonResourceURLs: [] +``` + +`ClusterRoles` hold the same information as a `Role` but can apply to any +namespace as well as non-namespaced resources (such as `Nodes`, +`PersistentVolume`, etc.). The following `ClusterRole` can grant permissions to +read secrets in any namespace. + +```yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + # "namespace" omitted since ClusterRoles are not namespaced. + name: secret-reader +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "watch", "list"] + nonResourceURLs: [] +``` + +`RoleBindings` perform the task of granting the permission to a user or set of +users. They hold a list of subjects which they apply to, and a reference to the +`Role` being assigned. + +The following `RoleBinding` assigns the "pod-reader" role to the user "jane" +within the "default" namespace, and allows jane to read pods. + +```yaml +# This role binding allows "jane" to read pods in the namespace "default" +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: read-pods + namespace: default +subjects: + - kind: User # May be "User", "Group" or "ServiceAccount" + name: jane +roleRef: + kind: Role + namespace: default + name: pod-reader + apiVersion: rbac.authorization.k8s.io/v1alpha1 +``` + +`RoleBindings` may also refer to a `ClusterRole`. However, a `RoleBinding` that +refers to a `ClusterRole` only applies in the `RoleBinding`'s namespace, not at +the cluster level. This allows admins to define a set of common roles for the +entire cluster, then reuse them in multiple namespaces. + +For instance, even though the following `RoleBinding` refers to a `ClusterRole`, +"dave" (the subject) will only be able read secrets in the "development" +namespace, the namespace of the `RoleBinding`. + +```yaml +# This role binding allows "dave" to read secrets in the namespace "development" +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: read-secrets + namespace: development # This binding only applies in the "development" namespace +subjects: + - kind: User # May be "User", "Group" or "ServiceAccount" + name: dave +roleRef: + kind: ClusterRole + name: secret-reader + apiVersion: rbac.authorization.k8s.io/v1alpha1 +``` + +Finally a `ClusterRoleBinding` may be used to grant permissions in all +namespaces. The following `ClusterRoleBinding` allows any user in the group +"manager" to read secrets in any namepsace. + +```yaml +# This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: read-secrets +subjects: + - kind: Group # May be "User", "Group" or "ServiceAccount" + name: manager +roleRef: + kind: ClusterRole + name: secret-reader + apiVersion: rbac.authorization.k8s.io/v1alpha1 +``` + +### Privilege Escalation Prevention and Bootstrapping + +The `rbac.authorization.k8s.io` API group inherently attempts to prevent users +from escalating privileges. Simply put, __a user can't grant permissions they +don't already have even when the RBAC authorizer it disabled__. If "user-1" +does not have the ability to read secrets in "namespace-a", they cannot create +a binding that would grant that permission to themselves or any other user. + +For bootstrapping the first roles, it becomes necessary for someone to get +around these limitations. For the alpha release of RBAC, an API Server flag was +added to allow one user to step around all RBAC authorization and privilege +escalation checks. NOTE: _This is subject to change with future releases._ + +``` +--authorization-rbac-super-user=admin +``` + +Once set the specified super user, in this case "admin", can be used to create +the roles and role bindings to initialize the system. + +This flag is optional and once the initial bootstrapping is performed can be +unset. + ## Webhook Mode When specified, mode `Webhook` causes Kubernetes to query an outside REST diff --git a/docs/admin/cluster-management.md b/docs/admin/cluster-management.md index 613ec8afa2..16d4a6dce2 100644 --- a/docs/admin/cluster-management.md +++ b/docs/admin/cluster-management.md @@ -66,37 +66,51 @@ Instance Group will take care of putting appropriate image on new machines and s In other environments you may need to configure the machine yourself and tell the Kubelet on which machine API server is running. -### Horizontal auto-scaling of nodes (GCE) +### Cluster autoscaling -If you are using GCE, you can configure your cluster so that the number of nodes will be automatically scaled based on: +If you are using GCE or GKE, you can configure your cluster so that it is automatically rescaled based on +pod needs. - * CPU and memory utilization. - * Amount of of CPU and memory requested by the pods (called also reservation). +As described in [Compute Resource](/docs/user-guide/compute-resources/), users can reserve how much CPU and memory is allocated to pods. +This information is used by the Kubernetes scheduler to find a place to run the pod. If there is +no node that has enough free capacity (or doesn't match other pod requirements) then the pod has +to wait until some pods are terminated or a new node is added. -Before setting up the cluster by `kube-up.sh`, you can set `KUBE_ENABLE_NODE_AUTOSCALER` environment variable to `true` and export it. -The script will create an autoscaler for the instance group managing your nodes. +Cluster autoscaler looks for the pods that cannot be scheduled and checks if adding a new node, similar +to the other in the cluster, would help. If yes, then it resizes the cluster to accomodate the waiting pods. -The autoscaler will try to maintain the average CPU/memory utilization and reservation of nodes within the cluster close to the target value. -The target value can be configured by `KUBE_TARGET_NODE_UTILIZATION` environment variable (default: 0.7) for ``kube-up.sh`` when creating the cluster. -Node utilization is the total node's CPU/memory usage (OS + k8s + user load) divided by the node's capacity. -Node reservation is the total CPU/memory requested by pods that are running on the node divided by the node's capacity. -If the desired numbers of nodes in the cluster resulting from CPU/memory utilization/reservation are different, -the autoscaler will choose the bigger number. The number of nodes in the cluster set by the autoscaler will be limited from `KUBE_AUTOSCALER_MIN_NODES` (default: 1) -to `KUBE_AUTOSCALER_MAX_NODES` (default: the initial number of nodes in the cluster). +Cluster autoscaler also scales down the cluster if it notices that some node is not needed anymore for +an extended period of time (10min but it may change in the future). -The autoscaler is implemented as a Compute Engine Autoscaler. -The initial values of the autoscaler parameters set by `kube-up.sh` and some more advanced options can be tweaked on -`Compute > Compute Engine > Instance groups > your group > Edit group`[Google Cloud Console page](https://console.developers.google.com) -or using gcloud CLI: +Cluster autoscaler is configured per instance group (GCE) or node pool (GKE). + +If you are using GCE then you can either enable it while creating a cluster with kube-up.sh script. +To configure cluser autoscaler you have to set 3 environment variables: + +* `KUBE_ENABLE_CLUSTER_AUTOSCALER` - it enables cluster autoscaler if set to true. +* `KUBE_AUTOSCALING_MIN_NODES` - minimum number of nodes in the cluster. +* `KUBE_AUTOSCALING_MAX_NODES` - maximum number of nodes in the cluster. + +Example: ```shell -gcloud alpha compute autoscaler --zone $ZONE +KUBE_ENABLE_CLUSTER_AUTOSCALER=true KUBE_AUTOSCALING_MIN_NODES=3 KUBE_AUTOSCALING_MAX_NODES=10 NUM_NODES=5 ./cluster/kube-up.sh ``` -Note that autoscaling will work properly only if node metrics are accessible in Google Cloud Monitoring. -To make the metrics accessible, you need to create your cluster with `KUBE_ENABLE_CLUSTER_MONITORING` -equal to `google` or `googleinfluxdb` (`googleinfluxdb` is the default value). Please also make sure -that you have Google Cloud Monitoring API enabled in Google Developer Console. +On GKE you configure cluster autoscaler either on cluster creation or update or when creating a particular node pool +(which you want to be autoscaled) by passing flags `--enable-autoscaling` `--min-nodes` and `--max-nodes` +to the corresponding `gcloud` commands. + +Examples: +```shell +gcloud container clusters create mytestcluster --zone=us-central1-b --enable-autoscaling=true --min-nodes=3 --max-nodes=10 --num-nodes=5 +``` + +```shell +gcloud container clusters update mytestcluster --enable-autoscaling=true --min-nodes=1 --max-nodes=15 +``` + +**Cluster autoscaler expects that nodes have not been manually modified (e.g. by adding labels via kubectl) as those properties would not be propagated to the new nodes within the same instance group.** ## Maintenance on a Node diff --git a/docs/admin/federation/index.md b/docs/admin/federation/index.md new file mode 100644 index 0000000000..fc077ec839 --- /dev/null +++ b/docs/admin/federation/index.md @@ -0,0 +1,195 @@ +--- +--- +This guide explains how to set up cluster federation that lets us control multiple Kubernetes clusters. + + +* TOC +{:toc} + +## Prerequisites + +This guide assumes that we have a running Kubernetes cluster. +If not, then head over to the [getting started guides](/docs/getting-started-guides/) to bring up a cluster. + +This guide also assumes that we have the Kubernetes source code that can be +[downloaded from here](/docs/getting-started-guides/binary_release/). + +## Setting up a federation control plane + +Setting up federation requires running the federation control plane which +consists of etcd, federation-apiserver and federation-controller-manager. +We can run these binaries as pods on an existing Kubernetes cluster. + +### Getting images + +To run these as pods, we first need images for all the components. We can use +official release images or we can build from HEAD. + +#### Using official release images + +As part of every release, images are pushed to `gcr.io/google_containers`. To use +these images, we set env var `FEDERATION_PUSH_REPO_BASE=gcr.io/google_containers` +This will always use the latest image. +To use federation-apiserver and federation-controller-manager images from a specific release, we can set `FEDERATION_IMAGE_TAG`. + +#### Building and pushing images from HEAD + +To run the code from HEAD, we need to build and push our own images. +We can build the images using the following command: + +```shell +$ FEDERATION=true KUBE_RELEASE_RUN_TESTS=n make quick-release +``` + +Next, we need to push these images to a registry such as Google Container Registry or Docker Hub, so that our cluster can pull them. +If Kubernetes cluster is running on Google Compute Engine (GCE), then we can push the images to `gcr.io/`. +The command to push the images will look like: + +```shell +$ FEDERATION=true FEDERATION_PUSH_REPO_BASE=gcr.io/ ./build/push-federation-images.sh +``` + +### Running the federation control plane + +Once we have the images, we can run these as pods on our existing kubernetes cluster. +The command to run these pods on an existing GCE cluster will look like: + +```shell +$ KUBERNETES_PROVIDER=gce FEDERATION_DNS_PROVIDER=google-clouddns FEDERATION_NAME=myfederation DNS_ZONE_NAME=myfederation.example FEDERATION_PUSH_REPO_BASE=gcr.io/google_containers ./federation/cluster/federation-up.sh +``` + +`KUBERNETES_PROVIDER` is the cloud provider. + +`FEDERATION_DNS_PROVIDER` can be `google-clouddns` or `aws-route53`. It will be +set appropriately if it is missing and `KUBERNETES_PROVIDER` is one of `gce`, `gke` and `aws`. +This is used to resolve DNS requests for federation services. The service +controller keeps DNS records with the provider updated as services/pods are +updated in underlying kubernetes clusters. + +`FEDERATION_NAME` is a name we can choose for our federation. This is the name that will appear in DNS routes. + +`DNS_ZONE_NAME` is the domain to be used for DNS records. This is a domain that we +need to buy and then configure it such that DNS queries for that domain are +routed to the appropriate provider as per `FEDERATION_DNS_PROVIDER`. + +Running that command creates a namespace `federation` and creates 2 deployments: `federation-apiserver` and `federation-controller-manager`. +We can verify that the pods are available by running the following command: + +```shell +$ kubectl get deployments --namespace=federation +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +federation-apiserver 1 1 1 1 1m +federation-controller-manager 1 1 1 1 1m +``` + +Running `federation-up.sh` also creates a new record in our kubeconfig for us +to be able to talk to federation apiserver. We can view this by running +`kubectl config view`. + +Note: `federation-up.sh` creates the federation-apiserver pod with an etcd +container that is backed by a persistent volume, so as to persist data. This +currently works only on AWS, GKE, and GCE. You can edit +`federation/manifests/federation-apiserver-deployment.yaml` to suit your needs, +if required. + +## Registering Kubernetes clusters for federation + +Now that we have the federation control plane up and running, we can start registering Kubernetes clusters. + +First of all, we need to create a secret containing kubeconfig for that Kubernetes cluster, which federation control plane will use to talk to that Kubernetes cluster. +For now, we create this secret in the host Kubernetes cluster (that hosts federation control plane). When we start supporting secrets in federation control plane, we will create this secret there. +Suppose that our kubeconfig for Kubernetes cluster is at `/cluster1/kubeconfig`, we can run the following command to create the secret: + +```shell +$ kubectl create secret generic cluster1 --namespace=federation --from-file=/cluster1/kubeconfig +``` + +Note that the file name should be `kubeconfig` since file name determines the name of the key in the secret. + +Now that the secret is created, we are ready to register the cluster. The YAML file for cluster will look like: + +```yaml +apiVersion: v1beta1 +kind: Cluster +metadata: + name: cluster1 +spec: + serverAddressByClientCIDRs: + - clientCIDR: + serverAddress: + secretRef: + name: +``` + +We need to insert the appropriate values for ``, `` and ``. +`` here is name of the secret that we just created. +serverAddressByClientCIDRs contains the various server addresses that clients +can use as per their CIDR. We can set the server's public IP address with CIDR +`"0.0.0.0/0"` which all clients will match. In addition, if we want internal +clients to use server's clusterIP, we can set that as serverAddress. The client +CIDR in that case will be a CIDR that only matches IPs of pods running in that +cluster. + +Assuming our YAML file is located at `/cluster1/cluster.yaml`, we can run the following command to register this cluster: + +```shell +$ kubectl create -f /cluster1/cluster.yaml --cluster=federation-cluster + +``` + +By specifying `--cluster=federation-cluster`, we direct the request to federation apiserver. +we can ensure that the cluster registration was successful by running: + +```shell +$ kubectl get clusters --context=federation-cluster +NAME STATUS VERSION AGE +cluster1 Ready 3m +``` + +### Updating KubeDNS + +Once the cluster is registered with the federation, we are all ready to use it. +But for the cluster to be able to route federation service requests, we need to restart +KubeDNS and pass it a `--federations` flag which tells it about valid federation DNS hostnames. +Format of the flag is like this: + +``` +--federations=${FEDERATION_NAME}=${DNS_DOMAIN_NAME} +``` + +To update KubeDNS with federations flag, we can edit the existing kubedns replication controller to +include that flag in pod template spec and then delete the existing pod. Replication controller will +recreate the pod with updated template. + +To find the name of existing kubedns replication controller, run + +```shell +$ kubectl get rc --namespace=kube-system +``` + +This will list all the replication controllers. Name of the kube-dns replication +controller will look like `kube-dns-v18`. You can then edit it by running: + +```shell +$ kubectl edit rc --namespace=kube-system +``` +Add the `--federations` flag as args to kube-dns container in the YAML file that +pops up after running the above command. + +To delete the existing kube dns pod, you can first find it by running: + +```shell +$ kubectl get pods --namespace=kube-system +``` + +And then delete it by running: + +```shell +$ kubectl delete pods --namespace=kube-system +``` + +We are now all set to start using federation. + +## For more information + + * [Federation proposal](https://github.com/kubernetes/kubernetes/blob/{{page.githubbranch}}/docs/proposals/federation.md) details use cases that motivated this work. diff --git a/docs/admin/multi-cluster.md b/docs/admin/multi-cluster.md index 47e433e29b..e549a4d3ca 100644 --- a/docs/admin/multi-cluster.md +++ b/docs/admin/multi-cluster.md @@ -5,9 +5,7 @@ You may want to set up multiple Kubernetes clusters, both to have clusters in different regions to be nearer to your users, and to tolerate failures and/or invasive maintenance. This document describes some of the issues to consider when making a decision about doing so. -Note that at present, -Kubernetes does not offer a mechanism to aggregate multiple clusters into a single virtual cluster. However, -we [plan to do this in the future](https://github.com/kubernetes/kubernetes/blob/{{page.githubbranch}}/docs/proposals/federation.md). +If you decide to have multiple clusters, kubernetes provides a way to [federate them](/docs/admin/federation/) ## Scope of a single cluster @@ -56,11 +54,10 @@ users in the event of a cluster failure), then you need to have `R * (U + 1)` cl (`U + 1` in each of `R` regions). In any case, try to put each cluster in a different zone. Finally, if any of your clusters would need more than the maximum recommended number of nodes for a Kubernetes cluster, then -you may need even more clusters. Kubernetes v1.0 currently supports clusters up to 100 nodes in size, but we are targeting -1000-node clusters by early 2016. +you may need even more clusters. Kubernetes v1.3 supports clusters up to 1000 nodes in size. ## Working with multiple clusters When you have multiple clusters, you would typically create services with the same config in each cluster and put each of those service instances behind a load balancer (AWS Elastic Load Balancer, GCE Forwarding Rule or HTTP Load Balancer) spanning all of them, so that -failures of a single cluster are not visible to end users. \ No newline at end of file +failures of a single cluster are not visible to end users. diff --git a/docs/admin/node-problem.md b/docs/admin/node-problem.md new file mode 100644 index 0000000000..5dc2d4cb52 --- /dev/null +++ b/docs/admin/node-problem.md @@ -0,0 +1,245 @@ +--- +--- + +* TOC +{:toc} + +## Node Problem Detector + +*Node problem detector* is a [DaemonSet](/docs/admin/daemons/) monitoring the +node health. It collects node problems from various daemons and reports them +to the apiserver as [NodeCondition](/docs/admin/node/#node-condition) and +[Event](/docs/api-reference/v1/definitions/#_v1_event). + +It supports some known kernel issue detection now, and will detect more and +more node problems over time. + +Currently Kubernetes won't take any action on the node conditions and events +generated by node problem detector. In the future, a remedy system could be +introduced to deal with node problems. + +See more information +[here](https://github.com/kubernetes/node-problem-detector). + +## Limitations + +* The kernel issue detection of node problem detector only supports file based +kernel log now. It doesn't support log tools like journald. + +* The kernel issue detection of node problem detector has assumption on kernel +log format, now it only works on Ubuntu and Debian. However, it is easy to extend +it to [support other log format](/docs/admin/node-problem/#support-other-log-format). + +## Enable/Disable in GCE cluster + +Node problem detector is running as a cluster +[addon](docs/admin/cluster-large/#addon-resources) enabled by default in the +gce cluster. + +You can enable/disable it by setting the environment variable +`KUBE_ENABLE_NODE_PROBLEM_DETECTOR` before `kube-up.sh`. + +## Use in Other Environment + +To enable node problem detector in other environment outside of GCE, you can use +either `kubectl` or addon pod. + +### Kubectl + +This is the recommanded way to start node problem detector outside of GCE. It +provides more flexible management, such as overwriting the default +configuration to fit it into your environment or detect +customized node problems. + +* **Step 1:** Create `node-problem-detector.yaml`: + +```yaml +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: node-problem-detector-v0.1 + namespace: kube-system + labels: + k8s-app: node-problem-detector + version: v0.1 + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: node-problem-detector + version: v0.1 + kubernetes.io/cluster-service: "true" + spec: + hostNetwork: true + containers: + - name: node-problem-detector + image: gcr.io/google_containers/node-problem-detector:v0.1 + securityContext: + privileged: true + resources: + limits: + cpu: "200m" + memory: "100Mi" + requests: + cpu: "20m" + memory: "20Mi" + volumeMounts: + - name: log + mountPath: /log + readOnly: true + volumes: + - name: log + hostPath: + path: /var/log/ +``` + +***Notice that you should make sure the system log directory is right for your +OS distro.*** + +* **Step 2:** Start node problem detector with `kubectl`: + +```shell +kubectl create -f node-problem-detector.yaml +``` + +### Addon Pod + +This is for those who have their own cluster bootstrap solution, and don't need +to overwrite the default configuration. They could leverage the addon pod to +further automate the deployment. + +Just create `node-problem-detector.yaml`, and put it under the addon pods directory +`/etc/kubernetes/addons/node-problem-detector` on master node. + +## Overwrite the Configuration + +The [default configuration](https://github.com/kubernetes/node-problem-detector/tree/v0.1/config) +is embedded when building the docker image of node problem detector. + +However, you can use [ConfigMap](/docs/user-guide/configmap/) to overwrite it +following the steps: + +* **Step 1:** Change the config files in `config/`. +* **Step 2:** Create the ConfigMap `node-problem-detector-config` with `kubectl create configmap +node-problem-detector-config --from-file=config/`. +* **Step 3:** Change the `node-problem-detector.yaml` to use the ConfigMap: + +```yaml +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: node-problem-detector-v0.1 + namespace: kube-system + labels: + k8s-app: node-problem-detector + version: v0.1 + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: node-problem-detector + version: v0.1 + kubernetes.io/cluster-service: "true" + spec: + hostNetwork: true + containers: + - name: node-problem-detector + image: gcr.io/google_containers/node-problem-detector:v0.1 + securityContext: + privileged: true + resources: + limits: + cpu: "200m" + memory: "100Mi" + requests: + cpu: "20m" + memory: "20Mi" + volumeMounts: + - name: log + mountPath: /log + readOnly: true + - name: config # Overwrite the config/ directory with ConfigMap volume + mountPath: /config + readOnly: true + volumes: + - name: log + hostPath: + path: /var/log/ + - name: config # Define ConfigMap volume + configMap: + name: node-problem-detector-config +``` + +* **Step 4:** Re-create the node problem detector with the new yaml file: + +```shell +kubectl delete -f node-problem-detector.yaml # If you have a node-problem-detector running +kubectl create -f node-problem-detector.yaml +``` + +***Notice that this approach only applies to node problem detector started with `kubectl`.*** + +For node problem detector running as cluster addon, because addon manager doesn't support +ConfigMap, configuration overwriting is not supported now. + +## Kernel Monitor + +*Kernel Monitor* is a problem daemon in node problem detector. It monitors kernel log +and detects known kernel issues following predefined rules. + +The Kernel Monitor matches kernel issues according to a set of predefined rule list in +[`config/kernel-monitor.json`](https://github.com/kubernetes/node-problem-detector/blob/v0.1/config/kernel-monitor.json). +The rule list is extensible, you can always extend it by [overwriting the +configuration](/docs/admin/node-problem/#overwrite-the-configuration). + +### Add New NodeConditions + +To support new node conditions, you can extend the `conditions` field in +`config/kernel-monitor.json` with new condition definition: + +```json +{ + "type": "NodeConditionType", + "reason": "CamelCaseDefaultNodeConditionReason", + "message": "arbitrary default node condition message" +} +``` + +### Detect New Problems + +To detect new problems, you can extend the `rules` field in `config/kernel-monitor.json` +with new rule definition: + +```json +{ + "type": "temporary/permanent", + "condition": "NodeConditionOfPermanentIssue", + "reason": "CamelCaseShortReason", + "message": "regexp matching the issue in the kernel log" +} +``` + +### Change Log Path + +Kernel log in different OS distros may locate in different path. The `log` +field in `config/kernel-monitor.json` is the log path inside the container. +You can always configure it to match your OS distro. + +### Support Other Log Format + +Kernel monitor uses [`Translator`](https://github.com/kubernetes/node-problem-detector/blob/v0.1/pkg/kernelmonitor/translator/translator.go) +plugin to translate kernel log the internal data structure. It is easy to +implement a new translator for a new log format. + +## Caveats + +It is recommanded to run the node problem detector in your cluster to monitor +the node health. However, you should be aware that this will introduce extra +resource overhead on each node. Usually this is fine, because: + +* The kernel log is generated relatively slowly. +* Resource limit is set for node problem detector. +* Even under high load, the resource usage is acceptable. +(see [benchmark result](https://github.com/kubernetes/node-problem-detector/issues/2#issuecomment-220255629)) diff --git a/docs/admin/out-of-resource.md b/docs/admin/out-of-resource.md new file mode 100644 index 0000000000..2b85f486d5 --- /dev/null +++ b/docs/admin/out-of-resource.md @@ -0,0 +1,220 @@ +--- +--- + +* TOC +{:toc} + +The `kubelet` needs to preserve node stability when available compute resources are low. + +This is especially important when dealing with incompressible resources such as memory or disk. + +If either resource is exhausted, the node would become unstable. + +## Eviction Policy + +The `kubelet` can pro-actively monitor for and prevent against total starvation of a compute resource. In +cases where it could appear to occur, the `kubelet` can pro-actively fail one or more pods in order to reclaim +the starved resource. When the `kubelet` fails a pod, it terminates all containers in the pod, and the `PodPhase` +is transitioned to `Failed`. + +### Eviction Signals + +The `kubelet` can support the ability to trigger eviction decisions on the signals described in the +table below. The value of each signal is described in the description column based on the `kubelet` +summary API. + +| Eviction Signal | Description | +|------------------|---------------------------------------------------------------------------------| +| `memory.available` | `memory.available` := `node.status.capacity[memory]` - `node.stats.memory.workingSet` | + +In future releases, the `kubelet` will support the ability to trigger eviction decisions based on disk pressure. + +Until that time, it is recommended users take advantage of [garbage collection](/docs/admin/garbage-collection/). + +### Eviction Thresholds + +The `kubelet` supports the ability to specify eviction thresholds that trigger the `kubelet` to reclaim resources. + +Each threshold is of the following form: + +`` + +* valid `eviction-signal` tokens as defined above. +* valid `operator` tokens are `<` +* valid `quantity` tokens must match the quantity representation used by Kubernetes + +#### Soft Eviction Thresholds + +A soft eviction threshold pairs an eviction threshold with a required +administrator specified grace period. No action is taken by the `kubelet` +to reclaim resources associated with the eviction signal until that grace +period has been exceeded. If no grace period is provided, the `kubelet` will +error on startup. + +In addition, if a soft eviction threshold has been met, an operator can +specify a maximum allowed pod termination grace period to use when evicting +pods from the node. If specified, the `kubelet` will use the lesser value among +the `pod.Spec.TerminationGracePeriodSeconds` and the max allowed grace period. +If not specified, the `kubelet` will kill pods immediately with no graceful +termination. + +To configure soft eviction thresholds, the following flags are supported: + +* `eviction-soft` describes a set of eviction thresholds (e.g. `memory.available<1.5Gi`) that if met over a +corresponding grace period would trigger a pod eviction. +* `eviction-soft-grace-period` describes a set of eviction grace periods (e.g. `memory.available=1m30s`) that +correspond to how long a soft eviction threshold must hold before triggering a pod eviction. +* `eviction-max-pod-grace-period` describes the maximum allowed grace period (in seconds) to use when terminating +pods in response to a soft eviction threshold being met. + +#### Hard Eviction Thresholds + +A hard eviction threshold has no grace period, and if observed, the `kubelet` +will take immediate action to reclaim the associated starved resource. If a +hard eviction threshold is met, the `kubelet` will kill the pod immediately +with no graceful termination. + +To configure hard eviction thresholds, the following flag is supported: + +* `eviction-hard` describes a set of eviction thresholds (e.g. `memory.available<1Gi`) that if met +would trigger a pod eviction. + +### Eviction Monitoring Interval + +The `kubelet` evaluates eviction thresholds per its configured housekeeping interval. + +* `housekeeping-interval` is the interval between container housekeepings. + +### Node Conditions + +The `kubelet` will map one or more eviction signals to a corresponding node condition. + +If a hard eviction threshold has been met, or a soft eviction threshold has been met +independent of its associated grace period, the `kubelet` will report a condition that +reflects the node is under pressure. + +The following node conditions are defined that correspond to the specified eviction signal. + +| Node Condition | Eviction Signal | Description | +|----------------|------------------|------------------------------------------------------------------| +| `MemoryPressure` | `memory.available` | Available memory on the node has satisfied an eviction threshold | + +The `kubelet` will continue to report node status updates at the frequency specified by +`--node-status-update-frequency` which defaults to `10s`. + +### Oscillation of node conditions + +If a node is oscillating above and below a soft eviction threshold, but not exceeding +its associated grace period, it would cause the corresponding node condition to +constantly oscillate between true and false, and could cause poor scheduling decisions +as a consequence. + +To protect against this oscillation, the following flag is defined to control how +long the `kubelet` must wait before transitioning out of a pressure condition. + +* `eviction-pressure-transition-period` is the duration for which the `kubelet` has +to wait before transitioning out of an eviction pressure condition. + +The `kubelet` would ensure that it has not observed an eviction threshold being met +for the specified pressure condition for the period specified before toggling the +condition back to `false`. + +### Eviction of Pods + +If an eviction threshold has been met and the grace period has passed, +the `kubelet` will initiate the process of evicting pods until it has observed +the signal has gone below its defined threshold. + +The `kubelet` ranks pods for eviction 1) by their quality of service, +2) and among those with the same quality of service by the consumption of the +starved compute resource relative to the pods scheduling request. + +* `BestEffort` pods that consume the most of the starved resource are failed +first. +* `Burstable` pods that consume the greatest amount of the starved resource +relative to their request for that resource are killed first. If no pod +has exceeded its request, the strategy targets the largest consumer of the +starved resource. +* `Guaranteed` pods that consume the greatest amount of the starved resource +relative to their request are killed first. If no pod has exceeded its request, +the strategy targets the largest consumer of the starved resource. + +A `Guaranteed` pod is guaranteed to never be evicted because of another pod's +resource consumption. If a system daemon (i.e. `kubelet`, `docker`, `journald`, etc.) +is consuming more resources than were reserved via `system-reserved` or `kube-reserved` allocations, +and the node only has `Guaranteed` pod(s) remaining, then the node must choose to evict a +`Guaranteed` pod in order to preserve node stability, and to limit the impact +of the unexpected consumption to other `Guaranteed` pod(s). + +### Scheduler + +The node will report a condition when a compute resource is under pressure. The +scheduler views that condition as a signal to dissuade placing additional +pods on the node. + +| Node Condition | Scheduler Behavior | +| ---------------- | ------------------------------------------------ | +| `MemoryPressure` | `BestEffort` pods are not scheduled to the node. | + +## Node OOM Behavior + +If the node experiences a system OOM (out of memory) event prior to the `kubelet` is able to reclaim memory, +the node depends on the [oom_killer](https://lwn.net/Articles/391222/) to respond. + +The `kubelet` sets a `oom_score_adj` value for each container based on the quality of service for the pod. + +| Quality of Service | oom_score_adj | +| ----------------- | ------------- | +| `Guaranteed` | -998 | +| `BestEffort` | 1000 | +| `Burstable` | min(max(2, 1000 - (1000 * memoryRequestBytes) / machineMemoryCapacityBytes), 999) | + +If the `kubelet` is unable to reclaim memory prior to a node experiencing system OOM, the `oom_killer` will calculate +an `oom_score` based on the percentage of memory its using on the node, and then add the `oom_score_adj` to get an +effective `oom_score` for the container, and then kills the container with the highest score. + +The intended behavior should be that containers with the lowest quality of service that +are consuming the largest amount of memory relative to the scheduling request should be killed first in order +to reclaim memory. + +Unlike pod eviction, if a pod container is OOM killed, it may be restarted by the `kubelet` based on its `RestartPolicy`. + +## Best Practices + +### Schedulable resources and eviction policies + +Let's imagine the following scenario: + +* Node memory capacity: `10Gi` +* Operator wants to reserve 10% of memory capacity for system daemons (kernel, `kubelet`, etc.) +* Operator wants to evict pods at 95% memory utilization to reduce thrashing and incidence of system OOM. + +To facilitate this scenario, the `kubelet` would be launched as follows: + +``` +--eviction-hard=memory.available<500Mi +--system-reserved=memory=1.5Gi +``` + +Implicit in this configuration is the understanding that "System reserved" should include the amount of memory +covered by the eviction threshold. + +To reach that capacity, either some pod is using more than its request, or the system is using more than `500Mi`. + +This configuration will ensure that the scheduler does not place pods on a node that immediately induce memory pressure +and trigger eviction assuming those pods use less than their configured request. + +### DaemonSet + +It is never desired for a `kubelet` to evict a pod that was derived from +a `DaemonSet` since the pod will immediately be recreated and rescheduled +back to the same node. + +At the moment, the `kubelet` has no ability to distinguish a pod created +from `DaemonSet` versus any other object. If/when that information is +available, the `kubelet` could pro-actively filter those pods from the +candidate set of pods provided to the eviction strategy. + +In general, it is strongly recommended that `DaemonSet` not +create `BestEffort` pods to avoid being identified as a candidate pod +for eviction. Instead `DaemonSet` should ideally launch `Guaranteed` pods. diff --git a/docs/admin/resourcequota/index.md b/docs/admin/resourcequota/index.md index f642da55f0..af96bc538d 100644 --- a/docs/admin/resourcequota/index.md +++ b/docs/admin/resourcequota/index.md @@ -74,6 +74,8 @@ are supported: | `replicationcontrollers` | The total number of replication controllers that can exist in the namespace. | | `resourcequotas` | The total number of [resource quotas](/docs/admin/admission-controllers/#resourcequota) that can exist in the namespace. | | `services` | The total number of services that can exist in the namespace. | +| `services.loadbalancers` | The total number of services of type load balancer that can exist in the namespace. | +| `services.nodeports` | The total number of services of type node port that can exist in the namespace. | | `secrets` | The total number of secrets that can exist in the namespace. | For example, `pods` quota counts and enforces a maximum on the number of `pods` @@ -153,6 +155,7 @@ spec: replicationcontrollers: "20" secrets: "10" services: "10" + services.loadbalancers: "2" EOF $ kubectl create -f ./object-counts.yaml --namespace=myspace @@ -182,6 +185,7 @@ persistentvolumeclaims 0 4 replicationcontrollers 0 20 secrets 1 10 services 0 10 +services.loadbalancers 0 2 ``` ## Quota and Cluster Capacity diff --git a/docs/admin/resourcequota/object-counts.yaml b/docs/admin/resourcequota/object-counts.yaml index 73ea172581..b36f14611a 100644 --- a/docs/admin/resourcequota/object-counts.yaml +++ b/docs/admin/resourcequota/object-counts.yaml @@ -5,3 +5,5 @@ metadata: spec: hard: persistentvolumeclaims: "2" + services.loadbalancers: "2" + services.nodeports: "0" \ No newline at end of file diff --git a/docs/admin/resourcequota/walkthrough.md b/docs/admin/resourcequota/walkthrough.md index 0502655fb4..dcae893d40 100644 --- a/docs/admin/resourcequota/walkthrough.md +++ b/docs/admin/resourcequota/walkthrough.md @@ -5,7 +5,7 @@ This example demonstrates a typical setup to control for resource usage in a nam It demonstrates using the following resources: -* [Namespace](/docs/admin/namespaces/) +* [Namespace](/docs/admin/namespaces) * [Resource Quota](/docs/admin/resourcequota/) * [Limit Range](/docs/admin/limitrange/) @@ -19,8 +19,10 @@ fair sharing of the cluster and control cost. The cluster-admin has the following goals: -* Limit the amount of compute resources for running pods +* Limit the amount of compute resource for running pods * Limit the number of persistent volume claims to control access to storage +* Limit the number of load balancers to control cost +* Prevent the use of node ports to preserve scarce resources * Provide default compute resource requests to enable better scheduling decisions ## Step 1: Create a namespace @@ -41,7 +43,11 @@ quota-example Active 39s ## Step 2: Apply an object-count quota to the namespace -The cluster-admin wants to control the amount of persistent volume claims. +The cluster-admin wants to control the following resources: + +* persistent volume claims +* load balancers +* node ports Let's create a simple quota that controls object counts for those resource types in this namespace. @@ -62,10 +68,13 @@ Namespace: quota-example Resource Used Hard -------- ---- ---- persistentvolumeclaims 0 2 +services.loadbalancers 0 2 +services.nodeports 0 0 ``` The quota system will now prevent users from creating more than the specified amount for each resource. + ## Step 3: Apply a compute-resource quota to the namespace To limit the amount of compute resource that can be consumed in this namespace, @@ -116,6 +125,7 @@ $ kubectl get pods --namespace=quota-example ``` What happened? I have no pods! Let's describe the deployment to get a view of what is happening. +<<<<<<< HEAD ```shell $ kubectl describe deployment nginx --namespace=quota-example @@ -154,7 +164,7 @@ Events: 4m 7s 11 {replicaset-controller } Warning FailedCreate Error creating: pods "nginx-3137573019-" is forbidden: Failed quota: compute-resources: must specify limits.cpu,limits.memory,requests.cpu,requests.memory ``` -The Kubernetes API server is rejecting the replica set's request to create a pod because our pods +The Kubernetes API server is rejecting the replica set requests to create a pod because our pods do not specify `requests` or `limits` for `cpu` and `memory`. So let's set some default values for the amount of `cpu` and `memory` a pod can consume: @@ -215,6 +225,8 @@ Namespace: quota-example Resource Used Hard -------- ---- ---- persistentvolumeclaims 0 2 +services.loadbalancers 0 2 +services.nodeports 0 0 ``` As you can see, the pod that was created is consuming explict amounts of compute resources, and the usage is being diff --git a/docs/api.md b/docs/api.md index e466962df2..ab8a6a3953 100644 --- a/docs/api.md +++ b/docs/api.md @@ -25,13 +25,15 @@ Complete API details are documented using [Swagger v1.2](http://swagger.io/). Th We also host a version of the [latest API documentation UI](http://kubernetes.io/kubernetes/third_party/swagger-ui/). This is updated with the latest release, so if you are using a different version of Kubernetes you will want to use the spec from your apiserver. +Kubernetes implements an alternative Protobuf based serialization format for the API that is primarily intended for intra-cluster communication, documented in the [design proposal](https://github.com/kubernetes/kubernetes/blob/{{ page.githubbranch }}/docs/proposals/protobuf.md) and the IDL files for each schema are located in the Go packages that define the API objects. + ## API versioning To make it easier to eliminate fields or restructure resource representations, Kubernetes supports multiple API versions, each at a different API path, such as `/api/v1` or `/apis/extensions/v1beta1`. -We chose to version at the API level rather than at the resource or field level to ensure that the API presents a clear, consistent view of system resources and behavior, and to enable controlling access to end-of-lifed and/or experimental APIs. +We chose to version at the API level rather than at the resource or field level to ensure that the API presents a clear, consistent view of system resources and behavior, and to enable controlling access to end-of-lifed and/or experimental APIs. The JSON and Protobuf serialization schemas follow the same guidelines for schema changes - all descriptions below cover both formats. Note that API versioning and Software versioning are only indirectly related. The [API and release versioning proposal](https://github.com/kubernetes/kubernetes/blob/{{page.githubbranch}}/docs/design/versioning.md) describes the relationship between API versioning and diff --git a/docs/user-guide/deployments.md b/docs/user-guide/deployments.md index 1cd688e8c8..a76dbff5e7 100644 --- a/docs/user-guide/deployments.md +++ b/docs/user-guide/deployments.md @@ -76,47 +76,39 @@ The created Replica Set will ensure that there are three nginx Pods at all times ## The Status of a Deployment -After creating or updating a Deployment, you would want to confirm whether it succeeded or not. The best way to do this is through checking its status. - -To verify if the above Deployment succeeded or not, first compare the `.metadata.generation` and `.status.observedGeneration` of the Deployment: +After creating or updating a Deployment, you would want to confirm whether it succeeded or not. The simplest way to do this is through `kubectl rollout status`. ```shell -$ kubectl get deployment/nginx-deployment -o yaml | grep [Gg]eneration - generation: 2 - observedGeneration: 2 +$ kubectl rollout status deployment/nginx-deployment +deployment nginx-deployment successfully rolled out ``` -When `observedGeneration` >= `generation`, the Deployment controller has observed current Deployment; if not, wait for a few more seconds. +This verifies the Deployment's `.status.observedGeneration` >= `.metadata.generation`, and its up-to-date replicas +(`.status.updatedReplicas`) matches the desired replicas (`.spec.replicas`) to determine if the rollout succeeded. +If the rollout is still in progress, it watches for Deployment status changes and prints related messages. -Once the above condition is met, check the Deployment's up-to-date replicas (`.status.updatedReplicas`) and see if it matches the desired replicas (`.spec.replicas`): - -```shell -$ kubectl get deployment/nginx-deployment -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -nginx-deployment 3 3 3 3 9m -``` +Note that it's impossible to know whether a Deployment will ever succeed, so if the above command doesn't return success, +you'll need to timeout and give up at some point. Additionally, if you set `.spec.minReadySeconds`, you would also want to check if the available replicas (`.status.availableReplicas`) matches the desired replicas too. -**Note:** It's impossible to know whether a Deployment will ever succeed, so one has to timeout and give up at some point. +```shell +$ kubectl get deployments +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +nginx-deployment 3 3 3 3 20s +``` ## Updating a Deployment -Suppose that we now want to update the nginx Pods to start using the `nginx:1.9.1` image -instead of the `nginx:1.7.9` image. - **Note:** a Deployment's rollout is triggered if and only if the Deployment's pod template (i.e. `.spec.template`) is changed, e.g. updating labels or container images of the template. Other updates, such as scaling the Deployment, will not trigger a rollout. -First, we update our Deployment configuration as follows: - -{% include code.html language="yaml" file="new-nginx-deployment.yaml" ghlink="/docs/user-guide/new-nginx-deployment.yaml" %} - -We can then `apply` the new Deployment: +Suppose that we now want to update the nginx Pods to start using the `nginx:1.9.1` image +instead of the `nginx:1.7.9` image. ```shell -$ kubectl apply -f docs/user-guide/new-nginx-deployment.yaml -deployment "nginx-deployment" configured +$ kubectl set image deployment/nginx-deployment nginx=nginx:1.9.1 +deployment "nginx-deployment" image updated ``` Alternatively, we can `edit` the Deployment and change `.spec.template.spec.containers[0].image` from `nginx:1.7.9` to `nginx:1.9.1`: @@ -126,17 +118,15 @@ $ kubectl edit deployment/nginx-deployment deployment "nginx-deployment" edited ``` -Running a `get` immediately will give: +To see its rollout status, simply run: ```shell -$ kubectl get deployments -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -nginx-deployment 3 3 0 3 20s +$ kubectl rollout status deployment/nginx-deployment +Waiting for rollout to finish: 2 out of 3 new replicas have been updated... +deployment nginx-deployment successfully rolled out ``` -The 0 number of up-to-date replicas indicates that the Deployment hasn't updated the replicas to the latest configuration. The current replicas indicates the total replicas (3 with old configuration and 0 with new configuration) this Deployment manages, and the available replicas indicates the number of current replicas that are available. - -The Deployment will update all the Pods in a few seconds. +After the rollout succeeds, you may want to `get` the Deployment: ```shell $ kubectl get deployments @@ -144,6 +134,10 @@ NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE nginx-deployment 3 3 3 3 36s ``` +The number of up-to-date replicas indicates that the Deployment has updated the replicas to the latest configuration. +The current replicas indicates the total replicas this Deployment manages, and the available replicas indicates the +number of current replicas that are available. + We can run `kubectl get rs` to see that the Deployment updated the Pods by creating a new Replica Set and scaling it up to 3 replicas, as well as scaling down the old Replica Set to 0 replicas. ```shell @@ -163,7 +157,7 @@ nginx-deployment-1564180365-nacti 1/1 Running 0 14s nginx-deployment-1564180365-z9gth 1/1 Running 0 14s ``` -Next time we want to update these Pods, we only need to update and re-apply the Deployment again. +Next time we want to update these Pods, we only need to update the Deployment's pod template again. Deployment can ensure that only a certain number of Pods may be down while they are being updated. By default, it ensures that at least 1 less than the desired number of Pods are @@ -237,14 +231,21 @@ auto-scaling. This implies that when you rollback to an earlier revision, only t Suppose that we made a typo while updating the Deployment, by putting the image name as `nginx:1.91` instead of `nginx:1.9.1`: -{% include code.html language="yaml" file="bad-nginx-deployment.yaml" ghlink="/docs/user-guide/bad-nginx-deployment.yaml" %} - ```shell -$ kubectl apply -f docs/user-guide/bad-nginx-deployment.yaml -deployment "nginx-deployment" configured +$ kubectl set image deployment/nginx-deployment nginx=nginx:1.91 +deployment "nginx-deployment" image updated ``` -You will see that both the number of old replicas (nginx-deployment-1564180365 and nginx-deployment-2035384211) and new replicas (nginx-deployment-3066724191) are 2. +The rollout will be stuck. + +``` +$ kubectl rollout status deployments nginx-deployment +Waiting for rollout to finish: 2 out of 3 new replicas have been updated... +``` + +Press Ctrl-C to stop the above rollout status watch. + +You will also see that both the number of old replicas (nginx-deployment-1564180365 and nginx-deployment-2035384211) and new replicas (nginx-deployment-3066724191) are 2. ```shell $ kubectl get rs @@ -305,8 +306,8 @@ $ kubectl rollout history deployment/nginx-deployment deployments "nginx-deployment": REVISION CHANGE-CAUSE 1 kubectl create -f docs/user-guide/nginx-deployment.yaml --record -2 kubectl apply -f docs/user-guide/new-nginx-deployment.yaml -3 kubectl apply -f docs/user-guide/bad-nginx-deployment.yaml +2 kubectl set image deployment/nginx-deployment nginx=nginx:1.9.1 +3 kubectl set image deployment/nginx-deployment nginx=nginx:1.91 ``` Because we recorded the command while creating this Deployment using `--record`, we can easily see the changes we made in each revision. @@ -316,10 +317,18 @@ To further see the details of each revision, run: ```shell $ kubectl rollout history deployment/nginx-deployment --revision=2 deployments "nginx-deployment" revision 2 -Labels: app=nginx,pod-template-hash=1564180365 -Annotations: kubernetes.io/change-cause=kubectl apply -f docs/user-guide/new-nginx-deployment.yaml -Image(s): nginx:1.9.1 -No volumes. + Labels: app=nginx + pod-template-hash=1159050644 + Annotations: kubernetes.io/change-cause=kubectl set image deployment/nginx-deployment nginx=nginx:1.9.1 + Containers: + nginx: + Image: nginx:1.9.1 + Port: 80/TCP + QoS Tier: + cpu: BestEffort + memory: BestEffort + Environment Variables: + No volumes. ``` ### Rolling Back to a Previous Revision @@ -346,6 +355,7 @@ The Deployment is now rolled back to a previous stable revision. As you can see, $ kubectl get deployment NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE nginx-deployment 3 3 3 3 30m + $ kubectl describe deployment Name: nginx-deployment Namespace: default @@ -381,8 +391,8 @@ You can also pause a Deployment mid-way and then resume it. A use case is to sup Update the Deployment again and then pause the Deployment with `kubectl rollout pause`: ```shell -$ kubectl apply -f docs/user-guide/new-nginx-deployment; kubectl rollout pause deployment/nginx-deployment -deployment "nginx-deployment" configured +$ kubectl set image deployment/nginx-deployment nginx=nginx:1.9.1; kubectl rollout pause deployment/nginx-deployment +deployment "nginx-deployment" image updated deployment "nginx-deployment" paused ``` @@ -398,6 +408,13 @@ nginx-deployment-2035384211 2 2 1h nginx-deployment-3066724191 0 0 1h ``` +In a separate terminal, watch for rollout status changes and you'll see the rollout won't continue: + +```shell +$ kubectl rollout status deployment/nginx-deployment +Waiting for rollout to finish: 2 out of 3 new replicas have been updated... +``` + To resume the Deployment, simply do `kubectl rollout resume`: ```shell @@ -407,6 +424,14 @@ deployment "nginx-deployment" resumed Then the Deployment will continue and finish the rollout: +```shell +$ kubectl rollout status deployment/nginx-deployment +Waiting for rollout to finish: 2 out of 3 new replicas have been updated... +Waiting for deployment spec update to be observed... +Waiting for rollout to finish: 2 out of 3 new replicas have been updated... +deployment nginx-deployment successfully rolled out +``` + ```shell $ kubectl get rs NAME DESIRED CURRENT AGE diff --git a/docs/user-guide/downward-api/README.md b/docs/user-guide/downward-api/README.md index 93c5e6f5dc..00a27bdcb0 100644 --- a/docs/user-guide/downward-api/README.md +++ b/docs/user-guide/downward-api/README.md @@ -1,5 +1,5 @@ -Following this example, you will create a pod with a container that consumes the pod's name and -namespace using the [downward API](http://kubernetes.io/docs/user-guide/downward-api/). +Following these examples, you will create a pod with a container that consumes the pod's name, +namespace, and resource values using the [downward API](http://kubernetes.io/docs/user-guide/downward-api/). ## Step Zero: Prerequisites @@ -29,4 +29,25 @@ $ kubectl logs dapi-test-pod | grep POD_ 2015-04-30T20:22:18.568024817Z MY_POD_NAME=dapi-test-pod 2015-04-30T20:22:18.568087688Z MY_POD_NAMESPACE=default 2015-04-30T20:22:18.568092435Z MY_POD_IP=10.0.1.6 -``` \ No newline at end of file +``` + +## Example of environment variables with container resources + +Use the [`dapi-container-resources.yaml`](dapi-container-resources.yaml) file to create a Pod +with a container that consumes the downward API exposing the container's resources. + +```shell +$ kubectl create -f docs/user-guide/downward-api/dapi-container-resources.yaml +``` + +### Examine the logs + +Grep through the pod logs to see that the pod was injected with the correct values: + +```shell +$ kubectl logs dapi-test-pod | grep MY_ +MY_MEM_LIMIT=67108864 +MY_CPU_LIMIT=1 +MY_MEM_REQUEST=33554432 +MY_CPU_REQUEST=1 +``` diff --git a/docs/user-guide/downward-api/dapi-container-resources.yaml b/docs/user-guide/downward-api/dapi-container-resources.yaml new file mode 100644 index 0000000000..2a3abb0145 --- /dev/null +++ b/docs/user-guide/downward-api/dapi-container-resources.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Pod +metadata: + name: dapi-test-pod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/busybox:1.24 + command: [ "/bin/sh", "-c", "env" ] + resources: + requests: + memory: "32Mi" + cpu: "125m" + limits: + memory: "64Mi" + cpu: "250m" + env: + - name: MY_CPU_REQUEST + valueFrom: + resourceFieldRef: + resource: requests.cpu + - name: MY_CPU_LIMIT + valueFrom: + resourceFieldRef: + resource: limits.cpu + - name: MY_MEM_REQUEST + valueFrom: + resourceFieldRef: + resource: requests.memory + - name: MY_MEM_LIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + restartPolicy: Never diff --git a/docs/user-guide/downward-api/index.md b/docs/user-guide/downward-api/index.md index 3227414cad..14b6a61836 100644 --- a/docs/user-guide/downward-api/index.md +++ b/docs/user-guide/downward-api/index.md @@ -21,6 +21,10 @@ The following information is available to a `Pod` through the downward API: * The pod's name * The pod's namespace * The pod's IP +* A container's cpu limit +* A container's cpu request +* A container's memory limit +* A container's memory request More information will be exposed through this same API over time. @@ -31,13 +35,15 @@ Containers consume information from the downward API using environment variables or using a volume plugin. -### Environment variables +## Environment variables Most environment variables in the Kubernetes API use the `value` field to carry simple values. However, the alternate `valueFrom` field allows you to specify -a `fieldRef` to select fields from the pod's definition. The `fieldRef` field -is a structure that has an `apiVersion` field and a `fieldPath` field. The -`fieldPath` field is an expression designating a field of the pod. The +a `fieldRef` to select fields from the pod's definition, and a `resourceFieldRef` +to select fields from one of its container's definition. + +The `fieldRef` field is a structure that has an `apiVersion` field and a `fieldPath` +field. The `fieldPath` field is an expression designating a field of the pod. The `apiVersion` field is the version of the API schema that the `fieldPath` is written in terms of. If the `apiVersion` field is not specified it is defaulted to the API version of the enclosing object. @@ -46,20 +52,39 @@ The `fieldRef` is evaluated and the resulting value is used as the value for the environment variable. This allows users to publish their pod's name in any environment variable they want. +The `resourceFieldRef` is a structure that has a `containerName` field, a `resource` +field, and a `divisor` field. The `containerName` is the name of a container, +whose resource (cpu or memory) information is to be exposed. The `containerName` is +optional for environment variables and defaults to the current container. The +`resource` field is an expression designating a resource in a container, and the `divisor` +field specifies an output format of the resource being exposed. If the `divisor` +is not specified, it defaults to "1" for cpu and memory. The table shows possible +values for cpu and memory resources for `resource` and `divisor` settings: -## Example + +| Setting | Cpu | Memory | +| ------------- |-------------| -----| +| resource | limits.cpu, requests.cpu| limits.memory, requests.memory| +| divisor | 1(cores), 1m(millicores) | 1(bytes), 1k(kilobytes), 1M(megabytes), 1G(gigabytes), 1T(terabytes), 1P(petabytes), 1E(exabytes), 1Ki(kibibyte), 1Mi(mebibyte), 1Gi(gibibyte), 1Ti(tebibyte), 1Pi(pebibyte), 1Ei(exbibyte)| + + +### Example This is an example of a pod that consumes its name and namespace via the downward API: {% include code.html language="yaml" file="dapi-pod.yaml" ghlink="/docs/user-guide/downward-api/dapi-pod.yaml" %} +This is an example of a pod that consumes its container's resources via the downward API: -### Downward API volume +{% include code.html language="yaml" file="dapi-container-resources.yaml" ghlink="/docs/user-guide/downward-api/dapi-container-resources.yaml" %} + +## Downward API volume Using a similar syntax it's possible to expose pod information to containers using plain text files. Downward API are dumped to a mounted volume. This is achieved using a `downwardAPI` volume type and the different items represent the files to be created. `fieldPath` references the field to be exposed. +For exposing a container's resources limits and requests, `containerName` must be specified with `resourceFieldRef`. Downward API volume permits to store more complex data like [`metadata.labels`](/docs/user-guide/labels) and [`metadata.annotations`](/docs/user-guide/annotations). Currently key/value pair set fields are saved using `key="value"` format: @@ -76,18 +101,31 @@ Downward API volumes can expose: * The pod's namespace * The pod's labels * The pod's annotations +* A container's cpu limit +* A container's cpu request +* A container's memory limit +* A container's memory request The downward API volume refreshes its data in step with the kubelet refresh loop. When labels will be modifiable on the fly without respawning the pod containers will be able to detect changes through mechanisms such as [inotify](https://en.wikipedia.org/wiki/Inotify). In future, it will be possible to specify a specific annotation or label. -## Example +### Example This is an example of a pod that consumes its labels and annotations via the downward API volume, labels and annotations are dumped in `/etc/labels` and in `/etc/annotations`, respectively: {% include code.html language="yaml" file="volume/dapi-volume.yaml" ghlink="/docs/user-guide/downward-api/volume/dapi-volume.yaml" %} +This is an example of a pod that consumes its container's resources via the downward API volume. + +{% include code.html language="yaml" file="volume/dapi-volume-resources.yaml" ghlink="/docs/user-guide/downward-api/volume/dapi-volume-resources.yaml" %} Some more thorough examples: + * [environment variables](/docs/user-guide/environment-guide/) + * [downward API](/docs/user-guide/downward-api/) + +## Default values for container resource limits + +If cpu and memory limits are not specified for a container, the downward API will default to node's cpu and memory capacities. diff --git a/docs/user-guide/downward-api/volume/dapi-volume-resources.yaml b/docs/user-guide/downward-api/volume/dapi-volume-resources.yaml new file mode 100644 index 0000000000..f28bb99e3f --- /dev/null +++ b/docs/user-guide/downward-api/volume/dapi-volume-resources.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kubernetes-downwardapi-volume-example +spec: + containers: + - name: client-container + image: gcr.io/google_containers/busybox:1.24 + command: ["sh", "-c", "while true; do echo; if [[ -e /etc/cpu_limit ]]; then cat /etc/cpu_limit; fi; if [[ -e /etc/cpu_request ]]; then cat /etc/cpu_request; fi; if [[ -e /etc/mem_limit ]]; then cat /etc/mem_limit; fi; if [[ -e /etc/mem_request ]]; then cat /etc/mem_request; fi; sleep 5; done"] + resources: + requests: + memory: "32Mi" + cpu: "125m" + limits: + memory: "64Mi" + cpu: "250m" + volumeMounts: + - name: podinfo + mountPath: /etc + readOnly: false + volumes: + - name: podinfo + downwardAPI: + items: + - path: "cpu_limit" + resourceFieldRef: + containerName: client-container + resource: limits.cpu + - path: "cpu_request" + resourceFieldRef: + containerName: client-container + resource: requests.cpu + - path: "mem_limit" + resourceFieldRef: + containerName: client-container + resource: limits.memory + - path: "mem_request" + resourceFieldRef: + containerName: client-container + resource: requests.memory diff --git a/docs/user-guide/downward-api/volume/index.md b/docs/user-guide/downward-api/volume/index.md index 446d8eb563..46f54d0776 100644 --- a/docs/user-guide/downward-api/volume/index.md +++ b/docs/user-guide/downward-api/volume/index.md @@ -2,7 +2,7 @@ --- Following this example, you will create a pod with a downward API volume. -A downward API volume is a k8s volume plugin with the ability to save some pod information in a plain text file. The pod information can be for example some [metadata](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/docs/devel/api-conventions.md#metadata). +A downward API volume is a k8s volume plugin with the ability to save some pod information in a plain text file. The pod information can be for example some [metadata](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/docs/devel/api-conventions.md#metadata) or a container's [resources](/docs/user-guide/compute-resources). Supported metadata fields: @@ -11,13 +11,20 @@ Supported metadata fields: 3. `metadata.name` 4. `metadata.labels` +Supported container's resources: + +1. `limits.cpu` +2. `limits.memory` +3. `requests.cpu` +4. `requests.memory` + ### Step Zero: Prerequisites This example assumes you have a Kubernetes cluster installed and running, and the `kubectl` command line tool somewhere in your path. Please see the [gettingstarted](/docs/getting-started-guides/) for installation instructions for your platform. ### Step One: Create the pod -Use the `docs/user-guide/downward-api/dapi-volume.yaml` file to create a Pod with a  downward API volume which stores pod labels and pod annotations to `/etc/labels` and  `/etc/annotations` respectively. +Use the [dapi-volume.yaml](/docs/user-guide/downward-api/volume/dapi-volume.yaml) file to create a Pod with a downward API volume which stores pod labels and pod annotations to `/etc/labels` and `/etc/annotations` respectively. ```shell $ kubectl create -f docs/user-guide/downward-api/volume/dapi-volume.yaml @@ -46,21 +53,66 @@ In pod's `/etc` directory one may find the file created by the plugin (system fi $ kubectl exec kubernetes-downwardapi-volume-example -i -t -- sh / # ls -laR /etc /etc: -total 32 -drwxrwxrwt 3 0 0 180 Aug 24 13:03 . -drwxr-xr-x 1 0 0 4096 Aug 24 13:05 .. -drwx------ 2 0 0 80 Aug 24 13:03 ..2015_08_24_13_03_44259413923 -lrwxrwxrwx 1 0 0 30 Aug 24 13:03 ..downwardapi -> ..2015_08_24_13_03_44259413923 -lrwxrwxrwx 1 0 0 25 Aug 24 13:03 annotations -> ..downwardapi/annotations -lrwxrwxrwx 1 0 0 20 Aug 24 13:03 labels -> ..downwardapi/labels +total 4 +drwxrwxrwt 3 0 0 120 Jun 1 19:55 . +drwxr-xr-x 17 0 0 4096 Jun 1 19:55 .. +drwxr-xr-x 2 0 0 80 Jun 1 19:55 ..6986_01_06_15_55_10.473583074 +lrwxrwxrwx 1 0 0 31 Jun 1 19:55 ..data -> ..6986_01_06_15_55_10.473583074 +lrwxrwxrwx 1 0 0 18 Jun 1 19:55 annotations -> ..data/annotations +lrwxrwxrwx 1 0 0 13 Jun 1 19:55 labels -> ..data/labels -/etc/..2015_08_24_13_03_44259413923: +/etc/..6986_01_06_15_55_10.473583074: total 8 -drwx------ 2 0 0 80 Aug 24 13:03 . -drwxrwxrwt 3 0 0 180 Aug 24 13:03 .. --rw-r--r-- 1 0 0 115 Aug 24 13:03 annotations --rw-r--r-- 1 0 0 53 Aug 24 13:03 labels +drwxr-xr-x 2 0 0 80 Jun 1 19:55 . +drwxrwxrwt 3 0 0 120 Jun 1 19:55 .. +-rw-r--r-- 1 0 0 129 Jun 1 19:55 annotations +-rw-r--r-- 1 0 0 59 Jun 1 19:55 labels / # ``` -The file `labels` is stored in a temporary directory (`..2015_08_24_13_03_44259413923` in the example above) which is symlinked to by `..downwardapi`. Symlinks for annotations and labels in `/etc` point to files containing the actual metadata through the `..downwardapi` indirection.  This structure allows for dynamic atomic refresh of the metadata: updates are written to a new temporary directory, and the `..downwardapi` symlink is updated atomically using `rename(2)`. \ No newline at end of file +The file `labels` is stored in a temporary directory (`..6986_01_06_15_55_10.473583074` in the example above) which is symlinked to by `..data`. Symlinks for annotations and labels in `/etc` point to files containing the actual metadata through the `..data` indirection.  This structure allows for dynamic atomic refresh of the metadata: updates are written to a new temporary directory, and the `..data` symlink is updated atomically using `rename(2)`. + +## Example of downward API volume with container resources + +Use the `docs/user-guide/downward-api/volume/dapi-volume-resources.yaml` file to create a Pod with a downward API volume which stores its container's limits and requests in /etc. + +```shell +$ kubectl create -f docs/user-guide/downward-api/volume/dapi-volume-resources.yaml +``` + +### Examine pod/container output + +In pod's `/etc` directory one may find the files created by the plugin: + +```shell +$ kubectl exec kubernetes-downwardapi-volume-example -i -t -- sh +/ # ls -alR /etc +/etc: +total 4 +drwxrwxrwt 3 0 0 160 Jun 1 19:47 . +drwxr-xr-x 17 0 0 4096 Jun 1 19:48 .. +drwxr-xr-x 2 0 0 120 Jun 1 19:47 ..6986_01_06_15_47_23.076909525 +lrwxrwxrwx 1 0 0 31 Jun 1 19:47 ..data -> ..6986_01_06_15_47_23.076909525 +lrwxrwxrwx 1 0 0 16 Jun 1 19:47 cpu_limit -> ..data/cpu_limit +lrwxrwxrwx 1 0 0 18 Jun 1 19:47 cpu_request -> ..data/cpu_request +lrwxrwxrwx 1 0 0 16 Jun 1 19:47 mem_limit -> ..data/mem_limit +lrwxrwxrwx 1 0 0 18 Jun 1 19:47 mem_request -> ..data/mem_request + +/etc/..6986_01_06_15_47_23.076909525: +total 16 +drwxr-xr-x 2 0 0 120 Jun 1 19:47 . +drwxrwxrwt 3 0 0 160 Jun 1 19:47 .. +-rw-r--r-- 1 0 0 1 Jun 1 19:47 cpu_limit +-rw-r--r-- 1 0 0 1 Jun 1 19:47 cpu_request +-rw-r--r-- 1 0 0 8 Jun 1 19:47 mem_limit +-rw-r--r-- 1 0 0 8 Jun 1 19:47 mem_request + +/ # cat /etc/cpu_limit +1 +/ # cat /etc/mem_limit +67108864 +/ # cat /etc/cpu_request +1 +/ # cat /etc/mem_request +33554432 +``` diff --git a/docs/user-guide/garbage-collector.md b/docs/user-guide/garbage-collector.md new file mode 100644 index 0000000000..e9d646e30c --- /dev/null +++ b/docs/user-guide/garbage-collector.md @@ -0,0 +1,26 @@ +--- +--- + +* TOC +{:toc} + +## WARNING: Garbage Collector is an alpha feature and is disabled by default. Use it at your own risk! + +### What is garbage collector for + +The garbage collector (GC) cascadingly deletes dependent API objects when the owner is deleted. One use case is if two objects have functional dependency, you can specify the dependency in their configuration file when creating them, and if one of them is deleted, the GC will delete the other one automatically. The other use case is if there is logical dependency among API objects, e.g., the pods created by a replicaset depending on the replicaset, Kubernetes will automatically set the dependency (this will be implemented in release 1.4) and the GC will delete the pods when the replicaset is deleted. + +### How does the garbage collector work + +In release 1.3, there is a new `ownerReferences` field in the `metadata` of every Kubernetes API objects. The GC monitors the cluster and checks the `metadata.ownerReferences` field of each object. If none of the owners present in `metadata.ownerReferences` exists in the cluster, the GC will request the API server to delete the object. + +Currently a user needs to manually set the `metadata.ownerReferences`. In release 1.4, controllers will automatically set the field for the objects it controls. For example, when the replicaset controller creates or adopts pods, it will automatically add the replicaset to the `metadata.ownerReferences` fields of the pods. + +### How to request the garbage collector to not delete dependents + +When deleting an object, you can prevent the GC from deleting that object's dependents by specifying `deleteOptions.orphanDependents=true` in the deletion request. It prevents garbage collection by removing the object from its dependents' metadata.ownerReferences field. + +### How to enable the garbage collector + +The garbage collector is an alpha feature so it is disabled by default. To enable it, you need to start the kube-apiserver and kube-controller-manager with flag `--enable-garbage-collector`. + diff --git a/docs/user-guide/ingress.md b/docs/user-guide/ingress.md index c6d63ea1f8..3803db0360 100644 --- a/docs/user-guide/ingress.md +++ b/docs/user-guide/ingress.md @@ -44,20 +44,12 @@ Before you start using the Ingress resource, there are a few things you should u On GCE/GKE there should be a [L7 cluster addon](https://github.com/kubernetes/contrib/blob/master/ingress/controllers/gce/README.md), deployed into the `kube-system` namespace: ```shell -$ kubectl get pods --namespace=kube-system -l name=glbc +$ kubectl get pods --namespace=kube-system -l k8s-app=glbc NAME READY STATUS RESTARTS AGE l7-lb-controller-v0.6.0-chnan 2/2 Running 0 1d ``` -Make sure you review the [beta limitations](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/gce/BETA_LIMITATIONS.md) of this controller. In particular, you need to create a single firewall-rule on your cloudprovider, to allow health checks. On GKE this would be: - -```shell -$ gcloud compute firewall-rules create allow-130-211-0-0-22 \ - --source-ranges 130.211.0.0/22 \ - --allow tcp:30000-32767 -``` - -In environments other than GCE/GKE, you need to [deploy a controller](https://github.com/kubernetes/contrib/tree/master/ingress/controllers) as a pod. +Make sure you review the [beta limitations](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/gce/BETA_LIMITATIONS.md) of this controller. In environments other than GCE/GKE, you need to [deploy a controller](https://github.com/kubernetes/contrib/tree/master/ingress/controllers) as a pod. ## The Ingress Resource @@ -94,6 +86,10 @@ __Global Parameters__: For the sake of simplicity the example Ingress has no glo In order for the Ingress resource to work, the cluster must have an Ingress controller running. This is unlike other types of controllers, which typically run as part of the `kube-controller-manager` binary, and which are typically started automatically as part of cluster creation. You need to choose the ingress controller implementation that is the best fit for your cluster, or implement one. Examples and instructions can be found [here](https://github.com/kubernetes/contrib/tree/master/ingress/controllers). +## Before you begin + +The following document describes a set of cross platform features exposed through the Ingress resource. Ideally, all Ingress controllers should fulfill this specification, but we're not there yet. The docs for the GCE and nginx controllers are [here](https://github.com/kubernetes/contrib/blob/master/ingress/controllers/gce/README.md) and [here](https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/README.md) respectively. **Make sure you review controller specific docs so you understand the caveats of each one**. + ## Types of Ingress ### Single Service Ingress @@ -221,11 +217,13 @@ spec: servicePort: 80 ``` +Note that there is a gap between TLS features supported by various Ingress controllers. Please refer to documentation on [nginx](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx#https), [GCE](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/gce#tls), or any other platform specific Ingress controller to understand how TLS works in your environment. + ### Loadbalancing An Ingress controller is bootstrapped with some loadbalancing policy settings that it applies to all Ingress, such as the loadbalancing algorithm, backend weight scheme etc. More advanced loadbalancing concepts (eg: persistent sessions, dynamic weights) are not yet exposed through the Ingress. You can still get these features through the [service loadbalancer](https://github.com/kubernetes/contrib/tree/master/service-loadbalancer). With time, we plan to distill loadbalancing patterns that are applicable cross platform into the Ingress resource. -It's also worth noting that even though health checks are not exposed directly through the Ingress, there exist parallel concepts in Kubernetes such as [readiness probes](https://github.com/kubernetes/kubernetes/blob/release-1.0/docs/user-guide/production-pods.md#liveness-and-readiness-probes-aka-health-checks) which allow you to achieve the same end result. +It's also worth noting that even though health checks are not exposed directly through the Ingress, there exist parallel concepts in Kubernetes such as [readiness probes](https://github.com/kubernetes/kubernetes/blob/release-1.0/docs/user-guide/production-pods.md#liveness-and-readiness-probes-aka-health-checks) which allow you to achieve the same end result. Please review the controller specific docs to see how they handle health checks ([nginx](https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/README.md), [GCE](https://github.com/kubernetes/contrib/blob/master/ingress/controllers/gce/README.md#health-checks)). ## Updating an Ingress diff --git a/docs/user-guide/managing-deployments.md b/docs/user-guide/managing-deployments.md index 611d56ed86..ee85850c0d 100644 --- a/docs/user-guide/managing-deployments.md +++ b/docs/user-guide/managing-deployments.md @@ -82,6 +82,49 @@ my-nginx-svc 10.0.0.208 80/TCP 0s With the above commands, we first create resources under docs/user-guide/nginx/ and print the resources created with `-o name` output format (print each resource as resource/name). Then we `grep` only the "service", and then print it with `kubectl get`. +If you happen to organize your resources across several subdirectories within a particular directory, you can recursively perform the operations on the subdirectories also, by specifying `--recursive` or `-R` alongside the `--filename,-f` flag. + +For instance, assume there is a directory `project/k8s/development` that holds all of the manifests needed for the development environment, organized by resource type: + +``` +project/k8s/development +├── configmap +│   └── my-configmap.yaml +├── deployment +│   └── my-deployment.yaml +└── pvc + └── my-pvc.yaml +``` + +By default, performing a bulk operation on `project/k8s/development` will stop at the first level of the directory, not processing any subdirectories. If we tried to create the resources in this directory using the following command, we'd encounter an error: + +```shell +$ kubectl create -f project/k8s/development +error: you must provide one or more resources by argument or filename (.json|.yaml|.yml|stdin) +``` + +Instead, specify the `--recursive` or `-R` flag with the `--filename,-f` flag as such: + +```shell +$ kubectl create -f project/k8s/development --recursive +configmap "my-config" created +deployment "my-deployment" created +persistentvolumeclaim "my-pvc" created +``` + +The `--recursive` flag works with any operation that accepts the `--filename,-f` flag such as: `kubectl {create,get,delete,describe,rollout} etc.` + +The `--recursive` flag also works when multiple `-f` arguments are provided: + +```shell +$ kubectl create -f project/k8s/namespaces -f project/k8s/development --recursive +namespace "development" created +namespace "staging" created +configmap "my-config" created +deployment "my-deployment" created +persistentvolumeclaim "my-pvc" created +``` + If you're interested in learning more about `kubectl`, go ahead and read [kubectl Overview](/docs/user-guide/kubectl-overview). ## Using labels effectively diff --git a/docs/user-guide/networkpolicies.md b/docs/user-guide/networkpolicies.md new file mode 100644 index 0000000000..e550657df6 --- /dev/null +++ b/docs/user-guide/networkpolicies.md @@ -0,0 +1,80 @@ +--- +--- + +* TOC +{:toc} + +A network policy is a specification of how selections of pods are allowed to communicate with each other and other network endpoints. + +`NetworkPolicy` resources use labels to select pods and define whitelist rules which allow traffic to the selected pods in addition to what is allowed by the isolation policy for a given namespace. + +## Prerequisites + +You must enable the `extensions/v1beta/networkpolicies` runtime config in your apiserver to enable this resource. + +You must also be using a networking solution which supports `NetworkPolicy` - simply creating the +resource without a controller to implement it will have no effect. + +## Configuring Namespace Isolation Policy + +Isolation can be configured on a per-namespace basis. Once isolation is configured on a namespace it will be applied to all pods in that namespace. Currently, only isolation policy on inbound traffic (ingress) can be defined. + +The following ingress isolation types being supported: + +- `DefaultDeny`: Pods in the namespace will be inaccessible from any source except the pod's local node. + +Ingress isolation can be enabled using an annotation on the Namespace. + +```yaml +kind: Namespace +apiVersion: v1 +metadata: + annotations: + net.beta.kubernetes.io/network-policy: | + { + "ingress": { + "isolation": "DefaultDeny" + } + } +``` + +To configure the annotation via `kubectl`: + +```shell{% raw %} +kubectl annotate ns "net.beta.kubernetes.io/networkpolicy={\"ingress\": {\"isolation\": \"DefaultDeny\"}}" +{% endraw %}``` + +## The `NetworkPolicy` Resource + +See the [api-reference](/docs/api-reference/extensions/v1beta1/definitions/#_v1beta1_networkpolicy) for a full definition of the resource. + +A minimal `NetworkPolicy` might look like this: + +```yaml +apiVersion: extensions/v1beta1 +kind: NetworkPolicy +metadata: + name: test-network-policy +spec: + podSelector: + matchLabels: + role: db + ingress: + - from: + podSelector: + matchLabels: + role: frontend + ports: + - protocol: tcp + port: 6379 +``` + +*POSTing this to the API server will have no effect unless your chosen networking solution supports network policy.* + +__Mandatory Fields__: As with all other Kubernetes config, a `NetworkPolicy` needs `apiVersion`, `kind`, and `metadata` fields. For general information about working with config files, see [here](/docs/user-guide/simple-yaml), [here](/docs/user-guide/configuring-containers), and [here](/docs/user-guide/working-with-resources). + +__spec__: `NetworkPolicy` [spec](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/docs/devel/api-conventions.md#spec-and-status) has all the information needed to define a network isolation policy in the deployed controller. + +__podSelector__: Each `NetworkPolicy` includes a `podSelector` which selects the grouping of pods to which the `ingress` rules in the policy apply. + +__ingress__: Each `NetworkPolicy` includes a list of whitelist `ingress` rules. Each rule allows traffic which matches both the `from` and `ports` sections. diff --git a/docs/user-guide/nginx-init-containers.yaml b/docs/user-guide/nginx-init-containers.yaml new file mode 100644 index 0000000000..34c20fa66a --- /dev/null +++ b/docs/user-guide/nginx-init-containers.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx + annotations: + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "install", + "image": "busybox", + "command": ["wget", "-O", "/work-dir/index.html", "http://kubernetes.io/index.html"], + "volumeMounts": [ + { + "name": "workdir", + "mountPath": "/work-dir" + } + ] + } + ]' +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 + volumeMounts: + - name: workdir + mountPath: /usr/share/nginx/html + dnsPolicy: Default + volumes: + - name: workdir + emptyDir: {} diff --git a/docs/user-guide/petset.md b/docs/user-guide/petset.md new file mode 100644 index 0000000000..77b132dea2 --- /dev/null +++ b/docs/user-guide/petset.md @@ -0,0 +1,322 @@ +--- +--- + +* TOC +{:toc} + +__Terminology__ + +Throughout this doc you will see a few terms that are sometimes used interchangeably elsewhere, that might cause confusion. This section attempts to clarify them. + +* Node: A single virtual or physical machine in a Kubernetes cluster. +* Cluster: A group of nodes in a single failure domain, unless mentioned otherwise. +* Persistent Volume Claim (PVC): A request for storage, typically a [persistent volume](/docs/user-guide/persistent-volumes/walkthrough/). +* Host name: The hostname attached to the UTS namespace of the pod, i.e the output of `hostname` in the pod. +* DNS/Domain name: A *cluster local* domain name resolvable using standard methods (eg: [gethostbyname](http://linux.die.net/man/3/gethostbyname)). +* Ordinality: the proprety of being "ordinal", or occupying a position in a sequence. +* Pet: a single member of a Pet Set; more generally, a stateful application. +* Peer: a process running a server, capable of communicating with other such processes. + +__Prerequisites__ + +This doc assumes familiarity with the following Kubernetes concepts: + +* [Pods](/docs/user-guide/pods/single-container/) +* [Cluster DNS](/docs/admin/dns/) +* [Headless Services](/docs/user-guide/services/#headless-services) +* [Persistent Volumes](/docs/user-guide/volumes/) +* [Dynamic volume provisioning](http://releases.k8s.io/{{page.githubbranch}}/examples/experimental/persistent-volume-provisioning/README.md) + +You need a working Kubernetes cluster at version >= 1.3, with a healthy DNS [cluster addon](http://releases.k8s.io/{{page.githubbranch}}/cluster/addons/README.md) at version >= 15. You cannot use Pet Set on a hosted Kubernetes provider that has disabled `alpha` resources. + +## What is a Pet Set? + +In Kubernetes, most pod management abstractions group them into disposable units of work that compose a micro service. Replication controllers for example, are designed with a weak guarantee - that there should be N replicas of a particular pod template. The pods are treated as stateless units, if one of them is unhealthy or superseded by a newer version, the system just disposes it. + +``` + foo.default.svc.cluster.local + |service| + / \ + | pod-asdf | | pod-zxcv | +``` + +A Pet Set, in contrast, is a group of stateful pods that require a stronger notion of identity. The document refers to these as "clustered applications". + +``` + *.foo.default.svc.cluster.local + | mysql-0 | <-> | mysql-1 | + [pv 0] [pv 1] +``` + +The co-ordinated deployment of clustered applications is notoriously hard. They require stronger notions of identity and membership, which they use in opaque internal protocols, and are especially prone to race conditions and deadlock. Traditionally administrators have deployed these applications by leveraging nodes as stable, long-lived entities with persistent storage and static ips. + +The goal of Pet Set is to decouple this dependency by assigning identities to individual instances of an application that are not anchored to the underlying physical infrastructure. For the rest of this document we will refer to these entities as "Pets". Our use of this term is predated by the "Pets vs Cattle" analogy. + +__Relationship between Pets and Pods__: PetSet requires there be {0..N-1} Pets. Each Pet has a deterministic name - PetSetName-Ordinal, and a unique identity. Each Pet has at most one pod, and each Pet Set has at most one Pet with a given identity. + +## When to use Pet Set? + +A Pet Set ensures that a specified number of "pets" with unique identities are running at any given time. The identity of a Pet is comprised of: + +* a stable hostname, available in DNS +* an ordinal index +* stable storage: linked to the ordinal & hostname + +These properties are useful in deploying stateful applications. However most stateful applications are also clustered, meaning they form groups with strict membership requirements that rely on stored state. Pet Set also helps with the 2 most common problems encountered managing such clustered applications: + +* discovery of peers for quorum +* startup/teardown ordering + +Only use Pet Set if your application requires some or all of these properties. Managing pods as stateless replicas is vastly easier. + +Example workloads for Pet Set: + +* Databases like MySQL or PostgreSQL that require a single instance attached to a NFS persistent volume at any time +* Clustered software like Zookeeper, Etcd, or Elasticsearch that require stable membership. + +## Alpha limitations + +Before you start deploying applications as Pet Sets, there are a few limitations you should understand. + +* Pet Set is an *alpha* resource, not available in any Kubernetes release prior to 1.3. +* As with all alpha/beta resources, it can be disable through the `--runtime-config` option passed to the apiserver, and in fact most likely will be disabled on hosted offerings of Kubernetes. +* The only updatable field on a Pet Set is `replicas` +* The storage for a given pet must either be provisioned by a [dynamic storage provisioner](http://releases.k8s.io/{{page.githubbranch}}/examples/experimental/persistent-volume-provisioning/README.md) based on the requested `storage class`, or pre-provisioned by an admin. Note that dynamic volume provisioning is also currently in alpha. +* Deleting the Pet Set *will not* delete any pets. You will either have to manually scale it down to 0 pets first, or delete the pets yourself. +* Deleting and/or scaling a Pet Set down will *not* delete the volumes associated with the Pet Set. This is done to ensure safety first, your data is more valuable than an auto purge of all related Pet Set resources. **Deleting the Persistent Volume Claims will result in a deletion of the associated volumes**. +* All Pet Sets currently require a "governing service", or a Service responsible for the network identity of the pets. The user is responsible for this Service. +* Updating an existing Pet Set is currently a manual process, meaning you either need to deploy a new Pet Set with the new image version, or orphan Pets one by one, update their image, and join them back to the cluster. + +## Example Pet Set + +We'll create a basic Pet Set to demonstrate how Pets are assigned unique and "sticky" identities. + +{% include code.html language="yaml" file="petset.yaml" ghlink="/docs/user-guide/petset.yaml" %} + +Saving this config into `petset.yaml` and submitting it to a Kubernetes cluster should creae the defined Pet Set and Pets it manages: + +```shell +$ kubectl create -f petset.yaml +service "nginx" created +petset "nginx" created +``` + +## Pet Identity + +The identity of a Pet sticks to it, regardless of which node it's (re) scheduled on. We can examine the identity of the pets we just created. + +### Ordinal index + +you should see 2 pods with predictable names formatted thus: `$(petset name)-$(ordinal index assigned by petset controller)` + +```shell +$ kubectl get po +NAME READY STATUS RESTARTS AGE +web-0 1/1 Running 0 10m +web-1 1/1 Running 0 10m +``` + +### Stable storage + +2 persistent volumes, one per pod. This is auto created by the Pet Set based on the `volumeTemplate` field + +```shell +$ kubectl get pv +NAME CAPACITY ACCESSMODES STATUS CLAIM REASON AGE +pvc-90234946-3717-11e6-a46e-42010af00002 1Gi RWO Bound default/www-web-0 11m +pvc-902733c2-3717-11e6-a46e-42010af00002 1Gi RWO Bound default/www-web-1 11m +``` + +### Network identity + +The network identity has 2 parts. First, we created a headless Service that controls the domain within which we create Pets. The domain managed by this Service takes the form: `$(service name).$(namespace).svc.cluster.local`, where "cluster.local" is the [cluster domain](http://releases.k8s.io/{{page.githubbranch}}/build/kube-dns/README.md#how-do-i-configure-it). As each pet is created, it gets a matching DNS subdomain, taking the form: `$(petname).$(governing service domain)`, where the governing service is defined by the `serviceName` field on the Pet Set. + +Here are some examples of choices for Cluster Domain, Service name, Pet Set name, and how that affects the DNS names for the Pets and the hostnames in the Pet's pods: + +Cluster Domain | Service (ns/name) | Pet Set (ns/name) | Pet Set Domain | Pet DNS | Pet Hostname | +-------------- | ----------------- | ----------------- | -------------- | ------- | ------------ | + cluster.local | default/nginx | default/web | nginx.default.svc.cluster.local | web-{0..N-1}.nginx.default.svc.cluster.local | web-{0..N-1} | + cluster.local | foo/nginx | foo/web | nginx.foo.svc.cluster.local | web-{0..N-1}.nginx.foo.svc.cluster.local | web-{0..N-1} | + kube.local | foo/nginx | foo/web | nginx.foo.svc.kube.local | web-{0..N-1}.nginx.foo.svc.kube.local | web-{0..N-1} | + +Note that Cluster Domain will be set to `cluster.local` unless [otherwise configured](http://releases.k8s.io/{{page.githubbranch}}/build/kube-dns/README.md#how-do-i-configure-it). + +Lets verify our assertion with a simple test. + +```shell +$ kubectl get svc +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +nginx None 80/TCP 12m +... +``` + +First, the PetSet gives provides a stable hostname: + +```shell +$ for i in 0 1; do kubectl exec web-$i -- sh -c 'hostname'; done +web-0 +web-1 +``` + +And the hostname is linked to the in-cluster DNS address: + +```shell +$ kubectl run -it --image busybox dns-test --restart=Never /bin/sh +dns-test # nslookup web-0.nginx +Server: 10.0.0.10 +Address 1: 10.0.0.10 kube-dns.kube-system.svc.cluster.local + +Name: web-0.nginx +Address 1: 10.180.3.5 + +dns-test # nslookup web-1.nginx +Server: 10.0.0.10 +Address 1: 10.0.0.10 kube-dns.kube-system.svc.cluster.local + +Name: web-1.nginx +Address 1: 10.180.0.9 +``` + +The containers are running nginx webservers, which by default will look for an index.html file in `/usr/share/nginx/html/index.html`. That directory is backed by a `PersistentVolume` created by the Pet Set. So lets write our hostname there: + +```shell +$ for i in 0 1; do + kubectl exec web-$i -- sh -c 'echo $(hostname) > /usr/share/nginx/html/index.html'; +done +``` + +And verify each webserver serves its own hostname: + +```shell +$ for i in 0 1; do kubectl exec -it web-$i -- curl localhost; done +web-0 +web-1 +``` + +Now delete all pods in the petset: + +```shell +$ kubectl delete po -l app=nginx +pod "web-0" deleted +pod "web-1" deleted +``` + +Wait for them to come back up, and try to retrieve the previously written hostname through the DNS name of the peer. They match, because the storage, DNS name, and hostname stick to the Pet no matter where it gets scheduled: + +```shell +$ kubectl exec -it web-1 -- curl web-0.nginx +web-0 +$ kubectl exec -it web-0 -- curl web-1.nginx +web-1 +``` + +## Peer discovery + +A pet can piece together its own identity: + +1. Use the [downward api](/docs/user-guide/downward-api/) to find its pod name +2. Run `hostname` to find its DNS name +3. Run `mount` or `df` to find its volumes (usually this is unnecessary) + +It's not necessary to "discover" the governing Service of a Pet Set, since it's known at creation time you can simply pass it down through an [environment variable](/docs/user-guide/environment-guide). + +Usually pets also need to find their peers. In the previous nginx example, we just used `kubectl` to get the names of existing pods, and as humans, we could tell which ones belonged to a given Pet Set. Another way to find peers is by contacting the API server, just like `kubectl`, but that has several disadvantages (you end up implementing a Kubernetes specific init system that runs as pid 1 in your application container). + +Pet Set gives you a way to disover your peers using DNS records. To illustrate this we can use the previous example (note: one usually doesn't `apt-get` in a container). + +```shell +$ kubectl exec -it web-0 /bin/sh +web-0 # apt-get update && apt-get install -y dnsutils +... + +web-0 # nslookup -type=srv nginx.default +Server: 10.0.0.10 +Address: 10.0.0.10#53 + +nginx.default.svc.cluster.local service = 10 50 0 web-1.ub.default.svc.cluster.local. +nginx.default.svc.cluster.local service = 10 50 0 web-0.ub.default.svc.cluster.local. +``` + +## Updating a Pet Set + +You cannot update any field of the PetSet except `spec.replicas`. You can update the replicas field using standard kubectl update commands like [patch](/docs/user-guide/kubectl/kubectl_patch) and [edit](/docs/user-guide/kubectl/kubectl_edit). Pet Set currently *does not* support image upgrade as noted in the section on [limitations](#alpha-limitations). + +## Scaling a Pet Set + +You can scale a Pet Set by updating the "replicas" field. Note however that the controller will only: + +1. Create one pet at a time, in order from {0..N-1}, and wait till each one is in [Running and Ready](/docs/user-guide/pod-states) before creating the next +2. Delete one pet at a time, in reverse order from {N-1..0}, and wait till each one is completely shutdown (past its [terminationGracePeriodSeconds](/docs/user-guide/pods/index#termination-of-pods)) before deleting the next + +## Deleting a Pet Set + +Cleaning up a Pet Set is somewhat manual, as noted in the [limitations section](#alpha-limitations). You can delete a Pet Set using Kubectl, but this will *not* scale it down to 0: + +```shell +$ kubectl delete -f petset.yaml +petset "web" deleted + +$ kubectl get po -l app=nginx +NAME READY STATUS RESTARTS AGE +web-0 1/1 Running 0 21h +web-1 1/1 Running 0 21h + +$ kubectl delete po -l app=nginx +pod "web-0" deleted +pod "web-1" deleted +``` + +Deleting the pods will *not* delete the volumes. Until we finalize the recycle policy for these volumes they will have to get cleaned up by an admin. This is to ensure that you have the chance to copy data off the volume before deleting it. Simply deleting the PVC after the pods have left the [terminating state](/docs/user-guide/pods/index#termination-of-pods) should trigger deletion of the backing Persistent Volumes. + +**Note: you will lose all your data once the PVC is deleted, do this with caution.** + +```shell +$ kubectl get po -l app=nginx +$ kubectl get pvc -l app=nginx +NAME STATUS VOLUME CAPACITY ACCESSMODES AGE +www-web-0 Bound pvc-62d271cd-3822-11e6-b1b7-42010af00002 0 21h +www-web-1 Bound pvc-62d6750e-3822-11e6-b1b7-42010af00002 0 21h + +$ kubectl delete pvc -l app=nginx +$ kubectl get pv +``` + +If you simply want to clean everything: + +```shell{% raw %} +$ grace=$(k get po web-0 --template '{{.spec.terminationGracePeriodSeconds}}') +$ kubectl delete petset,po -l app=nginx +$ sleep $grace +$ kubectl delete pvc -l app=nginx +{% endraw %}``` + +## Troubleshooting + +You might have noticed an `annotations` field in all the Pet Sets shown above. + +```yaml +annotations: + pod.alpha.kubernetes.io/initialized: "true" +``` + +This field is a debugging hook. It pauses any scale up/down operations on the entire Pet Set. If you'd like to pause a petset after each pet, set it to `false` in the template, wait for each pet to come up, verify it has initialized correctly, and then set it to `true` using `kubectl edit` on the pet (setting it to `false` on *any pet* is enough to pause the Pet Set). If you don't need it, create the Pet Set with it set to `true` as shown. This is surprisingly useful in debugging bootstrapping race conditions. + +## Future Work + +There are a LOT of planned improvements since Pet Set is still in alpha. + +* Data gravity and local storage +* Richer notification events +* Public network identities +* WAN cluster deployments (multi-AZ/region/cloud provider) +* Image and node upgrades + +This list goes on, if you have examples, ideas or thoughts, please contribute. + +## Alternatives + +Deploying one RC of size 1/Service per pod is a popular alternative, as is simply deploying a DaemonSet that utilizes the identity of a Node. + +## Next steps + +The deployment and maintenance of stateful applications is a vast topic. The next step is to explore cluster bootstrapping and initialization, [here](/docs/user-guide/petset/bootstrapping/). diff --git a/docs/user-guide/petset.yaml b/docs/user-guide/petset.yaml new file mode 100644 index 0000000000..017a65746c --- /dev/null +++ b/docs/user-guide/petset.yaml @@ -0,0 +1,56 @@ +# A headless service to create DNS records +apiVersion: v1 +kind: Service +metadata: + name: nginx + labels: + app: nginx +spec: + ports: + - port: 80 + name: web + # *.nginx.default.svc.cluster.local + clusterIP: None + selector: + app: nginx +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: web +spec: + serviceName: "nginx" + replicas: 2 + template: + metadata: + labels: + app: nginx + annotations: + pod.alpha.kubernetes.io/initialized: "true" + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: nginx + image: gcr.io/google_containers/nginx-slim:0.7 + ports: + - containerPort: 80 + name: web + command: + - nginx + args: + - -g + - "daemon off;" + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html + volumeClaimTemplates: + - metadata: + name: www + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + diff --git a/docs/user-guide/petset/bootstrapping/index.md b/docs/user-guide/petset/bootstrapping/index.md new file mode 100644 index 0000000000..2462c43691 --- /dev/null +++ b/docs/user-guide/petset/bootstrapping/index.md @@ -0,0 +1,234 @@ +--- +--- + +* TOC +{:toc} + +## Overview + +This purpose of this guide is to help you become familiar with the runtime initialization of [Pet Sets](/docs/user-guide/petset). This guide assumes the same prerequisites, and uses the same terminology as the [Pet Set user document](/docs/user-guide/petset). + +The most common way to initialize the runtime in a containerized environment, is through a custom [entrypoint](https://docs.docker.com/engine/reference/builder/#entrypoint). While this is not necessarily bad, making your application pid 1, and treating containers as processes in general is good for a few reasons outside the scope of this document. Doing so allows you to run docker images from third-party vendors without modification. We will not be writing custom entrypoints for this example, but using a feature called [init containers](http://releases.k8s.io/{{page.githubbranch}}/docs/proposals/container-init.md), to explain 2 common patterns that come up deploying Pet Sets. + +1. Transferring state across Pet restart, so that a future Pet is initialized with the computations of its past incarnation +2. Initializing the runtime environment of a Pet based on existing conditions, like a list of currently healthy peers + +## Example I: transferring state across Pet restart + +This example shows you how to "carry over" runtime state across Pet restart by simulating virtual machines with a Pet Set. + +### Background + +Applications that incrementally build state usually need strong guarantees that they will not restart for extended durations. This is tricky to achieve with containers, so instead, we will ensure that the results of previous computations are trasferred to future pets. Doing so is straight-forward using vanilla Persistent Volumes (which Pet Set already gives you), unless the volume mount point itself needs to be initialized for the Pet to start. This is exactly the case with "virtual machine" docker images, like those based on ubuntu or fedora. Such images embed the entier rootfs of the distro, including package managers like `apt-get` that assume a certain layout of the filesystem. Meaning: + +* If you mount an empty volume under `/usr`, you won't be able to `apt-get` +* If you mount an empty volume under `/lib`, all your `apt-gets` will fail because there are no system libraries +* If you clobber either of those, previous `apt-get` results will be dysfunctional + +### Simulating Virtual Machines + +Since Pet Set already gives each Pet a consistent identity, all we need is a way to initialize the user environment before allowing tools like `kubectl exec` to enter the application container. + +Download [this](petset_vm.yaml) petset into a file called petset_vm.yaml, and create it: + +```shell +$ kubectl create -f ./petset_vm.yaml +service "ub" created +petset "vm" created +``` + +This should give you 2 pods. + +```shell +$ kubectl get po +NAME READY STATUS RESTARTS AGE +vm-0 1/1 Running 0 37s +vm-1 1/1 Running 0 2m +``` + +We can exec into one and install nginx + +```shell +$ kubectl exec vm-0 /bin/sh +vm-0 # apt-get update +... +vm-0 # apt-get install nginx -y +``` + +On killing this pod we need it to come back with all the Pet Set properties, as well as the installed nginx packages. + +```shell +$ kubectl delete po vm-0 +pod "vm-0" deleted + +$ kubectl get po +NAME READY STATUS RESTARTS AGE +vm-0 1/1 Running 0 1m +vm-1 1/1 Running 0 4m +``` + +Now you can exec back into vm-0 and start nginx + +```shell +$ kubectl exec -it vm-0 /bin/sh +vm-0 # mkdir -p /var/log/nginx /var/lib/nginx; nginx -g 'daemon off;' + +``` + +And access it from anywhere in the cluster (and because this is an example that simulates vms, we're going to apt-get install netcat too) + +```shell +$ kubectl exec -it vm-1 /bin/sh +vm-1 # printf "GET / HTTP/1.0\r\n\r\n" | netcat vm-0.ub 80 +``` + +It's worth exploring what just happened. Init containers run sequentially *before* the application container. In this example we used the init container to copy shared libraries from the rootfs, while preserving user installed packages across container restart. + +```yaml +pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "rootfs", + "image": "ubuntu:15.10", + "command": [ + "/bin/sh", + "-c", + "for d in usr lib etc; do cp -vnpr /$d/* /${d}mnt; done;" + ], + "volumeMounts": [ + { + "name": "usr", + "mountPath": "/usrmnt" + }, + { + "name": "lib", + "mountPath": "/libmnt" + }, + { + "name": "etc", + "mountPath": "/etcmnt" + } + ] + } +]' +``` + +**It's important to note that the init container, when used this way, must be idempotent, or it'll end up clobbering data stored by a previous incarnation.** + + +## Example II: initializing state based on environment + +In this example we are going to setup a cluster of nginx servers, just like we did in the Pet Set [user guide](/docs/user-guide/petset), but make one of them a master. All the other nginx servers will simply proxy requests to the master. This is a common deployment pattern for databases like Mysql, but we're going to replace the database with a stateless webserver to simplify the problem. + +### Background + +Most clustered applications, such as mysql, require an admin to create a config file based on the current state of the world. The most common dynamic variable in such config files is a list of peers, or other Pets running similar database servers that are currently serving requests. The Pet Set user guide already [touched on this topic](/docs/user-guide/petset#peer-discovery), we'll explore it in greater depth in the context of writing a config file with a list of peers. + +Here's a tiny peer finder helper script that handles peer discovery, [available here](https://github.com/kubernetes/contrib/tree/master/pets/peer-finder). The peer finder takes 3 important arguments: + +* A DNS domain +* An `on-start` script to run with the initial constituency of the given domain as input +* An `on-change` script to run everytime the constituency of the given domain changes + +The role of the peer finder: + +* Poll DNS for SRV records of a given domain till the `hostname` of the pod it's running in shows up as a subdomain +* Pipe the sorted list of subdomains to the script specified by its `--on-start` argument +* Exit with the appropriate error code if no `--on-change` script is specified +* Loop invoking `--on-change` for every change + +You can invoke the peer finder inside the Pets we created in the last example: + +```shell +$ kubectl exec -it vm-0 /bin/bash +vm-0 # curl -sSL -o /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder +vm-0 # chmod -c 755 peer-finder + +vm-0 # ./peer-finder +2016/06/23 21:25:46 Incomplete args, require -on-change and/or -on-start, -service and -ns or an env var for POD_NAMESPACE. + +vm-0 # ./peer-finder -on-start 'tee' -service ub -ns default + +2016/06/23 21:30:21 Peer list updated +was [] +now [vm-0.ub.default.svc.cluster.local vm-1.ub.default.svc.cluster.local] +2016/06/23 21:30:21 execing: tee with stdin: vm-0.ub.default.svc.cluster.local +vm-1.ub.default.svc.cluster.local +2016/06/23 21:30:21 vm-0.ub.default.svc.cluster.local +vm-1.ub.default.svc.cluster.local +2016/06/23 21:30:22 Peer finder exiting +``` + +### Nginx master/slave cluster + +Lets create a Pet Set that writes out its own config based on a list of peers at initalization time, as described above. + +Download and create [this](petset_peers.yaml) petset. It will setup 2 nginx webservers, but the second one will proxy all requests to the first: + +```shell +$ kubectl create -f petset_peers.yaml +service "nginx" created +petset "web" created + +$ kubectl get po --watch-only +NAME READY STATUS RESTARTS AGE +web-0 0/1 Pending 0 7s +web-0 0/1 Init:0/1 0 18s +web-0 0/1 PodInitializing 0 20s +web-0 1/1 Running 0 21s +web-1 0/1 Pending 0 0s +web-1 0/1 Init:0/1 0 0s +web-1 0/1 PodInitializing 0 20s +web-1 1/1 Running 0 21s + +$ kubectl get po +NAME READY STATUS RESTARTS AGE +web-0 1/1 Running 0 1m +web-1 1/1 Running 0 47s +``` + +web-1 will redirect all requests to it's "master": + +```shell +$ kubectl exec -it web-1 -- curl localhost +web-0 +``` + +If you scale the cluster, the new pods parent themselves to the same master. To test this you can `kubectl edit` the petset and change the `replicas` field to 5: + +```shell +$ kubectl get po -l app=nginx +NAME READY STATUS RESTARTS AGE +web-0 1/1 Running 0 2h +web-1 1/1 Running 0 2h +web-2 1/1 Running 0 1h +web-3 1/1 Running 0 1h +web-4 1/1 Running 0 1h + +$ for i in $(seq 0 4); do kubectl exec -it web-$i -- curl localhost; done | sort | uniq +web-0 +``` + +Understanding how we generated the nginx config is important, we did so by passing an init script to the peer finder: + +```shell +echo ` +readarray PEERS; +if [ 1 = ${#PEERS[@]} ]; then + echo \"events{} http { server{ } }\"; +else + echo \"events{} http { server{ location / { proxy_pass http://${PEERS[0]}; } } }\"; +fi;` > /conf/nginx.conf +``` + +All that does is: + +* read in a list of peers from stdin +* if there's only 1, promote it to master +* if there's more than 1, proxy requests to the 0th member of the list +* write the config to a `hostPath` volume shared with the parent PetSet + +**It's important to note that in practice all Pets should query their peers for the current master, instead of making assumptions based on the index.** + +## Next Steps + +You can deploy some example Pet Sets found [here](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/petset), or write your own. diff --git a/docs/user-guide/petset/bootstrapping/petset_peers.yaml b/docs/user-guide/petset/bootstrapping/petset_peers.yaml new file mode 100644 index 0000000000..e12988dee2 --- /dev/null +++ b/docs/user-guide/petset/bootstrapping/petset_peers.yaml @@ -0,0 +1,103 @@ +# A headless service to create DNS records +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + name: nginx + labels: + app: nginx +spec: + ports: + - port: 80 + name: web + # *.nginx.default.svc.cluster.local + clusterIP: None + selector: + app: nginx +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: web +spec: + serviceName: "nginx" + replicas: 2 + template: + metadata: + labels: + app: nginx + annotations: + pod.alpha.kubernetes.io/initialized: "true" + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "peerfinder", + "image": "gcr.io/google_containers/peer-finder:0.1", + "args": [ + "-on-start=\"\"echo ` + readarray PEERS; + echo $(hostname) > /usr/share/nginx/html/index.html; + if [ 1 = ${#PEERS[@]} ]; then + echo \"events{} http { server{ } }\"; + else + echo \"events{} http { server{ location / { proxy_pass http://${PEERS[0]}; } } }\"; + fi;` > /conf/nginx.conf\"\"", + "-service=nginx" + ], + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "volumeMounts": [ + { + "name": "www", + "mountPath": "/usr/share/nginx/html" + }, + { + "name": "conf", + "mountPath": "/conf" + } + ] + } + ]' + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: nginx + image: gcr.io/google_containers/nginx-slim:0.7 + ports: + - containerPort: 80 + name: web + command: + - nginx + args: + - -g + - "daemon off;" + - -c + - "/conf/nginx.conf" + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html + - name: conf + mountPath: /conf + volumes: + - name: conf + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: www + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + diff --git a/docs/user-guide/petset/bootstrapping/petset_vm.yaml b/docs/user-guide/petset/bootstrapping/petset_vm.yaml new file mode 100644 index 0000000000..c506a1bac6 --- /dev/null +++ b/docs/user-guide/petset/bootstrapping/petset_vm.yaml @@ -0,0 +1,102 @@ +# A headless service to create DNS records +apiVersion: v1 +kind: Service +metadata: + name: ub + labels: + app: ub +spec: + ports: + - port: 80 + name: web + # *.ub.default.svc.cluster.local + clusterIP: None + selector: + app: ub +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: vm +spec: + serviceName: "ub" + replicas: 2 + template: + metadata: + labels: + app: ub + annotations: + pod.alpha.kubernetes.io/initialized: "true" + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "rootfs", + "image": "ubuntu:15.10", + "command": [ + "/bin/sh", + "-c", + "for d in usr lib etc; do cp -vnpr /$d/* /${d}mnt; done;" + ], + "volumeMounts": [ + { + "name": "usr", + "mountPath": "/usrmnt" + }, + { + "name": "lib", + "mountPath": "/libmnt" + }, + { + "name": "etc", + "mountPath": "/etcmnt" + } + ] + } + ]' + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: ub + image: ubuntu:15.10 + ports: + - containerPort: 80 + name: web + command: + - /bin/sh + - -c + - 'while true; do sleep 10; done' + volumeMounts: + - name: usr + mountPath: /usr + - name: lib + mountPath: /lib + - name: etc + mountPath: /etc + volumeClaimTemplates: + - metadata: + name: usr + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 300Mi + - metadata: + name: lib + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + - metadata: + name: etc + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 150Mi + diff --git a/docs/user-guide/production-pods.md b/docs/user-guide/production-pods.md index 13babe5b92..eb58086a2e 100644 --- a/docs/user-guide/production-pods.md +++ b/docs/user-guide/production-pods.md @@ -180,6 +180,26 @@ Other times, applications are only temporarily unable to serve, and will recover For more details (e.g., how to specify command-based probes), see the [example in the walkthrough](/docs/user-guide/walkthrough/k8s201/#health-checking), the [standalone example](/docs/user-guide/liveness/), and the [documentation](/docs/user-guide/pod-states/#container-probes). +## Handling initialization + +Applications often need a set of initialization steps prior to performing their day job. This may include: + +* Waiting for other components (like a database or web service) to be available +* Performing configuration templating from environment variables into a config file +* Registering the pod into a central database, or fetching remote configuration from that database +* Downloading application dependencies, seed data, or preconfiguring disk + +Kubernetes now includes an alpha feature known as **init containers**, which are one or more containers in a pod that get a chance to run and initialize shared volumes prior to the other application containers starting. An init container is exactly like a regular container, except that it always runs to completion and each init container must complete successfully before the next one is started. If the init container fails (exits with a non-zero exit code) on a `RestartNever` pod the pod will fail - otherwise it will be restarted until it succeeds or the user deletes the pod. + +Since init containers are an alpha feature, they are specified by setting the `pod.alpha.kubernetes.io/init-containers` annotation on a pod (or replica set, deployment, daemon set, pet set, or job). The value of the annotation must be a string containing a JSON array of container definitions: + +{% include code.html language="yaml" file="nginx-init-containers.yaml" ghlink="/docs/user-guide/nginx-init-containers.yaml" %} + +The status of the init containers is returned as another annotation - `pod.alpha.kubernetes.io/init-container-statuses` -- as an array of the container statuses (similar to the `status.containerStatuses` field). + +Init containers support all of the same features as normal containers, including resource limits, volumes, and security settings. The resource requests and limits for an init container are handled slightly different than normal containers since init containers are run one at a time instead of all at once - any limits or quotas will be applied based on the largest init container resource quantity, rather than as the sum of quantities. Init containers do not support readiness probes since they will run to completion before the pod can be ready. + + ## Lifecycle hooks and termination notice Of course, nodes and applications may fail at any time, but many applications benefit from clean shutdown, such as to complete in-flight requests, when the termination of the application is deliberate. To support such cases, Kubernetes supports two kinds of notifications: