From 84c75e2b6aaf382f692c3b1cf36ec6f4f1fcbbd2 Mon Sep 17 00:00:00 2001 From: johndmulhausen Date: Tue, 22 Mar 2016 18:37:38 -0700 Subject: [PATCH 01/31] Update _config.yml --- _config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_config.yml b/_config.yml index c3eaba8b58..02b865877c 100644 --- a/_config.yml +++ b/_config.yml @@ -16,8 +16,8 @@ defaults: scope: path: "" values: - version: "v1.2" - githubbranch: "release-1.2" + version: "v1.3" + githubbranch: "release-1.3" docsbranch: "master" - scope: From ebc680411046b3e364df160883a8166cb444cf2f Mon Sep 17 00:00:00 2001 From: johndmulhausen Date: Tue, 22 Mar 2016 18:38:32 -0700 Subject: [PATCH 02/31] Update _config.yml --- _config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_config.yml b/_config.yml index 02b865877c..5094499bfe 100644 --- a/_config.yml +++ b/_config.yml @@ -17,8 +17,8 @@ defaults: path: "" values: version: "v1.3" - githubbranch: "release-1.3" - docsbranch: "master" + githubbranch: "master" + docsbranch: "release-1.3" - scope: path: "docs" From 120359b8968e929250228c65f496706f86f20629 Mon Sep 17 00:00:00 2001 From: johndmulhausen Date: Sat, 30 Apr 2016 14:17:29 -0700 Subject: [PATCH 03/31] Create .travis.yml --- .travis.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..5fd825e6e3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,22 @@ +language: ruby +rvm: +- 2.1 +branches: + only: + - release-1.1 +script: +- cd $HOME +- git config --global user.email ${GIT_EMAIL} +- git config --global user.name "${GIT_NAME}" +- git clone https://${GIT_USERNAME}:${GH_TOKEN}@github.com/kubernetes-v1-3/kubernetes-v1-3.github.io.git +- cd kubernetes-v1-3.github.io +- git remote add --fetch --track release-1.3 homebase "https://${GIT_USERNAME}:${GH_TOKEN}@github.com/kubernetes/kubernetes.github.io.git" +- git merge -s recursive -X theirs homebase/release-1.3 -m "Sync from homebase" +- git push +env: + global: + - secure: Fd6wlE2mjPb1fAACxklQcJumpJWycYkaJQBfKRcjGCFlmw1XWVFGhpUC7Ni/MOyzTolqOvtb2rXnYpaujMlJP1UXqVFJ+zPbwur2lc8unQF8PcqPezl8DbPsr6HdceOjRdut/dN8zTw6+hZRDzw/mG4Rf8IVaozlYycAOnWZdAZsLXdbBpAvBp0WYHP9+8wn9xiet1L/QpSiawa/Q35Q9UM1tciPmfFBL4Fkq6Mm8/w6ECaxHyA2EX+eH5ea9EtykrzB5cMA/odJLptjnmfzsPeGS5F1MxuTqrH2Z09emZcqjfXW+fNNzioiZdD4BQe+rqOA1Ktpw645szJVJJenIMzM4P8dDiW+9cJyddGdTC9A7APgLlLVdG/tuoYcjiY1F068SWx8sUWW/xc7YenCzGj2nXIeHjUuWsjqjMorpBKe6Y1QSGfi7HsLq1DpJDR1xQZbPgM/FOaovVIPIozgAIBpS1ukdNNwadmrCw55tUKfr4s2SNv3SUmeprL37QEtIxtBpPDjL1z+qTLjc6JzRr7J57guPBdNbj6Ukg/uxW+z1CwDYb3uAIIa5e6eDxfxGQsUEj8NmkIHBLLGTnKb1WAyPXAIf9ZvZHUwQQzHvOWXGZRd32enRh00uvuwHEy3OuGEqTAOIs/31q5WAamZ1uqzW7KBTMU/nq4QA9W2yfM= + - secure: Lvj1MT/I2zaJp9mQi46us64O4JfeT9cz1m+nqKY/nuC828ULfqMYLlpw08jvrc25JhAaqHNawAwdtaxHTQqHztsmq1ixHFYA6G/1oll+YPhym29toFT9oIkXThf/L77FTJfM1gVFMyiHFbz11Ob8R15tRB1WF1+Uu1RBmtudJD7HSy0u6Uc6lPxpqtycWVRCyPdvZqF3e0KIZaTDRkRzJpcVMHa/5ZJDDcMtuJyjJYXZqS1WR3QHC1z44LlnqyB5ZM2PU4H3LyWnY8wHF2mutF0QtDDVdEuBqILBHiFuKHMxpLY92UgHm2n51RR63MxFjjEzE+iu5f9ComEm5JC0N/cc8sunIiol+d8SRC30/00Vs0tvmeAjRX9IMCExiP3mv7Tz6mqEVk6PyrVlg675hxRg0eVdaNNv92gWzCSIecZ5TuCaRG4JaWO0P8lQC4NreONt6gbnwMD350hMQZLpUUE0QoSEdafpudaD+agl4ZzTFVTOOcSOz2Sa/+RT96Msazq2YlddYXaEKZeyYzqHVkk60PiQzQcAuwCMFrAagqMe3bNI2aCFEWbc9CoR13K/wwRDwAeSzBq/UkylZ8AayJLnIpewr/iYBOQasrXLrorW969Rfr8d/nwhDN5VpRgItU0arDzngaJRoZEAtHx4zaCvZF/H/nvbORcD2gsFP9Q= + - secure: WGmgvSnT1OeHPYtsm/JzcRDnZYp+SeelgbSVNbtwOXGImkeqDo/sDIRH80HxVC3zkUu5wvRO/NkMA7N3LXN2BqENacWBFQ+j+Hf3lnDw9yETBaBpLQPO/MElKGd48GoSiukCipfaITUfwtjAQ3wx0W/z06dYC9BdUYvUM73OtsGtlrY1MEQnBfff/neDGZbzfvTdgDwg1n6d69WP+WSUJt5Aysz0ZoAbYO2rzBrKNwcL7KEhoI/ketXXQ4xEW0nnxU/qfxqhVfJbck1HpSz+HMsVGXF4tT3zdRXmU+P47KGbfYn0GmlviGrRSWCu9/elKTlX9fIqRnR1/UvdTkflwLsk4i3hWnEUflKpIT7soJrKjNQseyd06KI8qvVRXRcll92vCtsYDuOO1AlemxIqp2dIFduCf+5FWH06PbS8oHU8NBe5aVYaoBY8nK14EB3A98saJ8Un7ziB8moRaWZX9bMjfJWfxDF526NdHIusqLQrxvUthtBjf5aRULE2PeIyqYVY8/IKmBEfsy9IUENmlM5KItvgZZt4xdk8o//kceLrKabudHGVC1M1RLZo9ejkdLySz2i8nlJyJ03Hn30H9DfDjW/OMQTn/b1HU4CYSbd/yGR+6+5+tdPeKy/Z6gI6+EjHRYaaQrBLo0RXm2620cb7uDTBIsKVs2TetyhTq54= + - secure: lqboLQKFpfa6F+UkSCs9+NPv027EnSWZA4/cmAXj0hZvxc4W+qMFIlPKytcq+bbZrp7kx3cZgRd5NrgmOTqlJ9merVK1cgMsDegMv/YImHjhGVBH+3ATvqMDb7I7dLg4x/DjhKt/ogyZpoh8vsS77T9MQXaIJM8JMM3ISxYI6MMloCh0z/hbT2qv3IrPMFjpHohHdF1yorxKvSefNgPx6Y4wC+t1t+u8wS9RBhvrmOkVynlU2NgbSiMPyz3syJmnszHDTXXJlqciGT5Lc8lzpzt2OGOnZmAupEYVGok7Jx4aAxU9O8/bnoDwOcGGVrd/pGHTI5+rvT4TPy+WG1Aec55dgva4XZfrJQ88snHLvq0VrvdiI/fHJEPF18QnrBwKSni1Vd1jOzsuPIF6HL0Wwv8tklO8OX3D5wqVsXpeJd4Cj4HPPAtzxOVsV6Fz3On9vuLdV7StC/ZafQL40koaytqkNO0gX8zeUiaaVQtMCf/2MVTbX9x5m51gkGpwT0JBvqQpSMOlIM5S1fiN4X9DLBLqmsARYeZw1Jeiq7Zm39Sy1QeCgfUXS8+6t62BjWx70iftYGIkoljXtD7x/3pjKymdpwjcyUcS8KQ5W6vbMXFxmZe2phtgGTCxOIDuASPspD8zwm/ckIGArHj4qkrg94/mUpgDDIbVsaUpWXjYvSc= + - secure: bPZMNH79Lx0Wb2SCxTwyZek28w/keKxEAlfad8RDMwJrct6Bi2B0o4KjkrwFS5DyCUU7Ndk192145XnUKOWX2YVs6cQ8ge+5LvtgYhWLgX8g5Ycro+JyzOBskn+o1gQjvi8+3b42X31efcUmTEhfRdKVrrUpONEIcjG1NpLGk/mQJ6AM6hGWO2xdNfAezeWq5ISnpK0b6VUWZyTEDg3NivrTfCL2juWWCnjYm1BWSHUblXwRQ/Rl7Tcldl6cMMUVsalUQ1iG0h8YZDrxNz0cm3XTZZZJKuSHYeTCLd57RBeHD5/iMxjCmfzfq6ETNLONWLmtvA8yhWMQZ0DPFtLZzbqFIfOR6P1feBZFqP7/X5KZeFKBufN15JbcXIqHE8homLY9mS0LhyNffOs5G/P/x8ChE1DJaYiZIWCZ60umvpqibScZB3z5uFPTxLk9rJSOtT7hCWjcmg3EdJ+R4ExOiBDd62ZS5jH72WU4uysPXORRofUCL+zHycJoJxsFWQSW49GAGrohllrW45jnTgpalErxvjJFSKy8JW9w634eetz9ct2fObep7m8bfVMl8U2H3ITIoXHm+f8ooHUiNzHFLSl4wtcoAOtrCAvGtJPNfv8T2eTkznzj7Tk+XzTvFg03u+J99TmnC58Bs516Oc7E40NarmsZceOD2sN4BA3X9tE= From 4582d120dad0e067a24e3d3c92dfa94aaf24ec1a Mon Sep 17 00:00:00 2001 From: johndmulhausen Date: Sat, 30 Apr 2016 14:19:52 -0700 Subject: [PATCH 04/31] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 11b9717354..30e736cc6d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ ## Instructions for Contributing to the Docs/Website +**Note: Changes to this branch are automatically served as a website at [http://kubernetes-v1-3.github.io/](http://kubernetes-v1-3.github.io/). This branch is for the v1.3 docs. See the "master" branch to edit the current stable release, which is served at http://kubernetes.io.** + Welcome! We are very pleased you want to contribute to the documentation and/or website for Kubernetes. You can click the "Fork" button in the upper-right area of the screen to create a copy of our site on your GitHub account called a "fork." Make any changes you want in your fork, and when you are ready to send those changes to us, go to the index page for your fork and click "New Pull Request" to let us know about it. From 9dabfa4b62931e35628e3c1552db50373a446e78 Mon Sep 17 00:00:00 2001 From: johndmulhausen Date: Sat, 30 Apr 2016 14:20:36 -0700 Subject: [PATCH 05/31] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5fd825e6e3..df4d6c8710 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ rvm: - 2.1 branches: only: - - release-1.1 + - release-1.3 script: - cd $HOME - git config --global user.email ${GIT_EMAIL} From 37fe853b1a3ad728e49cc7334ae0ce03292eabad Mon Sep 17 00:00:00 2001 From: johndmulhausen Date: Sat, 30 Apr 2016 14:25:26 -0700 Subject: [PATCH 06/31] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 30e736cc6d..03fbbc16a5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## Instructions for Contributing to the Docs/Website -**Note: Changes to this branch are automatically served as a website at [http://kubernetes-v1-3.github.io/](http://kubernetes-v1-3.github.io/). This branch is for the v1.3 docs. See the "master" branch to edit the current stable release, which is served at http://kubernetes.io.** +**Note: This branch is for pre-release documentation for 1.3 of Kubernetes. Changes to this branch are automatically served as a website at [http://kubernetes-v1-3.github.io/](http://kubernetes-v1-3.github.io/). See the "master" branch to edit the current stable release, which is served at http://kubernetes.io.** Welcome! We are very pleased you want to contribute to the documentation and/or website for Kubernetes. From 6cf91c527c08dcc61693bf8ecdfb54c7832a76fb Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Fri, 20 May 2016 09:18:40 -0700 Subject: [PATCH 07/31] Document kubectl rollout status and kubectl set image for deployments --- docs/user-guide/deployments.md | 117 ++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 46 deletions(-) diff --git a/docs/user-guide/deployments.md b/docs/user-guide/deployments.md index 1cd688e8c8..a76dbff5e7 100644 --- a/docs/user-guide/deployments.md +++ b/docs/user-guide/deployments.md @@ -76,47 +76,39 @@ The created Replica Set will ensure that there are three nginx Pods at all times ## The Status of a Deployment -After creating or updating a Deployment, you would want to confirm whether it succeeded or not. The best way to do this is through checking its status. - -To verify if the above Deployment succeeded or not, first compare the `.metadata.generation` and `.status.observedGeneration` of the Deployment: +After creating or updating a Deployment, you would want to confirm whether it succeeded or not. The simplest way to do this is through `kubectl rollout status`. ```shell -$ kubectl get deployment/nginx-deployment -o yaml | grep [Gg]eneration - generation: 2 - observedGeneration: 2 +$ kubectl rollout status deployment/nginx-deployment +deployment nginx-deployment successfully rolled out ``` -When `observedGeneration` >= `generation`, the Deployment controller has observed current Deployment; if not, wait for a few more seconds. +This verifies the Deployment's `.status.observedGeneration` >= `.metadata.generation`, and its up-to-date replicas +(`.status.updatedReplicas`) matches the desired replicas (`.spec.replicas`) to determine if the rollout succeeded. +If the rollout is still in progress, it watches for Deployment status changes and prints related messages. -Once the above condition is met, check the Deployment's up-to-date replicas (`.status.updatedReplicas`) and see if it matches the desired replicas (`.spec.replicas`): - -```shell -$ kubectl get deployment/nginx-deployment -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -nginx-deployment 3 3 3 3 9m -``` +Note that it's impossible to know whether a Deployment will ever succeed, so if the above command doesn't return success, +you'll need to timeout and give up at some point. Additionally, if you set `.spec.minReadySeconds`, you would also want to check if the available replicas (`.status.availableReplicas`) matches the desired replicas too. -**Note:** It's impossible to know whether a Deployment will ever succeed, so one has to timeout and give up at some point. +```shell +$ kubectl get deployments +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +nginx-deployment 3 3 3 3 20s +``` ## Updating a Deployment -Suppose that we now want to update the nginx Pods to start using the `nginx:1.9.1` image -instead of the `nginx:1.7.9` image. - **Note:** a Deployment's rollout is triggered if and only if the Deployment's pod template (i.e. `.spec.template`) is changed, e.g. updating labels or container images of the template. Other updates, such as scaling the Deployment, will not trigger a rollout. -First, we update our Deployment configuration as follows: - -{% include code.html language="yaml" file="new-nginx-deployment.yaml" ghlink="/docs/user-guide/new-nginx-deployment.yaml" %} - -We can then `apply` the new Deployment: +Suppose that we now want to update the nginx Pods to start using the `nginx:1.9.1` image +instead of the `nginx:1.7.9` image. ```shell -$ kubectl apply -f docs/user-guide/new-nginx-deployment.yaml -deployment "nginx-deployment" configured +$ kubectl set image deployment/nginx-deployment nginx=nginx:1.9.1 +deployment "nginx-deployment" image updated ``` Alternatively, we can `edit` the Deployment and change `.spec.template.spec.containers[0].image` from `nginx:1.7.9` to `nginx:1.9.1`: @@ -126,17 +118,15 @@ $ kubectl edit deployment/nginx-deployment deployment "nginx-deployment" edited ``` -Running a `get` immediately will give: +To see its rollout status, simply run: ```shell -$ kubectl get deployments -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -nginx-deployment 3 3 0 3 20s +$ kubectl rollout status deployment/nginx-deployment +Waiting for rollout to finish: 2 out of 3 new replicas have been updated... +deployment nginx-deployment successfully rolled out ``` -The 0 number of up-to-date replicas indicates that the Deployment hasn't updated the replicas to the latest configuration. The current replicas indicates the total replicas (3 with old configuration and 0 with new configuration) this Deployment manages, and the available replicas indicates the number of current replicas that are available. - -The Deployment will update all the Pods in a few seconds. +After the rollout succeeds, you may want to `get` the Deployment: ```shell $ kubectl get deployments @@ -144,6 +134,10 @@ NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE nginx-deployment 3 3 3 3 36s ``` +The number of up-to-date replicas indicates that the Deployment has updated the replicas to the latest configuration. +The current replicas indicates the total replicas this Deployment manages, and the available replicas indicates the +number of current replicas that are available. + We can run `kubectl get rs` to see that the Deployment updated the Pods by creating a new Replica Set and scaling it up to 3 replicas, as well as scaling down the old Replica Set to 0 replicas. ```shell @@ -163,7 +157,7 @@ nginx-deployment-1564180365-nacti 1/1 Running 0 14s nginx-deployment-1564180365-z9gth 1/1 Running 0 14s ``` -Next time we want to update these Pods, we only need to update and re-apply the Deployment again. +Next time we want to update these Pods, we only need to update the Deployment's pod template again. Deployment can ensure that only a certain number of Pods may be down while they are being updated. By default, it ensures that at least 1 less than the desired number of Pods are @@ -237,14 +231,21 @@ auto-scaling. This implies that when you rollback to an earlier revision, only t Suppose that we made a typo while updating the Deployment, by putting the image name as `nginx:1.91` instead of `nginx:1.9.1`: -{% include code.html language="yaml" file="bad-nginx-deployment.yaml" ghlink="/docs/user-guide/bad-nginx-deployment.yaml" %} - ```shell -$ kubectl apply -f docs/user-guide/bad-nginx-deployment.yaml -deployment "nginx-deployment" configured +$ kubectl set image deployment/nginx-deployment nginx=nginx:1.91 +deployment "nginx-deployment" image updated ``` -You will see that both the number of old replicas (nginx-deployment-1564180365 and nginx-deployment-2035384211) and new replicas (nginx-deployment-3066724191) are 2. +The rollout will be stuck. + +``` +$ kubectl rollout status deployments nginx-deployment +Waiting for rollout to finish: 2 out of 3 new replicas have been updated... +``` + +Press Ctrl-C to stop the above rollout status watch. + +You will also see that both the number of old replicas (nginx-deployment-1564180365 and nginx-deployment-2035384211) and new replicas (nginx-deployment-3066724191) are 2. ```shell $ kubectl get rs @@ -305,8 +306,8 @@ $ kubectl rollout history deployment/nginx-deployment deployments "nginx-deployment": REVISION CHANGE-CAUSE 1 kubectl create -f docs/user-guide/nginx-deployment.yaml --record -2 kubectl apply -f docs/user-guide/new-nginx-deployment.yaml -3 kubectl apply -f docs/user-guide/bad-nginx-deployment.yaml +2 kubectl set image deployment/nginx-deployment nginx=nginx:1.9.1 +3 kubectl set image deployment/nginx-deployment nginx=nginx:1.91 ``` Because we recorded the command while creating this Deployment using `--record`, we can easily see the changes we made in each revision. @@ -316,10 +317,18 @@ To further see the details of each revision, run: ```shell $ kubectl rollout history deployment/nginx-deployment --revision=2 deployments "nginx-deployment" revision 2 -Labels: app=nginx,pod-template-hash=1564180365 -Annotations: kubernetes.io/change-cause=kubectl apply -f docs/user-guide/new-nginx-deployment.yaml -Image(s): nginx:1.9.1 -No volumes. + Labels: app=nginx + pod-template-hash=1159050644 + Annotations: kubernetes.io/change-cause=kubectl set image deployment/nginx-deployment nginx=nginx:1.9.1 + Containers: + nginx: + Image: nginx:1.9.1 + Port: 80/TCP + QoS Tier: + cpu: BestEffort + memory: BestEffort + Environment Variables: + No volumes. ``` ### Rolling Back to a Previous Revision @@ -346,6 +355,7 @@ The Deployment is now rolled back to a previous stable revision. As you can see, $ kubectl get deployment NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE nginx-deployment 3 3 3 3 30m + $ kubectl describe deployment Name: nginx-deployment Namespace: default @@ -381,8 +391,8 @@ You can also pause a Deployment mid-way and then resume it. A use case is to sup Update the Deployment again and then pause the Deployment with `kubectl rollout pause`: ```shell -$ kubectl apply -f docs/user-guide/new-nginx-deployment; kubectl rollout pause deployment/nginx-deployment -deployment "nginx-deployment" configured +$ kubectl set image deployment/nginx-deployment nginx=nginx:1.9.1; kubectl rollout pause deployment/nginx-deployment +deployment "nginx-deployment" image updated deployment "nginx-deployment" paused ``` @@ -398,6 +408,13 @@ nginx-deployment-2035384211 2 2 1h nginx-deployment-3066724191 0 0 1h ``` +In a separate terminal, watch for rollout status changes and you'll see the rollout won't continue: + +```shell +$ kubectl rollout status deployment/nginx-deployment +Waiting for rollout to finish: 2 out of 3 new replicas have been updated... +``` + To resume the Deployment, simply do `kubectl rollout resume`: ```shell @@ -407,6 +424,14 @@ deployment "nginx-deployment" resumed Then the Deployment will continue and finish the rollout: +```shell +$ kubectl rollout status deployment/nginx-deployment +Waiting for rollout to finish: 2 out of 3 new replicas have been updated... +Waiting for deployment spec update to be observed... +Waiting for rollout to finish: 2 out of 3 new replicas have been updated... +deployment nginx-deployment successfully rolled out +``` + ```shell $ kubectl get rs NAME DESIRED CURRENT AGE From cfaf885568d1239da98c8026455e48bb31ca4ced Mon Sep 17 00:00:00 2001 From: Avesh Agarwal Date: Wed, 1 Jun 2016 16:34:37 -0400 Subject: [PATCH 08/31] Fix the path to dapi-volume.yaml and update the example to reflect the current state --- docs/user-guide/downward-api/volume/index.md | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/user-guide/downward-api/volume/index.md b/docs/user-guide/downward-api/volume/index.md index 446d8eb563..3cfdc2c2d9 100644 --- a/docs/user-guide/downward-api/volume/index.md +++ b/docs/user-guide/downward-api/volume/index.md @@ -17,7 +17,7 @@ This example assumes you have a Kubernetes cluster installed and running, and th ### Step One: Create the pod -Use the `docs/user-guide/downward-api/dapi-volume.yaml` file to create a Pod with a  downward API volume which stores pod labels and pod annotations to `/etc/labels` and  `/etc/annotations` respectively. +Use the [dapi-volume.yaml](/docs/user-guide/downward-api/volume/dapi-volume.yaml) file to create a Pod with a downward API volume which stores pod labels and pod annotations to `/etc/labels` and `/etc/annotations` respectively. ```shell $ kubectl create -f docs/user-guide/downward-api/volume/dapi-volume.yaml @@ -46,21 +46,21 @@ In pod's `/etc` directory one may find the file created by the plugin (system fi $ kubectl exec kubernetes-downwardapi-volume-example -i -t -- sh / # ls -laR /etc /etc: -total 32 -drwxrwxrwt 3 0 0 180 Aug 24 13:03 . -drwxr-xr-x 1 0 0 4096 Aug 24 13:05 .. -drwx------ 2 0 0 80 Aug 24 13:03 ..2015_08_24_13_03_44259413923 -lrwxrwxrwx 1 0 0 30 Aug 24 13:03 ..downwardapi -> ..2015_08_24_13_03_44259413923 -lrwxrwxrwx 1 0 0 25 Aug 24 13:03 annotations -> ..downwardapi/annotations -lrwxrwxrwx 1 0 0 20 Aug 24 13:03 labels -> ..downwardapi/labels +total 4 +drwxrwxrwt 3 0 0 120 Jun 1 19:55 . +drwxr-xr-x 17 0 0 4096 Jun 1 19:55 .. +drwxr-xr-x 2 0 0 80 Jun 1 19:55 ..6986_01_06_15_55_10.473583074 +lrwxrwxrwx 1 0 0 31 Jun 1 19:55 ..data -> ..6986_01_06_15_55_10.473583074 +lrwxrwxrwx 1 0 0 18 Jun 1 19:55 annotations -> ..data/annotations +lrwxrwxrwx 1 0 0 13 Jun 1 19:55 labels -> ..data/labels -/etc/..2015_08_24_13_03_44259413923: +/etc/..6986_01_06_15_55_10.473583074: total 8 -drwx------ 2 0 0 80 Aug 24 13:03 . -drwxrwxrwt 3 0 0 180 Aug 24 13:03 .. --rw-r--r-- 1 0 0 115 Aug 24 13:03 annotations --rw-r--r-- 1 0 0 53 Aug 24 13:03 labels +drwxr-xr-x 2 0 0 80 Jun 1 19:55 . +drwxrwxrwt 3 0 0 120 Jun 1 19:55 .. +-rw-r--r-- 1 0 0 129 Jun 1 19:55 annotations +-rw-r--r-- 1 0 0 59 Jun 1 19:55 labels / # ``` -The file `labels` is stored in a temporary directory (`..2015_08_24_13_03_44259413923` in the example above) which is symlinked to by `..downwardapi`. Symlinks for annotations and labels in `/etc` point to files containing the actual metadata through the `..downwardapi` indirection.  This structure allows for dynamic atomic refresh of the metadata: updates are written to a new temporary directory, and the `..downwardapi` symlink is updated atomically using `rename(2)`. \ No newline at end of file +The file `labels` is stored in a temporary directory (`..6986_01_06_15_55_10.473583074` in the example above) which is symlinked to by `..data`. Symlinks for annotations and labels in `/etc` point to files containing the actual metadata through the `..data` indirection.  This structure allows for dynamic atomic refresh of the metadata: updates are written to a new temporary directory, and the `..data` symlink is updated atomically using `rename(2)`. From 47d7b0968bd26a40f9ffb2c3860d5c6018751769 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Thu, 16 Jun 2016 17:05:50 -0400 Subject: [PATCH 09/31] Add init container documentation --- docs/user-guide/nginx-init-containers.yaml | 31 ++++++++++++++++++++++ docs/user-guide/production-pods.md | 20 ++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 docs/user-guide/nginx-init-containers.yaml diff --git a/docs/user-guide/nginx-init-containers.yaml b/docs/user-guide/nginx-init-containers.yaml new file mode 100644 index 0000000000..34c20fa66a --- /dev/null +++ b/docs/user-guide/nginx-init-containers.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx + annotations: + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "install", + "image": "busybox", + "command": ["wget", "-O", "/work-dir/index.html", "http://kubernetes.io/index.html"], + "volumeMounts": [ + { + "name": "workdir", + "mountPath": "/work-dir" + } + ] + } + ]' +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 + volumeMounts: + - name: workdir + mountPath: /usr/share/nginx/html + dnsPolicy: Default + volumes: + - name: workdir + emptyDir: {} diff --git a/docs/user-guide/production-pods.md b/docs/user-guide/production-pods.md index 13babe5b92..eb58086a2e 100644 --- a/docs/user-guide/production-pods.md +++ b/docs/user-guide/production-pods.md @@ -180,6 +180,26 @@ Other times, applications are only temporarily unable to serve, and will recover For more details (e.g., how to specify command-based probes), see the [example in the walkthrough](/docs/user-guide/walkthrough/k8s201/#health-checking), the [standalone example](/docs/user-guide/liveness/), and the [documentation](/docs/user-guide/pod-states/#container-probes). +## Handling initialization + +Applications often need a set of initialization steps prior to performing their day job. This may include: + +* Waiting for other components (like a database or web service) to be available +* Performing configuration templating from environment variables into a config file +* Registering the pod into a central database, or fetching remote configuration from that database +* Downloading application dependencies, seed data, or preconfiguring disk + +Kubernetes now includes an alpha feature known as **init containers**, which are one or more containers in a pod that get a chance to run and initialize shared volumes prior to the other application containers starting. An init container is exactly like a regular container, except that it always runs to completion and each init container must complete successfully before the next one is started. If the init container fails (exits with a non-zero exit code) on a `RestartNever` pod the pod will fail - otherwise it will be restarted until it succeeds or the user deletes the pod. + +Since init containers are an alpha feature, they are specified by setting the `pod.alpha.kubernetes.io/init-containers` annotation on a pod (or replica set, deployment, daemon set, pet set, or job). The value of the annotation must be a string containing a JSON array of container definitions: + +{% include code.html language="yaml" file="nginx-init-containers.yaml" ghlink="/docs/user-guide/nginx-init-containers.yaml" %} + +The status of the init containers is returned as another annotation - `pod.alpha.kubernetes.io/init-container-statuses` -- as an array of the container statuses (similar to the `status.containerStatuses` field). + +Init containers support all of the same features as normal containers, including resource limits, volumes, and security settings. The resource requests and limits for an init container are handled slightly different than normal containers since init containers are run one at a time instead of all at once - any limits or quotas will be applied based on the largest init container resource quantity, rather than as the sum of quantities. Init containers do not support readiness probes since they will run to completion before the pod can be ready. + + ## Lifecycle hooks and termination notice Of course, nodes and applications may fail at any time, but many applications benefit from clean shutdown, such as to complete in-flight requests, when the termination of the application is deliberate. To support such cases, Kubernetes supports two kinds of notifications: From b99eaf0466919e66687deac3b728b6b7a4f1d12d Mon Sep 17 00:00:00 2001 From: Marcin Wielgus Date: Fri, 17 Jun 2016 17:26:27 +0200 Subject: [PATCH 10/31] Cluster autoscaler doc - part 1 (cherry picked from commit bf73c18603dbb28824199a7dfbe3cc796c0601b0) --- docs/admin/cluster-management.md | 58 ++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/docs/admin/cluster-management.md b/docs/admin/cluster-management.md index 613ec8afa2..16d4a6dce2 100644 --- a/docs/admin/cluster-management.md +++ b/docs/admin/cluster-management.md @@ -66,37 +66,51 @@ Instance Group will take care of putting appropriate image on new machines and s In other environments you may need to configure the machine yourself and tell the Kubelet on which machine API server is running. -### Horizontal auto-scaling of nodes (GCE) +### Cluster autoscaling -If you are using GCE, you can configure your cluster so that the number of nodes will be automatically scaled based on: +If you are using GCE or GKE, you can configure your cluster so that it is automatically rescaled based on +pod needs. - * CPU and memory utilization. - * Amount of of CPU and memory requested by the pods (called also reservation). +As described in [Compute Resource](/docs/user-guide/compute-resources/), users can reserve how much CPU and memory is allocated to pods. +This information is used by the Kubernetes scheduler to find a place to run the pod. If there is +no node that has enough free capacity (or doesn't match other pod requirements) then the pod has +to wait until some pods are terminated or a new node is added. -Before setting up the cluster by `kube-up.sh`, you can set `KUBE_ENABLE_NODE_AUTOSCALER` environment variable to `true` and export it. -The script will create an autoscaler for the instance group managing your nodes. +Cluster autoscaler looks for the pods that cannot be scheduled and checks if adding a new node, similar +to the other in the cluster, would help. If yes, then it resizes the cluster to accomodate the waiting pods. -The autoscaler will try to maintain the average CPU/memory utilization and reservation of nodes within the cluster close to the target value. -The target value can be configured by `KUBE_TARGET_NODE_UTILIZATION` environment variable (default: 0.7) for ``kube-up.sh`` when creating the cluster. -Node utilization is the total node's CPU/memory usage (OS + k8s + user load) divided by the node's capacity. -Node reservation is the total CPU/memory requested by pods that are running on the node divided by the node's capacity. -If the desired numbers of nodes in the cluster resulting from CPU/memory utilization/reservation are different, -the autoscaler will choose the bigger number. The number of nodes in the cluster set by the autoscaler will be limited from `KUBE_AUTOSCALER_MIN_NODES` (default: 1) -to `KUBE_AUTOSCALER_MAX_NODES` (default: the initial number of nodes in the cluster). +Cluster autoscaler also scales down the cluster if it notices that some node is not needed anymore for +an extended period of time (10min but it may change in the future). -The autoscaler is implemented as a Compute Engine Autoscaler. -The initial values of the autoscaler parameters set by `kube-up.sh` and some more advanced options can be tweaked on -`Compute > Compute Engine > Instance groups > your group > Edit group`[Google Cloud Console page](https://console.developers.google.com) -or using gcloud CLI: +Cluster autoscaler is configured per instance group (GCE) or node pool (GKE). + +If you are using GCE then you can either enable it while creating a cluster with kube-up.sh script. +To configure cluser autoscaler you have to set 3 environment variables: + +* `KUBE_ENABLE_CLUSTER_AUTOSCALER` - it enables cluster autoscaler if set to true. +* `KUBE_AUTOSCALING_MIN_NODES` - minimum number of nodes in the cluster. +* `KUBE_AUTOSCALING_MAX_NODES` - maximum number of nodes in the cluster. + +Example: ```shell -gcloud alpha compute autoscaler --zone $ZONE +KUBE_ENABLE_CLUSTER_AUTOSCALER=true KUBE_AUTOSCALING_MIN_NODES=3 KUBE_AUTOSCALING_MAX_NODES=10 NUM_NODES=5 ./cluster/kube-up.sh ``` -Note that autoscaling will work properly only if node metrics are accessible in Google Cloud Monitoring. -To make the metrics accessible, you need to create your cluster with `KUBE_ENABLE_CLUSTER_MONITORING` -equal to `google` or `googleinfluxdb` (`googleinfluxdb` is the default value). Please also make sure -that you have Google Cloud Monitoring API enabled in Google Developer Console. +On GKE you configure cluster autoscaler either on cluster creation or update or when creating a particular node pool +(which you want to be autoscaled) by passing flags `--enable-autoscaling` `--min-nodes` and `--max-nodes` +to the corresponding `gcloud` commands. + +Examples: +```shell +gcloud container clusters create mytestcluster --zone=us-central1-b --enable-autoscaling=true --min-nodes=3 --max-nodes=10 --num-nodes=5 +``` + +```shell +gcloud container clusters update mytestcluster --enable-autoscaling=true --min-nodes=1 --max-nodes=15 +``` + +**Cluster autoscaler expects that nodes have not been manually modified (e.g. by adding labels via kubectl) as those properties would not be propagated to the new nodes within the same instance group.** ## Maintenance on a Node From ad3009c822f0c33e39ff4b7158cdfff8b8fcc561 Mon Sep 17 00:00:00 2001 From: Mike Metral Date: Thu, 16 Jun 2016 14:49:10 -0700 Subject: [PATCH 11/31] add docs for recursive flag usage --- docs/user-guide/managing-deployments.md | 43 +++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/user-guide/managing-deployments.md b/docs/user-guide/managing-deployments.md index 611d56ed86..ee85850c0d 100644 --- a/docs/user-guide/managing-deployments.md +++ b/docs/user-guide/managing-deployments.md @@ -82,6 +82,49 @@ my-nginx-svc 10.0.0.208 80/TCP 0s With the above commands, we first create resources under docs/user-guide/nginx/ and print the resources created with `-o name` output format (print each resource as resource/name). Then we `grep` only the "service", and then print it with `kubectl get`. +If you happen to organize your resources across several subdirectories within a particular directory, you can recursively perform the operations on the subdirectories also, by specifying `--recursive` or `-R` alongside the `--filename,-f` flag. + +For instance, assume there is a directory `project/k8s/development` that holds all of the manifests needed for the development environment, organized by resource type: + +``` +project/k8s/development +├── configmap +│   └── my-configmap.yaml +├── deployment +│   └── my-deployment.yaml +└── pvc + └── my-pvc.yaml +``` + +By default, performing a bulk operation on `project/k8s/development` will stop at the first level of the directory, not processing any subdirectories. If we tried to create the resources in this directory using the following command, we'd encounter an error: + +```shell +$ kubectl create -f project/k8s/development +error: you must provide one or more resources by argument or filename (.json|.yaml|.yml|stdin) +``` + +Instead, specify the `--recursive` or `-R` flag with the `--filename,-f` flag as such: + +```shell +$ kubectl create -f project/k8s/development --recursive +configmap "my-config" created +deployment "my-deployment" created +persistentvolumeclaim "my-pvc" created +``` + +The `--recursive` flag works with any operation that accepts the `--filename,-f` flag such as: `kubectl {create,get,delete,describe,rollout} etc.` + +The `--recursive` flag also works when multiple `-f` arguments are provided: + +```shell +$ kubectl create -f project/k8s/namespaces -f project/k8s/development --recursive +namespace "development" created +namespace "staging" created +configmap "my-config" created +deployment "my-deployment" created +persistentvolumeclaim "my-pvc" created +``` + If you're interested in learning more about `kubectl`, go ahead and read [kubectl Overview](/docs/user-guide/kubectl-overview). ## Using labels effectively From 57abef3e780b52a87420b1843e2c30b4cf6c143f Mon Sep 17 00:00:00 2001 From: derekwaynecarr Date: Thu, 23 Jun 2016 10:48:05 -0400 Subject: [PATCH 12/31] Update 1.3 documentation for quota --- docs/admin/resourcequota/index.md | 189 ++++++++++++++++++++---------- 1 file changed, 126 insertions(+), 63 deletions(-) diff --git a/docs/admin/resourcequota/index.md b/docs/admin/resourcequota/index.md index 40559f058e..59be97e38e 100644 --- a/docs/admin/resourcequota/index.md +++ b/docs/admin/resourcequota/index.md @@ -4,29 +4,26 @@ When several users or teams share a cluster with a fixed number of nodes, there is a concern that one team could use more than its fair share of resources. -Resource quotas are a tool for administrators to address this concern. Resource quotas -work like this: +Resource quotas are a tool for administrators to address this concern. + +A resource quota, defined by a `ResourceQuota` object, provides constraints that limit +aggregate resource consumption per namespace. It can limit the quantity of objects that can +be created in a namespace by type, as well as the total amount of compute resources that may +be consumed by resources in that project. + +Resource quotas work like this: - Different teams work in different namespaces. Currently this is voluntary, but support for making this mandatory via ACLs is planned. -- The administrator creates a Resource Quota for each namespace. -- Users put compute resource requests on their pods. The sum of all resource requests across - all pods in the same namespace must not exceed any hard resource limit in any Resource Quota - document for the namespace. Note that we used to verify Resource Quota by taking the sum of - resource limits of the pods, but this was altered to use resource requests. Backwards compatibility - for those pods previously created is preserved because pods that only specify a resource limit have - their resource requests defaulted to match their defined limits. The user is only charged for the - resources they request in the Resource Quota versus their limits because the request is the minimum - amount of resource guaranteed by the cluster during scheduling. For more information on over commit, - see [compute-resources](/docs/user-guide/compute-resources). -- If creating a pod would cause the namespace to exceed any of the limits specified in the - the Resource Quota for that namespace, then the request will fail with HTTP status - code `403 FORBIDDEN`. -- If quota is enabled in a namespace and the user does not specify *requests* on the pod for each - of the resources for which quota is enabled, then the POST of the pod will fail with HTTP - status code `403 FORBIDDEN`. Hint: Use the LimitRange admission controller to force default - values of *limits* (then resource *requests* would be equal to *limits* by default, see - [admission controller](/docs/admin/admission-controllers)) before the quota is checked to avoid this problem. +- The administrator creates one or more Resource Quota objects for each namespace. +- Users create resources (pods, services, etc.) in the namespace, and the quota system + tracks usage to ensure it does not exceed hard resource limits defined in a Resource Quota. +- If creating or updating a resource violates a quota constraint, the request will fail with HTTP + status code `403 FORBIDDEN` with a message explaining the constraint that would have been violated. +- If quota is enabled in a namespace for compute resources like `cpu` and `memory`, users must specify + requests or limits for those values; otherwise, the quota system may reject pod creation. Hint: Use + the LimitRange admission controller to force defaults for pods that make no compute resource requirements. + See the [walkthrough](/docs/admin/resourcequota/walkthrough.md) for an example to avoid this problem. Examples of policies that could be created using namespaces and quotas are: @@ -38,7 +35,7 @@ Examples of policies that could be created using namespaces and quotas are: In the case where the total capacity of the cluster is less than the sum of the quotas of the namespaces, there may be contention for resources. This is handled on a first-come-first-served basis. -Neither contention nor changes to quota will affect already-running pods. +Neither contention nor changes to quota will affect already created resources. ## Enabling Resource Quota @@ -57,11 +54,12 @@ in a namespace can be limited. The following compute resource types are support | ResourceName | Description | | ------------ | ----------- | -| cpu | Total cpu requests of containers | -| memory | Total memory requests of containers - -For example, `cpu` quota sums up the `resources.requests.cpu` fields of every -container of every pod in the namespace, and enforces a maximum on that sum. +| cpu | Across all pods in a non-terminal state, the sum of CPU requests cannot exceed this value. | +| limits.cpu | Across all pods in a non-terminal state, the sum of CPU limits cannot exceed this value. | +| limits.memory | Across all pods in a non-terminal state, the sum of memory limits cannot exceed this value. | +| memory | Across all pods in a non-terminal state, the sum of memory requests cannot exceed this value. | +| requests.cpu | Across all pods in a non-terminal state, the sum of CPU requests cannot exceed this value. | +| requests.memory | Across all pods in a non-terminal state, the sum of memory requests cannot exceed this value. | ## Object Count Quota @@ -70,12 +68,15 @@ are supported: | ResourceName | Description | | ------------ | ----------- | -| pods | Total number of pods | -| services | Total number of services | -| replicationcontrollers | Total number of replication controllers | -| resourcequotas | Total number of [resource quotas](/docs/admin/admission-controllers/#resourcequota) | -| secrets | Total number of secrets | -| persistentvolumeclaims | Total number of [persistent volume claims](/docs/user-guide/persistent-volumes/#persistentvolumeclaims) | +| configmaps | The total number of config maps that can exist in the namespace. | +| persistentvolumeclaims | The total number of [persistent volume claims](/docs/user-guide/persistent-volumes/#persistentvolumeclaims) that can exist in the namespace. | +| pods | The total number of pods in a non-terminal state that can exist in the namespace. A pod is in a terminal state if `status.phase in (Failed, Succeeded)` is true. | +| replicationcontrollers | The total number of replication controllers that can exist in the namespace. | +| resourcequotas | The total number of [resource quotas](/docs/admin/admission-controllers/#resourcequota) that can exist in the namespace. | +| services | The total number of services that can exist in the namespace. | +| services.loadbalancers | The total number of services of type load balancer that can exist in the namespace. | +| services.nodeports | The total number of services of type node port that can exist in the namespace. | +| secrets | The total number of secrets that can exist in the namespace. | For example, `pods` quota counts and enforces a maximum on the number of `pods` created in a single namespace. @@ -84,45 +85,107 @@ You might want to set a pods quota on a namespace to avoid the case where a user creates many small pods and exhausts the cluster's supply of Pod IPs. +## Quota Scopes + +Each quota can have an associated set of scopes. A quota will only measure usage for a resource if it matches +the intersection of enumerated scopes. + +When a scope is added to the quota, it limits the number of resources it supports to those that pertain to the scope. +Resources specified on the quota outside of the allowed set results in a validation error. + +| Scope | Description | +| ----- | ----------- | +| Terminating | Match pods where `spec.activeDeadlineSeconds >= 0` | +| NotTerminating | Match pods where `spec.activeDeadlineSeconds is nil` | +| BestEffort | Match pods that have best effort quality of service. | +| NotBestEffort | Match pods that do not have best effort quality of service. | + +The `BestEffort` scope restricts a quota to tracking the following resources: +* pods + +The `Terminating`, `NotTerminating`, and `NotBestEffort` scopes restrict a quota to tracking the following resources: +* cpu +* limits.cpu +* limits.memory +* memory +* pods +* requests.cpu +* requests.memory + +## Requests vs Limits + +When allocating compute resources, each container may specify a request and a limit value for either CPU or memory. +The quota can be configured to quota either value. + +If the quota has a value specified for `requests.cpu` or `requests.memory`, then it requires that every incoming +container makes an explicit request for those resources. If the quota has a value specified for `limits.cpu` or `limits.memory`, +then it requires that every incoming container specifies an explict limit for those resources. + ## Viewing and Setting Quotas Kubectl supports creating, updating, and viewing quotas: ```shell -$ kubectl namespace myspace -$ cat < quota.json -{ - "apiVersion": "v1", - "kind": "ResourceQuota", - "metadata": { - "name": "quota" - }, - "spec": { - "hard": { - "memory": "1Gi", - "cpu": "20", - "pods": "10", - "services": "5", - "replicationcontrollers":"20", - "resourcequotas":"1" - } - } -} +$ kubectl create namespace myspace + +$ cat < compute-resources.yaml +apiVersion: v1 +kind: ResourceQuota +metadata: + name: compute-resources +spec: + hard: + pods: "4" + requests.cpu: "1" + requests.memory: 1Gi + limits.cpu: "2" + limits.memory: 2Gi EOF -$ kubectl create -f ./quota.json -$ kubectl get quota -NAME -quota -$ kubectl describe quota quota -Name: quota +$ kubectl create -f ./compute-resources.yaml --namespace=myspace + +$ cat < object-counts.yaml +apiVersion: v1 +kind: ResourceQuota +metadata: + name: object-counts +spec: + hard: + configmaps: "10" + persistentvolumeclaims: "4" + replicationcontrollers: "20" + secrets: "10" + services: "10" + services.loadbalancers: "2" +EOF +$ kubectl create -f ./object-counts.yaml --namespace=myspace + +$ kubectl get quota --namespace=myspace +NAME AGE +compute-resources 30s +object-counts 32s + +$ kubectl describe quota compute-resources --namespace=myspace +Name: compute-resources +Namespace: myspace +Resource Used Hard +-------- ---- ---- +limits.cpu 0 2 +limits.memory 0 2Gi +pods 0 4 +requests.cpu 0 1 +requests.memory 0 1Gi + +$ kubectl describe quota object-counts --namespace=myspace +Name: object-counts +Namespace: myspace Resource Used Hard -------- ---- ---- -cpu 0m 20 -memory 0 1Gi -pods 5 10 -replicationcontrollers 5 20 -resourcequotas 1 1 -services 3 5 +configmaps 0 10 +persistentvolumeclaims 0 4 +replicationcontrollers 0 20 +secrets 1 10 +services 0 10 +services.loadbalancers 0 2 ``` ## Quota and Cluster Capacity From 212341002aca8667b6101caf07fc7e055d65299b Mon Sep 17 00:00:00 2001 From: Chao Xu Date: Fri, 17 Jun 2016 16:23:55 -0700 Subject: [PATCH 13/31] add a doc for garbage collector --- _data/guides.yml | 4 +++- docs/user-guide/garbage-collector.md | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 docs/user-guide/garbage-collector.md diff --git a/_data/guides.yml b/_data/guides.yml index f5654a6efd..6b99f48574 100644 --- a/_data/guides.yml +++ b/_data/guides.yml @@ -58,6 +58,8 @@ toc: path: /docs/user-guide/config-best-practices/ - title: Using kubectl to Manage Resources path: /docs/user-guide/working-with-resources/ + - title: Garbage collection + path: /docs/user-guide/garbage-collector/ - title: Service Discovery and Load Balancing section: @@ -261,7 +263,7 @@ toc: path: /docs/admin/network-plugins/ - title: Static Pods path: /docs/admin/static-pods/ - - title: Configuring Garbage Collection + - title: Configuring kubelet Garbage Collection path: /docs/admin/garbage-collection/ - title: Configuring Kubernetes with Salt path: /docs/admin/salt/ diff --git a/docs/user-guide/garbage-collector.md b/docs/user-guide/garbage-collector.md new file mode 100644 index 0000000000..e9d646e30c --- /dev/null +++ b/docs/user-guide/garbage-collector.md @@ -0,0 +1,26 @@ +--- +--- + +* TOC +{:toc} + +## WARNING: Garbage Collector is an alpha feature and is disabled by default. Use it at your own risk! + +### What is garbage collector for + +The garbage collector (GC) cascadingly deletes dependent API objects when the owner is deleted. One use case is if two objects have functional dependency, you can specify the dependency in their configuration file when creating them, and if one of them is deleted, the GC will delete the other one automatically. The other use case is if there is logical dependency among API objects, e.g., the pods created by a replicaset depending on the replicaset, Kubernetes will automatically set the dependency (this will be implemented in release 1.4) and the GC will delete the pods when the replicaset is deleted. + +### How does the garbage collector work + +In release 1.3, there is a new `ownerReferences` field in the `metadata` of every Kubernetes API objects. The GC monitors the cluster and checks the `metadata.ownerReferences` field of each object. If none of the owners present in `metadata.ownerReferences` exists in the cluster, the GC will request the API server to delete the object. + +Currently a user needs to manually set the `metadata.ownerReferences`. In release 1.4, controllers will automatically set the field for the objects it controls. For example, when the replicaset controller creates or adopts pods, it will automatically add the replicaset to the `metadata.ownerReferences` fields of the pods. + +### How to request the garbage collector to not delete dependents + +When deleting an object, you can prevent the GC from deleting that object's dependents by specifying `deleteOptions.orphanDependents=true` in the deletion request. It prevents garbage collection by removing the object from its dependents' metadata.ownerReferences field. + +### How to enable the garbage collector + +The garbage collector is an alpha feature so it is disabled by default. To enable it, you need to start the kube-apiserver and kube-controller-manager with flag `--enable-garbage-collector`. + From c2e4bd9147cfdb374a088426fcfab849758eda9c Mon Sep 17 00:00:00 2001 From: Eric Chiang Date: Tue, 7 Jun 2016 16:56:17 -0700 Subject: [PATCH 14/31] docs/admin: document RBAC authorizer --- docs/admin/authorization.md | 159 ++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/docs/admin/authorization.md b/docs/admin/authorization.md index 3b521b5df1..81eabc658a 100644 --- a/docs/admin/authorization.md +++ b/docs/admin/authorization.md @@ -19,9 +19,20 @@ need authorization. - `--authorization-mode=ABAC`allows for user-configured authorization policy. ABAC stands for Attribute-Based Access Control. + - `--authorization-mode=RBAC` is an experimental implementation which allows +for authorization to be driven by the Kubernetes API. +RBAC stands for Roles-Based Access Control. - `--authorization-mode=Webhook` allows for authorization to be driven by a remote service using REST. +If multiple modes are provided the set is unioned, and only a single authorizer is required to admit the action. This means the flag: + +``` +--authorization-mode=AlwaysDeny,AlwaysAllow +``` + +will always allow. + ## ABAC Mode ### Request Attributes @@ -152,6 +163,154 @@ file: The apiserver will need to be restarted to pickup the new policy lines. +## RBAC Mode + +When specified "RBAC" (Role-Based Access Control) uses the +"rbac.authorization.k8s.io" API group to drive authorization decisions, +allowing admins to dynamically configure permission policies through the +Kubernetes API. + +As of 1.3 RBAC mode is in alpha and considered experimental. + +### Roles, RolesBindings, ClusterRoles, and ClusterRoleBindings + +The RBAC API Group declares four top level types which will be covered in this +section. Users can interact with these resources as they would with any other +API resource. Through `kubectl`, direct calls to the API, etc. For instance, +`kubectl create -f (resource).yml` can be used with any of these examples, +though readers who wish to follow along should review the following section on +bootstrapping first. + +In the RBAC API Group, roles hold a logical grouping of permissions. These +permissions map very closely to ABAC policies, but only contain information +about requests being made. Permission are purely additive, rules may only omit +permissions they do not wish to grant. + +Here's an example of a role which grants read access to pods within the +"default" namespace. + +```yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + namespace: default + name: pod-reader +rules: + - apiGroups: [""] # The API group "" indicates the default API Group. + resources: ["pods"] + verbs: ["get", "watch", "list"] + nonResourceURLs: [] +``` + +`ClusterRoles` hold the same information as a `Role` but can apply to any +namespace as well as non-namespaced resources (such as `Nodes`, +`PersistentVolume`, etc.). The following `ClusterRole` can grant permissions to +read secrets in any namespace. + +```yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + # "namespace" omitted since ClusterRoles are not namespaced. + name: secret-reader +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "watch", "list"] + nonResourceURLs: [] +``` + +`RoleBindings` perform the task of granting the permission to a user or set of +users. They hold a list of subjects which they apply to, and a reference to the +`Role` being assigned. + +The following `RoleBinding` assigns the "pod-reader" role to the user "jane" +within the "default" namespace, and allows jane to read pods. + +```yaml +# This role binding allows "jane" to read pods in the namespace "default" +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: read-pods + namespace: default +subjects: + - kind: User # May be "User", "Group" or "ServiceAccount" + name: jane +roleRef: + kind: Role + namespace: default + name: pod-reader + apiVersion: rbac.authorization.k8s.io/v1alpha1 +``` + +`RoleBindings` may also refer to a `ClusterRole`. However, a `RoleBinding` that +refers to a `ClusterRole` only applies in the `RoleBinding`'s namespace, not at +the cluster level. This allows admins to define a set of common roles for the +entire cluster, then reuse them in multiple namespaces. + +For instance, even though the following `RoleBinding` refers to a `ClusterRole`, +"dave" (the subject) will only be able read secrets in the "development" +namespace, the namespace of the `RoleBinding`. + +```yaml +# This role binding allows "dave" to read secrets in the namespace "development" +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: read-secrets + namespace: development # This binding only applies in the "development" namespace +subjects: + - kind: User # May be "User", "Group" or "ServiceAccount" + name: dave +roleRef: + kind: ClusterRole + name: secret-reader + apiVersion: rbac.authorization.k8s.io/v1alpha1 +``` + +Finally a `ClusterRoleBinding` may be used to grant permissions in all +namespaces. The following `ClusterRoleBinding` allows any user in the group +"manager" to read secrets in any namepsace. + +```yaml +# This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: read-secrets +subjects: + - kind: Group # May be "User", "Group" or "ServiceAccount" + name: manager +roleRef: + kind: ClusterRole + name: secret-reader + apiVersion: rbac.authorization.k8s.io/v1alpha1 +``` + +### Privilege Escalation Prevention and Bootstrapping + +The `rbac.authorization.k8s.io` API group inherently attempts to prevent users +from escalating privileges. Simply put, __a user can't grant permissions they +don't already have even when the RBAC authorizer it disabled__. If "user-1" +does not have the ability to read secrets in "namespace-a", they cannot create +a binding that would grant that permission to themselves or any other user. + +For bootstrapping the first roles, it becomes necessary for someone to get +around these limitations. For the alpha release of RBAC, an API Server flag was +added to allow one user to step around all RBAC authorization and privilege +escalation checks. NOTE: _This is subject to change with future releases._ + +``` +--authorization-rbac-super-user=admin +``` + +Once set the specified super user, in this case "admin", can be used to create +the roles and role bindings to initialize the system. + +This flag is optional and once the initial bootstrapping is performed can be +unset. + ## Webhook Mode When specified, mode `Webhook` causes Kubernetes to query an outside REST From d45e0ce4df68870e363ea72e4dc582681caa897b Mon Sep 17 00:00:00 2001 From: Avesh Agarwal Date: Wed, 1 Jun 2016 16:15:50 -0400 Subject: [PATCH 15/31] Document examples of downward API exposing container resources via environment variables and volumes. --- docs/user-guide/downward-api/README.md | 27 ++++++++-- .../dapi-container-resources.yaml | 34 ++++++++++++ docs/user-guide/downward-api/index.md | 50 ++++++++++++++--- .../volume/dapi-volume-resources.yaml | 40 ++++++++++++++ docs/user-guide/downward-api/volume/index.md | 54 ++++++++++++++++++- 5 files changed, 194 insertions(+), 11 deletions(-) create mode 100644 docs/user-guide/downward-api/dapi-container-resources.yaml create mode 100644 docs/user-guide/downward-api/volume/dapi-volume-resources.yaml diff --git a/docs/user-guide/downward-api/README.md b/docs/user-guide/downward-api/README.md index 93c5e6f5dc..00a27bdcb0 100644 --- a/docs/user-guide/downward-api/README.md +++ b/docs/user-guide/downward-api/README.md @@ -1,5 +1,5 @@ -Following this example, you will create a pod with a container that consumes the pod's name and -namespace using the [downward API](http://kubernetes.io/docs/user-guide/downward-api/). +Following these examples, you will create a pod with a container that consumes the pod's name, +namespace, and resource values using the [downward API](http://kubernetes.io/docs/user-guide/downward-api/). ## Step Zero: Prerequisites @@ -29,4 +29,25 @@ $ kubectl logs dapi-test-pod | grep POD_ 2015-04-30T20:22:18.568024817Z MY_POD_NAME=dapi-test-pod 2015-04-30T20:22:18.568087688Z MY_POD_NAMESPACE=default 2015-04-30T20:22:18.568092435Z MY_POD_IP=10.0.1.6 -``` \ No newline at end of file +``` + +## Example of environment variables with container resources + +Use the [`dapi-container-resources.yaml`](dapi-container-resources.yaml) file to create a Pod +with a container that consumes the downward API exposing the container's resources. + +```shell +$ kubectl create -f docs/user-guide/downward-api/dapi-container-resources.yaml +``` + +### Examine the logs + +Grep through the pod logs to see that the pod was injected with the correct values: + +```shell +$ kubectl logs dapi-test-pod | grep MY_ +MY_MEM_LIMIT=67108864 +MY_CPU_LIMIT=1 +MY_MEM_REQUEST=33554432 +MY_CPU_REQUEST=1 +``` diff --git a/docs/user-guide/downward-api/dapi-container-resources.yaml b/docs/user-guide/downward-api/dapi-container-resources.yaml new file mode 100644 index 0000000000..2a3abb0145 --- /dev/null +++ b/docs/user-guide/downward-api/dapi-container-resources.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Pod +metadata: + name: dapi-test-pod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/busybox:1.24 + command: [ "/bin/sh", "-c", "env" ] + resources: + requests: + memory: "32Mi" + cpu: "125m" + limits: + memory: "64Mi" + cpu: "250m" + env: + - name: MY_CPU_REQUEST + valueFrom: + resourceFieldRef: + resource: requests.cpu + - name: MY_CPU_LIMIT + valueFrom: + resourceFieldRef: + resource: limits.cpu + - name: MY_MEM_REQUEST + valueFrom: + resourceFieldRef: + resource: requests.memory + - name: MY_MEM_LIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + restartPolicy: Never diff --git a/docs/user-guide/downward-api/index.md b/docs/user-guide/downward-api/index.md index bbc0bf58f0..eb9819ef14 100644 --- a/docs/user-guide/downward-api/index.md +++ b/docs/user-guide/downward-api/index.md @@ -21,6 +21,10 @@ The following information is available to a `Pod` through the downward API: * The pod's name * The pod's namespace * The pod's IP +* A container's cpu limit +* A container's cpu request +* A container's memory limit +* A container's memory request More information will be exposed through this same API over time. @@ -31,13 +35,15 @@ Containers consume information from the downward API using environment variables or using a volume plugin. -### Environment variables +## Environment variables Most environment variables in the Kubernetes API use the `value` field to carry simple values. However, the alternate `valueFrom` field allows you to specify -a `fieldRef` to select fields from the pod's definition. The `fieldRef` field -is a structure that has an `apiVersion` field and a `fieldPath` field. The -`fieldPath` field is an expression designating a field of the pod. The +a `fieldRef` to select fields from the pod's definition, and a `resourceFieldRef` +to select fields from one of its container's definition. + +The `fieldRef` field is a structure that has an `apiVersion` field and a `fieldPath` +field. The `fieldPath` field is an expression designating a field of the pod. The `apiVersion` field is the version of the API schema that the `fieldPath` is written in terms of. If the `apiVersion` field is not specified it is defaulted to the API version of the enclosing object. @@ -46,20 +52,39 @@ The `fieldRef` is evaluated and the resulting value is used as the value for the environment variable. This allows users to publish their pod's name in any environment variable they want. +The `resourceFieldRef` is a structure that has a `containerName` field, a `resource` +field, and a `divisor` field. The `containerName` is the name of a container, +whose resource (cpu or memory) information is to be exposed. The `containerName` is +optional for environment variables and defaults to the current container. The +`resource` field is an expression designating a resource in a container, and the `divisor` +field specifies an output format of the resource being exposed. If the `divisor` +is not specified, it defaults to "1" for cpu and memory. The table shows possible +values for cpu and memory resources for `resource` and `divisor` settings: -## Example + +| Setting | Cpu | Memory | +| ------------- |-------------| -----| +| resource | limits.cpu, requests.cpu| limits.memory, requests.memory| +| divisor | 1(cores), 1m(millicores) | 1(bytes), 1k(kilobytes), 1M(megabytes), 1G(gigabytes), 1T(terabytes), 1P(petabytes), 1E(exabytes), 1Ki(kibibyte), 1Mi(mebibyte), 1Gi(gibibyte), 1Ti(tebibyte), 1Pi(pebibyte), 1Ei(exbibyte)| + + +### Example This is an example of a pod that consumes its name and namespace via the downward API: {% include code.html language="yaml" file="dapi-pod.yaml" ghlink="/docs/user-guide/downward-api/dapi-pod.yaml" %} +This is an example of a pod that consumes its container's resources via the downward API: -### Downward API volume +{% include code.html language="yaml" file="dapi-container-resources.yaml" ghlink="/docs/user-guide/downward-api/dapi-container-resources.yaml" %} + +## Downward API volume Using a similar syntax it's possible to expose pod information to containers using plain text files. Downward API are dumped to a mounted volume. This is achieved using a `downwardAPI` volume type and the different items represent the files to be created. `fieldPath` references the field to be exposed. +For exposing a container's resources limits and requests, `containerName` must be specified with `resourceFieldRef`. Downward API volume permits to store more complex data like [`metadata.labels`](/docs/user-guide/labels) and [`metadata.annotations`](/docs/user-guide/annotations). Currently key/value pair set fields are saved using `key="value"` format: @@ -76,19 +101,30 @@ Downward API volumes can expose: * The pod's namespace * The pod's labels * The pod's annotations +* A container's cpu limit +* A container's cpu request +* A container's memory limit +* A container's memory request The downward API volume refreshes its data in step with the kubelet refresh loop. When labels will be modifiable on the fly without respawning the pod containers will be able to detect changes through mechanisms such as [inotify](https://en.wikipedia.org/wiki/Inotify). In future, it will be possible to specify a specific annotation or label. -## Example +### Example This is an example of a pod that consumes its labels and annotations via the downward API volume, labels and annotations are dumped in `/etc/labels` and in `/etc/annotations`, respectively: {% include code.html language="yaml" file="volume/dapi-volume.yaml" ghlink="/docs/user-guide/downward-api/volume/dapi-volume.yaml" %} +This is an example of a pod that consumes its container's resources via the downward API volume. + +{% include code.html language="yaml" file="volume/dapi-volume-resources.yaml" ghlink="/docs/user-guide/downward-api/volume/dapi-volume-resources.yaml" %} Some more thorough examples: * [environment variables](/docs/user-guide/environment-guide/) * [downward API](/docs/user-guide/downward-api/) + +## Default values for container resource limits + +If cpu and memory limits are not specified for a container, the downward API will default to node's cpu and memory capacities. diff --git a/docs/user-guide/downward-api/volume/dapi-volume-resources.yaml b/docs/user-guide/downward-api/volume/dapi-volume-resources.yaml new file mode 100644 index 0000000000..f28bb99e3f --- /dev/null +++ b/docs/user-guide/downward-api/volume/dapi-volume-resources.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kubernetes-downwardapi-volume-example +spec: + containers: + - name: client-container + image: gcr.io/google_containers/busybox:1.24 + command: ["sh", "-c", "while true; do echo; if [[ -e /etc/cpu_limit ]]; then cat /etc/cpu_limit; fi; if [[ -e /etc/cpu_request ]]; then cat /etc/cpu_request; fi; if [[ -e /etc/mem_limit ]]; then cat /etc/mem_limit; fi; if [[ -e /etc/mem_request ]]; then cat /etc/mem_request; fi; sleep 5; done"] + resources: + requests: + memory: "32Mi" + cpu: "125m" + limits: + memory: "64Mi" + cpu: "250m" + volumeMounts: + - name: podinfo + mountPath: /etc + readOnly: false + volumes: + - name: podinfo + downwardAPI: + items: + - path: "cpu_limit" + resourceFieldRef: + containerName: client-container + resource: limits.cpu + - path: "cpu_request" + resourceFieldRef: + containerName: client-container + resource: requests.cpu + - path: "mem_limit" + resourceFieldRef: + containerName: client-container + resource: limits.memory + - path: "mem_request" + resourceFieldRef: + containerName: client-container + resource: requests.memory diff --git a/docs/user-guide/downward-api/volume/index.md b/docs/user-guide/downward-api/volume/index.md index 3cfdc2c2d9..46f54d0776 100644 --- a/docs/user-guide/downward-api/volume/index.md +++ b/docs/user-guide/downward-api/volume/index.md @@ -2,7 +2,7 @@ --- Following this example, you will create a pod with a downward API volume. -A downward API volume is a k8s volume plugin with the ability to save some pod information in a plain text file. The pod information can be for example some [metadata](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/docs/devel/api-conventions.md#metadata). +A downward API volume is a k8s volume plugin with the ability to save some pod information in a plain text file. The pod information can be for example some [metadata](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/docs/devel/api-conventions.md#metadata) or a container's [resources](/docs/user-guide/compute-resources). Supported metadata fields: @@ -11,6 +11,13 @@ Supported metadata fields: 3. `metadata.name` 4. `metadata.labels` +Supported container's resources: + +1. `limits.cpu` +2. `limits.memory` +3. `requests.cpu` +4. `requests.memory` + ### Step Zero: Prerequisites This example assumes you have a Kubernetes cluster installed and running, and the `kubectl` command line tool somewhere in your path. Please see the [gettingstarted](/docs/getting-started-guides/) for installation instructions for your platform. @@ -64,3 +71,48 @@ drwxrwxrwt 3 0 0 120 Jun 1 19:55 .. ``` The file `labels` is stored in a temporary directory (`..6986_01_06_15_55_10.473583074` in the example above) which is symlinked to by `..data`. Symlinks for annotations and labels in `/etc` point to files containing the actual metadata through the `..data` indirection.  This structure allows for dynamic atomic refresh of the metadata: updates are written to a new temporary directory, and the `..data` symlink is updated atomically using `rename(2)`. + +## Example of downward API volume with container resources + +Use the `docs/user-guide/downward-api/volume/dapi-volume-resources.yaml` file to create a Pod with a downward API volume which stores its container's limits and requests in /etc. + +```shell +$ kubectl create -f docs/user-guide/downward-api/volume/dapi-volume-resources.yaml +``` + +### Examine pod/container output + +In pod's `/etc` directory one may find the files created by the plugin: + +```shell +$ kubectl exec kubernetes-downwardapi-volume-example -i -t -- sh +/ # ls -alR /etc +/etc: +total 4 +drwxrwxrwt 3 0 0 160 Jun 1 19:47 . +drwxr-xr-x 17 0 0 4096 Jun 1 19:48 .. +drwxr-xr-x 2 0 0 120 Jun 1 19:47 ..6986_01_06_15_47_23.076909525 +lrwxrwxrwx 1 0 0 31 Jun 1 19:47 ..data -> ..6986_01_06_15_47_23.076909525 +lrwxrwxrwx 1 0 0 16 Jun 1 19:47 cpu_limit -> ..data/cpu_limit +lrwxrwxrwx 1 0 0 18 Jun 1 19:47 cpu_request -> ..data/cpu_request +lrwxrwxrwx 1 0 0 16 Jun 1 19:47 mem_limit -> ..data/mem_limit +lrwxrwxrwx 1 0 0 18 Jun 1 19:47 mem_request -> ..data/mem_request + +/etc/..6986_01_06_15_47_23.076909525: +total 16 +drwxr-xr-x 2 0 0 120 Jun 1 19:47 . +drwxrwxrwt 3 0 0 160 Jun 1 19:47 .. +-rw-r--r-- 1 0 0 1 Jun 1 19:47 cpu_limit +-rw-r--r-- 1 0 0 1 Jun 1 19:47 cpu_request +-rw-r--r-- 1 0 0 8 Jun 1 19:47 mem_limit +-rw-r--r-- 1 0 0 8 Jun 1 19:47 mem_request + +/ # cat /etc/cpu_limit +1 +/ # cat /etc/mem_limit +67108864 +/ # cat /etc/cpu_request +1 +/ # cat /etc/mem_request +33554432 +``` From 53e888aa9c78254ed6b3598ac29b713df98e5147 Mon Sep 17 00:00:00 2001 From: Yifan Gu Date: Fri, 24 Jun 2016 11:04:09 -0700 Subject: [PATCH 16/31] rkt: Update the getting-started-guide and known issues. --- _data/guides.yml | 4 +- docs/getting-started-guides/rkt/index.md | 217 ++++++++++------------- docs/getting-started-guides/rkt/notes.md | 131 +++++++------- 3 files changed, 153 insertions(+), 199 deletions(-) diff --git a/_data/guides.yml b/_data/guides.yml index f5654a6efd..6f21a5749e 100644 --- a/_data/guides.yml +++ b/_data/guides.yml @@ -190,9 +190,9 @@ toc: path: /docs/getting-started-guides/fedora/fedora-calico/ - title: rkt section: - - title: Running Kubernetes on rkt + - title: Running Kubernetes with rkt path: /docs/getting-started-guides/rkt/ - - title: Notes on Different UX with rkt Container Runtime + - title: Known Issues when Using rkt path: /docs/getting-started-guides/rkt/notes/ - title: Kubernetes on Mesos path: /docs/getting-started-guides/mesos/ diff --git a/docs/getting-started-guides/rkt/index.md b/docs/getting-started-guides/rkt/index.md index e644b52462..764ed2e0e5 100644 --- a/docs/getting-started-guides/rkt/index.md +++ b/docs/getting-started-guides/rkt/index.md @@ -2,108 +2,100 @@ --- This document describes how to run Kubernetes using [rkt](https://github.com/coreos/rkt) as a container runtime. -We still have [a bunch of work](http://issue.k8s.io/8262) to do to make the experience with rkt wonderful, please stay tuned! ### **Prerequisite** -- [systemd](http://www.freedesktop.org/wiki/Software/systemd/) should be installed on the machine and should be enabled. The minimum version required at this moment (2015/09/01) is 219 +- [systemd](http://www.freedesktop.org/wiki/Software/systemd/) should be installed on the machine and should be enabled. + The minimum version required for Kubernetes 1.3 is `219`. *(Note that systemd is not required by rkt itself, we are using it here to monitor and manage the pods launched by kubelet.)* - Install the latest rkt release according to the instructions [here](https://github.com/coreos/rkt). - The minimum version required for now is [v0.8.0](https://github.com/coreos/rkt/releases/tag/v0.8.0). + The minimum version required is [v1.9.1](https://github.com/coreos/rkt/releases/tag/v1.9.1). -- Note that for rkt version later than v0.7.0, `metadata service` is not required for running pods in private networks. So now rkt pods will not register the metadata service be default. +- The [rkt API service](http://coreos.com/rkt/docs/latest/subcommands/api-service.html) must be running on the node. -- Since release [v1.2.0-alpha.5](https://github.com/kubernetes/kubernetes/releases/tag/v1.2.0-alpha.5), -the [rkt API service](https://github.com/coreos/rkt/blob/master/api/v1alpha/README.md) -must be running on the node. +### Setup network -### Network Setup +You can configure the Kubernetes networking using its own `kubenet` and `CNI` [network +plugins](http://kubernetes.io/docs/admin/network-plugins/) by setting the kubelet's `--network-plugin` and `--network-plugin-dir` flag. +In addition, rkt supports using rkt's [Contained Networking](https://coreos.com/rkt/docs/latest/networking.html#contained-mode). -rkt uses the [Container Network Interface (CNI)](https://github.com/appc/cni) -to manage container networking. By default, all pods attempt to join a network -called `rkt.kubernetes.io`, which is currently defined [in -`rkt.go`](https://github.com/kubernetes/kubernetes/blob/v1.2.0-alpha.6/pkg/kubelet/rkt/rkt.go#L91). -In order for pods to get correct IP addresses, the CNI config file must be -edited to add this `rkt.kubernetes.io` network: +##### Use rkt's Contained Networking -#### Using flannel +In this mode, rkt will attempt to join pods into a network named `rkt.kubernetes.io`. +To use rkt's contained networking, you can leave the `--network-plugin` to empty, and put a network config file under one of the rkt's [config directories](https://github.com/coreos/rkt/blob/master/Documentation/configuration.md#command-line-flags), for example: -In addition to the basic prerequisites above, each node must be running -a [flannel](https://github.com/coreos/flannel) daemon. This implies -that a flannel-supporting etcd service must be available to the cluster -as well, apart from the Kubernetes etcd, which will not yet be -available at flannel configuration time. Once it's running, flannel can -be set up with a CNI config like: - -```shell -$ cat </etc/rkt/net.d/k8s_cluster.conf +```shell +$ cat </etc/rkt/net.d/k8s_network_example.conf { - "name": "rkt.kubernetes.io", - "type": "flannel" -} -EOF -``` - -While `k8s_cluster.conf` is a rather arbitrary name for the config file itself, -and can be adjusted to suit local conventions, the keys and values should be exactly -as shown above. `name` must be `rkt.kubernetes.io` and `type` should be `flannel`. -More details about the flannel CNI plugin can be found -[in the CNI documentation](https://github.com/appc/cni/blob/master/Documentation/flannel.md). - -#### On GCE - -Each VM on GCE has an additional 256 IP addresses routed to it, so -it is possible to forego flannel in smaller clusters. This makes the -necessary CNI config file a bit more verbose: - -```shell -$ cat </etc/rkt/net.d/k8s_cluster.conf -{ - "name": "rkt.kubernetes.io", - "type": "bridge", - "bridge": "cbr0", - "isGateway": true, - "ipam": { - "type": "host-local", - "subnet": "10.255.228.1/24", - "gateway": "10.255.228.1" - }, + "name": "rkt.kubernetes.io", + "type": "bridge", + "bridge": "mybridge", + "mtu": 1460, + "addIf": "true", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "10.22.0.0/16", + "gateway": "10.22.0.1", "routes": [ { "dst": "0.0.0.0/0" } ] + } } EOF ``` -This example creates a `bridge` plugin configuration for the CNI network, specifying -the bridge name `cbr0`. It also specifies the CIDR, in the `ipam` field. +However, there are a small number of caveats you should be aware of when using rkt's networking: -Creating these files for any moderately-sized cluster is at best inconvenient. -Work is in progress to -[enable Kubernetes to use the CNI by default] -(https://github.com/kubernetes/kubernetes/pull/18795/files). -As that work matures, such manual CNI config munging will become unnecessary -for primary use cases. For early adopters, an initial example shows one way to -[automatically generate these CNI configurations] -(https://gist.github.com/yifan-gu/fbb911db83d785915543) -for rkt. +* You must create an appropriate CNI configuration file with a network name of `rkt.kubernetes.io`. +* The downwards API and environment variable substitution will not contain the pod IP. +* The `/etc/hosts` file will not contain your own hostname (though `/etc/hostname` is populated). -### Local cluster +##### Use flannel + +While it's recommended that you configure flannel using kubernetes' CNI support, you can also configure it using rkt's contained networking. +An example flannel/CNI config file looks like this: + +```shell +$ cat </etc/rkt/net.d/k8s_flannel_example.conf +{ + "name": "rkt.kubernetes.io", + "type": "flannel", + "delegate": { + "isDefaultGateway": true + } +} +EOF +``` + +For more information on flannel configuration, please read [CNI/flannel README](https://github.com/containernetworking/cni/blob/master/Documentation/flannel.md). + +##### Use Google Compute Engine (GCE) network + +Each VM on GCE has an additional 256 IP addresses routed to it, so it is possible to forego flannel in smaller clusters. +This can most easily be done by using the builtin kubenet plugin. +This can be done by setting `--network-plugin=kubenet`. + +### Launch a local cluster To use rkt as the container runtime, we need to supply the following flags to kubelet: -- `--container-runtime=rkt` chooses the container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'. -- `--rkt-path=$PATH_TO_RKT_BINARY` sets the path of rkt binary. Leave empty to use the first rkt in $PATH. -- `--rkt-stage1-image` sets the path of the stage1 image. Local paths and http/https URLs are supported. Leave empty to use the 'stage1.aci' that locates in the same directory as the rkt binary. +- `--container-runtime=rkt` chooses the container runtime to use. +- `--rkt-api-endpoint=HOST:PORT` sets the endpoint of the rkt API service. + Leave empty to use the default one (`localhost:15441`). +- `--rkt-path=$PATH_TO_RKT_BINARY` sets the path of rkt binary. + Leave empty to use the first rkt in $PATH. +- `--rkt-stage1-image` sets the name of the stage1 image, e.g. coreos.com/rkt/stage1-coreos. + Leave empty to use the default stage1 image in the rkt's configuration. -If you are using the [hack/local-up-cluster.sh](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/hack/local-up-cluster.sh) script to launch the local cluster, then you can edit the environment variable `CONTAINER_RUNTIME`, `RKT_PATH` and `RKT_STAGE1_IMAGE` to -set these flags: +If you are using the [hack/local-up-cluster.sh](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/hack/local-up-cluster.sh) script to launch the local cluster, then you can edit the environment variable `CONTAINER_RUNTIME`, `RKT_PATH` and `RKT_STAGE1_IMAGE` to set these flags, the `RKT_PATH` and `RKT_STAGE1_IMAGE` are optional if you have `rkt` in your `$PATH` with appropriate configuration. ```shell $ export CONTAINER_RUNTIME=rkt $ export RKT_PATH=$PATH_TO_RKT_BINARY -$ export RKT_STAGE1_IMAGE=PATH=$PATH_TO_STAGE1_IMAGE +$ export RKT_STAGE1_IMAGE=$NAME_OF_THE_STAGE1_IMAGE ``` Then we can launch the local cluster using the script: @@ -112,21 +104,26 @@ Then we can launch the local cluster using the script: $ hack/local-up-cluster.sh ``` -### CoreOS cluster on Google Compute Engine (GCE) +We are also working on set up rkt as the container runtime for [minikube](https://github.com/kubernetes/minikube/issues/168). -To use rkt as the container runtime for your CoreOS cluster on GCE, you need to specify the OS distribution, project, image: +### Launch a CoreOS/rkt cluster on Google Compute Engine (GCE) + +Here we provide instruction on how to use the `kube-up` script to launch a CoreOS/rkt cluster on GCE. +In order to do that, you need to specify the OS distribution, project, image: ```shell $ export KUBE_OS_DISTRIBUTION=coreos -$ export KUBE_GCE_NODE_IMAGE= +$ export KUBE_GCE_MASTER_PROJECT=coreos-cloud +$ export KUBE_GCE_MASTER_IMAGE= $ export KUBE_GCE_NODE_PROJECT=coreos-cloud +$ export KUBE_GCE_NODE_IMAGE= $ export KUBE_CONTAINER_RUNTIME=rkt ``` You can optionally choose the version of rkt used by setting `KUBE_RKT_VERSION`: ```shell -$ export KUBE_RKT_VERSION=0.15.0 +$ export KUBE_RKT_VERSION=1.9.1 ``` Then you can launch the cluster by: @@ -135,52 +132,25 @@ Then you can launch the cluster by: $ cluster/kube-up.sh ``` -Note that we are still working on making all containerized the master components run smoothly in rkt. Before that we are not able to run the master node with rkt yet. +### Launch a CoreOS/rkt cluster on AWS -### CoreOS cluster on AWS +`kube-up` for AWS is currently unsupported. +Instead, we recommend you to refer the [Kubernetes on AWS guide](https://coreos.com/kubernetes/docs/latest/kubernetes-on-aws.html) to launch a CoreOS/rkt cluster on AWS. -To use rkt as the container runtime for your CoreOS cluster on AWS, you need to specify the provider and OS distribution: +### Deploy apps to your cluster -```shell -$ export KUBERNETES_PROVIDER=aws -$ export KUBE_OS_DISTRIBUTION=coreos -$ export KUBE_CONTAINER_RUNTIME=rkt -``` +After you created the cluster, you can start deploying apps to the cluster. For example here is how you can [deploy a simgle nginx app](/docs/user-guide/simple-nginx). +More examples can be found in the [examples directory](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/). -You can optionally choose the version of rkt used by setting `KUBE_RKT_VERSION`: +### Known Issues and Differences -```shell -$ export KUBE_RKT_VERSION=0.8.0 -``` - -You can optionally choose the CoreOS channel by setting `COREOS_CHANNEL`: - -```shell -$ export COREOS_CHANNEL=stable -``` - -Then you can launch the cluster by: - -```shell -$ kube-up.sh -``` - -Note: CoreOS is not supported as the master using the automated launch -scripts. The master node is always Ubuntu. - -### Getting started with your cluster - -See [a simple nginx example](/docs/user-guide/simple-nginx) to try out your new cluster. - -For more complete applications, please look in the [examples directory](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/). - -### Different UX with rkt container runtime - -rkt and Docker have very different designs, as well as ACI and Docker image format. Users might experience some different experience when switching from one to the other. More information can be found [here](/docs/getting-started-guides/rkt/notes/). +rkt and Docker have very different designs, as well as ACI and Docker image format. +Users might experience some different experience when switching from one to the other. +More information can be found [here](/docs/getting-started-guides/rkt/notes/). ### Debugging -Here are several tips for you when you run into any issues. +Here are several tips in case you run into any issues. ##### Check logs @@ -191,29 +161,20 @@ If the cluster is using salt, we can edit the [logging.sls](https://github.com/k ##### Check rkt pod status To check the pods' status, we can use rkt command, such as `rkt list`, `rkt status`, `rkt image list`, etc. -More information about rkt command line can be found [here](https://github.com/coreos/rkt/blob/master/Documentation/commands.md) +More information about rkt command line can be found [here](https://github.com/coreos/rkt/blob/master/Documentation/commands.md). ##### Check journal logs -As we use systemd to launch rkt pods(by creating service files which will run `rkt run-prepared`, we can check the pods' log -using `journalctl`: +As we use systemd to launch/manage rkt pods, we can check the pods' log using `journalctl`: - Check the running state of the systemd service: ```shell -$ sudo journalctl -u $SERVICE_FILE +$ sudo journalctl -u ${SERVICE_NAME} ``` -where `$SERVICE_FILE` is the name of the service file created for the pod, you can find it in the kubelet logs. - -##### Check the log of the container in the pod: - -```shell -$ sudo journalctl -M rkt-$UUID -u $CONTAINER_NAME -``` - -where `$UUID` is the rkt pod's UUID, which you can find via `rkt list --full`, and `$CONTAINER_NAME` is the container's name. +where `${SERVICE_NAME}` is the name of the service file created for the pod, typically the format is `k8s_${RKT_UUID}`. ##### Check Kubernetes events, logs. -Besides above tricks, Kubernetes also provides us handy tools for debugging the pods. More information can be found [here](/docs/user-guide/application-troubleshooting) +Kubernetes also provides various tools for debugging. More information can be found [here](/docs/user-guide/application-troubleshooting). diff --git a/docs/getting-started-guides/rkt/notes.md b/docs/getting-started-guides/rkt/notes.md index 44eb57f90e..61e9676ba4 100644 --- a/docs/getting-started-guides/rkt/notes.md +++ b/docs/getting-started-guides/rkt/notes.md @@ -1,99 +1,92 @@ --- --- -# Notes on Different UX with rkt container runtime +# Known issues -### Doesn't support ENTRYPOINT + CMD feature +The following features either are not supported or have large caveats when using the rkt container runtime. -To run a Docker image, rkt will convert it into [App Container Image (ACI) format](https://github.com/appc/spec/blob/master/SPEC.md) first. -However, during the conversion, the `ENTRYPOINT` and `CMD` are concatentated to construct ACI's `Exec` field. -This means after the conversion, we are not able to replace only `ENTRYPOINT` or `CMD` without touching the other part. -So for now, users are recommended to specify the **executable path** in `Command` and **arguments** in `Args`. -(This has the same effect if users specify the **executable path + arguments** in `Command` or `Args` alone). +##### Non-existent host volume paths -For example: +When mounting a host volume path that does not exist, rkt will error out. Under the Docker runtime, an empty directory will be created at the referenced path. + +An example of a pod which will error out: ```yaml apiVersion: v1 kind: Pod metadata: - name: nginx labels: - name: nginx + name: mount-dne + name: mount-dne spec: + volumes: + - name: does-not-exist + hostPath: + path: /does/not/exist containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 + - name: exit + image: busybox + command: ["sh", "-c", "ls /test; sleep 60"] + volumeMounts: + - mountPath: /test + name: does-not-exist ``` -The above pod yaml file is valid as it's not specifying `Command` or `Args`, so the default `ENTRYPOINT` and `CMD` of the image will be used. +##### Kubectl attach + +The `kubectl attach` command does not work under the rkt container runtime. +And because of this, some flags in `kubectl run` are not supported, including: + +- --attach=true +- --leave-stdin-open=true +- --rm=true + +##### Init containers + +The alpha [init container](https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/container-init.md) feature is currently not supported. + +##### Container restart back-off + +Exponential restart back-off for a failing container is currently not supported. + +##### Experimental NVIDIA GPU support + +The `--experimental-nvidia-gpus` flag, and related [GPU features](https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/gpu-support.md) are not supported. + +##### QoS Classes + +Under rkt, QoS classes do not result in the `OOM Score` of containers being adjusted as occurs under Docker. + +##### HostPID and HostIPC namespaces + +Setting a the hostPID or hostIPC flag on a pod is not supported. + +For example, the following pod will not run correctly: ```yaml apiVersion: v1 kind: Pod metadata: - name: busybox labels: - name: busybox + name: host-ipc-pid + name: host-ipc-pid spec: + hostIPC: true + hostPID: true containers: - - name: busybox - image: busybox - command: - - /bin/sleep - - 1000 + ... ``` -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: busybox - labels: - name: busybox -spec: - containers: - - name: busybox - image: busybox - command: - - /bin/sleep - args: - - 1000 -``` +On the other hand, when running the pod with [stage1-fly](https://coreos.com/rkt/docs/latest/running-fly-stage1.html), the pod will be run in the host namespace. -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: busybox - labels: - name: busybox -spec: - containers: - - name: busybox - image: busybox - args: - - /bin/sleep - - 1000 -``` -All the three examples above are valid as they contain both the executable path and the arguments. +##### Container image updates (patch) -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: busybox - labels: - name: busybox -spec: - containers: - - name: busybox - image: busybox - args: - - 1000 -``` +Patching a pod to change the image will result in the entire pod restarting, not just the container that was changed. -The last example is invalid, as we cannot override just the `CMD` of the image alone. +##### Volume mounts specifying a subPath + +The [subPath](https://github.com/kubernetes/kubernetes/pull/22575) feature does not work correctly under rkt. +In addition, the above-issue of Non-existent host volume paths being invalid would make many common use-cases for subPaths fail in that way as well. + +In some cases, this issue can be worked around by creating and using subdirectories from within the container rather than relying on Kubernetes to do so. From be2d47401decd0b1277ccffb4d8a1ebdad86d60b Mon Sep 17 00:00:00 2001 From: Yifan Gu Date: Fri, 24 Jun 2016 11:30:14 -0700 Subject: [PATCH 17/31] rkt: Minor refactoring. --- docs/getting-started-guides/rkt/index.md | 9 ++++----- docs/getting-started-guides/rkt/notes.md | 10 ++++------ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/docs/getting-started-guides/rkt/index.md b/docs/getting-started-guides/rkt/index.md index 764ed2e0e5..ac79c3ac7b 100644 --- a/docs/getting-started-guides/rkt/index.md +++ b/docs/getting-started-guides/rkt/index.md @@ -3,7 +3,7 @@ This document describes how to run Kubernetes using [rkt](https://github.com/coreos/rkt) as a container runtime. -### **Prerequisite** +### Prerequisite - [systemd](http://www.freedesktop.org/wiki/Software/systemd/) should be installed on the machine and should be enabled. The minimum version required for Kubernetes 1.3 is `219`. @@ -25,7 +25,7 @@ In addition, rkt supports using rkt's [Contained Networking](https://coreos.com/ In this mode, rkt will attempt to join pods into a network named `rkt.kubernetes.io`. To use rkt's contained networking, you can leave the `--network-plugin` to empty, and put a network config file under one of the rkt's [config directories](https://github.com/coreos/rkt/blob/master/Documentation/configuration.md#command-line-flags), for example: -```shell +```shell $ cat </etc/rkt/net.d/k8s_network_example.conf { "name": "rkt.kubernetes.io", @@ -75,8 +75,7 @@ For more information on flannel configuration, please read [CNI/flannel README]( ##### Use Google Compute Engine (GCE) network Each VM on GCE has an additional 256 IP addresses routed to it, so it is possible to forego flannel in smaller clusters. -This can most easily be done by using the builtin kubenet plugin. -This can be done by setting `--network-plugin=kubenet`. +This can most easily be done by using the builtin kubenet plugin, by setting the kubelet flag `--network-plugin=kubenet`. ### Launch a local cluster @@ -104,7 +103,7 @@ Then we can launch the local cluster using the script: $ hack/local-up-cluster.sh ``` -We are also working on set up rkt as the container runtime for [minikube](https://github.com/kubernetes/minikube/issues/168). +We are also working on setting up rkt as the container runtime for [minikube](https://github.com/kubernetes/minikube/issues/168). ### Launch a CoreOS/rkt cluster on Google Compute Engine (GCE) diff --git a/docs/getting-started-guides/rkt/notes.md b/docs/getting-started-guides/rkt/notes.md index 61e9676ba4..8e53481046 100644 --- a/docs/getting-started-guides/rkt/notes.md +++ b/docs/getting-started-guides/rkt/notes.md @@ -1,8 +1,6 @@ --- --- -# Known issues - The following features either are not supported or have large caveats when using the rkt container runtime. ##### Non-existent host volume paths @@ -37,9 +35,9 @@ spec: The `kubectl attach` command does not work under the rkt container runtime. And because of this, some flags in `kubectl run` are not supported, including: -- --attach=true -- --leave-stdin-open=true -- --rm=true +- `--attach=true` +- `--leave-stdin-open=true` +- `--rm=true` ##### Init containers @@ -84,7 +82,7 @@ On the other hand, when running the pod with [stage1-fly](https://coreos.com/rkt Patching a pod to change the image will result in the entire pod restarting, not just the container that was changed. -##### Volume mounts specifying a subPath +##### Volume mounts specifying a subPath The [subPath](https://github.com/kubernetes/kubernetes/pull/22575) feature does not work correctly under rkt. In addition, the above-issue of Non-existent host volume paths being invalid would make many common use-cases for subPaths fail in that way as well. From c65d7400fd13ccbbb4ab75c90effaa566f519e4a Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 20 Jun 2016 16:24:35 -0700 Subject: [PATCH 18/31] Add document for node problem detector. --- _data/guides.yml | 2 + docs/admin/node-problem.md | 245 +++++++++++++++++++++++++++++++++++++ 2 files changed, 247 insertions(+) create mode 100644 docs/admin/node-problem.md diff --git a/_data/guides.yml b/_data/guides.yml index f5654a6efd..91ce1bef50 100644 --- a/_data/guides.yml +++ b/_data/guides.yml @@ -265,3 +265,5 @@ toc: path: /docs/admin/garbage-collection/ - title: Configuring Kubernetes with Salt path: /docs/admin/salt/ + - title: Monitoring Node Health + path: /docs/admin/node-problem/ diff --git a/docs/admin/node-problem.md b/docs/admin/node-problem.md new file mode 100644 index 0000000000..5dc2d4cb52 --- /dev/null +++ b/docs/admin/node-problem.md @@ -0,0 +1,245 @@ +--- +--- + +* TOC +{:toc} + +## Node Problem Detector + +*Node problem detector* is a [DaemonSet](/docs/admin/daemons/) monitoring the +node health. It collects node problems from various daemons and reports them +to the apiserver as [NodeCondition](/docs/admin/node/#node-condition) and +[Event](/docs/api-reference/v1/definitions/#_v1_event). + +It supports some known kernel issue detection now, and will detect more and +more node problems over time. + +Currently Kubernetes won't take any action on the node conditions and events +generated by node problem detector. In the future, a remedy system could be +introduced to deal with node problems. + +See more information +[here](https://github.com/kubernetes/node-problem-detector). + +## Limitations + +* The kernel issue detection of node problem detector only supports file based +kernel log now. It doesn't support log tools like journald. + +* The kernel issue detection of node problem detector has assumption on kernel +log format, now it only works on Ubuntu and Debian. However, it is easy to extend +it to [support other log format](/docs/admin/node-problem/#support-other-log-format). + +## Enable/Disable in GCE cluster + +Node problem detector is running as a cluster +[addon](docs/admin/cluster-large/#addon-resources) enabled by default in the +gce cluster. + +You can enable/disable it by setting the environment variable +`KUBE_ENABLE_NODE_PROBLEM_DETECTOR` before `kube-up.sh`. + +## Use in Other Environment + +To enable node problem detector in other environment outside of GCE, you can use +either `kubectl` or addon pod. + +### Kubectl + +This is the recommanded way to start node problem detector outside of GCE. It +provides more flexible management, such as overwriting the default +configuration to fit it into your environment or detect +customized node problems. + +* **Step 1:** Create `node-problem-detector.yaml`: + +```yaml +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: node-problem-detector-v0.1 + namespace: kube-system + labels: + k8s-app: node-problem-detector + version: v0.1 + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: node-problem-detector + version: v0.1 + kubernetes.io/cluster-service: "true" + spec: + hostNetwork: true + containers: + - name: node-problem-detector + image: gcr.io/google_containers/node-problem-detector:v0.1 + securityContext: + privileged: true + resources: + limits: + cpu: "200m" + memory: "100Mi" + requests: + cpu: "20m" + memory: "20Mi" + volumeMounts: + - name: log + mountPath: /log + readOnly: true + volumes: + - name: log + hostPath: + path: /var/log/ +``` + +***Notice that you should make sure the system log directory is right for your +OS distro.*** + +* **Step 2:** Start node problem detector with `kubectl`: + +```shell +kubectl create -f node-problem-detector.yaml +``` + +### Addon Pod + +This is for those who have their own cluster bootstrap solution, and don't need +to overwrite the default configuration. They could leverage the addon pod to +further automate the deployment. + +Just create `node-problem-detector.yaml`, and put it under the addon pods directory +`/etc/kubernetes/addons/node-problem-detector` on master node. + +## Overwrite the Configuration + +The [default configuration](https://github.com/kubernetes/node-problem-detector/tree/v0.1/config) +is embedded when building the docker image of node problem detector. + +However, you can use [ConfigMap](/docs/user-guide/configmap/) to overwrite it +following the steps: + +* **Step 1:** Change the config files in `config/`. +* **Step 2:** Create the ConfigMap `node-problem-detector-config` with `kubectl create configmap +node-problem-detector-config --from-file=config/`. +* **Step 3:** Change the `node-problem-detector.yaml` to use the ConfigMap: + +```yaml +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: node-problem-detector-v0.1 + namespace: kube-system + labels: + k8s-app: node-problem-detector + version: v0.1 + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: node-problem-detector + version: v0.1 + kubernetes.io/cluster-service: "true" + spec: + hostNetwork: true + containers: + - name: node-problem-detector + image: gcr.io/google_containers/node-problem-detector:v0.1 + securityContext: + privileged: true + resources: + limits: + cpu: "200m" + memory: "100Mi" + requests: + cpu: "20m" + memory: "20Mi" + volumeMounts: + - name: log + mountPath: /log + readOnly: true + - name: config # Overwrite the config/ directory with ConfigMap volume + mountPath: /config + readOnly: true + volumes: + - name: log + hostPath: + path: /var/log/ + - name: config # Define ConfigMap volume + configMap: + name: node-problem-detector-config +``` + +* **Step 4:** Re-create the node problem detector with the new yaml file: + +```shell +kubectl delete -f node-problem-detector.yaml # If you have a node-problem-detector running +kubectl create -f node-problem-detector.yaml +``` + +***Notice that this approach only applies to node problem detector started with `kubectl`.*** + +For node problem detector running as cluster addon, because addon manager doesn't support +ConfigMap, configuration overwriting is not supported now. + +## Kernel Monitor + +*Kernel Monitor* is a problem daemon in node problem detector. It monitors kernel log +and detects known kernel issues following predefined rules. + +The Kernel Monitor matches kernel issues according to a set of predefined rule list in +[`config/kernel-monitor.json`](https://github.com/kubernetes/node-problem-detector/blob/v0.1/config/kernel-monitor.json). +The rule list is extensible, you can always extend it by [overwriting the +configuration](/docs/admin/node-problem/#overwrite-the-configuration). + +### Add New NodeConditions + +To support new node conditions, you can extend the `conditions` field in +`config/kernel-monitor.json` with new condition definition: + +```json +{ + "type": "NodeConditionType", + "reason": "CamelCaseDefaultNodeConditionReason", + "message": "arbitrary default node condition message" +} +``` + +### Detect New Problems + +To detect new problems, you can extend the `rules` field in `config/kernel-monitor.json` +with new rule definition: + +```json +{ + "type": "temporary/permanent", + "condition": "NodeConditionOfPermanentIssue", + "reason": "CamelCaseShortReason", + "message": "regexp matching the issue in the kernel log" +} +``` + +### Change Log Path + +Kernel log in different OS distros may locate in different path. The `log` +field in `config/kernel-monitor.json` is the log path inside the container. +You can always configure it to match your OS distro. + +### Support Other Log Format + +Kernel monitor uses [`Translator`](https://github.com/kubernetes/node-problem-detector/blob/v0.1/pkg/kernelmonitor/translator/translator.go) +plugin to translate kernel log the internal data structure. It is easy to +implement a new translator for a new log format. + +## Caveats + +It is recommanded to run the node problem detector in your cluster to monitor +the node health. However, you should be aware that this will introduce extra +resource overhead on each node. Usually this is fine, because: + +* The kernel log is generated relatively slowly. +* Resource limit is set for node problem detector. +* Even under high load, the resource usage is acceptable. +(see [benchmark result](https://github.com/kubernetes/node-problem-detector/issues/2#issuecomment-220255629)) From 99f6734342aae44b888b666ea801db678a94089d Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Tue, 21 Jun 2016 10:25:46 -0700 Subject: [PATCH 19/31] Petset 1.3 user guide --- _data/reference.yml | 2 + docs/user-guide/petset.md | 281 ++++++++++++++++++++++++++++++++++++ docs/user-guide/petset.yaml | 56 +++++++ 3 files changed, 339 insertions(+) create mode 100644 docs/user-guide/petset.md create mode 100644 docs/user-guide/petset.yaml diff --git a/_data/reference.yml b/_data/reference.yml index 3b9b864cc7..e2439cc78b 100644 --- a/_data/reference.yml +++ b/_data/reference.yml @@ -224,6 +224,8 @@ toc: path: /docs/admin/resource-quota/ - title: Replica Sets path: /docs/user-guide/replicasets/ + - title: Pet Sets + path: /docs/user-guide/petset/ - title: Kubernetes Design Docs section: diff --git a/docs/user-guide/petset.md b/docs/user-guide/petset.md new file mode 100644 index 0000000000..324e1f1764 --- /dev/null +++ b/docs/user-guide/petset.md @@ -0,0 +1,281 @@ +--- +--- + +* TOC +{:toc} + +__Terminology__ + +Throughout this doc you will see a few terms that are sometimes used interchangeably elsewhere, that might cause confusion. This section attempts to clarify them. + +* Node: A single virtual or physical machine in a Kubernetes cluster. +* Cluster: A group of nodes in a single failure domain, unless mentioned otherwise. +* Persistent Volume Claim (PVC): A request for storage, typically a [persistent volume](/docs/user-guide/persistent-volumes/walkthrough/). +* Host name: The hostname attached to the UTS namespace of the pod, i.e the output of `hostname` in the pod. +* DNS/Domain name: A *cluster local* domain name resolvable using standard methods (eg: [gethostbyname](http://linux.die.net/man/3/gethostbyname)). +* Ordinality: the proprety of being "ordinal", or occupying a position in a sequence. +* Pet: a single member of a Pet Set; more generally, a stateful application. +* Peer: a process running a server, capable of communicating with other such processes. + +__Prerequisites__ + +This doc assumes familiarity with the following Kubernetes concepts: + +* [Pods](/docs/user-guide/pods/single-container/) +* [Cluster DNS](/docs/admin/dns/) +* [Headless Services](/docs/user-guide/services/#headless-services) +* [Persistent Volumes](/docs/user-guide/volumes/) +* [Dynamic volume provisioning](http://releases.k8s.io/{{page.githubbranch}}/examples/experimental/persistent-volume-provisioning/README.md) + +You need a working Kubernetes cluster at version >= 1.3, with a healthy DNS [cluster addon](http://releases.k8s.io/{{page.githubbranch}}/cluster/addons/README.md) at version >= 15. You cannot use Pet Set on a hosted Kubernetes provider that has disabled `alpha` resources. + +## What is a Pet Set? + +In Kubernetes, most pod management abstractions group them into disposable units of work that compose a micro service. Replication controllers for example, are designed with a weak guarantee - that there should be N replicas of a particular pod template. The pods are treated as stateless units, if one of them is unhealthy or superseded by a newer version, the system just disposes it. + +``` + foo.default.svc.cluster.local + |service| + / \ + | pod-asdf | | pod-zxcv | +``` + +A Pet Set, in contrast, is a group of stateful pods that require a stronger notion of identity. The document refers to these as "clustered applications". + +``` + *.foo.default.svc.cluster.local + | mysql-0 | <-> | mysql-1 | + [pv 0] [pv 1] +``` + +The co-ordinated deployment of clustered applications is notoriously hard. They require stronger notions of identity and membership, which they use in opaque internal protocols, and are especially prone to race conditions and deadlock. Traditionally administrators have deployed these applications by leveraging nodes as stable, long-lived entities with persistent storage and static ips. + +The goal of Pet Set is to decouple this dependency by assigning identities to individual instances of an application that are not anchored to the underlying physical infrastructure. For the rest of this document we will refer to these entities as "Pets". Our use of this term is predated by the "Pets vs Cattle" analogy. + +__Relationship between Pets and Pods__: PetSet requires there be {0..N-1} Pets. Each Pet has a deterministic name - PetSetName-Ordinal, and a unique identity. Each Pet has at most one pod, and each Pet Set has at most one Pet with a given identity. + +## When to use Pet Set? + +A Pet Set ensures that a specified number of "pets" with unique identities are running at any given time. The identity of a Pet is comprised of: + +* a stable hostname, available in DNS +* an ordinal index +* stable storage: linked to the ordinal & hostname + +These properties are useful in deploying stateful applications. However most stateful applications are also clustered, meaning they form groups with strict membership requirements that rely on stored state. Pet Set also helps with the 2 most common problems encountered managing such clustered applications: + +* discovery of peers for quorum +* startup/teardown ordering + +Only use Pet Set if your application requires some or all of these properties. Managing pods as stateless replicas is vastly easier. + +Example workloads for Pet Set: + +* Databases like MySQL or PostgreSQL that require a single instance attached to a NFS persistent volume at any time +* Clustered software like Zookeeper, Etcd, or Elasticsearch that require stable membership. + +## Alpha limitations + +Before you start deploying applications as Pet Sets, there are a few limitations you should understand. + +* Pet Set is an *alpha* resource, not available in any Kubernetes release prior to 1.3. +* As with all alpha/beta resources, it can be disable through the `--runtime-config` option passed to the apiserver, and in fact most likely will be disabled on hosted offerings of Kubernetes. +* The only updatable field on a Pet Set is `replicas` +* The storage for a given pet must either be provisioned by a [dynamic storage provisioner](http://releases.k8s.io/{{page.githubbranch}}/examples/experimental/persistent-volume-provisioning/README.md) based on the requested `storage class`, or pre-provisioned by an admin. Note that dynamic volume provisioning is also currently in alpha. +* Deleting the Pet Set *will not* delete any pets. You will either have to manually scale it down to 0 pets first, or delete the pets yourself. +* Deleting and/or scaling a Pet Set down will *not* delete the volumes associated with the Pet Set. This is done to ensure safety first, your data is more valuable than an auto purge of all related Pet Set resources. **Deleting the Persistent Volume Claims will result in a deletion of the associated volumes**. +* All Pet Sets currently require a "governing service", or a Service responsible for the network identity of the pets. The user is responsible for this Service. +* Updating an existing Pet Set is currently a manual process, meaning you either need to deploy a new Pet Set with the new image version, or orphan Pets one by one, update their image, and join them back to the cluster. + +## Example Pet Set + +We'll create a basic Pet Set to demonstrate how Pets are assigned unique and "sticky" identities. + +{% include code.html language="yaml" file="petset.yaml" ghlink="/docs/user-guide/petset.yaml" %} + +Saving this config into `petset.yaml` and submitting it to a Kubernetes cluster should creae the defined Pet Set and Pets it manages: + +```shell +$ kubectl create -f petset.yaml +service "nginx" created +petset "nginx" deleted +``` + +## Pet Identity + +The identity of a Pet sticks to it, regardless of which node it's (re) scheduled on. We can examine the identity of the pets we just created. + +### Ordinal index + +you should see 2 pods with predictable names formatted thus: `$(petset name)-$(ordinal index assigned by petset controller)` + +```shell +$ kubectl get po +NAME READY STATUS RESTARTS AGE +web-0 1/1 Running 0 10m +web-1 1/1 Running 0 10m +``` + +### Stable storage + +2 persistent volumes, one per pod. This is auto created by the Pet Set based on the `volumeTemplate` field + +```shell +$ kubectl get pv +NAME CAPACITY ACCESSMODES STATUS CLAIM REASON AGE +pvc-90234946-3717-11e6-a46e-42010af00002 1Gi RWO Bound default/www-web-0 11m +pvc-902733c2-3717-11e6-a46e-42010af00002 1Gi RWO Bound default/www-web-1 11m +``` + +### Network identity + +The network identity has 2 parts. First, we created a headless Service that controls the domain within which we create Pets. The domain managed by this Service takes the form: `$(service name).$(namespace).svc.cluster.local`, where "cluster.local" is the [cluster domain](http://releases.k8s.io/{{page.githubbranch}}/build/kube-dns/README.md#how-do-i-configure-it). As each pet is created, it gets a matching DNS subdomain, taking the form: `$(petname).$(governing service domain)`, where the governing service is defined by the `serviceName` field on the Pet Set. + +Cluster Domain | Service (ns/name) | Pet Set (ns/name) | Pet Set Domain | Pet DNS | Pet Hostname | +-------------- | ----------------- | ----------------- | -------------- | ------- | ------------ | + cluster.local | default/nginx | default/web | nginx.default.svc.cluster.local | web-{0..N-1}.nginx.default.svc.cluster.local | web-{0..N-1} | + cluster.local | foo/nginx | foo/web | nginx.foo.svc.cluster.local | web-{0..N-1}.nginx.foo.svc.cluster.local | web-{0..N-1} | + kube.local | foo/nginx | foo/web | nginx.foo.svc.kube.local | web-{0..N-1}.nginx.foo.svc.kube.local | web-{0..N-1} | + +Lets verify this assertion with a simple test. + +```shell +$ kubectl get svc +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +nginx None 80/TCP 12m +``` + +The containers are running nginx webservers, which by default will look for an index.html file in `/usr/share/nginx/html/index.html`. That directory is backed by a PersistentVolume created by the Pet Set. So lets write our hostname there (remember the Pet Set gives us a stable hostname): + +```shell +$ for i in 0 1; do + kubectl exec web-$i -- sh -c 'echo $(hostname) > /usr/share/nginx/html/index.html'; +done +``` + +Now delete all pods in the petset: + +```shell +$ kubectl delete po -l app=nginx +pod "web-0" deleted +pod "web-1" deleted +``` + +Wait for them to come back up, and try to retrieve the previously written hostname through the DNS name of the peer (remember the Pet Set also gives us stable storage, and that the hostname is linked to the DNS name). + +```shell +$ kubectl exec -it web-1 -- curl web-0.nginx +web-0 +$ kubectl exec -it web-0 -- curl web-1.nginx +web-1 +``` + +## Peer discovery + +A pet can piece together its own identity: + +1. Use the [downward api](/docs/user-guide/downward-api/) to find its pod name +2. Run `hostname` to find its DNS name +3. Run `mount` or `df` to find its volumes (usually this is unnecessary) + +It's not necessary to "discover" the governing Service of a Pet Set, since it's known at creation time you can simply pass it down through an [environment variable](/docs/user-guide/environment-guide). + +Usually pets also need to find their peers. In the previous nginx example, we just used `kubectl` to get the names of existing pods, and as humans, we could tell which ones belonged to a given Pet Set. Another way to find peers is by contacting the API server, just like `kubectl`, but that has several disadvantages (you end up implementing a Kubernetes specific init system that runs as pid 1 in your application container). + +Pet Set gives you a way to disover your peers using DNS records. To illustrate this we can use the previous example (note: one usually doesn't `apt-get` in a container). + +```shell +$ kubectl exec -it web-0 /bin/sh +web-0 # apt-get update && apt-get install -y dnsutils +... + +web-0 # nslookup -type=srv nginx.default +Server: 10.0.0.10 +Address: 10.0.0.10#53 + +nginx.default.svc.cluster.local service = 10 50 0 web-1.ub.default.svc.cluster.local. +nginx.default.svc.cluster.local service = 10 50 0 web-0.ub.default.svc.cluster.local. +``` + +## Updating a Pet Set + +You cannot update any field of the PetSet except `spec.replicas`. You can update the replicas field using standard kubectl update commands like [patch](/docs/user-guide/kubectl/kubectl_patch) and [edit](/docs/user-guide/kubectl/kubectl_edit). Pet Set currently *does not* support image upgrade as noted in the section on [limitations](#alpha-limitations). + +## Scaling a Pet Set + +You can scale a Pet Set by updating the "replicas" field. Note however that the controller will only: + +1. Create one pet at a time, in order from {0..N-1}, and wait till each one is in [Running and Ready](/docs/user-guide/pod-states) before creating the next +2. Delete one pet at a time, in reverse order from {N-1..0}, and wait till each one is completely shutdown (past its [terminationGracePeriodSeconds](/docs/user-guide/pods/index#termination-of-pods)) before deleting the next + +## Deleting a Pet Set + +Cleaning up a Pet Set is somewhat manual, as noted in the [limitations section](#alpha-limitations). You can delete a Pet Set using Kubectl, but this will *not* scale it down to 0: + +```shell +$ kubectl delete -f petset.yaml +petset "web" deleted + +$ kubectl get po -l app=nginx +NAME READY STATUS RESTARTS AGE +web-0 1/1 Running 0 21h +web-1 1/1 Running 0 21h + +$ kubectl delete po -l app=nginx +pod "web-0" deleted +pod "web-1" deleted +``` + +Deleting the pods will *not* delete the volumes. Until we finalize the recycle policy for these volumes they will have to get cleaned up by an admin. This is to ensure that you have the chance to copy data off the volume before deleting it. Simply deleting the PVC after the pods have left the [terminating state](/docs/user-guide/pods/index#termination-of-pods) should trigger deletion of the backing Persistent Volumes. + +**Note: you will lose all your data once the PVC is deleted, do this with caution.** + +```shell +$ kubectl get po -l app=nginx +$ kubectl get pvc -l app=nginx +NAME STATUS VOLUME CAPACITY ACCESSMODES AGE +www-web-0 Bound pvc-62d271cd-3822-11e6-b1b7-42010af00002 0 21h +www-web-1 Bound pvc-62d6750e-3822-11e6-b1b7-42010af00002 0 21h + +$ kubectl delete pvc -l app=nginx +$ kubectl get pv +``` + +If you simply want to clean everything: + +```shell{% raw %} +$ grace=$(k get po web-0 --template '{{.spec.terminationGracePeriodSeconds}}') +$ kubectl delete petset,po -l app=nginx +$ sleep $grace +$ kubectl delete pvc -l app=nginx +{% endraw %}``` + +## Troubleshooting + +You might have noticed an `annotations` field in all the Pet Sets shown above. + +```yaml +annotations: + pod.alpha.kubernetes.io/initialized: "true" +``` + +This field is a debugging hook. It pauses any scale up/down operations on the entire Pet Set. If you'd like to pause a petset after each pet, set it to `false` in the template, wait for each pet to come up, verify it has initialized correctly, and then set it to `true` using `kubectl edit` on the pet (setting it to `false` on *any pet* is enough to pause the Pet Set). If you don't need it, create the Pet Set with it set to `true` as shown. This is surprisingly useful in debugging bootstrapping race conditions. + +## Future Work + +There are a LOT of planned improvements since Pet Set is still in alpha. + +* Data gravity and local storage +* Richer notification events +* Public network identities +* WAN cluster deployments (multi-AZ/region/cloud provider) +* Image and node upgrades + +This list goes on, if you have examples, ideas or thoughts, please contribute. + +## Alternatives + +Deploying one RC of size 1/Service per pod is a popular alternative, as is simply deploying a DaemonSet that utilizes the identity of a Node. + + + diff --git a/docs/user-guide/petset.yaml b/docs/user-guide/petset.yaml new file mode 100644 index 0000000000..017a65746c --- /dev/null +++ b/docs/user-guide/petset.yaml @@ -0,0 +1,56 @@ +# A headless service to create DNS records +apiVersion: v1 +kind: Service +metadata: + name: nginx + labels: + app: nginx +spec: + ports: + - port: 80 + name: web + # *.nginx.default.svc.cluster.local + clusterIP: None + selector: + app: nginx +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: web +spec: + serviceName: "nginx" + replicas: 2 + template: + metadata: + labels: + app: nginx + annotations: + pod.alpha.kubernetes.io/initialized: "true" + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: nginx + image: gcr.io/google_containers/nginx-slim:0.7 + ports: + - containerPort: 80 + name: web + command: + - nginx + args: + - -g + - "daemon off;" + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html + volumeClaimTemplates: + - metadata: + name: www + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + From e4afede634184527540d964e3712ed6be5ba08e8 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Wed, 15 Jun 2016 15:35:20 -0400 Subject: [PATCH 20/31] Add description of protobuf and clarify versioning policy --- docs/api.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index e466962df2..ab8a6a3953 100644 --- a/docs/api.md +++ b/docs/api.md @@ -25,13 +25,15 @@ Complete API details are documented using [Swagger v1.2](http://swagger.io/). Th We also host a version of the [latest API documentation UI](http://kubernetes.io/kubernetes/third_party/swagger-ui/). This is updated with the latest release, so if you are using a different version of Kubernetes you will want to use the spec from your apiserver. +Kubernetes implements an alternative Protobuf based serialization format for the API that is primarily intended for intra-cluster communication, documented in the [design proposal](https://github.com/kubernetes/kubernetes/blob/{{ page.githubbranch }}/docs/proposals/protobuf.md) and the IDL files for each schema are located in the Go packages that define the API objects. + ## API versioning To make it easier to eliminate fields or restructure resource representations, Kubernetes supports multiple API versions, each at a different API path, such as `/api/v1` or `/apis/extensions/v1beta1`. -We chose to version at the API level rather than at the resource or field level to ensure that the API presents a clear, consistent view of system resources and behavior, and to enable controlling access to end-of-lifed and/or experimental APIs. +We chose to version at the API level rather than at the resource or field level to ensure that the API presents a clear, consistent view of system resources and behavior, and to enable controlling access to end-of-lifed and/or experimental APIs. The JSON and Protobuf serialization schemas follow the same guidelines for schema changes - all descriptions below cover both formats. Note that API versioning and Software versioning are only indirectly related. The [API and release versioning proposal](https://github.com/kubernetes/kubernetes/blob/{{page.githubbranch}}/docs/design/versioning.md) describes the relationship between API versioning and From 03dddefd01244c3077b62baa36d31f44566c2f76 Mon Sep 17 00:00:00 2001 From: derekwaynecarr Date: Thu, 23 Jun 2016 11:51:35 -0400 Subject: [PATCH 21/31] Update 1.3 example for quota --- docs/admin/resourcequota/best-effort.yaml | 9 + .../resourcequota/compute-resources.yaml | 11 + docs/admin/resourcequota/index.md | 62 +-- docs/admin/resourcequota/not-best-effort.yaml | 13 + docs/admin/resourcequota/object-counts.yaml | 9 + docs/admin/resourcequota/quota.yaml | 14 - docs/admin/resourcequota/walkthrough.md | 373 +++++++++++++----- 7 files changed, 358 insertions(+), 133 deletions(-) create mode 100644 docs/admin/resourcequota/best-effort.yaml create mode 100644 docs/admin/resourcequota/compute-resources.yaml create mode 100644 docs/admin/resourcequota/not-best-effort.yaml create mode 100644 docs/admin/resourcequota/object-counts.yaml delete mode 100644 docs/admin/resourcequota/quota.yaml diff --git a/docs/admin/resourcequota/best-effort.yaml b/docs/admin/resourcequota/best-effort.yaml new file mode 100644 index 0000000000..5bd6efdcfd --- /dev/null +++ b/docs/admin/resourcequota/best-effort.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: best-effort +spec: + hard: + pods: "10" + scopes: + - BestEffort diff --git a/docs/admin/resourcequota/compute-resources.yaml b/docs/admin/resourcequota/compute-resources.yaml new file mode 100644 index 0000000000..9757018f19 --- /dev/null +++ b/docs/admin/resourcequota/compute-resources.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: compute-resources +spec: + hard: + pods: "4" + requests.cpu: "1" + requests.memory: 1Gi + limits.cpu: "2" + limits.memory: 2Gi diff --git a/docs/admin/resourcequota/index.md b/docs/admin/resourcequota/index.md index 59be97e38e..af96bc538d 100644 --- a/docs/admin/resourcequota/index.md +++ b/docs/admin/resourcequota/index.md @@ -23,7 +23,7 @@ Resource quotas work like this: - If quota is enabled in a namespace for compute resources like `cpu` and `memory`, users must specify requests or limits for those values; otherwise, the quota system may reject pod creation. Hint: Use the LimitRange admission controller to force defaults for pods that make no compute resource requirements. - See the [walkthrough](/docs/admin/resourcequota/walkthrough.md) for an example to avoid this problem. + See the [walkthrough](/docs/admin/resourcequota/walkthrough/) for an example to avoid this problem. Examples of policies that could be created using namespaces and quotas are: @@ -52,31 +52,31 @@ Resource Quota is enforced in a particular namespace when there is a The total sum of [compute resources](/docs/user-guide/compute-resources) requested by pods in a namespace can be limited. The following compute resource types are supported: -| ResourceName | Description | +| Resource Name | Description | | ------------ | ----------- | -| cpu | Across all pods in a non-terminal state, the sum of CPU requests cannot exceed this value. | -| limits.cpu | Across all pods in a non-terminal state, the sum of CPU limits cannot exceed this value. | -| limits.memory | Across all pods in a non-terminal state, the sum of memory limits cannot exceed this value. | -| memory | Across all pods in a non-terminal state, the sum of memory requests cannot exceed this value. | -| requests.cpu | Across all pods in a non-terminal state, the sum of CPU requests cannot exceed this value. | -| requests.memory | Across all pods in a non-terminal state, the sum of memory requests cannot exceed this value. | +| `cpu` | Across all pods in a non-terminal state, the sum of CPU requests cannot exceed this value. | +| `limits.cpu` | Across all pods in a non-terminal state, the sum of CPU limits cannot exceed this value. | +| `limits.memory` | Across all pods in a non-terminal state, the sum of memory limits cannot exceed this value. | +| `memory` | Across all pods in a non-terminal state, the sum of memory requests cannot exceed this value. | +| `requests.cpu` | Across all pods in a non-terminal state, the sum of CPU requests cannot exceed this value. | +| `requests.memory` | Across all pods in a non-terminal state, the sum of memory requests cannot exceed this value. | ## Object Count Quota The number of objects of a given type can be restricted. The following types are supported: -| ResourceName | Description | +| Resource Name | Description | | ------------ | ----------- | -| configmaps | The total number of config maps that can exist in the namespace. | -| persistentvolumeclaims | The total number of [persistent volume claims](/docs/user-guide/persistent-volumes/#persistentvolumeclaims) that can exist in the namespace. | -| pods | The total number of pods in a non-terminal state that can exist in the namespace. A pod is in a terminal state if `status.phase in (Failed, Succeeded)` is true. | -| replicationcontrollers | The total number of replication controllers that can exist in the namespace. | -| resourcequotas | The total number of [resource quotas](/docs/admin/admission-controllers/#resourcequota) that can exist in the namespace. | -| services | The total number of services that can exist in the namespace. | -| services.loadbalancers | The total number of services of type load balancer that can exist in the namespace. | -| services.nodeports | The total number of services of type node port that can exist in the namespace. | -| secrets | The total number of secrets that can exist in the namespace. | +| `configmaps` | The total number of config maps that can exist in the namespace. | +| `persistentvolumeclaims` | The total number of [persistent volume claims](/docs/user-guide/persistent-volumes/#persistentvolumeclaims) that can exist in the namespace. | +| `pods` | The total number of pods in a non-terminal state that can exist in the namespace. A pod is in a terminal state if `status.phase in (Failed, Succeeded)` is true. | +| `replicationcontrollers` | The total number of replication controllers that can exist in the namespace. | +| `resourcequotas` | The total number of [resource quotas](/docs/admin/admission-controllers/#resourcequota) that can exist in the namespace. | +| `services` | The total number of services that can exist in the namespace. | +| `services.loadbalancers` | The total number of services of type load balancer that can exist in the namespace. | +| `services.nodeports` | The total number of services of type node port that can exist in the namespace. | +| `secrets` | The total number of secrets that can exist in the namespace. | For example, `pods` quota counts and enforces a maximum on the number of `pods` created in a single namespace. @@ -95,22 +95,22 @@ Resources specified on the quota outside of the allowed set results in a validat | Scope | Description | | ----- | ----------- | -| Terminating | Match pods where `spec.activeDeadlineSeconds >= 0` | -| NotTerminating | Match pods where `spec.activeDeadlineSeconds is nil` | -| BestEffort | Match pods that have best effort quality of service. | -| NotBestEffort | Match pods that do not have best effort quality of service. | +| `Terminating` | Match pods where `spec.activeDeadlineSeconds >= 0` | +| `NotTerminating` | Match pods where `spec.activeDeadlineSeconds is nil` | +| `BestEffort` | Match pods that have best effort quality of service. | +| `NotBestEffort` | Match pods that do not have best effort quality of service. | -The `BestEffort` scope restricts a quota to tracking the following resources: -* pods +The `BestEffort` scope restricts a quota to tracking the following resource: `pods` The `Terminating`, `NotTerminating`, and `NotBestEffort` scopes restrict a quota to tracking the following resources: -* cpu -* limits.cpu -* limits.memory -* memory -* pods -* requests.cpu -* requests.memory + +* `cpu` +* `limits.cpu` +* `limits.memory` +* `memory` +* `pods` +* `requests.cpu` +* `requests.memory` ## Requests vs Limits diff --git a/docs/admin/resourcequota/not-best-effort.yaml b/docs/admin/resourcequota/not-best-effort.yaml new file mode 100644 index 0000000000..82666058e1 --- /dev/null +++ b/docs/admin/resourcequota/not-best-effort.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: not-best-effort +spec: + hard: + pods: "4" + requests.cpu: "1" + requests.memory: 1Gi + limits.cpu: "2" + limits.memory: 2Gi + scopes: + - NotBestEffort diff --git a/docs/admin/resourcequota/object-counts.yaml b/docs/admin/resourcequota/object-counts.yaml new file mode 100644 index 0000000000..604b909614 --- /dev/null +++ b/docs/admin/resourcequota/object-counts.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: object-counts +spec: + hard: + persistentvolumeclaims: "2" + services.loadbalancers: "2" + services.nodeports: "0" diff --git a/docs/admin/resourcequota/quota.yaml b/docs/admin/resourcequota/quota.yaml deleted file mode 100644 index 6885efc6d1..0000000000 --- a/docs/admin/resourcequota/quota.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: ResourceQuota -metadata: - name: quota -spec: - hard: - cpu: "20" - memory: 1Gi - persistentvolumeclaims: "10" - pods: "10" - replicationcontrollers: "20" - resourcequotas: "1" - secrets: "10" - services: "5" diff --git a/docs/admin/resourcequota/walkthrough.md b/docs/admin/resourcequota/walkthrough.md index eeacbddabf..651f8ae15d 100644 --- a/docs/admin/resourcequota/walkthrough.md +++ b/docs/admin/resourcequota/walkthrough.md @@ -1,12 +1,30 @@ --- --- -This example demonstrates how [resource quota](/docs/admin/admission-controllers/#resourcequota) and -[limitsranger](/docs/admin/admission-controllers/#limitranger) can be applied to a Kubernetes namespace. -See [ResourceQuota design doc](https://github.com/kubernetes/kubernetes/blob/{{page.githubbranch}}/docs/design/admission_control_resource_quota.md) for more information. +This example demonstrates a typical setup to control for resource usage in a namespace. + +It demonstrates using the following resources: + +* [Namespace](/docs/admin/namespaces) +* [Resource Quota](/docs/admin/resourcequota/) +* [Limit Range](/docs/admin/limitrange/) This example assumes you have a functional Kubernetes setup. +## Scenario + +The cluster-admin is operating a cluster on behalf of a user population and the cluster-admin +wants to control the amount of resources that can be consumed in a particular namespace to promote +fair sharing of the cluster and control cost. + +The cluster-admin has the following goals: + +* Limit the amount of compute resource for running pods +* Limit the number of persistent volume claims to control access to storage +* Limit the number of load balancers to control cost +* Prevent the use of node ports to preserve scarce resources +* Provide default compute resource requests to enable better scheduling decisions + ## Step 1: Create a namespace This example will work in a custom namespace to demonstrate the concepts involved. @@ -14,151 +32,330 @@ This example will work in a custom namespace to demonstrate the concepts involve Let's create a new namespace called quota-example: ```shell -$ kubectl create namespace quota-example +$ kubectl create -f docs/admin/resourcequota/namespace.yaml namespace "quota-example" created -``` - -Note that `kubectl` commands will print the type and name of the resource created or mutated, which can then be used in subsequent commands: - -```shell $ kubectl get namespaces NAME STATUS AGE -default Active 50m -quota-example Active 2s +default Active 2m +kube-system Active 2m +quota-example Active 39s ``` -## Step 2: Apply a quota to the namespace +## Step 2: Apply an object-count quota to the namespace -By default, a pod will run with unbounded CPU and memory requests/limits. This means that any pod in the -system will be able to consume as much CPU and memory on the node that executes the pod. +The cluster-admin wants to control the following resources: -Users may want to restrict how much of the cluster resources a given namespace may consume -across all of its pods in order to manage cluster usage. To do this, a user applies a quota to -a namespace. A quota lets the user set hard limits on the total amount of node resources (cpu, memory) -and API resources (pods, services, etc.) that a namespace may consume. In term of resources, Kubernetes -checks the total resource *requests*, not resource *limits* of all containers/pods in the namespace. +* persistent volume claims +* load balancers +* node ports -Let's create a simple quota in our namespace: +Let's create a simple quota that controls object counts for those resource types in this namespace. ```shell -$ kubectl create -f docs/admin/resourcequota/quota.yaml --namespace=quota-example -resourcequota "quota" created +$ kubectl create -f docs/admin/resourcequota/object-counts.yaml --namespace=quota-example +resourcequota "object-counts" created ``` -Once your quota is applied to a namespace, the system will restrict any creation of content -in the namespace until the quota usage has been calculated. This should happen quickly. +The quota system will observe that a quota has been created, and will calculate consumption +in the namespace in response. This should happen quickly. -You can describe your current quota usage to see what resources are being consumed in your -namespace. +Let's describe the quota to see what is currently being consumed in this namespace: ```shell -$ kubectl describe quota quota --namespace=quota-example -Name: quota -Namespace: quota-example -Resource Used Hard --------- ---- ---- -cpu 0 20 -memory 0 1Gi -persistentvolumeclaims 0 10 -pods 0 10 -replicationcontrollers 0 20 -resourcequotas 1 1 -secrets 1 10 -services 0 5 +$ kubectl describe quota object-counts --namespace=quota-example +Name: object-counts +Namespace: quota-example +Resource Used Hard +-------- ---- ---- +persistentvolumeclaims 0 2 +services.loadbalancers 0 2 +services.nodeports 0 0 ``` -## Step 3: Applying default resource requests and limits +The quota system will now prevent users from creating more than the specified amount for each resource. + +## Step 3: Apply a compute-resource quota to the namespace + +To limit the amount of compute resource that can be consumed in this namespace, +let's create a quota that tracks compute resources. + +```shell +$ kubectl create -f docs/admin/resourcequota/compute-resources.yaml --namespace=quota-example +resourcequota "compute-resources" created +``` + +Let's describe the quota to see what is currently being consumed in this namespace: + +```shell +$ kubectl describe quota compute-resources --namespace=quota-example +Name: compute-resources +Namespace: quota-example +Resource Used Hard +-------- ---- ---- +limits.cpu 0 2 +limits.memory 0 2Gi +pods 0 4 +requests.cpu 0 1 +requests.memory 0 1Gi +``` + +The quota system will now prevent the namespace from having more than 4 non-terminal pods. In +addition, it will enforce that each container in a pod makes a `request` and defines a `limit` for +`cpu` and `memory`. + +## Step 4: Applying default resource requests and limits Pod authors rarely specify resource requests and limits for their pods. Since we applied a quota to our project, let's see what happens when an end-user creates a pod that has unbounded cpu and memory by creating an nginx container. -To demonstrate, lets create a Deployment that runs nginx: +To demonstrate, lets create a deployment that runs nginx: ```shell $ kubectl run nginx --image=nginx --replicas=1 --namespace=quota-example deployment "nginx" created ``` -This creates a Deployment "nginx" with its underlying resource, a ReplicaSet, which handles the creation and deletion of Pod replicas. Now let's look at the pods that were created. +Now let's look at the pods that were created. ```shell $ kubectl get pods --namespace=quota-example -NAME READY STATUS RESTARTS AGE ``` -What happened? I have no pods! Let's describe the ReplicaSet managed by the nginx Deployment to get a view of what is happening. -Note that `kubectl describe rs` works only on kubernetes cluster >= v1.2. If you are running older versions, use `kubectl describe rc` instead. -If you want to obtain the old behavior, use `--generator=run/v1` to create replication controllers. See [`kubectl run`](/docs/user-guide/kubectl/kubectl_run/) for more details. +What happened? I have no pods! Let's describe the deployment to get a view of what is happening. ```shell -$ kubectl describe rs -l run=nginx --namespace=quota-example -Name: nginx-2040093540 -Namespace: quota-example -Image(s): nginx -Selector: pod-template-hash=2040093540,run=nginx -Labels: pod-template-hash=2040093540,run=nginx -Replicas: 0 current / 1 desired -Pods Status: 0 Running / 0 Waiting / 0 Succeeded / 0 Failed -No volumes. -Events: - FirstSeen LastSeen Count From SubobjectPath Type Reason Message - --------- -------- ----- ---- ------------- -------- ------ ------- - 48s 26s 4 {replicaset-controller } Warning FailedCreate Error creating: pods "nginx-2040093540-" is forbidden: Failed quota: quota: must specify cpu,memory +$ kubectl describe deployment nginx --namespace=quota-example +Name: nginx +Namespace: quota-example +CreationTimestamp: Mon, 06 Jun 2016 16:11:37 -0400 +Labels: run=nginx +Selector: run=nginx +Replicas: 0 updated | 1 total | 0 available | 1 unavailable +StrategyType: RollingUpdate +MinReadySeconds: 0 +RollingUpdateStrategy: 1 max unavailable, 1 max surge +OldReplicaSets: +NewReplicaSet: nginx-3137573019 (0/1 replicas created) +... ``` -The Kubernetes API server is rejecting the ReplicaSet requests to create a pod because our pods -do not specify any memory usage *request*. +A deployment created a corresponding replica set and attempted to size it to create a single pod. -So let's set some default values for the amount of cpu and memory a pod can consume: +Let's look at the replica set to get more detail. + +```shell +$ kubectl describe rs nginx-3137573019 --namespace=quota-example +Name: nginx-3137573019 +Namespace: quota-example +Image(s): nginx +Selector: pod-template-hash=3137573019,run=nginx +Labels: pod-template-hash=3137573019 + run=nginx +Replicas: 0 current / 1 desired +Pods Status: 0 Running / 0 Waiting / 0 Succeeded / 0 Failed +No volumes. +Events: + FirstSeen LastSeen Count From SubobjectPath Type Reason Message + --------- -------- ----- ---- ------------- -------- ------ ------- + 4m 7s 11 {replicaset-controller } Warning FailedCreate Error creating: pods "nginx-3137573019-" is forbidden: Failed quota: compute-resources: must specify limits.cpu,limits.memory,requests.cpu,requests.memory +``` + +The Kubernetes API server is rejecting the replica set requests to create a pod because our pods +do not specify `requests` or `limits` for `cpu` and `memory`. + +So let's set some default values for the amount of `cpu` and `memory` a pod can consume: ```shell $ kubectl create -f docs/admin/resourcequota/limits.yaml --namespace=quota-example limitrange "limits" created $ kubectl describe limits limits --namespace=quota-example -Name: limits -Namespace: quota-example -Type Resource Min Max Default Request Default Limit Max Limit/Request Ratio ----- -------- --- --- --------------- ------------- ----------------------- -Container cpu - - 100m 200m - -Container memory - - 256Mi 512Mi - +Name: limits +Namespace: quota-example +Type Resource Min Max Default Request Default Limit Max Limit/Request Ratio +---- -------- --- --- --------------- ------------- ----------------------- +Container memory - - 256Mi 512Mi - +Container cpu - - 100m 200m - ``` -Now any time a pod is created in this namespace, if it has not specified any resource request/limit, the default -amount of cpu and memory per container will be applied, and the request will be used as part of admission control. +If the Kubernetes API server observes a request to create a pod in this namespace, and the containers +in that pod do not make any compute resource requests, a default request and default limit will be applied +as part of admission control. -Now that we have applied default resource *request* for our namespace, our Deployment should be able to -create its pods. +In this example, each pod created will have compute resources equivalent to the following: + +```shell +$ kubectl run nginx \ + --image=nginx \ + --replicas=1 \ + --requests=cpu=100m,memory=256Mi \ + --limits=cpu=200m,memory=512Mi \ + --namespace=quota-example +``` + +Now that we have applied default compute resources for our namespace, our replica set should be able to create +its pods. ```shell $ kubectl get pods --namespace=quota-example NAME READY STATUS RESTARTS AGE -nginx-2040093540-miohp 1/1 Running 0 5s +nginx-3137573019-fvrig 1/1 Running 0 6m ``` And if we print out our quota usage in the namespace: ```shell -$ kubectl describe quota quota --namespace=quota-example -Name: quota -Namespace: quota-example -Resource Used Hard --------- ---- ---- -cpu 100m 20 -memory 256Mi 1Gi -persistentvolumeclaims 0 10 -pods 1 10 -replicationcontrollers 1 20 -resourcequotas 1 1 -secrets 1 10 -services 0 5 +$ kubectl describe quota --namespace=quota-example +Name: compute-resources +Namespace: quota-example +Resource Used Hard +-------- ---- ---- +limits.cpu 200m 2 +limits.memory 512Mi 2Gi +pods 1 4 +requests.cpu 100m 1 +requests.memory 256Mi 1Gi + + +Name: object-counts +Namespace: quota-example +Resource Used Hard +-------- ---- ---- +persistentvolumeclaims 0 2 +services.loadbalancers 0 2 +services.nodeports 0 0 ``` -You can now see the pod that was created is consuming explicit amounts of resources (specified by resource *request*), and the usage is being tracked by the Kubernetes system properly. +As you can see, the pod that was created is consuming explict amounts of compute resources, and the usage is being +tracked by Kubernetes properly. + +## Step 5: Advanced quota scopes + +Let's imagine you did not want to specify default compute resource consumption in your namespace. + +Instead, you want to let users run a specific number of `BestEffort` pods in their namespace to take +advantage of slack compute resources, and then require that users make an explicit resource request for +pods that require a higher quality of service. + +Let's create a new namespace with two quotas to demonstrate this behavior: + +```shell +$ kubectl create namespace quota-scopes +namespace "quota-scopes" created +$ kubectl create -f docs/admin/resourcequota/best-effort.yaml --namespace=quota-scopes +resourcequota "best-effort" created +$ kubectl create -f docs/admin/resourcequota/not-best-effort.yaml --namespace=quota-scopes +resourcequota "not-best-effort" created +$ kubectl describe quota --namespace=quota-scopes +Name: best-effort +Namespace: quota-scopes +Scopes: BestEffort + * Matches all pods that have best effort quality of service. +Resource Used Hard +-------- ---- ---- +pods 0 10 + + +Name: not-best-effort +Namespace: quota-scopes +Scopes: NotBestEffort + * Matches all pods that do not have best effort quality of service. +Resource Used Hard +-------- ---- ---- +limits.cpu 0 2 +limits.memory 0 2Gi +pods 0 4 +requests.cpu 0 1 +requests.memory 0 1Gi +``` + +In this scenario, a pod that makes no compute resource requests will be tracked by the `best-effort` quota. + +A pod that does make compute resource requests will be tracked by the `not-best-effort` quota. + +Let's demonstrate this by creating two deployments: + +```shell +$ kubectl run best-effort-nginx --image=nginx --replicas=8 --namespace=quota-scopes +deployment "best-effort-nginx" created +$ kubectl run not-best-effort-nginx \ + --image=nginx \ + --replicas=2 \ + --requests=cpu=100m,memory=256Mi \ + --limits=cpu=200m,memory=512Mi \ + --namespace=quota-scopes +deployment "not-best-effort-nginx" created +``` + +Even though no default limits were specified, the `best-effort-nginx` deployment will create +all 8 pods. This is because it is tracked by the `best-effort` quota, and the `not-best-effort` +quota will just ignore it. The `not-best-effort` quota will track the `not-best-effort-nginx` +deployment since it creates pods with `Burstable` quality of service. + +Let's list the pods in the namespace: + +```shell +$ kubectl get pods --namespace=quota-scopes +NAME READY STATUS RESTARTS AGE +best-effort-nginx-3488455095-2qb41 1/1 Running 0 51s +best-effort-nginx-3488455095-3go7n 1/1 Running 0 51s +best-effort-nginx-3488455095-9o2xg 1/1 Running 0 51s +best-effort-nginx-3488455095-eyg40 1/1 Running 0 51s +best-effort-nginx-3488455095-gcs3v 1/1 Running 0 51s +best-effort-nginx-3488455095-rq8p1 1/1 Running 0 51s +best-effort-nginx-3488455095-udhhd 1/1 Running 0 51s +best-effort-nginx-3488455095-zmk12 1/1 Running 0 51s +not-best-effort-nginx-2204666826-7sl61 1/1 Running 0 23s +not-best-effort-nginx-2204666826-ke746 1/1 Running 0 23s +``` + +As you can see, all 10 pods have been allowed to be created. + +Let's describe current quota usage in the namespace: + +```shell +$ kubectl describe quota --namespace=quota-scopes +Name: best-effort +Namespace: quota-scopes +Scopes: BestEffort + * Matches all pods that have best effort quality of service. +Resource Used Hard +-------- ---- ---- +pods 8 10 + + +Name: not-best-effort +Namespace: quota-scopes +Scopes: NotBestEffort + * Matches all pods that do not have best effort quality of service. +Resource Used Hard +-------- ---- ---- +limits.cpu 400m 2 +limits.memory 1Gi 2Gi +pods 2 4 +requests.cpu 200m 1 +requests.memory 512Mi 1Gi +``` + +As you can see, the `best-effort` quota has tracked the usage for the 8 pods we created in +the `best-effort-nginx` deployment, and the `not-best-effort` quota has tracked the usage for +the 2 pods we created in the `not-best-effort-nginx` quota. + +Scopes provide a mechanism to subdivide the set of resources that are tracked by +any quota document to allow greater flexibility in how operators deploy and track resource +consumption. + +In addition to `BestEffort` and `NotBestEffort` scopes, there are scopes to restrict +long-running versus time-bound pods. The `Terminating` scope will match any pod +where `spec.activeDeadlineSeconds is not nil`. The `NotTerminating` scope will match any pod +where `spec.activeDeadlineSeconds is nil`. These scopes allow you to quota pods based on their +anticipated permanence on a node in your cluster. ## Summary -Actions that consume node resources for cpu and memory can be subject to hard quota limits defined by the namespace quota. The resource consumption is measured by resource *request* in pod specification. +Actions that consume node resources for cpu and memory can be subject to hard quota limits defined by the namespace quota. Any action that consumes those resources can be tweaked, or can pick up namespace level defaults to meet your end goal. + +Quota can be apportioned based on quality of service and anticipated permanence on a node in your cluster. \ No newline at end of file From c1206d7b5398eca1c3b036deff0adee29403a708 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Sat, 4 Jun 2016 10:23:57 -0700 Subject: [PATCH 22/31] First commit of network policy user guide --- docs/user-guide/networkpolicies.md | 82 ++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 docs/user-guide/networkpolicies.md diff --git a/docs/user-guide/networkpolicies.md b/docs/user-guide/networkpolicies.md new file mode 100644 index 0000000000..06824c692c --- /dev/null +++ b/docs/user-guide/networkpolicies.md @@ -0,0 +1,82 @@ +--- +--- + +* TOC +{:toc} + +## What is a _Network Policy_? + +A Network Policy is a specification of how groupings of pods are allowed to communicate with each other and other network endpoints. + +NetworkPolicy resources use labels to select pods and define whitelist rules which allow traffic to the selected pods in addition to what is allowed by the ingress isolation policy for a given namespace. + +## Prerequisites +Before you start using the NetworkPolicy resource, there are a few things to understand. The NetworkPolicy resource is a beta resource and is +not available in any Kubernetes release prior to 1.3. + +You must enable the `extensions/v1beta/networkpolicies` runtime config in your apiserver to enable this resource. + +You must also be using a networking solution which supports Network Policy - simply creating the +resource without a controller to implement it will have no effect. + +## Configuring Namespace Isolation Policy +Ingress isolation can be configured on a per-namespace basis. Once ingress isolation is configured on a namespace it will be applied to all pods in that namespace. + +Currently the following ingress isolation types are supported: + +- _DefaultDeny_: Pods in the namespace will be inaccessible from any source except the pod's local node. + +Ingress isolation can be enabled using an annotation on the Namespace. + +```yaml +kind: Namespace +apiVersion: v1 +metadata: + annotations: + net.beta.kubernetes.io/network-policy: | + { + "ingress": { + "isolation": "DefaultDeny" + } + } +``` + +To configure the annotation via `kubectl`: + +``` +kubectl annotate ns "net.beta.kubernetes.io/networkpolicy={\"ingress\": {\"isolation\": \"DefaultDeny\"}}" +``` + +## The NetworkPolicy Resource +A minimal `NetworkPolicy` might look like this: + +```yaml +01. apiVersion: extensions/v1beta1 +02. kind: NetworkPolicy +03. metadata: +04. name: test-network-policy +05. spec: +06. podSelector: +07. matchLabels: +08. role: db +09. ingress: +10. - from: +11. podSelector: +12. matchLabels: +13. role: frontend +14. ports: +15. - protocol: tcp +16. port: 6379 +``` + +*POSTing this to the API server will have no effect unless your chosen networking solution supports network policy.* + +__Lines 1-4__: As with all other Kubernetes config, a NetworkPolicy needs `apiVersion`, `kind`, and `metadata` fields. For general information about working with config files, see [here](/docs/user-guide/simple-yaml), [here](/docs/user-guide/configuring-containers), and [here](/docs/user-guide/working-with-resources). + +__Lines 5-9__: NetworkPolicy [spec](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/docs/devel/api-conventions.md#spec-and-status) has all the information needed to configure a loadbalancer or proxy server. Most importantly, it contains a list of rules matched against all incoming requests. Currently the Ingress resource only supports http rules. + +__Lines 6-8__: Each NetworkPolicy includes a `podSelector` which selects the grouping of pods to which the `ingress` rules in the policy apply. + +__Lines 9-16__: Each NetworkPolicy includes a list of whitelist `ingress` rules. Each rule allows traffic which matches both the `from` and `ports` sections. + +__Complete Specification__: See the [api-reference](https://kubernetes.github.io/docs/api-reference/extensions/v1beta1/definitions/#_v1beta1_networkpolicy) for a full definition of the resource. From cb6f85b484f88308300a191c93bdaa857beb96c7 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Mon, 27 Jun 2016 15:58:08 -0700 Subject: [PATCH 23/31] Code review comments --- _data/reference.yml | 2 + docs/user-guide/networkpolicies.md | 68 +++++++++++++++--------------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/_data/reference.yml b/_data/reference.yml index e2439cc78b..f337e158b1 100644 --- a/_data/reference.yml +++ b/_data/reference.yml @@ -226,6 +226,8 @@ toc: path: /docs/user-guide/replicasets/ - title: Pet Sets path: /docs/user-guide/petset/ + - title: NetworkPolicy Resources + path: /docs/user-guide/networkpolicies/ - title: Kubernetes Design Docs section: diff --git a/docs/user-guide/networkpolicies.md b/docs/user-guide/networkpolicies.md index 06824c692c..7050b9df81 100644 --- a/docs/user-guide/networkpolicies.md +++ b/docs/user-guide/networkpolicies.md @@ -4,27 +4,24 @@ * TOC {:toc} -## What is a _Network Policy_? +A network policy is a specification of how selections of pods are allowed to communicate with each other and other network endpoints. -A Network Policy is a specification of how groupings of pods are allowed to communicate with each other and other network endpoints. - -NetworkPolicy resources use labels to select pods and define whitelist rules which allow traffic to the selected pods in addition to what is allowed by the ingress isolation policy for a given namespace. +`NetworkPolicy` resources use labels to select pods and define whitelist rules which allow traffic to the selected pods in addition to what is allowed by the isolation policy for a given namespace. ## Prerequisites -Before you start using the NetworkPolicy resource, there are a few things to understand. The NetworkPolicy resource is a beta resource and is -not available in any Kubernetes release prior to 1.3. You must enable the `extensions/v1beta/networkpolicies` runtime config in your apiserver to enable this resource. -You must also be using a networking solution which supports Network Policy - simply creating the +You must also be using a networking solution which supports `NetworkPolicy` - simply creating the resource without a controller to implement it will have no effect. ## Configuring Namespace Isolation Policy -Ingress isolation can be configured on a per-namespace basis. Once ingress isolation is configured on a namespace it will be applied to all pods in that namespace. -Currently the following ingress isolation types are supported: +Isolation can be configured on a per-namespace basis. Once isolation is configured on a namespace it will be applied to all pods in that namespace. Currently, only isolation policy on inbound traffic (ingress) can be defined. -- _DefaultDeny_: Pods in the namespace will be inaccessible from any source except the pod's local node. +The following ingress isolation types being supported: + +- `DefaultDeny`: Pods in the namespace will be inaccessible from any source except the pod's local node. Ingress isolation can be enabled using an annotation on the Namespace. @@ -43,40 +40,41 @@ metadata: To configure the annotation via `kubectl`: -``` +```shell{% raw %} kubectl annotate ns "net.beta.kubernetes.io/networkpolicy={\"ingress\": {\"isolation\": \"DefaultDeny\"}}" -``` +{% endraw %}``` + +## The `NetworkPolicy` Resource + +See the [api-reference](docs/api-reference/extensions/v1beta1/definitions/#_v1beta1_networkpolicy) for a full definition of the resource. -## The NetworkPolicy Resource A minimal `NetworkPolicy` might look like this: ```yaml -01. apiVersion: extensions/v1beta1 -02. kind: NetworkPolicy -03. metadata: -04. name: test-network-policy -05. spec: -06. podSelector: -07. matchLabels: -08. role: db -09. ingress: -10. - from: -11. podSelector: -12. matchLabels: -13. role: frontend -14. ports: -15. - protocol: tcp -16. port: 6379 +apiVersion: extensions/v1beta1 +kind: NetworkPolicy +metadata: + name: test-network-policy +spec: + podSelector: + matchLabels: + role: db + ingress: + - from: + podSelector: + matchLabels: + role: frontend + ports: + - protocol: tcp + port: 6379 ``` *POSTing this to the API server will have no effect unless your chosen networking solution supports network policy.* -__Lines 1-4__: As with all other Kubernetes config, a NetworkPolicy needs `apiVersion`, `kind`, and `metadata` fields. For general information about working with config files, see [here](/docs/user-guide/simple-yaml), [here](/docs/user-guide/configuring-containers), and [here](/docs/user-guide/working-with-resources). +__Mandatory Fields__: As with all other Kubernetes config, a `NetworkPolicy` needs `apiVersion`, `kind`, and `metadata` fields. For general information about working with config files, see [here](/docs/user-guide/simple-yaml), [here](/docs/user-guide/configuring-containers), and [here](/docs/user-guide/working-with-resources). -__Lines 5-9__: NetworkPolicy [spec](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/docs/devel/api-conventions.md#spec-and-status) has all the information needed to configure a loadbalancer or proxy server. Most importantly, it contains a list of rules matched against all incoming requests. Currently the Ingress resource only supports http rules. +__spec__: `NetworkPolicy` [spec](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/docs/devel/api-conventions.md#spec-and-status) has all the information needed to define a network isolation policy in the deployed controller. -__Lines 6-8__: Each NetworkPolicy includes a `podSelector` which selects the grouping of pods to which the `ingress` rules in the policy apply. +__podSelector__: Each `NetworkPolicy` includes a `podSelector` which selects the grouping of pods to which the `ingress` rules in the policy apply. -__Lines 9-16__: Each NetworkPolicy includes a list of whitelist `ingress` rules. Each rule allows traffic which matches both the `from` and `ports` sections. - -__Complete Specification__: See the [api-reference](https://kubernetes.github.io/docs/api-reference/extensions/v1beta1/definitions/#_v1beta1_networkpolicy) for a full definition of the resource. +__ingress__: Each `NetworkPolicy` includes a list of whitelist `ingress` rules. Each rule allows traffic which matches both the `from` and `ports` sections. From ad53cd7423f5f98f1ee3a4fd4f404b58345cd6bf Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 28 Jun 2016 13:23:30 -0700 Subject: [PATCH 24/31] Fix title --- _data/reference.yml | 2 +- docs/user-guide/networkpolicies.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/reference.yml b/_data/reference.yml index f337e158b1..a11214565a 100644 --- a/_data/reference.yml +++ b/_data/reference.yml @@ -226,7 +226,7 @@ toc: path: /docs/user-guide/replicasets/ - title: Pet Sets path: /docs/user-guide/petset/ - - title: NetworkPolicy Resources + - title: Network Policies path: /docs/user-guide/networkpolicies/ - title: Kubernetes Design Docs diff --git a/docs/user-guide/networkpolicies.md b/docs/user-guide/networkpolicies.md index 7050b9df81..e550657df6 100644 --- a/docs/user-guide/networkpolicies.md +++ b/docs/user-guide/networkpolicies.md @@ -46,7 +46,7 @@ kubectl annotate ns "net.beta.kubernetes.io/networkpolicy={\"ingress ## The `NetworkPolicy` Resource -See the [api-reference](docs/api-reference/extensions/v1beta1/definitions/#_v1beta1_networkpolicy) for a full definition of the resource. +See the [api-reference](/docs/api-reference/extensions/v1beta1/definitions/#_v1beta1_networkpolicy) for a full definition of the resource. A minimal `NetworkPolicy` might look like this: From 7f5008d1b99534d5e4f69ba01f70da9d35e213e7 Mon Sep 17 00:00:00 2001 From: nikhiljindal Date: Mon, 13 Jun 2016 21:13:22 -0700 Subject: [PATCH 25/31] Adding a doc with instructions to setup federation --- _data/guides.yml | 2 + docs/admin/federation/index.md | 194 +++++++++++++++++++++++++++++++++ docs/admin/multi-cluster.md | 9 +- 3 files changed, 199 insertions(+), 6 deletions(-) create mode 100644 docs/admin/federation/index.md diff --git a/_data/guides.yml b/_data/guides.yml index f5654a6efd..6ba05ee6a0 100644 --- a/_data/guides.yml +++ b/_data/guides.yml @@ -243,6 +243,8 @@ toc: path: /docs/admin/cluster-components/ - title: Configuring Kubernetes Use of etcd path: /docs/admin/etcd/ + - title: Federating Clusters + path: /docs/admin/federation/ - title: Using Multiple Clusters path: /docs/admin/multi-cluster/ - title: Changing Cluster Size diff --git a/docs/admin/federation/index.md b/docs/admin/federation/index.md new file mode 100644 index 0000000000..dee50e33b4 --- /dev/null +++ b/docs/admin/federation/index.md @@ -0,0 +1,194 @@ +--- +--- +This guide explains how to set up cluster federation that lets us control multiple Kubernetes clusters. + + +* TOC +{:toc} + +## Prerequisites + +This guide assumes that we have a running Kubernetes cluster. +If not, then head over to the [getting started guides](/docs/getting-started-guides/) to bring up a cluster. + +This guide also assumes that we have the Kubernetes source code that can be +[downloaded from here](/docs/getting-started-guides/binary_release/). + +## Setting up a federation control plane + +Setting up federation requires running the federation control plane which +consists of etcd, federation-apiserver and federation-controller-manager. +We can run these binaries as pods on an existing Kubernetes cluster. + +### Getting images + +To run these as pods, we first need images for all the components. We can use +official release images or we can build from HEAD. + +#### Using official release images + +As part of every release, images are pushed to `gcr.io/google_containers`. To use +these images, we set env var `FEDERATION_PUSH_REPO_BASE=gcr.io/google_containers` +This will always use the latest image. +To use federation-apiserver and federation-controller-manager images from a specific release, we can set `FEDERATION_IMAGE_TAG`. + +#### Building and pushing images from HEAD + +To run the code from HEAD, we need to build and push our own images. +We can build the images using the following command: + +```shell +$ FEDERATION=true KUBE_RELEASE_RUN_TESTS=n make quick-release +``` + +Next, we need to push these images to a registry such as Google Container Registry or Docker Hub, so that our cluster can pull them. +If Kubernetes cluster is running on Google Compute Engine (GCE), then we can push the images to `gcr.io/`. +The command to push the images will look like: + +```shell +$ FEDERATION=true FEDERATION_PUSH_REPO_BASE=gcr.io/ ./build/push-federation-images.sh +``` + +### Running the federation control plane + +Once we have the images, we can run these as pods on our existing kubernetes cluster. +The command to run these pods on an existing GCE cluster will look like: + +```shell +$ KUBERNETES_PROVIDER=gce FEDERATION_DNS_PROVIDER=google-clouddns FEDERATION_NAME=myfederation DNS_ZONE_NAME=myfederation.example FEDERATION_PUSH_REPO_BASE=gcr.io/google_containers ./federation/cluster/federation-up.sh +``` + +`KUBERNETES_PROVIDER` is the cloud provider. + +`FEDERATION_DNS_PROVIDER` can be `google-clouddns` or `aws-route53`. It will be +set appropriately if it is missing and `KUBERNETES_PROVIDER` is one of `gce`, `gke` and `aws`. +This is used to resolve DNS requests for federation services. The service +controller keeps DNS records with the provider updated as services/pods are +updated in underlying kubernetes clusters. + +`FEDERATION_NAME` is a name we can choose for our federation. This is the name that will appear in DNS routes. + +`DNS_ZONE_NAME` is the domain to be used for DNS records. This is a domain that we +need to buy and then configure it such that DNS queries for that domain are +routed to the appropriate provider as per `FEDERATION_DNS_PROVIDER`. + +Running that command creates a namespace `federation` and creates 2 deployments: `federation-apiserver` and `federation-controller-manager`. +We can verify that the pods are available by running the following command: + +```shell +$ kubectl get deployments --namespace=federation +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +federation-apiserver 1 1 1 1 1m +federation-controller-manager 1 1 1 1 1m +``` + +Running `federation-up.sh` also creates a new record in our kubeconfig for us +to be able to talk to federation apiserver. We can view this by running +`kubectl config view`. + +Note: `federation-up.sh` creates the federation-apiserver pod with an etcd container that is not backed by any +persistent volume, so it is possible to lose data. You can edit +`federation/manifests/federation-apiserver-deployment.yaml` to suit your needs, +if required. + +## Registering Kubernetes clusters for federation + +Now that we have the federation control plane up and running, we can start registering Kubernetes clusters. + +First of all, we need to create a secret containing kubeconfig for that Kubernetes cluster, which federation control plane will use to talk to that Kubernetes cluster. +For now, we create this secret in the host Kubernetes cluster (that hosts federation control plane). When we start supporting secrets in federation control plane, we will create this secret there. +Suppose that our kubeconfig for Kubernetes cluster is at `/cluster1/kubeconfig`, we can run the following command to create the secret: + +```shell +$ kubectl create secret generic cluster1 --namespace=federation --from-file=/cluster1/kubeconfig +``` + +Note that the file name should be `kubeconfig` since file name determines the name of the key in the secret. + +Now that the secret is created, we are ready to register the cluster. The YAML file for cluster will look like: + +```yaml +apiVersion: v1beta1 +kind: Cluster +metadata: + name: cluster1 +spec: + serverAddressByClientCIDRs: + - clientCIDR: + serverAddress: + secretRef: + name: +``` + +We need to insert the appropriate values for ``, `` and ``. +`` here is name of the secret that we just created. +serverAddressByClientCIDRs contains the various server addresses that clients +can use as per their CIDR. We can set the server's public IP address with CIDR +`"0.0.0.0/0"` which all clients will match. In addition, if we want internal +clients to use server's clusterIP, we can set that as serverAddress. The client +CIDR in that case will be a CIDR that only matches IPs of pods running in that +cluster. + +Assuming our YAML file is located at `/cluster1/cluster.yaml`, we can run the following command to register this cluster: + +```shell +$ kubectl create -f /cluster1/cluster.yaml --cluster=federation-cluster + +``` + +By specifying `--cluster=federation-cluster`, we direct the request to federation apiserver. +we can ensure that the cluster registration was successful by running: + +```shell +$ kubectl get clusters --context=federation-cluster +NAME STATUS VERSION AGE +cluster1 Ready 3m +``` + +### Updating KubeDNS + +Once the cluster is registered with the federation, we are all ready to use it. +But for the cluster to be able to route federation service requests, we need to restart +KubeDNS and pass it a `--federations` flag which tells it about valid federation DNS hostnames. +Format of the flag is like this: + +``` +--federations=${FEDERATION_NAME}=${DNS_DOMAIN_NAME} +``` + +To update KubeDNS with federations flag, we can edit the existing kubedns replication controller to +include that flag in pod template spec and then delete the existing pod. Replication controller will +recreate the pod with updated template. + +To find the name of existing kubedns replication controller, run + +```shell +$ kubectl get rc --namespace=kube-system +``` + +This will list all the replication controllers. Name of the kube-dns replication +controller will look like `kube-dns-v18`. You can then edit it by running: + +```shell +$ kubectl edit rc --namespace=kube-system +``` +Add the `--federations` flag as args to kube-dns container in the YAML file that +pops up after running the above command. + +To delete the existing kube dns pod, you can first find it by running: + +```shell +$ kubectl get pods --namespace=kube-system +``` + +And then delete it by running: + +```shell +$ kubectl delete pods --namespace=kube-system +``` + +We are now all set to start using federation. + +## For more information + + * [Federation proposal](https://github.com/kubernetes/kubernetes/blob/{{page.githubbranch}}/docs/proposals/federation.md) details use cases that motivated this work. diff --git a/docs/admin/multi-cluster.md b/docs/admin/multi-cluster.md index 47e433e29b..e549a4d3ca 100644 --- a/docs/admin/multi-cluster.md +++ b/docs/admin/multi-cluster.md @@ -5,9 +5,7 @@ You may want to set up multiple Kubernetes clusters, both to have clusters in different regions to be nearer to your users, and to tolerate failures and/or invasive maintenance. This document describes some of the issues to consider when making a decision about doing so. -Note that at present, -Kubernetes does not offer a mechanism to aggregate multiple clusters into a single virtual cluster. However, -we [plan to do this in the future](https://github.com/kubernetes/kubernetes/blob/{{page.githubbranch}}/docs/proposals/federation.md). +If you decide to have multiple clusters, kubernetes provides a way to [federate them](/docs/admin/federation/) ## Scope of a single cluster @@ -56,11 +54,10 @@ users in the event of a cluster failure), then you need to have `R * (U + 1)` cl (`U + 1` in each of `R` regions). In any case, try to put each cluster in a different zone. Finally, if any of your clusters would need more than the maximum recommended number of nodes for a Kubernetes cluster, then -you may need even more clusters. Kubernetes v1.0 currently supports clusters up to 100 nodes in size, but we are targeting -1000-node clusters by early 2016. +you may need even more clusters. Kubernetes v1.3 supports clusters up to 1000 nodes in size. ## Working with multiple clusters When you have multiple clusters, you would typically create services with the same config in each cluster and put each of those service instances behind a load balancer (AWS Elastic Load Balancer, GCE Forwarding Rule or HTTP Load Balancer) spanning all of them, so that -failures of a single cluster are not visible to end users. \ No newline at end of file +failures of a single cluster are not visible to end users. From 456daa13911688da8a8c16ba17c74154c35974cd Mon Sep 17 00:00:00 2001 From: Josh Wood Date: Mon, 27 Jun 2016 15:22:21 -0700 Subject: [PATCH 26/31] docs/getting-started/rkt: Add stage1 section, update, rewrite Update the rkt getting started guide: Add information about rkt stage1 images. Fix broken/outdated links. Fix the heading hierarchy. Edit language/clarity. Reformat markdown source for plaintext legibility. Addresses prior reviews on and supersedes #725. --- docs/getting-started-guides/rkt/index.md | 203 ++++++++++++++--------- docs/getting-started-guides/rkt/notes.md | 50 +++--- 2 files changed, 152 insertions(+), 101 deletions(-) diff --git a/docs/getting-started-guides/rkt/index.md b/docs/getting-started-guides/rkt/index.md index ac79c3ac7b..00d5764c83 100644 --- a/docs/getting-started-guides/rkt/index.md +++ b/docs/getting-started-guides/rkt/index.md @@ -1,29 +1,38 @@ --- --- -This document describes how to run Kubernetes using [rkt](https://github.com/coreos/rkt) as a container runtime. +This document describes how to run Kubernetes using [rkt](https://github.com/coreos/rkt) as the container runtime. -### Prerequisite +* TOC +{:toc} -- [systemd](http://www.freedesktop.org/wiki/Software/systemd/) should be installed on the machine and should be enabled. - The minimum version required for Kubernetes 1.3 is `219`. - *(Note that systemd is not required by rkt itself, we are using it here to monitor and manage the pods launched by kubelet.)* +## Prerequisites -- Install the latest rkt release according to the instructions [here](https://github.com/coreos/rkt). - The minimum version required is [v1.9.1](https://github.com/coreos/rkt/releases/tag/v1.9.1). +* [Systemd](http://www.freedesktop.org/wiki/Software/systemd/) must be installed and enabled. The minimum systemd version required for Kubernetes v1.3 is `219`. Systemd is used to monitor and manage the pods on each node. -- The [rkt API service](http://coreos.com/rkt/docs/latest/subcommands/api-service.html) must be running on the node. +* [Install the latest rkt release](https://coreos.com/rkt/docs/latest/trying-out-rkt.html). The minimum rkt version required is [v1.9.1](https://github.com/coreos/rkt/releases/tag/v1.9.1). The [CoreOS Linux alpha channel](https://coreos.com/releases/) ships with a recent rkt release, and you can easily [upgrade rkt on CoreOS](https://coreos.com/rkt/docs/latest/install-rkt-in-coreos.html), if necessary. -### Setup network +* The [rkt API service](https://coreos.com/rkt/docs/latest/subcommands/api-service.html) must be running on the node. -You can configure the Kubernetes networking using its own `kubenet` and `CNI` [network -plugins](http://kubernetes.io/docs/admin/network-plugins/) by setting the kubelet's `--network-plugin` and `--network-plugin-dir` flag. -In addition, rkt supports using rkt's [Contained Networking](https://coreos.com/rkt/docs/latest/networking.html#contained-mode). +## Pod networking in rktnetes -##### Use rkt's Contained Networking +### Kubernetes CNI networking -In this mode, rkt will attempt to join pods into a network named `rkt.kubernetes.io`. -To use rkt's contained networking, you can leave the `--network-plugin` to empty, and put a network config file under one of the rkt's [config directories](https://github.com/coreos/rkt/blob/master/Documentation/configuration.md#command-line-flags), for example: +You can configure Kubernetes pod networking with the usual Container Network Interface (CNI) [network plugins](/docs/admin/network-plugins/) by setting the kubelet's `--network-plugin` and `--network-plugin-dir` options appropriately. Configured in this fashion, the rkt container engine will be unaware of network details, and expects to connect pods to the provided subnet. + +#### kubenet: Google Compute Engine (GCE) network + +The `kubenet` plugin can be selected with the kubelet option `--network-plugin=kubenet`. This plugin is currently only supported on GCE. When using kubenet, Kubernetes CNI creates and manages the network, and rkt is provided with a subnet from a bridge device connected to the GCE network. + +### rkt contained network + +Rather than delegating pod networking to Kubernetes, rkt can configure connectivity directly with its own [*contained network*](https://coreos.com/rkt/docs/latest/networking/overview.html#contained-mode) on a subnet provided by a bridge device, the flannel SDN, or another CNI plugin. Configured this way, rkt looks in its [config directories](https://coreos.com/rkt/docs/latest/configuration.html#command-line-flags), usually `/etc/rkt/net.d`, to discover the CNI configuration and invoke the appropriate plugins to create the pod network. + +#### rkt contained network with bridge + +The *contained network* is rkt's default, so you can leave the kubelet's `--network-plugin` option empty to select this network. The contained network can be backed by any CNI plugin. With the *contained network*, rkt will attempt to join pods to a network named `rkt.kubernetes.io`, so this network name must be used for whatever desired CNI configuration. + +When using the contained network, create a network configuration file beneath the rkt network config directory that defines how to create this `rkt.kubernetes.io` network in your environment. This example sets up a bridge device with the `bridge` CNI plugin: ```shell $ cat </etc/rkt/net.d/k8s_network_example.conf @@ -47,16 +56,9 @@ $ cat </etc/rkt/net.d/k8s_network_example.conf EOF ``` -However, there are a small number of caveats you should be aware of when using rkt's networking: +#### rkt contained network with flannel -* You must create an appropriate CNI configuration file with a network name of `rkt.kubernetes.io`. -* The downwards API and environment variable substitution will not contain the pod IP. -* The `/etc/hosts` file will not contain your own hostname (though `/etc/hostname` is populated). - -##### Use flannel - -While it's recommended that you configure flannel using kubernetes' CNI support, you can also configure it using rkt's contained networking. -An example flannel/CNI config file looks like this: +While it is recommended to operate flannel through the Kubernetes CNI support, you can alternatively configure the flannel plugin directly to provide the subnet for rkt's contained network. An example CNI/flannel config file looks like this: ```shell $ cat </etc/rkt/net.d/k8s_flannel_example.conf @@ -70,45 +72,46 @@ $ cat </etc/rkt/net.d/k8s_flannel_example.conf EOF ``` -For more information on flannel configuration, please read [CNI/flannel README](https://github.com/containernetworking/cni/blob/master/Documentation/flannel.md). +For more information on flannel configuration, see the [CNI/flannel README](https://github.com/containernetworking/cni/blob/master/Documentation/flannel.md). -##### Use Google Compute Engine (GCE) network +#### Contained network caveats: -Each VM on GCE has an additional 256 IP addresses routed to it, so it is possible to forego flannel in smaller clusters. -This can most easily be done by using the builtin kubenet plugin, by setting the kubelet flag `--network-plugin=kubenet`. +* You must create an appropriate CNI configuration file with a network name of `rkt.kubernetes.io`. +* The downwards API and environment variable substitution will not contain the pod IP address. +* The `/etc/hosts` file will not contain the pod's own hostname, although `/etc/hostname` is populated. -### Launch a local cluster +## Running rktnetes -To use rkt as the container runtime, we need to supply the following flags to kubelet: +### Spin up a local Kubernetes cluster with the rkt runtime -- `--container-runtime=rkt` chooses the container runtime to use. -- `--rkt-api-endpoint=HOST:PORT` sets the endpoint of the rkt API service. - Leave empty to use the default one (`localhost:15441`). -- `--rkt-path=$PATH_TO_RKT_BINARY` sets the path of rkt binary. - Leave empty to use the first rkt in $PATH. -- `--rkt-stage1-image` sets the name of the stage1 image, e.g. coreos.com/rkt/stage1-coreos. - Leave empty to use the default stage1 image in the rkt's configuration. +To use rkt as the container runtime in a local Kubernetes cluster, supply the following flags to the kubelet: -If you are using the [hack/local-up-cluster.sh](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/hack/local-up-cluster.sh) script to launch the local cluster, then you can edit the environment variable `CONTAINER_RUNTIME`, `RKT_PATH` and `RKT_STAGE1_IMAGE` to set these flags, the `RKT_PATH` and `RKT_STAGE1_IMAGE` are optional if you have `rkt` in your `$PATH` with appropriate configuration. +* `--container-runtime=rkt` Set the node's container runtime to rkt. +* `--rkt-api-endpoint=HOST:PORT` Set the endpoint of the rkt API service. Default: `localhost:15441`. +* `--rkt-path=PATH_TO_RKT_BINARY` Set the path of the rkt binary. Optional. If empty, look for `rkt` in `$PATH`. +* `--rkt-stage1-image=STAGE1` Set the name of the stage1 image, e.g. `coreos.com/rkt/stage1-coreos`. Optional. If not set, the default Linux kernel software isolation stage1 is used. + +If you are using the [hack/local-up-cluster.sh](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/hack/local-up-cluster.sh) script to launch the cluster, you can edit the environment variables `CONTAINER_RUNTIME`, `RKT_PATH`, and `RKT_STAGE1_IMAGE` to set these flags. `RKT_PATH` and `RKT_STAGE1_IMAGE` are optional if `rkt` is in your $PATH` with appropriate configuration. ```shell $ export CONTAINER_RUNTIME=rkt -$ export RKT_PATH=$PATH_TO_RKT_BINARY -$ export RKT_STAGE1_IMAGE=$NAME_OF_THE_STAGE1_IMAGE +$ export RKT_PATH= +$ export RKT_STAGE1_IMAGE= ``` -Then we can launch the local cluster using the script: +Now you can launch the cluster using the `local-up-cluster.sh` script: ```shell $ hack/local-up-cluster.sh ``` -We are also working on setting up rkt as the container runtime for [minikube](https://github.com/kubernetes/minikube/issues/168). +We are also working on getting rkt working as the container runtime in [minikube](https://github.com/kubernetes/minikube/issues/168). -### Launch a CoreOS/rkt cluster on Google Compute Engine (GCE) +### Launch a rktnetes cluster on Google Compute Engine (GCE) -Here we provide instruction on how to use the `kube-up` script to launch a CoreOS/rkt cluster on GCE. -In order to do that, you need to specify the OS distribution, project, image: +This section outlines using the `kube-up` script to launch a CoreOS/rkt cluster on GCE. + +Specify the OS distribution, the GCE distributor's master project, and the instance images for the Kubernetes master and nodes. Set the `KUBE_CONTAINER_RUNTIME` to `rkt`: ```shell $ export KUBE_OS_DISTRIBUTION=coreos @@ -119,61 +122,101 @@ $ export KUBE_GCE_NODE_IMAGE= $ export KUBE_CONTAINER_RUNTIME=rkt ``` -You can optionally choose the version of rkt used by setting `KUBE_RKT_VERSION`: +Optionally, set the version of rkt by setting `KUBE_RKT_VERSION`: ```shell $ export KUBE_RKT_VERSION=1.9.1 ``` -Then you can launch the cluster by: +Optionally, select an alternative [stage1 isolator](#modular-isolation-with-interchangeable-stage1-images) for the container runtime by setting `KUBE_RKT_STAGE1_IMAGE`: + +```shell +$ export KUBE_RKT_STAGE1_IMAGE= +``` + +Then you can launch the cluster with: ```shell $ cluster/kube-up.sh ``` -### Launch a CoreOS/rkt cluster on AWS +### Launch a rktnetes cluster on AWS -`kube-up` for AWS is currently unsupported. -Instead, we recommend you to refer the [Kubernetes on AWS guide](https://coreos.com/kubernetes/docs/latest/kubernetes-on-aws.html) to launch a CoreOS/rkt cluster on AWS. +The `kube-up` script is not yet supported on AWS. Instead, we recommend following the [Kubernetes on AWS guide](https://coreos.com/kubernetes/docs/latest/kubernetes-on-aws.html) to launch a CoreOS Kubernetes cluster on AWS, then setting kubelet options as above. -### Deploy apps to your cluster +### Deploy apps to the cluster -After you created the cluster, you can start deploying apps to the cluster. For example here is how you can [deploy a simgle nginx app](/docs/user-guide/simple-nginx). -More examples can be found in the [examples directory](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/). +After creating the cluster, you can start deploying applications. For an introductory example, [deploy a simple nginx web server](/docs/user-guide/simple-nginx). Note that this example did not have to be modified for use with a "rktnetes" cluster. More examples can be found in the [Kubernetes examples directory](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/). -### Known Issues and Differences +## Modular isolation with interchangeable stage1 images -rkt and Docker have very different designs, as well as ACI and Docker image format. -Users might experience some different experience when switching from one to the other. -More information can be found [here](/docs/getting-started-guides/rkt/notes/). +rkt executes containers in an interchangeable isolation environment. This facility is called the [*stage1* image](https://coreos.com/rkt/docs/latest/devel/architecture.html#stage-1). There are currently three supported rkt stage1 images: -### Debugging +* `systemd-nspawn` stage1, the default. Isolates running containers with Linux kernel namespaces and cgroups in a manner similar to the default container runtime. +* [`KVM` stage1](https://coreos.com/rkt/docs/latest/running-lkvm-stage1.html), runs containers inside a KVM hypervisor-managed virtual machine. Experimental in the Kubernetes v1.3 release. +* [`fly stage1`](https://coreos.com/rkt/docs/latest/running-fly-stage1.html), which isolates containers with only a `chroot`, giving host-level access to mount and network namespaces for specially-privileged utilities. -Here are several tips in case you run into any issues. +In addition to the three provided stage1 images, you can [create your own](https://coreos.com/rkt/docs/latest/devel/stage1-implementors-guide.html) for specific isolation requirements. If no configuration is set, the [default stage1](https://coreos.com/rkt/docs/latest/build-configure.html#parameters-for-setting-up-default-stage1-image) is used. There are two ways to select a different stage1; either per-node, or per-pod: -##### Check logs +* Set the kubelet's `--rkt-stage1-image` flag, which tells the kubelet the stage1 image to use for every pod on the node. For example, `--rkt-stage1-image=coreos/rkt/stage1-coreos` selects the default systemd-nspawn stage1. +* Set the annotation `rkt.alpha.kubernetes.io/stage1-name-override` to override the stage1 used to execute a given pod. This allows for mixing different container isolation mechanisms on the same cluster or on the same node. For example, the following (shortened) pod manifest will run its pod with the `fly stage1` to give the application -- the `kubelet` in this case -- access to the host's namespace: -By default, the log verbose level is 2. In order to see more logs related to rkt, we can set the verbose level to 4. -For local cluster, we can set the environment variable: `LOG_LEVEL=4`. -If the cluster is using salt, we can edit the [logging.sls](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/cluster/saltbase/pillar/logging.sls) in the saltbase. - -##### Check rkt pod status - -To check the pods' status, we can use rkt command, such as `rkt list`, `rkt status`, `rkt image list`, etc. -More information about rkt command line can be found [here](https://github.com/coreos/rkt/blob/master/Documentation/commands.md). - -##### Check journal logs - -As we use systemd to launch/manage rkt pods, we can check the pods' log using `journalctl`: - -- Check the running state of the systemd service: - -```shell -$ sudo journalctl -u ${SERVICE_NAME} +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: kubelet + namespace: kube-system + labels: + k8s-app: kubelet + annotations: + rkt.alpha.kubernetes.io/stage1-name-override: coreos.com/rkt/stage1-fly +spec: + containers: + - name: kubelet + image: quay.io/coreos/hyperkube:v1.3.0-beta.2_coreos.0 + command: + - kubelet + - --api-servers=127.0.0.1:8080 + - --config=/etc/kubernetes/manifests + - --allow-privileged + - --kubeconfig=/etc/kubernetes/kubeconfig + securityContext: + privileged: true +[...] ``` -where `${SERVICE_NAME}` is the name of the service file created for the pod, typically the format is `k8s_${RKT_UUID}`. +### Notes on using different stage1 images -##### Check Kubernetes events, logs. +Setting the stage1 annotation could potentially give the pod root privileges. Because of this, the `privileged` boolean in the pod's `securityContext` must be set to `true`. -Kubernetes also provides various tools for debugging. More information can be found [here](/docs/user-guide/application-troubleshooting). +Use rkt's [*contained network*](#rkt-contained-network) with the KVM stage1, because the CNI plugin driver does not yet fully support the hypervisor-based runtime. + +## Known issues and differences between rkt and Docker + +rkt and the default node container engine have very different designs, as do rkt's native ACI and the Docker container image format. Users may experience different behaviors when switching from one container engine to the other. More information can be found [in the Kubernetes rkt notes](/docs/getting-started-guides/rkt/notes/). + +## Troubleshooting + +Here are a few tips for troubleshooting Kubernetes with the rkt container engine: + +### Check rkt pod status + +To check the status of running pods, use the rkt subcommands [`rkt list`](https://coreos.com/rkt/docs/latest/subcommands/list.html), [`rkt status`](https://coreos.com/rkt/docs/latest/subcommands/status.html), and [`rkt image list`](https://coreos.com/rkt/docs/latest/subcommands/image.html#rkt-image-list). See the [rkt commands documentation](https://coreos.com/rkt/docs/latest/commands.html) for more information about rkt subcommands. + +### Check journal logs + +Check a pod's log using `journalctl` on the node. Pods are managed and named as systemd units. The pod's unit name is formed by concatenating a `k8s_` prefix with the pod UUID, in a format like `k8s_${RKT_UUID}`. Find the pod's UUID with `rkt list` to assemble its service name, then ask journalctl for the logs: + + +```shell +$ sudo journalctl -u k8s_ad623346 +``` + +#### Log verbosity + +By default, the log verbosity level is 2. In order to see more log messages related to rkt, set this level to 4 or above. For a local cluster, set the environment variable: `LOG_LEVEL=4`. + +### Check Kubernetes events and logs. + +Kubernetes provides various tools for troubleshooting and examination. More information can be found [in the app troubleshooting guide](/docs/user-guide/application-troubleshooting). diff --git a/docs/getting-started-guides/rkt/notes.md b/docs/getting-started-guides/rkt/notes.md index 8e53481046..30a022dc24 100644 --- a/docs/getting-started-guides/rkt/notes.md +++ b/docs/getting-started-guides/rkt/notes.md @@ -1,9 +1,9 @@ --- --- -The following features either are not supported or have large caveats when using the rkt container runtime. +The following features either are not supported or have large caveats when using the rkt container runtime. Increasing support for these items and others, including reasonable feature parity with the default container engine, is planned through future releases. -##### Non-existent host volume paths +## Non-existent host volume paths When mounting a host volume path that does not exist, rkt will error out. Under the Docker runtime, an empty directory will be created at the referenced path. @@ -30,34 +30,46 @@ spec: name: does-not-exist ``` -##### Kubectl attach +## Kubectl attach The `kubectl attach` command does not work under the rkt container runtime. -And because of this, some flags in `kubectl run` are not supported, including: +Because of this, some flags in `kubectl run` are not supported, including: -- `--attach=true` -- `--leave-stdin-open=true` -- `--rm=true` +* `--attach=true` +* `--leave-stdin-open=true` +* `--rm=true` -##### Init containers +## Port forwarding for kvm and fly stage1s + +`kubectl port-forward` is not supported for pods that are executed with `stage1-kvm` or `stage1-fly`. + +## Volume relabeling + +Currently rkt supports only *per-pod* volume relabeling. After relabeling, the mounted volume is shared by all containers in the pod. There is not yet a way to make the relabeled volume accessible to only one, or some subset, of containers in the pod. [Kubernetes issue # 28187](https://github.com/kubernetes/kubernetes/issues/28187) has the details. + +## kubectl get logs + +Under rktnetes, `kubectl get logs` currently cannot get logs from applications that write them to directly to `/dev/stdout`. Currently such log messages are printed on the node's console. + +## Init containers The alpha [init container](https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/container-init.md) feature is currently not supported. -##### Container restart back-off +## Container restart back-off Exponential restart back-off for a failing container is currently not supported. -##### Experimental NVIDIA GPU support +## Experimental NVIDIA GPU support The `--experimental-nvidia-gpus` flag, and related [GPU features](https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/gpu-support.md) are not supported. -##### QoS Classes +## QoS Classes -Under rkt, QoS classes do not result in the `OOM Score` of containers being adjusted as occurs under Docker. +Under rkt, QoS classes do not adjust the `OOM Score` of containers as occurs under Docker. -##### HostPID and HostIPC namespaces +## HostPID and HostIPC namespaces -Setting a the hostPID or hostIPC flag on a pod is not supported. +Setting the hostPID or hostIPC flags on a pod is not supported. For example, the following pod will not run correctly: @@ -77,14 +89,10 @@ spec: On the other hand, when running the pod with [stage1-fly](https://coreos.com/rkt/docs/latest/running-fly-stage1.html), the pod will be run in the host namespace. - -##### Container image updates (patch) +## Container image updates (patch) Patching a pod to change the image will result in the entire pod restarting, not just the container that was changed. -##### Volume mounts specifying a subPath +## Volume mounts specifying a subPath -The [subPath](https://github.com/kubernetes/kubernetes/pull/22575) feature does not work correctly under rkt. -In addition, the above-issue of Non-existent host volume paths being invalid would make many common use-cases for subPaths fail in that way as well. - -In some cases, this issue can be worked around by creating and using subdirectories from within the container rather than relying on Kubernetes to do so. +The [subPath](https://github.com/kubernetes/kubernetes/pull/22575) feature does not work correctly under rkt. In addition, the issue of non-existent host volume paths being invalid, mentioned above, would cause many common use cases for subPaths to fail in the same way. In some cases, this issue can be worked around by creating and using subdirectories from within the container, rather than relying on Kubernetes to do so. From 6b9412830a8ff5fc928d65f34d2d853a519bf6cf Mon Sep 17 00:00:00 2001 From: Matt Liggett Date: Thu, 30 Jun 2016 13:45:13 -0700 Subject: [PATCH 27/31] Update details now that we try to make a PV for etcd. --- docs/admin/federation/index.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/admin/federation/index.md b/docs/admin/federation/index.md index dee50e33b4..fc077ec839 100644 --- a/docs/admin/federation/index.md +++ b/docs/admin/federation/index.md @@ -86,8 +86,9 @@ Running `federation-up.sh` also creates a new record in our kubeconfig for us to be able to talk to federation apiserver. We can view this by running `kubectl config view`. -Note: `federation-up.sh` creates the federation-apiserver pod with an etcd container that is not backed by any -persistent volume, so it is possible to lose data. You can edit +Note: `federation-up.sh` creates the federation-apiserver pod with an etcd +container that is backed by a persistent volume, so as to persist data. This +currently works only on AWS, GKE, and GCE. You can edit `federation/manifests/federation-apiserver-deployment.yaml` to suit your needs, if required. From 8513dfb9db1e56cf95362f0502e0e87d63eb31b8 Mon Sep 17 00:00:00 2001 From: derekwaynecarr Date: Thu, 23 Jun 2016 16:57:05 -0400 Subject: [PATCH 28/31] Out of resource handling for 1.3 --- _data/guides.yml | 2 + docs/admin/out-of-resource.md | 220 ++++++++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 docs/admin/out-of-resource.md diff --git a/_data/guides.yml b/_data/guides.yml index f5654a6efd..82ffea11da 100644 --- a/_data/guides.yml +++ b/_data/guides.yml @@ -263,5 +263,7 @@ toc: path: /docs/admin/static-pods/ - title: Configuring Garbage Collection path: /docs/admin/garbage-collection/ + - title: Configuring Out Of Resource Handling + path: /docs/admin/out-of-resource/ - title: Configuring Kubernetes with Salt path: /docs/admin/salt/ diff --git a/docs/admin/out-of-resource.md b/docs/admin/out-of-resource.md new file mode 100644 index 0000000000..2b85f486d5 --- /dev/null +++ b/docs/admin/out-of-resource.md @@ -0,0 +1,220 @@ +--- +--- + +* TOC +{:toc} + +The `kubelet` needs to preserve node stability when available compute resources are low. + +This is especially important when dealing with incompressible resources such as memory or disk. + +If either resource is exhausted, the node would become unstable. + +## Eviction Policy + +The `kubelet` can pro-actively monitor for and prevent against total starvation of a compute resource. In +cases where it could appear to occur, the `kubelet` can pro-actively fail one or more pods in order to reclaim +the starved resource. When the `kubelet` fails a pod, it terminates all containers in the pod, and the `PodPhase` +is transitioned to `Failed`. + +### Eviction Signals + +The `kubelet` can support the ability to trigger eviction decisions on the signals described in the +table below. The value of each signal is described in the description column based on the `kubelet` +summary API. + +| Eviction Signal | Description | +|------------------|---------------------------------------------------------------------------------| +| `memory.available` | `memory.available` := `node.status.capacity[memory]` - `node.stats.memory.workingSet` | + +In future releases, the `kubelet` will support the ability to trigger eviction decisions based on disk pressure. + +Until that time, it is recommended users take advantage of [garbage collection](/docs/admin/garbage-collection/). + +### Eviction Thresholds + +The `kubelet` supports the ability to specify eviction thresholds that trigger the `kubelet` to reclaim resources. + +Each threshold is of the following form: + +`` + +* valid `eviction-signal` tokens as defined above. +* valid `operator` tokens are `<` +* valid `quantity` tokens must match the quantity representation used by Kubernetes + +#### Soft Eviction Thresholds + +A soft eviction threshold pairs an eviction threshold with a required +administrator specified grace period. No action is taken by the `kubelet` +to reclaim resources associated with the eviction signal until that grace +period has been exceeded. If no grace period is provided, the `kubelet` will +error on startup. + +In addition, if a soft eviction threshold has been met, an operator can +specify a maximum allowed pod termination grace period to use when evicting +pods from the node. If specified, the `kubelet` will use the lesser value among +the `pod.Spec.TerminationGracePeriodSeconds` and the max allowed grace period. +If not specified, the `kubelet` will kill pods immediately with no graceful +termination. + +To configure soft eviction thresholds, the following flags are supported: + +* `eviction-soft` describes a set of eviction thresholds (e.g. `memory.available<1.5Gi`) that if met over a +corresponding grace period would trigger a pod eviction. +* `eviction-soft-grace-period` describes a set of eviction grace periods (e.g. `memory.available=1m30s`) that +correspond to how long a soft eviction threshold must hold before triggering a pod eviction. +* `eviction-max-pod-grace-period` describes the maximum allowed grace period (in seconds) to use when terminating +pods in response to a soft eviction threshold being met. + +#### Hard Eviction Thresholds + +A hard eviction threshold has no grace period, and if observed, the `kubelet` +will take immediate action to reclaim the associated starved resource. If a +hard eviction threshold is met, the `kubelet` will kill the pod immediately +with no graceful termination. + +To configure hard eviction thresholds, the following flag is supported: + +* `eviction-hard` describes a set of eviction thresholds (e.g. `memory.available<1Gi`) that if met +would trigger a pod eviction. + +### Eviction Monitoring Interval + +The `kubelet` evaluates eviction thresholds per its configured housekeeping interval. + +* `housekeeping-interval` is the interval between container housekeepings. + +### Node Conditions + +The `kubelet` will map one or more eviction signals to a corresponding node condition. + +If a hard eviction threshold has been met, or a soft eviction threshold has been met +independent of its associated grace period, the `kubelet` will report a condition that +reflects the node is under pressure. + +The following node conditions are defined that correspond to the specified eviction signal. + +| Node Condition | Eviction Signal | Description | +|----------------|------------------|------------------------------------------------------------------| +| `MemoryPressure` | `memory.available` | Available memory on the node has satisfied an eviction threshold | + +The `kubelet` will continue to report node status updates at the frequency specified by +`--node-status-update-frequency` which defaults to `10s`. + +### Oscillation of node conditions + +If a node is oscillating above and below a soft eviction threshold, but not exceeding +its associated grace period, it would cause the corresponding node condition to +constantly oscillate between true and false, and could cause poor scheduling decisions +as a consequence. + +To protect against this oscillation, the following flag is defined to control how +long the `kubelet` must wait before transitioning out of a pressure condition. + +* `eviction-pressure-transition-period` is the duration for which the `kubelet` has +to wait before transitioning out of an eviction pressure condition. + +The `kubelet` would ensure that it has not observed an eviction threshold being met +for the specified pressure condition for the period specified before toggling the +condition back to `false`. + +### Eviction of Pods + +If an eviction threshold has been met and the grace period has passed, +the `kubelet` will initiate the process of evicting pods until it has observed +the signal has gone below its defined threshold. + +The `kubelet` ranks pods for eviction 1) by their quality of service, +2) and among those with the same quality of service by the consumption of the +starved compute resource relative to the pods scheduling request. + +* `BestEffort` pods that consume the most of the starved resource are failed +first. +* `Burstable` pods that consume the greatest amount of the starved resource +relative to their request for that resource are killed first. If no pod +has exceeded its request, the strategy targets the largest consumer of the +starved resource. +* `Guaranteed` pods that consume the greatest amount of the starved resource +relative to their request are killed first. If no pod has exceeded its request, +the strategy targets the largest consumer of the starved resource. + +A `Guaranteed` pod is guaranteed to never be evicted because of another pod's +resource consumption. If a system daemon (i.e. `kubelet`, `docker`, `journald`, etc.) +is consuming more resources than were reserved via `system-reserved` or `kube-reserved` allocations, +and the node only has `Guaranteed` pod(s) remaining, then the node must choose to evict a +`Guaranteed` pod in order to preserve node stability, and to limit the impact +of the unexpected consumption to other `Guaranteed` pod(s). + +### Scheduler + +The node will report a condition when a compute resource is under pressure. The +scheduler views that condition as a signal to dissuade placing additional +pods on the node. + +| Node Condition | Scheduler Behavior | +| ---------------- | ------------------------------------------------ | +| `MemoryPressure` | `BestEffort` pods are not scheduled to the node. | + +## Node OOM Behavior + +If the node experiences a system OOM (out of memory) event prior to the `kubelet` is able to reclaim memory, +the node depends on the [oom_killer](https://lwn.net/Articles/391222/) to respond. + +The `kubelet` sets a `oom_score_adj` value for each container based on the quality of service for the pod. + +| Quality of Service | oom_score_adj | +| ----------------- | ------------- | +| `Guaranteed` | -998 | +| `BestEffort` | 1000 | +| `Burstable` | min(max(2, 1000 - (1000 * memoryRequestBytes) / machineMemoryCapacityBytes), 999) | + +If the `kubelet` is unable to reclaim memory prior to a node experiencing system OOM, the `oom_killer` will calculate +an `oom_score` based on the percentage of memory its using on the node, and then add the `oom_score_adj` to get an +effective `oom_score` for the container, and then kills the container with the highest score. + +The intended behavior should be that containers with the lowest quality of service that +are consuming the largest amount of memory relative to the scheduling request should be killed first in order +to reclaim memory. + +Unlike pod eviction, if a pod container is OOM killed, it may be restarted by the `kubelet` based on its `RestartPolicy`. + +## Best Practices + +### Schedulable resources and eviction policies + +Let's imagine the following scenario: + +* Node memory capacity: `10Gi` +* Operator wants to reserve 10% of memory capacity for system daemons (kernel, `kubelet`, etc.) +* Operator wants to evict pods at 95% memory utilization to reduce thrashing and incidence of system OOM. + +To facilitate this scenario, the `kubelet` would be launched as follows: + +``` +--eviction-hard=memory.available<500Mi +--system-reserved=memory=1.5Gi +``` + +Implicit in this configuration is the understanding that "System reserved" should include the amount of memory +covered by the eviction threshold. + +To reach that capacity, either some pod is using more than its request, or the system is using more than `500Mi`. + +This configuration will ensure that the scheduler does not place pods on a node that immediately induce memory pressure +and trigger eviction assuming those pods use less than their configured request. + +### DaemonSet + +It is never desired for a `kubelet` to evict a pod that was derived from +a `DaemonSet` since the pod will immediately be recreated and rescheduled +back to the same node. + +At the moment, the `kubelet` has no ability to distinguish a pod created +from `DaemonSet` versus any other object. If/when that information is +available, the `kubelet` could pro-actively filter those pods from the +candidate set of pods provided to the eviction strategy. + +In general, it is strongly recommended that `DaemonSet` not +create `BestEffort` pods to avoid being identified as a candidate pod +for eviction. Instead `DaemonSet` should ideally launch `Guaranteed` pods. From 37510560b6ab0fdf87946c7d88d3f01378b716d8 Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Tue, 21 Jun 2016 10:25:46 -0700 Subject: [PATCH 29/31] Petset 1.3 user guide --- docs/user-guide/ingress.md | 2 ++ docs/user-guide/petset.md | 53 +++++++++++++++++++++++++++++++++----- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/docs/user-guide/ingress.md b/docs/user-guide/ingress.md index 78a0d201a9..61bae15d59 100644 --- a/docs/user-guide/ingress.md +++ b/docs/user-guide/ingress.md @@ -221,6 +221,8 @@ spec: servicePort: 80 ``` +Please refer to documentation on [nginx](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx#https), [GCE](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/gce#tls), or any other platform specific Ingress controller to understand how TLS works in your environment. + ### Loadbalancing An Ingress controller is bootstrapped with some loadbalancing policy settings that it applies to all Ingress, such as the loadbalancing algorithm, backend weight scheme etc. More advanced loadbalancing concepts (eg: persistent sessions, dynamic weights) are not yet exposed through the Ingress. You can still get these features through the [service loadbalancer](https://github.com/kubernetes/contrib/tree/master/service-loadbalancer). With time, we plan to distill loadbalancing patterns that are applicable cross platform into the Ingress resource. diff --git a/docs/user-guide/petset.md b/docs/user-guide/petset.md index 324e1f1764..1a14f4ee26 100644 --- a/docs/user-guide/petset.md +++ b/docs/user-guide/petset.md @@ -98,7 +98,7 @@ Saving this config into `petset.yaml` and submitting it to a Kubernetes cluster ```shell $ kubectl create -f petset.yaml service "nginx" created -petset "nginx" deleted +petset "nginx" created ``` ## Pet Identity @@ -131,21 +131,53 @@ pvc-902733c2-3717-11e6-a46e-42010af00002 1Gi RWO Bound de The network identity has 2 parts. First, we created a headless Service that controls the domain within which we create Pets. The domain managed by this Service takes the form: `$(service name).$(namespace).svc.cluster.local`, where "cluster.local" is the [cluster domain](http://releases.k8s.io/{{page.githubbranch}}/build/kube-dns/README.md#how-do-i-configure-it). As each pet is created, it gets a matching DNS subdomain, taking the form: `$(petname).$(governing service domain)`, where the governing service is defined by the `serviceName` field on the Pet Set. +Here are some examples of choices for Cluster Domain, Service name, Pet Set name, and how that affects the DNS names for the Pets and the hostnames in the Pet's pods: + Cluster Domain | Service (ns/name) | Pet Set (ns/name) | Pet Set Domain | Pet DNS | Pet Hostname | -------------- | ----------------- | ----------------- | -------------- | ------- | ------------ | cluster.local | default/nginx | default/web | nginx.default.svc.cluster.local | web-{0..N-1}.nginx.default.svc.cluster.local | web-{0..N-1} | cluster.local | foo/nginx | foo/web | nginx.foo.svc.cluster.local | web-{0..N-1}.nginx.foo.svc.cluster.local | web-{0..N-1} | kube.local | foo/nginx | foo/web | nginx.foo.svc.kube.local | web-{0..N-1}.nginx.foo.svc.kube.local | web-{0..N-1} | -Lets verify this assertion with a simple test. +Note that Cluster Domain will be set to `cluster.local` unless [otherwise configured](http://releases.k8s.io/{{page.githubbranch}}/build/kube-dns/README.md#how-do-i-configure-it). + +Lets verify our assertion with a simple test. ```shell $ kubectl get svc NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE nginx None 80/TCP 12m +... ``` -The containers are running nginx webservers, which by default will look for an index.html file in `/usr/share/nginx/html/index.html`. That directory is backed by a PersistentVolume created by the Pet Set. So lets write our hostname there (remember the Pet Set gives us a stable hostname): +First, the PetSet gives provides a stable hostname: + +```shell +$ for i in 0 1; do kubectl exec web-$i -- sh -c 'hostname'; done +web-0 +web-1 +``` + +And the hostname is linked to the in-cluster DNS address: + +```shell +$ kubectl run -it --image busybox dns-test --restart=Never /bin/sh +dns-test # nslookup web-0.nginx +Server: 10.0.0.10 +Address 1: 10.0.0.10 kube-dns.kube-system.svc.cluster.local + +Name: web-0.nginx +Address 1: 10.180.3.5 + +dns-test # nslookup web-1.nginx +Server: 10.0.0.10 +Address 1: 10.0.0.10 kube-dns.kube-system.svc.cluster.local + +Name: web-1.nginx +Address 1: 10.180.0.9 +``` + +The containers are running nginx webservers, which by default will look for an index.html file in `/usr/share/nginx/html/index.html`. That directory is backed by a `PersistentVolume` created by the Pet Set. So lets write our hostname there: ```shell $ for i in 0 1; do @@ -153,6 +185,14 @@ $ for i in 0 1; do done ``` +And verify each webserver serves its own hostname: + +```shell +$ for i in 0 1; do kubectl exec -it web-$i -- curl localhost; done +web-0 +web-1 +``` + Now delete all pods in the petset: ```shell @@ -161,7 +201,7 @@ pod "web-0" deleted pod "web-1" deleted ``` -Wait for them to come back up, and try to retrieve the previously written hostname through the DNS name of the peer (remember the Pet Set also gives us stable storage, and that the hostname is linked to the DNS name). +Wait for them to come back up, and try to retrieve the previously written hostname through the DNS name of the peer. They match, because the storage, DNS name, and hostname stick to the Pet no matter where it gets scheduled: ```shell $ kubectl exec -it web-1 -- curl web-0.nginx @@ -250,7 +290,7 @@ $ sleep $grace $ kubectl delete pvc -l app=nginx {% endraw %}``` -## Troubleshooting +## Troubelshooting You might have noticed an `annotations` field in all the Pet Sets shown above. @@ -277,5 +317,6 @@ This list goes on, if you have examples, ideas or thoughts, please contribute. Deploying one RC of size 1/Service per pod is a popular alternative, as is simply deploying a DaemonSet that utilizes the identity of a Node. +## Next steps - +The deployment and maintenance of stateful applications is a vast topic. The next step is to explore cluster bootstrapping and initialization, [here](/docs/user-guide/petset/bootstrapping/). From 4df7d96369964a267466bdffc73f7d343c77584f Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Thu, 23 Jun 2016 22:43:11 -0700 Subject: [PATCH 30/31] Petset bootstrapping --- _data/guides.yml | 2 + docs/user-guide/petset.md | 2 +- docs/user-guide/petset/bootstrapping/index.md | 234 ++++++++++++++++++ .../petset/bootstrapping/petset_peers.yaml | 103 ++++++++ .../petset/bootstrapping/petset_vm.yaml | 102 ++++++++ 5 files changed, 442 insertions(+), 1 deletion(-) create mode 100644 docs/user-guide/petset/bootstrapping/index.md create mode 100644 docs/user-guide/petset/bootstrapping/petset_peers.yaml create mode 100644 docs/user-guide/petset/bootstrapping/petset_vm.yaml diff --git a/_data/guides.yml b/_data/guides.yml index 46a988505f..108359f4fe 100644 --- a/_data/guides.yml +++ b/_data/guides.yml @@ -104,6 +104,8 @@ toc: path: /docs/user-guide/downward-api/volume - title: Persistent Volumes Walkthrough path: /docs/user-guide/persistent-volumes/walkthrough/ + - title: Bootstrapping Pet Sets + path: /docs/user-guide/petset/bootstrapping/ - title: Monitoring, Logging, and Debugging Containers section: diff --git a/docs/user-guide/petset.md b/docs/user-guide/petset.md index 1a14f4ee26..77b132dea2 100644 --- a/docs/user-guide/petset.md +++ b/docs/user-guide/petset.md @@ -290,7 +290,7 @@ $ sleep $grace $ kubectl delete pvc -l app=nginx {% endraw %}``` -## Troubelshooting +## Troubleshooting You might have noticed an `annotations` field in all the Pet Sets shown above. diff --git a/docs/user-guide/petset/bootstrapping/index.md b/docs/user-guide/petset/bootstrapping/index.md new file mode 100644 index 0000000000..2462c43691 --- /dev/null +++ b/docs/user-guide/petset/bootstrapping/index.md @@ -0,0 +1,234 @@ +--- +--- + +* TOC +{:toc} + +## Overview + +This purpose of this guide is to help you become familiar with the runtime initialization of [Pet Sets](/docs/user-guide/petset). This guide assumes the same prerequisites, and uses the same terminology as the [Pet Set user document](/docs/user-guide/petset). + +The most common way to initialize the runtime in a containerized environment, is through a custom [entrypoint](https://docs.docker.com/engine/reference/builder/#entrypoint). While this is not necessarily bad, making your application pid 1, and treating containers as processes in general is good for a few reasons outside the scope of this document. Doing so allows you to run docker images from third-party vendors without modification. We will not be writing custom entrypoints for this example, but using a feature called [init containers](http://releases.k8s.io/{{page.githubbranch}}/docs/proposals/container-init.md), to explain 2 common patterns that come up deploying Pet Sets. + +1. Transferring state across Pet restart, so that a future Pet is initialized with the computations of its past incarnation +2. Initializing the runtime environment of a Pet based on existing conditions, like a list of currently healthy peers + +## Example I: transferring state across Pet restart + +This example shows you how to "carry over" runtime state across Pet restart by simulating virtual machines with a Pet Set. + +### Background + +Applications that incrementally build state usually need strong guarantees that they will not restart for extended durations. This is tricky to achieve with containers, so instead, we will ensure that the results of previous computations are trasferred to future pets. Doing so is straight-forward using vanilla Persistent Volumes (which Pet Set already gives you), unless the volume mount point itself needs to be initialized for the Pet to start. This is exactly the case with "virtual machine" docker images, like those based on ubuntu or fedora. Such images embed the entier rootfs of the distro, including package managers like `apt-get` that assume a certain layout of the filesystem. Meaning: + +* If you mount an empty volume under `/usr`, you won't be able to `apt-get` +* If you mount an empty volume under `/lib`, all your `apt-gets` will fail because there are no system libraries +* If you clobber either of those, previous `apt-get` results will be dysfunctional + +### Simulating Virtual Machines + +Since Pet Set already gives each Pet a consistent identity, all we need is a way to initialize the user environment before allowing tools like `kubectl exec` to enter the application container. + +Download [this](petset_vm.yaml) petset into a file called petset_vm.yaml, and create it: + +```shell +$ kubectl create -f ./petset_vm.yaml +service "ub" created +petset "vm" created +``` + +This should give you 2 pods. + +```shell +$ kubectl get po +NAME READY STATUS RESTARTS AGE +vm-0 1/1 Running 0 37s +vm-1 1/1 Running 0 2m +``` + +We can exec into one and install nginx + +```shell +$ kubectl exec vm-0 /bin/sh +vm-0 # apt-get update +... +vm-0 # apt-get install nginx -y +``` + +On killing this pod we need it to come back with all the Pet Set properties, as well as the installed nginx packages. + +```shell +$ kubectl delete po vm-0 +pod "vm-0" deleted + +$ kubectl get po +NAME READY STATUS RESTARTS AGE +vm-0 1/1 Running 0 1m +vm-1 1/1 Running 0 4m +``` + +Now you can exec back into vm-0 and start nginx + +```shell +$ kubectl exec -it vm-0 /bin/sh +vm-0 # mkdir -p /var/log/nginx /var/lib/nginx; nginx -g 'daemon off;' + +``` + +And access it from anywhere in the cluster (and because this is an example that simulates vms, we're going to apt-get install netcat too) + +```shell +$ kubectl exec -it vm-1 /bin/sh +vm-1 # printf "GET / HTTP/1.0\r\n\r\n" | netcat vm-0.ub 80 +``` + +It's worth exploring what just happened. Init containers run sequentially *before* the application container. In this example we used the init container to copy shared libraries from the rootfs, while preserving user installed packages across container restart. + +```yaml +pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "rootfs", + "image": "ubuntu:15.10", + "command": [ + "/bin/sh", + "-c", + "for d in usr lib etc; do cp -vnpr /$d/* /${d}mnt; done;" + ], + "volumeMounts": [ + { + "name": "usr", + "mountPath": "/usrmnt" + }, + { + "name": "lib", + "mountPath": "/libmnt" + }, + { + "name": "etc", + "mountPath": "/etcmnt" + } + ] + } +]' +``` + +**It's important to note that the init container, when used this way, must be idempotent, or it'll end up clobbering data stored by a previous incarnation.** + + +## Example II: initializing state based on environment + +In this example we are going to setup a cluster of nginx servers, just like we did in the Pet Set [user guide](/docs/user-guide/petset), but make one of them a master. All the other nginx servers will simply proxy requests to the master. This is a common deployment pattern for databases like Mysql, but we're going to replace the database with a stateless webserver to simplify the problem. + +### Background + +Most clustered applications, such as mysql, require an admin to create a config file based on the current state of the world. The most common dynamic variable in such config files is a list of peers, or other Pets running similar database servers that are currently serving requests. The Pet Set user guide already [touched on this topic](/docs/user-guide/petset#peer-discovery), we'll explore it in greater depth in the context of writing a config file with a list of peers. + +Here's a tiny peer finder helper script that handles peer discovery, [available here](https://github.com/kubernetes/contrib/tree/master/pets/peer-finder). The peer finder takes 3 important arguments: + +* A DNS domain +* An `on-start` script to run with the initial constituency of the given domain as input +* An `on-change` script to run everytime the constituency of the given domain changes + +The role of the peer finder: + +* Poll DNS for SRV records of a given domain till the `hostname` of the pod it's running in shows up as a subdomain +* Pipe the sorted list of subdomains to the script specified by its `--on-start` argument +* Exit with the appropriate error code if no `--on-change` script is specified +* Loop invoking `--on-change` for every change + +You can invoke the peer finder inside the Pets we created in the last example: + +```shell +$ kubectl exec -it vm-0 /bin/bash +vm-0 # curl -sSL -o /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder +vm-0 # chmod -c 755 peer-finder + +vm-0 # ./peer-finder +2016/06/23 21:25:46 Incomplete args, require -on-change and/or -on-start, -service and -ns or an env var for POD_NAMESPACE. + +vm-0 # ./peer-finder -on-start 'tee' -service ub -ns default + +2016/06/23 21:30:21 Peer list updated +was [] +now [vm-0.ub.default.svc.cluster.local vm-1.ub.default.svc.cluster.local] +2016/06/23 21:30:21 execing: tee with stdin: vm-0.ub.default.svc.cluster.local +vm-1.ub.default.svc.cluster.local +2016/06/23 21:30:21 vm-0.ub.default.svc.cluster.local +vm-1.ub.default.svc.cluster.local +2016/06/23 21:30:22 Peer finder exiting +``` + +### Nginx master/slave cluster + +Lets create a Pet Set that writes out its own config based on a list of peers at initalization time, as described above. + +Download and create [this](petset_peers.yaml) petset. It will setup 2 nginx webservers, but the second one will proxy all requests to the first: + +```shell +$ kubectl create -f petset_peers.yaml +service "nginx" created +petset "web" created + +$ kubectl get po --watch-only +NAME READY STATUS RESTARTS AGE +web-0 0/1 Pending 0 7s +web-0 0/1 Init:0/1 0 18s +web-0 0/1 PodInitializing 0 20s +web-0 1/1 Running 0 21s +web-1 0/1 Pending 0 0s +web-1 0/1 Init:0/1 0 0s +web-1 0/1 PodInitializing 0 20s +web-1 1/1 Running 0 21s + +$ kubectl get po +NAME READY STATUS RESTARTS AGE +web-0 1/1 Running 0 1m +web-1 1/1 Running 0 47s +``` + +web-1 will redirect all requests to it's "master": + +```shell +$ kubectl exec -it web-1 -- curl localhost +web-0 +``` + +If you scale the cluster, the new pods parent themselves to the same master. To test this you can `kubectl edit` the petset and change the `replicas` field to 5: + +```shell +$ kubectl get po -l app=nginx +NAME READY STATUS RESTARTS AGE +web-0 1/1 Running 0 2h +web-1 1/1 Running 0 2h +web-2 1/1 Running 0 1h +web-3 1/1 Running 0 1h +web-4 1/1 Running 0 1h + +$ for i in $(seq 0 4); do kubectl exec -it web-$i -- curl localhost; done | sort | uniq +web-0 +``` + +Understanding how we generated the nginx config is important, we did so by passing an init script to the peer finder: + +```shell +echo ` +readarray PEERS; +if [ 1 = ${#PEERS[@]} ]; then + echo \"events{} http { server{ } }\"; +else + echo \"events{} http { server{ location / { proxy_pass http://${PEERS[0]}; } } }\"; +fi;` > /conf/nginx.conf +``` + +All that does is: + +* read in a list of peers from stdin +* if there's only 1, promote it to master +* if there's more than 1, proxy requests to the 0th member of the list +* write the config to a `hostPath` volume shared with the parent PetSet + +**It's important to note that in practice all Pets should query their peers for the current master, instead of making assumptions based on the index.** + +## Next Steps + +You can deploy some example Pet Sets found [here](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/petset), or write your own. diff --git a/docs/user-guide/petset/bootstrapping/petset_peers.yaml b/docs/user-guide/petset/bootstrapping/petset_peers.yaml new file mode 100644 index 0000000000..e12988dee2 --- /dev/null +++ b/docs/user-guide/petset/bootstrapping/petset_peers.yaml @@ -0,0 +1,103 @@ +# A headless service to create DNS records +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + name: nginx + labels: + app: nginx +spec: + ports: + - port: 80 + name: web + # *.nginx.default.svc.cluster.local + clusterIP: None + selector: + app: nginx +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: web +spec: + serviceName: "nginx" + replicas: 2 + template: + metadata: + labels: + app: nginx + annotations: + pod.alpha.kubernetes.io/initialized: "true" + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "peerfinder", + "image": "gcr.io/google_containers/peer-finder:0.1", + "args": [ + "-on-start=\"\"echo ` + readarray PEERS; + echo $(hostname) > /usr/share/nginx/html/index.html; + if [ 1 = ${#PEERS[@]} ]; then + echo \"events{} http { server{ } }\"; + else + echo \"events{} http { server{ location / { proxy_pass http://${PEERS[0]}; } } }\"; + fi;` > /conf/nginx.conf\"\"", + "-service=nginx" + ], + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "volumeMounts": [ + { + "name": "www", + "mountPath": "/usr/share/nginx/html" + }, + { + "name": "conf", + "mountPath": "/conf" + } + ] + } + ]' + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: nginx + image: gcr.io/google_containers/nginx-slim:0.7 + ports: + - containerPort: 80 + name: web + command: + - nginx + args: + - -g + - "daemon off;" + - -c + - "/conf/nginx.conf" + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html + - name: conf + mountPath: /conf + volumes: + - name: conf + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: www + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + diff --git a/docs/user-guide/petset/bootstrapping/petset_vm.yaml b/docs/user-guide/petset/bootstrapping/petset_vm.yaml new file mode 100644 index 0000000000..c506a1bac6 --- /dev/null +++ b/docs/user-guide/petset/bootstrapping/petset_vm.yaml @@ -0,0 +1,102 @@ +# A headless service to create DNS records +apiVersion: v1 +kind: Service +metadata: + name: ub + labels: + app: ub +spec: + ports: + - port: 80 + name: web + # *.ub.default.svc.cluster.local + clusterIP: None + selector: + app: ub +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: vm +spec: + serviceName: "ub" + replicas: 2 + template: + metadata: + labels: + app: ub + annotations: + pod.alpha.kubernetes.io/initialized: "true" + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "rootfs", + "image": "ubuntu:15.10", + "command": [ + "/bin/sh", + "-c", + "for d in usr lib etc; do cp -vnpr /$d/* /${d}mnt; done;" + ], + "volumeMounts": [ + { + "name": "usr", + "mountPath": "/usrmnt" + }, + { + "name": "lib", + "mountPath": "/libmnt" + }, + { + "name": "etc", + "mountPath": "/etcmnt" + } + ] + } + ]' + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: ub + image: ubuntu:15.10 + ports: + - containerPort: 80 + name: web + command: + - /bin/sh + - -c + - 'while true; do sleep 10; done' + volumeMounts: + - name: usr + mountPath: /usr + - name: lib + mountPath: /lib + - name: etc + mountPath: /etc + volumeClaimTemplates: + - metadata: + name: usr + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 300Mi + - metadata: + name: lib + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + - metadata: + name: etc + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 150Mi + From 1c47cd800cf1122ab8de8ccdbfe1b77c8351c0fd Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Wed, 6 Jul 2016 17:16:59 -0700 Subject: [PATCH 31/31] Update ingress docs --- docs/user-guide/ingress.md | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/docs/user-guide/ingress.md b/docs/user-guide/ingress.md index 61bae15d59..09728d8501 100644 --- a/docs/user-guide/ingress.md +++ b/docs/user-guide/ingress.md @@ -44,20 +44,12 @@ Before you start using the Ingress resource, there are a few things you should u On GCE/GKE there should be a [L7 cluster addon](https://github.com/kubernetes/contrib/blob/master/ingress/controllers/gce/README.md), deployed into the `kube-system` namespace: ```shell -$ kubectl get pods --namespace=kube-system -l name=glbc +$ kubectl get pods --namespace=kube-system -l k8s-app=glbc NAME READY STATUS RESTARTS AGE l7-lb-controller-v0.6.0-chnan 2/2 Running 0 1d ``` -Make sure you review the [beta limitations](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/gce/BETA_LIMITATIONS.md) of this controller. In particular, you need to create a single firewall-rule on your cloudprovider, to allow health checks. On GKE this would be: - -```shell -$ gcloud compute firewall-rules create allow-130-211-0-0-22 \ - --source-ranges 130.211.0.0/22 \ - --allow tcp:30000-32767 -``` - -In environments other than GCE/GKE, you need to [deploy a controller](https://github.com/kubernetes/contrib/tree/master/ingress/controllers) as a pod. +Make sure you review the [beta limitations](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/gce/BETA_LIMITATIONS.md) of this controller. In environments other than GCE/GKE, you need to [deploy a controller](https://github.com/kubernetes/contrib/tree/master/ingress/controllers) as a pod. ## The Ingress Resource @@ -94,6 +86,10 @@ __Global Parameters__: For the sake of simplicity the example Ingress has no glo In order for the Ingress resource to work, the cluster must have an Ingress controller running. This is unlike other types of controllers, which typically run as part of the `kube-controller-manager` binary, and which are typically started automatically as part of cluster creation. You need to choose the ingress controller implementation that is the best fit for your cluster, or implement one. Examples and instructions can be found [here](https://github.com/kubernetes/contrib/tree/master/ingress/controllers). +## Before you begin + +The following document describes a set of cross platform features exposed through the Ingress resource. Ideally, all Ingress controllers should fulfill this specification, but we're not there yet. The docs for the GCE and nginx controllers are [here](https://github.com/kubernetes/contrib/blob/master/ingress/controllers/gce/README.md) and [here](https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/README.md) respectively. **Make sure you review controller specific docs so you understand the caveats of each one**. + ## Types of Ingress ### Single Service Ingress @@ -221,13 +217,13 @@ spec: servicePort: 80 ``` -Please refer to documentation on [nginx](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx#https), [GCE](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/gce#tls), or any other platform specific Ingress controller to understand how TLS works in your environment. +Note that there is a gap between TLS features supported by various Ingress controllers. Please refer to documentation on [nginx](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx#https), [GCE](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/gce#tls), or any other platform specific Ingress controller to understand how TLS works in your environment. ### Loadbalancing An Ingress controller is bootstrapped with some loadbalancing policy settings that it applies to all Ingress, such as the loadbalancing algorithm, backend weight scheme etc. More advanced loadbalancing concepts (eg: persistent sessions, dynamic weights) are not yet exposed through the Ingress. You can still get these features through the [service loadbalancer](https://github.com/kubernetes/contrib/tree/master/service-loadbalancer). With time, we plan to distill loadbalancing patterns that are applicable cross platform into the Ingress resource. -It's also worth noting that even though health checks are not exposed directly through the Ingress, there exist parallel concepts in Kubernetes such as [readiness probes](https://github.com/kubernetes/kubernetes/blob/release-1.0/docs/user-guide/production-pods.md#liveness-and-readiness-probes-aka-health-checks) which allow you to achieve the same end result. +It's also worth noting that even though health checks are not exposed directly through the Ingress, there exist parallel concepts in Kubernetes such as [readiness probes](https://github.com/kubernetes/kubernetes/blob/release-1.0/docs/user-guide/production-pods.md#liveness-and-readiness-probes-aka-health-checks) which allow you to achieve the same end result. Please review the controller specific docs to see how they handle health checks ([nginx](https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/README.md), [GCE](https://github.com/kubernetes/contrib/blob/master/ingress/controllers/gce/README.md#health-checks)). ## Updating an Ingress