From 09bf6df3d32bd6aac9cdfbc3a48ee7b567be052f Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Tue, 22 Nov 2016 16:57:44 -0800 Subject: [PATCH 1/3] Add task: scaling stateful sets --- _data/tasks.yml | 2 + docs/tasks/index.md | 1 + .../manage-stateful-set/scale-stateful-set.md | 83 +++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 docs/tasks/manage-stateful-set/scale-stateful-set.md diff --git a/_data/tasks.yml b/_data/tasks.yml index 24e19c35c1..65cd74db91 100644 --- a/_data/tasks.yml +++ b/_data/tasks.yml @@ -37,3 +37,5 @@ toc: section: - title: Upgrading from PetSets to StatefulSets path: /docs/tasks/manage-stateful-set/upgrade-pet-set-to-stateful-set/ + - title: Scaling a Stateful Set + path: /docs/tasks/manage-stateful-set/scale-stateful-set/ diff --git a/docs/tasks/index.md b/docs/tasks/index.md index d8e46cff09..14fc254b60 100644 --- a/docs/tasks/index.md +++ b/docs/tasks/index.md @@ -31,6 +31,7 @@ single thing, typically by giving a short sequence of steps. #### Managing Stateful Applications * [Upgrading from PetSets to StatefulSets](/docs/tasks/manage-stateful-set/upgrade-pet-set-to-stateful-set/) +* [Scaling a Stateful Set](/docs/tasks/manage-stateful-set/scale-stateful-set/) ### What's next diff --git a/docs/tasks/manage-stateful-set/scale-stateful-set.md b/docs/tasks/manage-stateful-set/scale-stateful-set.md new file mode 100644 index 0000000000..f326165b59 --- /dev/null +++ b/docs/tasks/manage-stateful-set/scale-stateful-set.md @@ -0,0 +1,83 @@ +--- +assignees: +- bprashanth +- enisoc +- erictune +- foxish +- janetkuo +- kow3ns +- smarterclayton + +--- + +{% capture overview %} +This page shows how to scale a Stateful Set. +{% endcapture %} + +{% capture prerequisites %} + +* Stateful Sets are only available in Kubernetes release >= 1.5. +* Stateful Sets are previously known as Pet Sets in Kubernetes release 1.3-1.4. You can either upgrade your Pet Sets to Stateful Sets, +or just change all `statefulset` references to `petset`. *TODO: link to upgrade from Pet Sets to Stateful Sets.* +* **Not all stateful applications scale nicely.** You need to understand your Stateful Sets well before continue. If you're unsure, remember that it may not be safe to scale your Stateful Sets. + +{% endcapture %} + +{% capture steps %} + +### Use `kubectl` to scale Stateful Sets + +#### `kubectl scale` (>= 1.4 release) + +First, find the Stateful Set you want to scale. Remember, you need to first understand if you can scale it or not. + +```shell +kubectl get statefulsets +``` + +If you wish to change the number of replicas of your Stateful Set, just use this command: + +```shell +kubectl scale statefulsets --replicas= +``` + +Note that `kubectl scale` only works on Stateful Set with Kubernetes release >= 1.4. + +#### Alternative: `kubectl apply` / `kubectl edit` / `kubectl patch` (>= 1.3 release) + +Alternatively, you may do [in-place updates](/docs/user-guide/managing-deployments/#in-place-updates-of-resources) on your Stateful Sets. + +If your Stateful Set was initially created with `kubectl apply` or `kubectl create --save-config`, +you may update `.spec.replicas` of the Stateful Set manifests, and then do a `kubectl apply`: + +```shell +kubectl apply -f +``` + +Otherwise, you can just edit that field with `kubectl edit`: + +```shell +kubectl edit statefulsets +``` + +Or use `kubectl patch`: + +```shell +kubectl patch statefulsets -p '{"spec":{"replicas":}}' +``` + +### Troubleshooting + +#### Scaling down doesn't not work right + +You cannot scale down a Stateful Set when some of the stateful pods it manages are unhealthy. Scaling down only takes place +after those stateful pods become running and ready. See discussions [here](https://github.com/kubernetes/kubernetes/issues/36333). + + +{% endcapture %} + +{% capture whatsnext %} +*TODO: link to other docs about Stateful Set?* +{% endcapture %} + +{% include templates/task.md %} From d43cbbe58df60f10dbacdffed0b3792dcd88c487 Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Wed, 30 Nov 2016 15:29:06 -0800 Subject: [PATCH 2/3] Address comments from foxish, enisoc, and erictune --- .../manage-stateful-set/scale-stateful-set.md | 54 ++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/docs/tasks/manage-stateful-set/scale-stateful-set.md b/docs/tasks/manage-stateful-set/scale-stateful-set.md index f326165b59..53bdab4a08 100644 --- a/docs/tasks/manage-stateful-set/scale-stateful-set.md +++ b/docs/tasks/manage-stateful-set/scale-stateful-set.md @@ -11,50 +11,52 @@ assignees: --- {% capture overview %} -This page shows how to scale a Stateful Set. +This page shows how to scale a StatefulSet. {% endcapture %} {% capture prerequisites %} -* Stateful Sets are only available in Kubernetes release >= 1.5. -* Stateful Sets are previously known as Pet Sets in Kubernetes release 1.3-1.4. You can either upgrade your Pet Sets to Stateful Sets, -or just change all `statefulset` references to `petset`. *TODO: link to upgrade from Pet Sets to Stateful Sets.* -* **Not all stateful applications scale nicely.** You need to understand your Stateful Sets well before continue. If you're unsure, remember that it may not be safe to scale your Stateful Sets. +* StatefulSets are only available in Kubernetes version 1.5 or later. +* **Not all stateful applications scale nicely.** You need to understand your StatefulSets well before continuing. If you're unsure, remember that it might not be safe to scale your StatefulSets. +* You should perform scaling only when you're sure that your stateful application + cluster is completely healthy. {% endcapture %} {% capture steps %} -### Use `kubectl` to scale Stateful Sets +### Use `kubectl` to scale StatefulSets -#### `kubectl scale` (>= 1.4 release) +Make sure you have `kubectl` upgraded to Kubernetes version 1.5 or later before +continuing. If you're unsure, run `kubectl version` and check `Client Version` +for which kubectl you're using. -First, find the Stateful Set you want to scale. Remember, you need to first understand if you can scale it or not. +#### `kubectl scale` + +First, find the StatefulSet you want to scale. Remember, you need to first understand if you can scale it or not. ```shell kubectl get statefulsets ``` -If you wish to change the number of replicas of your Stateful Set, just use this command: +Change the number of replicas of your StatefulSet: ```shell kubectl scale statefulsets --replicas= ``` -Note that `kubectl scale` only works on Stateful Set with Kubernetes release >= 1.4. +#### Alternative: `kubectl apply` / `kubectl edit` / `kubectl patch` -#### Alternative: `kubectl apply` / `kubectl edit` / `kubectl patch` (>= 1.3 release) +Alternatively, you can do [in-place updates](/docs/user-guide/managing-deployments/#in-place-updates-of-resources) on your StatefulSets. -Alternatively, you may do [in-place updates](/docs/user-guide/managing-deployments/#in-place-updates-of-resources) on your Stateful Sets. - -If your Stateful Set was initially created with `kubectl apply` or `kubectl create --save-config`, -you may update `.spec.replicas` of the Stateful Set manifests, and then do a `kubectl apply`: +If your StatefulSet was initially created with `kubectl apply` or `kubectl create --save-config`, +update `.spec.replicas` of the StatefulSet manifests, and then do a `kubectl apply`: ```shell kubectl apply -f ``` -Otherwise, you can just edit that field with `kubectl edit`: +Otherwise, edit that field with `kubectl edit`: ```shell kubectl edit statefulsets @@ -70,14 +72,28 @@ kubectl patch statefulsets -p '{"spec":{"replicas": 1, if there is an unhealthy Pod, there is no way +for Kubernetes to know (yet) if it is due to a permanent fault or a transient +one (upgrade/maintenance/node reboot). If it were a permanent fault, scaling +without paying heed to it may lead to a state where the StatefulSet membership +drops below a certain minimum number of "replicas" that are needed to function +correctly, leading to unavailability (or worse). + +If it were a transient one and the Pod were coming back up shortly, you won't +want that to interleave with your scale-up/scale-down operation. Some distributed +databases have issues when nodes join and leave at the same time. It is better +to reason about scaling operations at the application level in these cases, and +perform scaling only when you're sure that your stateful application cluster is +completely healthy. {% endcapture %} {% capture whatsnext %} -*TODO: link to other docs about Stateful Set?* +*TODO: link to other docs about StatefulSet?* {% endcapture %} {% include templates/task.md %} From b77034987fb2ff187d83b00b1c592d704e0bb0a5 Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Fri, 2 Dec 2016 13:35:50 -0800 Subject: [PATCH 3/3] Address Devin's comments --- _data/tasks.yml | 2 +- docs/tasks/index.md | 2 +- docs/tasks/manage-stateful-set/scale-stateful-set.md | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/_data/tasks.yml b/_data/tasks.yml index 65cd74db91..75c60e32e2 100644 --- a/_data/tasks.yml +++ b/_data/tasks.yml @@ -37,5 +37,5 @@ toc: section: - title: Upgrading from PetSets to StatefulSets path: /docs/tasks/manage-stateful-set/upgrade-pet-set-to-stateful-set/ - - title: Scaling a Stateful Set + - title: Scaling a StatefulSet path: /docs/tasks/manage-stateful-set/scale-stateful-set/ diff --git a/docs/tasks/index.md b/docs/tasks/index.md index 14fc254b60..e89610777c 100644 --- a/docs/tasks/index.md +++ b/docs/tasks/index.md @@ -31,7 +31,7 @@ single thing, typically by giving a short sequence of steps. #### Managing Stateful Applications * [Upgrading from PetSets to StatefulSets](/docs/tasks/manage-stateful-set/upgrade-pet-set-to-stateful-set/) -* [Scaling a Stateful Set](/docs/tasks/manage-stateful-set/scale-stateful-set/) +* [Scaling a StatefulSet](/docs/tasks/manage-stateful-set/scale-stateful-set/) ### What's next diff --git a/docs/tasks/manage-stateful-set/scale-stateful-set.md b/docs/tasks/manage-stateful-set/scale-stateful-set.md index 53bdab4a08..a74b8f80d7 100644 --- a/docs/tasks/manage-stateful-set/scale-stateful-set.md +++ b/docs/tasks/manage-stateful-set/scale-stateful-set.md @@ -77,13 +77,13 @@ after those stateful Pods become running and ready. With a StatefulSet of size > 1, if there is an unhealthy Pod, there is no way for Kubernetes to know (yet) if it is due to a permanent fault or a transient -one (upgrade/maintenance/node reboot). If it were a permanent fault, scaling -without paying heed to it may lead to a state where the StatefulSet membership +one (upgrade/maintenance/node reboot). If the Pod is unhealthy due to a permanent fault, scaling +without correcting the fault may lead to a state where the StatefulSet membership drops below a certain minimum number of "replicas" that are needed to function -correctly, leading to unavailability (or worse). +correctly. This may cause your StatefulSet to become unavailable. -If it were a transient one and the Pod were coming back up shortly, you won't -want that to interleave with your scale-up/scale-down operation. Some distributed +If the Pod is unhealthy due to a transient fault and the Pod might become available again, +the transient error may interfere with your scale-up/scale-down operation. Some distributed databases have issues when nodes join and leave at the same time. It is better to reason about scaling operations at the application level in these cases, and perform scaling only when you're sure that your stateful application cluster is