Merge pull request #1722 from enisoc/mysql-stateful-set

Add replicated MySQL tutorial

_data/tutorials.yml

@@ -55,3 +55,5 @@ toc:
   section:
   - title: Running a Single-Instance Stateful Application
     path: /docs/tutorials/stateful-application/run-stateful-application/
+  - title: Running a Replicated Stateful Application
+    path: /docs/tutorials/stateful-application/run-replicated-stateful-application/

_includes/default-storage-class-prereqs.md

@@ -0,0 +1,6 @@
+You need to either have a dynamic PersistentVolume provisioner with a default
+[StorageClass](/docs/user-guide/persistent-volumes/#storageclasses),
+or [statically provision PersistentVolumes](/docs/user-guide/persistent-volumes/#provisioning)
+yourself to satisfy the [PersistentVolumeClaims](/docs/user-guide/persistent-volumes/#persistentvolumeclaims)
+used here.
+

docs/tutorials/index.md

@@ -21,6 +21,7 @@ each of which has a sequence of steps.
 #### Stateful Applications
 
 * [Running a Single-Instance Stateful Application](/docs/tutorials/stateful-application/run-stateful-application/)
+* [Running a Replicated Stateful Application](/docs/tutorials/stateful-application/run-replicated-stateful-application/)
 
 ### What's next
 

docs/tutorials/stateful-application/Dockerfile

@@ -0,0 +1,17 @@
+# This is an image with Percona XtraBackup, mysql-client and ncat installed.
+FROM debian:jessie
+
+RUN \
+  echo "deb http://repo.percona.com/apt jessie main" > /etc/apt/sources.list.d/percona.list \
+  && echo "deb-src http://repo.percona.com/apt jessie main" >> /etc/apt/sources.list.d/percona.list \
+  && apt-key adv --keyserver keys.gnupg.net --recv-keys 8507EFA5
+
+RUN \
+  apt-get update && apt-get install -y --no-install-recommends \
+    percona-xtrabackup-24 \
+    mysql-client \
+    nmap \
+  && rm -rf /var/lib/apt/lists/*
+
+CMD ["bash"]
+

docs/tutorials/stateful-application/mysql-configmap.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+data:
+  master.cnf: |
+    # Apply this config only on the master.
+    [mysqld]
+    log-bin
+  slave.cnf: |
+    # Apply this config only on slaves.
+    [mysqld]
+    super-read-only
+

docs/tutorials/stateful-application/mysql-services.yaml

@@ -0,0 +1,30 @@
+# Headless service for stable DNS entries of StatefulSet members.
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+spec:
+  ports:
+  - name: mysql
+    port: 3306
+  clusterIP: None
+  selector:
+    app: mysql
+---
+# Client service for connecting to any MySQL instance for reads.
+# For writes, you must instead connect to the master: mysql-0.mysql.
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql-read
+  labels:
+    app: mysql
+spec:
+  ports:
+  - name: mysql
+    port: 3306
+  selector:
+    app: mysql
+

docs/tutorials/stateful-application/mysql-statefulset.yaml

@@ -0,0 +1,165 @@
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: mysql
+spec:
+  serviceName: mysql
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: mysql
+      annotations:
+        pod.beta.kubernetes.io/init-containers: '[
+          {
+            "name": "init-mysql",
+            "image": "mysql:5.7",
+            "command": ["bash", "-c", "
+              set -ex\n
+              # mysqld --initialize expects an empty data dir.\n
+              rm -rf /mnt/data/lost+found\n
+              # Generate mysql server-id from pod ordinal index.\n
+              [[ `hostname` =~ -([0-9]+)$ ]] || exit 1\n
+              ordinal=${BASH_REMATCH[1]}\n
+              echo [mysqld] > /mnt/conf.d/server-id.cnf\n
+              # Add an offset to avoid reserved server-id=0 value.\n
+              echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf\n
+              # Copy appropriate conf.d files from config-map to emptyDir.\n
+              if [[ $ordinal -eq 0 ]]; then\n
+                cp /mnt/config-map/master.cnf /mnt/conf.d/\n
+              else\n
+                cp /mnt/config-map/slave.cnf /mnt/conf.d/\n
+              fi\n
+            "],
+            "volumeMounts": [
+              {"name": "data", "mountPath": "/mnt/data"},
+              {"name": "conf", "mountPath": "/mnt/conf.d"},
+              {"name": "config-map", "mountPath": "/mnt/config-map"}
+            ]
+          },
+          {
+            "name": "clone-mysql",
+            "image": "gcr.io/google-samples/xtrabackup:1.0",
+            "command": ["bash", "-c", "
+              set -ex\n
+              # Skip the clone if data already exists.\n
+              [[ -d /var/lib/mysql/mysql ]] && exit 0\n
+              # Skip the clone on master (ordinal index 0).\n
+              [[ `hostname` =~ -([0-9]+)$ ]] || exit 1\n
+              ordinal=${BASH_REMATCH[1]}\n
+              [[ $ordinal -eq 0 ]] && exit 0\n
+              # Clone data from previous peer.\n
+              ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql\n
+              # Prepare the backup.\n
+              xtrabackup --prepare --target-dir=/var/lib/mysql\n
+            "],
+            "volumeMounts": [
+              {"name": "data", "mountPath": "/var/lib/mysql"},
+              {"name": "conf", "mountPath": "/etc/mysql/conf.d"}
+            ]
+          }
+        ]'
+    spec:
+      containers:
+      - name: mysql
+        image: mysql:5.7
+        env:
+        - name: MYSQL_ALLOW_EMPTY_PASSWORD
+          value: "1"
+        ports:
+        - name: mysql
+          containerPort: 3306
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+        resources:
+          requests:
+            cpu: 1
+            memory: 1Gi
+        livenessProbe:
+          exec:
+            command: ["mysqladmin", "ping"]
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            # Check we can execute queries over TCP (skip-networking is off).
+            command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"]
+          initialDelaySeconds: 5
+          timeoutSeconds: 1
+      - name: xtrabackup
+        image: gcr.io/google-samples/xtrabackup:1.0
+        ports:
+        - name: xtrabackup
+          containerPort: 3307
+        command:
+        - bash
+        - "-c"
+        - |
+          set -ex
+          cd /var/lib/mysql
+
+          # Determine binlog position of cloned data, if any.
+          if [[ -f xtrabackup_slave_info ]]; then
+            # XtraBackup already generated a partial "CHANGE MASTER TO" query
+            # because we're cloning from an existing slave.
+            mv xtrabackup_slave_info change_master_to.sql.in
+            # Ignore xtrabackup_binlog_info in this case (it's useless).
+            rm -f xtrabackup_binlog_info
+          elif [[ -f xtrabackup_binlog_info ]]; then
+            # We're cloning directly from master. Parse binlog position.
+            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
+            rm xtrabackup_binlog_info
+            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
+                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
+          fi
+
+          # Check if we need to complete a clone by starting replication.
+          if [[ -f change_master_to.sql.in ]]; then
+            echo "Waiting for mysqld to be ready (accepting connections)"
+            until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
+
+            echo "Initializing replication from clone position"
+            # In case of container restart, attempt this at-most-once.
+            mv change_master_to.sql.in change_master_to.sql.orig
+            mysql -h 127.0.0.1 <<EOF
+          $(<change_master_to.sql.orig),
+            MASTER_HOST='mysql-0.mysql',
+            MASTER_USER='root',
+            MASTER_PASSWORD='',
+            MASTER_CONNECT_RETRY=10;
+          START SLAVE;
+          EOF
+          fi
+
+          # Start a server to send backups when requested by peers.
+          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
+            "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+      volumes:
+      - name: conf
+        emptyDir: {}
+      - name: config-map
+        configMap:
+          name: mysql
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+      annotations:
+        volume.alpha.kubernetes.io/storage-class: default
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      resources:
+        requests:
+          storage: 10Gi
+

docs/tutorials/stateful-application/run-replicated-stateful-application.md

@@ -0,0 +1,535 @@
+---
+assignees:
+- bprashanth
+- enisoc
+- erictune
+- foxish
+- janetkuo
+- kow3ns
+- smarterclayton
+
+---
+
+{% capture overview %}
+
+This page shows how to run a replicated stateful application using a
+[StatefulSet](/docs/concepts/abstractions/controllers/statefulsets/) controller.
+The example is a MySQL single-master topology with multiple slaves running
+asynchronous replication.
+
+Note that **this is not a production configuration**.
+In particular, MySQL settings remain on insecure defaults to keep the focus
+on general patterns for running stateful applications in Kubernetes.
+
+{% endcapture %}
+
+{% capture prerequisites %}
+
+* {% include task-tutorial-prereqs.md %}
+* {% include default-storage-class-prereqs.md %}
+* This tutorial assumes you are familiar with
+  [PersistentVolumes](/docs/user-guide/persistent-volumes/)
+  and [StatefulSets](/docs/concepts/abstractions/controllers/statefulsets/),
+  as well as other core concepts like [Pods](/docs/user-guide/pods/),
+  [Services](/docs/user-guide/services/), and
+  [ConfigMaps](/docs/user-guide/configmap/).
+* Some familiarity with MySQL helps, but this tutorial aims to present
+  general patterns that should be useful for other systems.
+
+{% endcapture %}
+
+{% capture objectives %}
+
+* Deploy a replicated MySQL topology with a StatefulSet controller.
+* Send MySQL client traffic.
+* Observe resistance to downtime.
+* Scale the StatefulSet up and down.
+
+{% endcapture %}
+
+{% capture lessoncontent %}
+
+### Deploying MySQL
+
+The example MySQL deployment consists of a ConfigMap, two Services,
+and a StatefulSet.
+
+#### ConfigMap
+
+Create the ConfigMap from the following YAML configuration file:
+
+```shell
+kubectl create -f http://k8s.io/docs/tutorials/stateful-application/mysql-configmap.yaml
+```
+
+{% include code.html language="yaml" file="mysql-configmap.yaml" ghlink="/docs/tutorials/stateful-application/mysql-configmap.yaml" %}
+
+This ConfigMap provides `my.cnf` overrides that let you independently control
+configuration on the MySQL master and slaves.
+In this case, you want the master to be able to serve replication logs to slaves
+and you want slaves to reject any writes that don't come via replication.
+
+There's nothing special about the ConfigMap itself that causes different
+portions to apply to different Pods.
+Each Pod decides which portion to look at as it's initializing,
+based on information provided by the StatefulSet controller.
+
+#### Services
+
+Create the Services from the following YAML configuration file:
+
+```shell
+kubectl create -f http://k8s.io/docs/tutorials/stateful-application/mysql-services.yaml
+```
+
+{% include code.html language="yaml" file="mysql-services.yaml" ghlink="/docs/tutorials/stateful-application/mysql-services.yaml" %}
+
+The Headless Service provides a home for the DNS entries that the StatefulSet
+controller creates for each Pod that's part of the set.
+Because the Headless Service is named `mysql`, the Pods are accessible by
+resolving `<pod-name>.mysql` from within any other Pod in the same Kubernetes
+cluster and namespace.
+
+The Client Service, called `mysql-read`, is a normal Service with its own
+cluster IP that distributes connections across all MySQL Pods that report
+being Ready. The set of potential endpoints includes the MySQL master and all
+slaves.
+
+Note that only read queries can use the load-balanced Client Service.
+Because there is only one MySQL master, clients should connect directly to the
+MySQL master Pod (through its DNS entry within the Headless Service) to execute
+writes.
+
+#### StatefulSet
+
+Finally, create the StatefulSet from the following YAML configuration file:
+
+```shell
+kubectl create -f http://k8s.io/docs/tutorials/stateful-application/mysql-statefulset.yaml
+```
+
+{% include code.html language="yaml" file="mysql-statefulset.yaml" ghlink="/docs/tutorials/stateful-application/mysql-statefulset.yaml" %}
+
+You can watch the startup progress by running:
+
+```shell
+kubectl get pods -l app=mysql --watch
+```
+
+After a while, you should see all 3 Pods become Running:
+
+```
+NAME      READY     STATUS    RESTARTS   AGE
+mysql-0   2/2       Running   0          2m
+mysql-1   2/2       Running   0          1m
+mysql-2   2/2       Running   0          1m
+```
+
+Press **Ctrl+C** to cancel the watch.
+If you don't see any progress, make sure you have a dynamic PersistentVolume
+provisioner enabled as mentioned in the [prerequisites](#before-you-begin).
+
+This manifest uses a variety of techniques for managing stateful Pods as part of
+a StatefulSet. The next section highlights some of these techniques to explain
+what happens as the StatefulSet creates Pods.
+
+### Understanding stateful Pod initialization
+
+The StatefulSet controller starts Pods one at a time, in order by their
+ordinal index.
+It waits until each Pod reports being Ready before starting the next one.
+
+In addition, the controller assigns each Pod a unique, stable name of the form
+`<statefulset-name>-<ordinal-index>`.
+In this case, that results in Pods named `mysql-0`, `mysql-1`, and `mysql-2`.
+
+The Pod template in the above StatefulSet manifest takes advantage of these
+properties to perform orderly startup of MySQL replication.
+
+#### Generating configuration
+
+Before starting any of the containers in the Pod spec, the Pod first runs any
+[Init Containers](/docs/user-guide/production-pods/#handling-initialization)
+in the order defined.
+In the StatefulSet manifest, you can find these defined within the
+`pod.beta.kubernetes.io/init-containers` annotation.
+
+The first Init Container, named `init-mysql`, generates special MySQL config
+files based on the ordinal index.
+
+The script determines its own ordinal index by extracting it from the end of
+the Pod name, which is returned by the `hostname` command.
+Then it saves the ordinal (with a numeric offset to avoid reserved values)
+into a file called `server-id.cnf` in the MySQL `conf.d` directory.
+This translates the unique, stable identity provided by the StatefulSet
+controller into the domain of MySQL server IDs, which require the same
+properties.
+
+The script in the `init-mysql` container also applies either `master.cnf` or
+`slave.cnf` from the ConfigMap by copying the contents into `conf.d`.
+Because the example topology consists of a single MySQL master and any number of
+slaves, the script simply assigns ordinal `0` to be the master, and everyone
+else to be slaves.
+Combined with the StatefulSet controller's
+[deployment order guarantee](/docs/concepts/abstractions/controllers/statefulsets/#deployment-and-scaling-guarantee),
+this ensures the MySQL master is Ready before creating slaves, so they can begin
+replicating.
+
+#### Cloning existing data
+
+In general, when a new Pod joins the set as a slave, it must assume the MySQL
+master might already have data on it. It also must assume that the replication
+logs might not go all the way back to the beginning of time.
+These conservative assumptions are the key to allowing a running StatefulSet
+to scale up and down over time, rather than being fixed at its initial size.
+
+The second Init Container, named `clone-mysql`, performs a clone operation on
+a slave Pod the first time it starts up on an empty PersistentVolume.
+That means it copies all existing data from another running Pod,
+so its local state is consistent enough to begin replicating from the master.
+
+MySQL itself does not provide a mechanism to do this, so the example uses a
+popular open-source tool called Percona XtraBackup.
+During the clone, the source MySQL server might suffer reduced performance.
+To minimize impact on the MySQL master, the script instructs each Pod to clone
+from the Pod whose ordinal index is one lower.
+This works because the StatefulSet controller always ensures Pod `N` is
+Ready before starting Pod `N+1`.
+
+#### Starting replication
+
+After the Init Containers complete successfully, the regular containers run.
+The MySQL Pods consist of a `mysql` container that runs the actual `mysqld`
+server, and an `xtrabackup` container that acts as a
+[sidecar](http://blog.kubernetes.io/2015/06/the-distributed-system-toolkit-patterns.html).
+
+The `xtrabackup` sidecar looks at the cloned data files and determines if
+it's necessary to initialize MySQL replication on the slave.
+If so, it waits for `mysqld` to be ready and then executes the
+`CHANGE MASTER TO` and `START SLAVE` commands with replication parameters
+extracted from the XtraBackup clone files.
+
+Once a slave begins replication, it remembers its MySQL master and
+reconnects automatically if the server restarts or the connection dies.
+Also, because slaves look for the master at its stable DNS name
+(`mysql-0.mysql`), they automatically find the master even if it gets a new
+Pod IP due to being rescheduled.
+
+Lastly, after starting replication, the `xtrabackup` container listens for
+connections from other Pods requesting a data clone.
+This server remains up indefinitely in case the StatefulSet scales up, or in
+case the next Pod loses its PersistentVolumeClaim and needs to redo the clone.
+
+### Sending client traffic
+
+You can send test queries to the MySQL master (hostname `mysql-0.mysql`)
+by running a temporary container with the `mysql:5.7` image and running the
+`mysql` client binary.
+
+```shell
+kubectl run mysql-client --image=mysql:5.7 -i -t --rm --restart=Never --\
+  mysql -h mysql-0.mysql <<EOF
+CREATE DATABASE test;
+CREATE TABLE test.messages (message VARCHAR(250));
+INSERT INTO test.messages VALUES ('hello');
+EOF
+```
+
+Use the hostname `mysql-read` to send test queries to any server that reports
+being Ready:
+
+```shell
+kubectl run mysql-client --image=mysql:5.7 -i -t --rm --restart=Never --\
+  mysql -h mysql-read -e "SELECT * FROM test.messages"
+```
+
+You should get output like this:
+
+```
+Waiting for pod default/mysql-client to be running, status is Pending, pod ready: false
++---------+
+| message |
++---------+
+| hello   |
++---------+
+pod "mysql-client" deleted
+```
+
+To demonstrate that the `mysql-read` Service distributes connections across
+servers, you can run `SELECT @@server_id` in a loop:
+
+```shell
+kubectl run mysql-client-loop --image=mysql:5.7 -i -t --rm --restart=Never --\
+  bash -ic "while sleep 1; do mysql -h mysql-read -e 'SELECT @@server_id,NOW()'; done"
+```
+
+You should see the reported `@@server_id` change randomly, because a different
+endpoint might be selected upon each connection attempt:
+
+```
++-------------+---------------------+
+| @@server_id | NOW()               |
++-------------+---------------------+
+|         100 | 2006-01-02 15:04:05 |
++-------------+---------------------+
++-------------+---------------------+
+| @@server_id | NOW()               |
++-------------+---------------------+
+|         102 | 2006-01-02 15:04:06 |
++-------------+---------------------+
++-------------+---------------------+
+| @@server_id | NOW()               |
++-------------+---------------------+
+|         101 | 2006-01-02 15:04:07 |
++-------------+---------------------+
+```
+
+You can press **Ctrl+C** when you want to stop the loop, but it's useful to keep
+it running in another window so you can see the effects of the following steps.
+
+### Simulating Pod and Node downtime
+
+To demonstrate the increased availability of reading from the pool of slaves
+instead of a single server, keep the `SELECT @@server_id` loop from above
+running while you force a Pod out of the Ready state.
+
+#### Break the Readiness Probe
+
+The [readiness probe](/docs/user-guide/production-pods/#liveness-and-readiness-probes-aka-health-checks)
+for the `mysql` container runs the command `mysql -h 127.0.0.1 -e 'SELECT 1'`
+to make sure the server is up and able to execute queries.
+
+One way to force this readiness probe to fail is to break that command:
+
+```shell
+kubectl exec mysql-2 -c mysql -- mv /usr/bin/mysql /usr/bin/mysql.off
+```
+
+This reaches into the actual container's filesystem for Pod `mysql-2` and
+renames the `mysql` command so the readiness probe can't find it.
+After a few seconds, the Pod should report one of its containers as not Ready,
+which you can check by running:
+
+```shell
+kubectl get pod mysql-2
+```
+
+Look for `1/2` in the `READY` column:
+
+```
+NAME      READY     STATUS    RESTARTS   AGE
+mysql-2   1/2       Running   0          3m
+```
+
+At this point, you should see your `SELECT @@server_id` loop continue to run,
+although it never reports `102` anymore.
+Recall that the `init-mysql` script defined `server-id` as `100 + $ordinal`,
+so server ID `102` corresponds to Pod `mysql-2`.
+
+Now repair the Pod and it should reappear in the loop output
+after a few seconds:
+
+```shell
+kubectl exec mysql-2 -c mysql -- mv /usr/bin/mysql.off /usr/bin/mysql
+```
+
+#### Delete Pods
+
+The StatefulSet also recreates Pods if they're deleted, similar to what a
+ReplicaSet does for stateless Pods.
+
+```shell
+kubectl delete pod mysql-2
+```
+
+The StatefulSet controller notices that no `mysql-2` Pod exists anymore,
+and creates a new one with the same name and linked to the same
+PersistentVolumeClaim.
+You should see server ID `102` disappear from the loop output for a while
+and then return on its own.
+
+#### Drain a Node
+
+If your Kubernetes cluster has multiple Nodes, you can simulate Node downtime
+(such as when Nodes are upgraded) by issuing a
+[drain](http://kubernetes.io/docs/user-guide/kubectl/kubectl_drain/).
+
+First determine which Node one of the MySQL Pods is on:
+
+```shell
+kubectl get pod mysql-2 -o wide
+```
+
+The Node name should show up in the last column:
+
+```
+NAME      READY     STATUS    RESTARTS   AGE       IP            NODE
+mysql-2   2/2       Running   0          15m       10.244.5.27   kubernetes-minion-group-9l2t
+```
+
+Then drain the Node by running the following command, which cordons it so
+no new Pods may schedule there, and then evicts any existing Pods.
+Replace `<node-name>` with the name of the Node you found in the last step.
+
+This might impact other applications on the Node, so it's best to
+**only do this in a test cluster**.
+
+```shell
+kubectl drain <node-name> --force --delete-local-data --ignore-daemonsets
+```
+
+Now you can watch as the Pod reschedules on a different Node:
+
+```shell
+kubectl get pod mysql-2 -o wide --watch
+```
+
+It should look something like this:
+
+```
+NAME      READY   STATUS          RESTARTS   AGE       IP            NODE
+mysql-2   2/2     Terminating     0          15m       10.244.1.56   kubernetes-minion-group-9l2t
+[...]
+mysql-2   0/2     Pending         0          0s        <none>        kubernetes-minion-group-fjlm
+mysql-2   0/2     Init:0/2        0          0s        <none>        kubernetes-minion-group-fjlm
+mysql-2   0/2     Init:1/2        0          20s       10.244.5.32   kubernetes-minion-group-fjlm
+mysql-2   0/2     PodInitializing 0          21s       10.244.5.32   kubernetes-minion-group-fjlm
+mysql-2   1/2     Running         0          22s       10.244.5.32   kubernetes-minion-group-fjlm
+mysql-2   2/2     Running         0          30s       10.244.5.32   kubernetes-minion-group-fjlm
+```
+
+And again, you should see server ID `102` disappear from the
+`SELECT @@server_id` loop output for a while and then return.
+
+Now uncordon the Node to return it to a normal state:
+
+```shell
+kubectl uncordon <node-name>
+```
+
+### Scaling the number of slaves
+
+With MySQL replication, you can scale your read query capacity by adding slaves.
+With StatefulSet, you can do this with a single command:
+
+```shell
+kubectl scale --replicas=5 statefulset mysql
+```
+
+Watch the new Pods come up by running:
+
+```shell
+kubectl get pods -l app=mysql --watch
+```
+
+Once they're up, you should see server IDs `103` and `104` start appearing in
+the `SELECT @@server_id` loop output.
+
+You can also verify that these new servers have the data you added before they
+existed:
+
+```shell
+kubectl run mysql-client --image=mysql:5.7 -i -t --rm --restart=Never --\
+  mysql -h mysql-3.mysql -e "SELECT * FROM test.messages"
+```
+
+```
+Waiting for pod default/mysql-client to be running, status is Pending, pod ready: false
++---------+
+| message |
++---------+
+| hello   |
++---------+
+pod "mysql-client" deleted
+```
+
+Scaling back down is also seamless:
+
+```shell
+kubectl scale --replicas=3 statefulset mysql
+```
+
+Note, however, that while scaling up creates new PersistentVolumeClaims
+automatically, scaling down does not automatically delete these PVCs.
+This gives you the choice to keep those initialized PVCs around to make
+scaling back up quicker, or to extract data before deleting them.
+
+You can see this by running:
+
+```shell
+kubectl get pvc -l app=mysql
+```
+
+Which shows that all 5 PVCs still exist, despite having scaled the
+StatefulSet down to 3:
+
+```
+NAME           STATUS    VOLUME                                     CAPACITY   ACCESSMODES   AGE
+data-mysql-0   Bound     pvc-8acbf5dc-b103-11e6-93fa-42010a800002   10Gi       RWO           20m
+data-mysql-1   Bound     pvc-8ad39820-b103-11e6-93fa-42010a800002   10Gi       RWO           20m
+data-mysql-2   Bound     pvc-8ad69a6d-b103-11e6-93fa-42010a800002   10Gi       RWO           20m
+data-mysql-3   Bound     pvc-50043c45-b1c5-11e6-93fa-42010a800002   10Gi       RWO           2m
+data-mysql-4   Bound     pvc-500a9957-b1c5-11e6-93fa-42010a800002   10Gi       RWO           2m
+```
+
+If you don't intend to reuse the extra PVCs, you can delete them:
+
+```shell
+kubectl delete pvc data-mysql-3
+kubectl delete pvc data-mysql-4
+```
+
+{% endcapture %}
+
+{% capture cleanup %}
+
+1. Cancel the `SELECT @@server_id` loop by pressing **Ctrl+C** in its terminal,
+   or running the following from another terminal:
+
+   ```shell
+   kubectl delete pod mysql-client-loop --now
+   ```
+
+1. Delete the StatefulSet. This also begins terminating the Pods.
+
+   ```shell
+   kubectl delete statefulset mysql
+   ```
+
+1. Verify that the Pods disappear.
+   They might take some time to finish terminating.
+
+   ```shell
+   kubectl get pods -l app=mysql
+   ```
+
+   You'll know the Pods have terminated when the above returns:
+
+   ```
+   No resources found.
+   ```
+
+1. Delete the ConfigMap, Services, and PersistentVolumeClaims.
+
+   ```shell
+   kubectl delete configmap,service,pvc -l app=mysql
+   ```
+
+1. If you manually provisioned PersistentVolumes, you also need to manually
+   delete them, as well as release the underlying resources.
+   If you used a dynamic provisioner, it automatically deletes the
+   PersistentVolumes when it sees that you deleted the PersistentVolumeClaims.
+   Some dynamic provisioners (such as those for EBS and PD) also release the
+   underlying resources upon deleting the PersistentVolumes.
+
+{% endcapture %}
+
+{% capture whatsnext %}
+
+* Look in the [Helm Charts repository](https://github.com/kubernetes/charts)
+  for other stateful application examples.
+
+{% endcapture %}
+
+{% include templates/tutorial.md %}
+

docs/user-guide/petset.yaml

@@ -14,8 +14,8 @@ spec:
   selector:
     app: nginx
 ---
-apiVersion: apps/v1alpha1
+apiVersion: apps/v1beta1
-kind: PetSet
+kind: StatefulSet
 metadata:
   name: web
 spec:

test/examples_test.go

@@ -17,7 +17,10 @@ limitations under the License.
 package examples_test
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -28,6 +31,8 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/testapi"
 	"k8s.io/kubernetes/pkg/api/validation"
+	"k8s.io/kubernetes/pkg/apis/apps"
+	apps_validation "k8s.io/kubernetes/pkg/apis/apps/validation"
 	"k8s.io/kubernetes/pkg/apis/batch"
 	batch_validation "k8s.io/kubernetes/pkg/apis/batch/validation"
 	"k8s.io/kubernetes/pkg/apis/extensions"
@@ -132,6 +137,16 @@ func validateObject(obj runtime.Object) (errors field.ErrorList) {
 			t.Namespace = api.NamespaceDefault
 		}
 		errors = batch_validation.ValidateCronJob(t)
+	case *api.ConfigMap:
+		if t.Namespace == "" {
+			t.Namespace = api.NamespaceDefault
+		}
+		errors = validation.ValidateConfigMap(t)
+	case *apps.StatefulSet:
+		if t.Namespace == "" {
+			t.Namespace = api.NamespaceDefault
+		}
+		errors = apps_validation.ValidateStatefulSet(t)
 	default:
 		errors = field.ErrorList{}
 		errors = append(errors, field.InternalError(field.NewPath(""), fmt.Errorf("no validation defined for %#v", obj)))
@@ -141,7 +156,7 @@ func validateObject(obj runtime.Object) (errors field.ErrorList) {
 
 // Walks inDir for any json/yaml files. Converts yaml to json, and calls fn for
 // each file found with the contents in data.
-func walkConfigFiles(inDir string, fn func(name, path string, data []byte)) error {
+func walkConfigFiles(inDir string, fn func(name, path string, data [][]byte)) error {
 	return filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -160,132 +175,153 @@ func walkConfigFiles(inDir string, fn func(name, path string, data []byte)) erro
 			}
 			name := strings.TrimSuffix(file, ext)
 
+			var docs [][]byte
 			if ext == ".yaml" {
-				out, err := yaml.ToJSON(data)
+				// YAML can contain multiple documents.
-				if err != nil {
+				splitter := yaml.NewYAMLReader(bufio.NewReader(bytes.NewBuffer(data)))
-					return fmt.Errorf("%s: %v", path, err)
+				for {
+					doc, err := splitter.Read()
+					if err == io.EOF {
+						break
+					}
+					if err != nil {
+						return fmt.Errorf("%s: %v", path, err)
+					}
+					out, err := yaml.ToJSON(doc)
+					if err != nil {
+						return fmt.Errorf("%s: %v", path, err)
+					}
+					docs = append(docs, out)
 				}
-				data = out
+			} else {
+				docs = append(docs, data)
 			}
 
-			fn(name, path, data)
+			fn(name, path, docs)
 		}
 		return nil
 	})
 }
 
 func TestExampleObjectSchemas(t *testing.T) {
-	cases := map[string]map[string]runtime.Object{
+	cases := map[string]map[string][]runtime.Object{
 		"../docs/user-guide/walkthrough": {
-			"deployment":                &extensions.Deployment{},
+			"deployment":                {&extensions.Deployment{}},
-			"deployment-update":         &extensions.Deployment{},
+			"deployment-update":         {&extensions.Deployment{}},
-			"pod-nginx":                 &api.Pod{},
+			"pod-nginx":                 {&api.Pod{}},
-			"pod-nginx-with-label":      &api.Pod{},
+			"pod-nginx-with-label":      {&api.Pod{}},
-			"pod-redis":                 &api.Pod{},
+			"pod-redis":                 {&api.Pod{}},
-			"pod-with-http-healthcheck": &api.Pod{},
+			"pod-with-http-healthcheck": {&api.Pod{}},
-			"podtemplate":               &api.PodTemplate{},
+			"podtemplate":               {&api.PodTemplate{}},
-			"service":                   &api.Service{},
+			"service":                   {&api.Service{}},
 		},
 		"../docs/user-guide/update-demo": {
-			"kitten-rc":   &api.ReplicationController{},
+			"kitten-rc":   {&api.ReplicationController{}},
-			"nautilus-rc": &api.ReplicationController{},
+			"nautilus-rc": {&api.ReplicationController{}},
 		},
 		"../docs/user-guide/persistent-volumes/volumes": {
-			"local-01": &api.PersistentVolume{},
+			"local-01": {&api.PersistentVolume{}},
-			"local-02": &api.PersistentVolume{},
+			"local-02": {&api.PersistentVolume{}},
-			"gce":      &api.PersistentVolume{},
+			"gce":      {&api.PersistentVolume{}},
-			"nfs":      &api.PersistentVolume{},
+			"nfs":      {&api.PersistentVolume{}},
 		},
 		"../docs/user-guide/persistent-volumes/claims": {
-			"claim-01": &api.PersistentVolumeClaim{},
+			"claim-01": {&api.PersistentVolumeClaim{}},
-			"claim-02": &api.PersistentVolumeClaim{},
+			"claim-02": {&api.PersistentVolumeClaim{}},
-			"claim-03": &api.PersistentVolumeClaim{},
+			"claim-03": {&api.PersistentVolumeClaim{}},
 		},
 		"../docs/user-guide/persistent-volumes/simpletest": {
-			"namespace": &api.Namespace{},
+			"namespace": {&api.Namespace{}},
-			"pod":       &api.Pod{},
+			"pod":       {&api.Pod{}},
-			"service":   &api.Service{},
+			"service":   {&api.Service{}},
 		},
 		"../docs/user-guide/liveness": {
-			"exec-liveness":            &api.Pod{},
+			"exec-liveness":            {&api.Pod{}},
-			"http-liveness":            &api.Pod{},
+			"http-liveness":            {&api.Pod{}},
-			"http-liveness-named-port": &api.Pod{},
+			"http-liveness-named-port": {&api.Pod{}},
 		},
 		"../docs/user-guide/jobs/work-queue-1": {
-			"job": &batch.Job{},
+			"job": {&batch.Job{}},
 		},
 		"../docs/user-guide/jobs/work-queue-2": {
-			"job":           &batch.Job{},
+			"job":           {&batch.Job{}},
-			"redis-pod":     &api.Pod{},
+			"redis-pod":     {&api.Pod{}},
-			"redis-service": &api.Service{},
+			"redis-service": {&api.Service{}},
 		},
 		"../docs/user-guide": {
-			"bad-nginx-deployment":       &extensions.Deployment{},
+			"bad-nginx-deployment":       {&extensions.Deployment{}},
-			"counter-pod":                &api.Pod{},
+			"counter-pod":                {&api.Pod{}},
-			"curlpod":                    &extensions.Deployment{},
+			"curlpod":                    {&extensions.Deployment{}},
-			"deployment":                 &extensions.Deployment{},
+			"deployment":                 {&extensions.Deployment{}},
-			"ingress":                    &extensions.Ingress{},
+			"ingress":                    {&extensions.Ingress{}},
-			"job":                        &batch.Job{},
+			"job":                        {&batch.Job{}},
-			"multi-pod":                  &api.Pod{},
+			"multi-pod":                  {&api.Pod{}, &api.Pod{}},
-			"new-nginx-deployment":       &extensions.Deployment{},
+			"new-nginx-deployment":       {&extensions.Deployment{}},
-			"nginx-app":                  &api.Service{},
+			"nginx-app":                  {&api.Service{}, &extensions.Deployment{}},
-			"nginx-deployment":           &extensions.Deployment{},
+			"nginx-deployment":           {&extensions.Deployment{}},
-			"nginx-init-containers":      &api.Pod{},
+			"nginx-init-containers":      {&api.Pod{}},
-			"nginx-lifecycle-deployment": &extensions.Deployment{},
+			"nginx-lifecycle-deployment": {&extensions.Deployment{}},
-			"nginx-probe-deployment":     &extensions.Deployment{},
+			"nginx-probe-deployment":     {&extensions.Deployment{}},
-			"nginx-secure-app":           &api.Service{},
+			"nginx-secure-app":           {&api.Service{}, &extensions.Deployment{}},
-			"nginx-svc":                  &api.Service{},
+			"nginx-svc":                  {&api.Service{}},
-			"petset":                     &api.Service{},
+			"petset":                     {&api.Service{}, &apps.StatefulSet{}},
-			"pod":                        &api.Pod{},
+			"pod":                        {&api.Pod{}},
-			"pod-w-message":              &api.Pod{},
+			"pod-w-message":              {&api.Pod{}},
-			"redis-deployment":           &extensions.Deployment{},
+			"redis-deployment":           {&extensions.Deployment{}},
-			"redis-resource-deployment":  &extensions.Deployment{},
+			"redis-resource-deployment":  {&extensions.Deployment{}},
-			"redis-secret-deployment":    &extensions.Deployment{},
+			"redis-secret-deployment":    {&extensions.Deployment{}},
-			"run-my-nginx":               &extensions.Deployment{},
+			"run-my-nginx":               {&extensions.Deployment{}},
-			"cronjob":                    &batch.CronJob{},
+			"cronjob":                    {&batch.CronJob{}},
 		},
 		"../docs/admin": {
-			"daemon": &extensions.DaemonSet{},
+			"daemon": {&extensions.DaemonSet{}},
 		},
 		"../docs/user-guide/downward-api": {
-			"dapi-pod":                 &api.Pod{},
+			"dapi-pod":                 {&api.Pod{}},
-			"dapi-container-resources": &api.Pod{},
+			"dapi-container-resources": {&api.Pod{}},
 		},
 		"../docs/user-guide/downward-api/volume/": {
-			"dapi-volume":           &api.Pod{},
+			"dapi-volume":           {&api.Pod{}},
-			"dapi-volume-resources": &api.Pod{},
+			"dapi-volume-resources": {&api.Pod{}},
 		},
 		"../docs/admin/namespaces": {
-			"namespace-dev":  &api.Namespace{},
+			"namespace-dev":  {&api.Namespace{}},
-			"namespace-prod": &api.Namespace{},
+			"namespace-prod": {&api.Namespace{}},
 		},
 		"../docs/admin/limitrange": {
-			"invalid-pod": &api.Pod{},
+			"invalid-pod": {&api.Pod{}},
-			"limits":      &api.LimitRange{},
+			"limits":      {&api.LimitRange{}},
-			"namespace":   &api.Namespace{},
+			"namespace":   {&api.Namespace{}},
-			"valid-pod":   &api.Pod{},
+			"valid-pod":   {&api.Pod{}},
 		},
 		"../docs/user-guide/logging-demo": {
-			"synthetic_0_25lps": &api.Pod{},
+			"synthetic_0_25lps": {&api.Pod{}},
-			"synthetic_10lps":   &api.Pod{},
+			"synthetic_10lps":   {&api.Pod{}},
 		},
 		"../docs/user-guide/node-selection": {
-			"pod": &api.Pod{},
+			"pod": {&api.Pod{}},
-			"pod-with-node-affinity": &api.Pod{},
+			"pod-with-node-affinity": {&api.Pod{}},
-			"pod-with-pod-affinity":  &api.Pod{},
+			"pod-with-pod-affinity":  {&api.Pod{}},
 		},
 		"../docs/admin/resourcequota": {
-			"best-effort":       &api.ResourceQuota{},
+			"best-effort":       {&api.ResourceQuota{}},
-			"compute-resources": &api.ResourceQuota{},
+			"compute-resources": {&api.ResourceQuota{}},
-			"limits":            &api.LimitRange{},
+			"limits":            {&api.LimitRange{}},
-			"namespace":         &api.Namespace{},
+			"namespace":         {&api.Namespace{}},
-			"not-best-effort":   &api.ResourceQuota{},
+			"not-best-effort":   {&api.ResourceQuota{}},
-			"object-counts":     &api.ResourceQuota{},
+			"object-counts":     {&api.ResourceQuota{}},
 		},
 		"../docs/user-guide/secrets": {
-			"secret-pod":     &api.Pod{},
+			"secret-pod":     {&api.Pod{}},
-			"secret":         &api.Secret{},
+			"secret":         {&api.Secret{}},
-			"secret-env-pod": &api.Pod{},
+			"secret-env-pod": {&api.Pod{}},
+		},
+		"../docs/tutorials/stateful-application": {
+			"gce-volume":        {&api.PersistentVolume{}},
+			"mysql-deployment":  {&api.Service{}, &api.PersistentVolumeClaim{}, &extensions.Deployment{}},
+			"mysql-services":    {&api.Service{}, &api.Service{}},
+			"mysql-configmap":   {&api.ConfigMap{}},
+			"mysql-statefulset": {&apps.StatefulSet{}},
 		},
 	}
 
@@ -295,43 +331,52 @@ func TestExampleObjectSchemas(t *testing.T) {
 
 	for path, expected := range cases {
 		tested := 0
-		err := walkConfigFiles(path, func(name, path string, data []byte) {
+		numExpected := 0
-			expectedType, found := expected[name]
+		err := walkConfigFiles(path, func(name, path string, docs [][]byte) {
+			expectedTypes, found := expected[name]
 			if !found {
 				t.Errorf("%s: %s does not have a test case defined", path, name)
 				return
 			}
-			tested++
+			numExpected += len(expectedTypes)
-			if expectedType == nil {
+			if len(expectedTypes) != len(docs) {
-				t.Logf("skipping : %s/%s\n", path, name)
+				t.Errorf("%s: number of expected types (%v) doesn't match number of docs in YAML (%v)", path, len(expectedTypes), len(docs))
 				return
 			}
-			if strings.Contains(name, "scheduler-policy-config") {
+			for i, data := range docs {
-				if err := runtime.DecodeInto(schedulerapilatest.Codec, data, expectedType); err != nil {
+				expectedType := expectedTypes[i]
-					t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
+				tested++
+				if expectedType == nil {
+					t.Logf("skipping : %s/%s\n", path, name)
 					return
 				}
-				// TODO: Add validate method for
+				if strings.Contains(name, "scheduler-policy-config") {
-				// &schedulerapi.Policy, and remove this
+					if err := runtime.DecodeInto(schedulerapilatest.Codec, data, expectedType); err != nil {
-				// special case
+						t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
-			} else {
+						return
-				codec, err := testapi.GetCodecForObject(expectedType)
+					}
-				if err != nil {
+					// TODO: Add validate method for
-					t.Errorf("Could not get codec for %s: %s", expectedType, err)
+					// &schedulerapi.Policy, and remove this
-				}
+					// special case
-				if err := runtime.DecodeInto(codec, data, expectedType); err != nil {
+				} else {
-					t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
+					codec, err := testapi.GetCodecForObject(expectedType)
-					return
+					if err != nil {
-				}
+						t.Errorf("Could not get codec for %s: %s", expectedType, err)
-				if errors := validateObject(expectedType); len(errors) > 0 {
+					}
-					t.Errorf("%s did not validate correctly: %v", path, errors)
+					if err := runtime.DecodeInto(codec, data, expectedType); err != nil {
+						t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
+						return
+					}
+					if errors := validateObject(expectedType); len(errors) > 0 {
+						t.Errorf("%s did not validate correctly: %v", path, errors)
+					}
 				}
 			}
 		})
 		if err != nil {
 			t.Errorf("Expected no error, Got %v on Path %v", err, path)
 		}
-		if tested != len(expected) {
+		if tested != numExpected {
 			t.Errorf("Directory %v: Expected %d examples, Got %d", path, len(expected), tested)
 		}
 	}