From 14d17fdfda81491de7c2253b2881fc5927622095 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Thu, 17 Nov 2016 17:41:58 -0800 Subject: [PATCH 01/17] Add replicated MySQL tutorial --- .../Dockerfile | 17 +++ .../replicated-stateful-application/my.cnf | 5 + .../mysql-headless-service.yaml | 14 ++ .../mysql-statefulset.yaml | 144 ++++++++++++++++++ 4 files changed, 180 insertions(+) create mode 100644 docs/tutorials/replicated-stateful-application/Dockerfile create mode 100644 docs/tutorials/replicated-stateful-application/my.cnf create mode 100644 docs/tutorials/replicated-stateful-application/mysql-headless-service.yaml create mode 100644 docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml diff --git a/docs/tutorials/replicated-stateful-application/Dockerfile b/docs/tutorials/replicated-stateful-application/Dockerfile new file mode 100644 index 0000000000..8016958d83 --- /dev/null +++ b/docs/tutorials/replicated-stateful-application/Dockerfile @@ -0,0 +1,17 @@ +# This is an image with Percona XtraBackup, mysql-client and ncat installed. +FROM debian:jessie + +RUN \ + echo "deb http://repo.percona.com/apt jessie main" > /etc/apt/sources.list.d/percona.list \ + && echo "deb-src http://repo.percona.com/apt jessie main" >> /etc/apt/sources.list.d/percona.list \ + && apt-key adv --keyserver keys.gnupg.net --recv-keys 8507EFA5 + +RUN \ + apt-get update && apt-get install -y --no-install-recommends \ + percona-xtrabackup-24 \ + mysql-client \ + nmap \ + && rm -rf /var/lib/apt/lists/* + +CMD ["bash"] + diff --git a/docs/tutorials/replicated-stateful-application/my.cnf b/docs/tutorials/replicated-stateful-application/my.cnf new file mode 100644 index 0000000000..c15357b9fd --- /dev/null +++ b/docs/tutorials/replicated-stateful-application/my.cnf @@ -0,0 +1,5 @@ +# Create a ConfigMap resource from this file: +# kubectl create configmap mysql --from-file=./my.cnf +[mysqld] +log-bin + diff --git a/docs/tutorials/replicated-stateful-application/mysql-headless-service.yaml b/docs/tutorials/replicated-stateful-application/mysql-headless-service.yaml new file mode 100644 index 0000000000..5f9fdcbc23 --- /dev/null +++ b/docs/tutorials/replicated-stateful-application/mysql-headless-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: mysql + labels: + app: mysql +spec: + ports: + - name: mysql + port: 3306 + clusterIP: None + selector: + app: mysql + diff --git a/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml b/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml new file mode 100644 index 0000000000..4687eff9b0 --- /dev/null +++ b/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml @@ -0,0 +1,144 @@ +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: mysql +spec: + serviceName: mysql + replicas: 3 + template: + metadata: + labels: + app: mysql + annotations: + pod.beta.kubernetes.io/init-containers: '[ + { + "name": "init-mysql", + "image": "mysql:5.7", + "command": ["bash", "-c", " + set -ex\n + # mysqld --initialize expects an empty data dir.\n + rm -rf /mnt/data/lost+found\n + # Copy conf.d from config-map to emptyDir.\n + cp /mnt/config-map/* /mnt/conf.d/\n + # Generate mysql server-id from pod ordinal index.\n + [[ `hostname` =~ -([0-9]+)$ ]] || exit 1\n + echo [mysqld] > /mnt/conf.d/server-id.cnf\n + echo server-id=$((100 + ${BASH_REMATCH[1]})) >> /mnt/conf.d/server-id.cnf\n + "], + "volumeMounts": [ + {"name": "data", "mountPath": "/mnt/data"}, + {"name": "conf", "mountPath": "/mnt/conf.d"}, + {"name": "config-map", "mountPath": "/mnt/config-map"} + ] + }, + { + "name": "clone-mysql", + "image": "enisoc/xtrabackup", + "command": ["bash", "-c", " + set -ex\n + # Skip the clone if data already exists.\n + [[ -d /var/lib/mysql/mysql ]] && exit 0\n + # Skip the clone on master (ordinal index 0).\n + [[ `hostname` =~ -0$ ]] && exit 0\n + # Clone data from master.\n + ncat --recv-only mysql-0.mysql 3307 | xbstream -x -C /var/lib/mysql\n + # Prepare the backup.\n + xtrabackup --prepare --target-dir=/var/lib/mysql\n + "], + "volumeMounts": [ + {"name": "data", "mountPath": "/var/lib/mysql"}, + {"name": "conf", "mountPath": "/etc/mysql/conf.d"} + ] + } + ]' + spec: + containers: + - name: mysql + image: mysql:5.7 + env: + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "1" + ports: + - name: mysql + containerPort: 3306 + volumeMounts: + - name: data + mountPath: /var/lib/mysql + - name: conf + mountPath: /etc/mysql/conf.d + resources: + requests: + cpu: 1 + memory: 1Gi + livenessProbe: + exec: + command: ["mysqladmin", "ping"] + initialDelaySeconds: 30 + timeoutSeconds: 5 + readinessProbe: + exec: + # Check we can execute queries over TCP (skip-networking is off). + command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"] + initialDelaySeconds: 5 + timeoutSeconds: 1 + - name: xtrabackup + image: enisoc/xtrabackup:latest + ports: + - name: xtrabackup + containerPort: 3307 + command: + - bash + - "-c" + - | + set -ex + + # Check if we need to complete a clone by starting replication. + cd /var/lib/mysql + if [[ -f xtrabackup_binlog_info ]]; then + echo "Waiting for mysqld to accept connections" + until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done + + echo "Initializing replication from clone position" + [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1 + mv xtrabackup_binlog_info xtrabackup_binlog_info.orig + mysql -h 127.0.0.1 < Date: Fri, 18 Nov 2016 11:19:15 -0800 Subject: [PATCH 02/17] Demystify server-id offset. --- .../replicated-stateful-application/mysql-statefulset.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml b/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml index 4687eff9b0..f4a6c34df8 100644 --- a/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml +++ b/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml @@ -23,6 +23,7 @@ spec: # Generate mysql server-id from pod ordinal index.\n [[ `hostname` =~ -([0-9]+)$ ]] || exit 1\n echo [mysqld] > /mnt/conf.d/server-id.cnf\n + # Add an offset to avoid reserved server-id=0 value.\n echo server-id=$((100 + ${BASH_REMATCH[1]})) >> /mnt/conf.d/server-id.cnf\n "], "volumeMounts": [ From 5e4b37e07239c44c0db3bb43af89c20496c675d0 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Fri, 18 Nov 2016 16:10:05 -0800 Subject: [PATCH 03/17] Clone from previous peer instead of from master. --- .../mysql-statefulset.yaml | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml b/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml index f4a6c34df8..accefb2fe2 100644 --- a/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml +++ b/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml @@ -40,9 +40,11 @@ spec: # Skip the clone if data already exists.\n [[ -d /var/lib/mysql/mysql ]] && exit 0\n # Skip the clone on master (ordinal index 0).\n - [[ `hostname` =~ -0$ ]] && exit 0\n - # Clone data from master.\n - ncat --recv-only mysql-0.mysql 3307 | xbstream -x -C /var/lib/mysql\n + [[ `hostname` =~ -([0-9]+)$ ]] || exit 1\n + ordinal=${BASH_REMATCH[1]}\n + [[ $ordinal -eq 0 ]] && exit 0\n + # Clone data from previous peer.\n + ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql\n # Prepare the backup.\n xtrabackup --prepare --target-dir=/var/lib/mysql\n "], @@ -92,31 +94,44 @@ spec: - "-c" - | set -ex + cd /var/lib/mysql + + # Determine binlog position of cloned data, if any. + if [[ -f xtrabackup_slave_info ]]; then + # XtraBackup already generated a partial "CHANGE MASTER TO" query + # because we're cloning from an existing slave. + mv xtrabackup_slave_info change_master_to.sql.in + # Ignore xtrabackup_binlog_info in this case (it's useless). + rm xtrabackup_binlog_info + elif [[ -f xtrabackup_binlog_info ]]; then + # We're cloning directly from master. Parse binlog position. + [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1 + rm xtrabackup_binlog_info + echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\ + MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in + fi # Check if we need to complete a clone by starting replication. - cd /var/lib/mysql - if [[ -f xtrabackup_binlog_info ]]; then - echo "Waiting for mysqld to accept connections" + if [[ -f change_master_to.sql.in ]]; then + echo "Waiting for mysqld to be ready (accepting connections)" until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done echo "Initializing replication from clone position" - [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1 - mv xtrabackup_binlog_info xtrabackup_binlog_info.orig + # In case of container restart, attempt this at-most-once. + mv change_master_to.sql.in change_master_to.sql.orig mysql -h 127.0.0.1 < Date: Fri, 18 Nov 2016 16:28:08 -0800 Subject: [PATCH 04/17] Add MySQL client service for reads. --- .../mysql-headless-service.yaml | 14 --------- .../mysql-services.yaml | 30 +++++++++++++++++++ 2 files changed, 30 insertions(+), 14 deletions(-) delete mode 100644 docs/tutorials/replicated-stateful-application/mysql-headless-service.yaml create mode 100644 docs/tutorials/replicated-stateful-application/mysql-services.yaml diff --git a/docs/tutorials/replicated-stateful-application/mysql-headless-service.yaml b/docs/tutorials/replicated-stateful-application/mysql-headless-service.yaml deleted file mode 100644 index 5f9fdcbc23..0000000000 --- a/docs/tutorials/replicated-stateful-application/mysql-headless-service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: mysql - labels: - app: mysql -spec: - ports: - - name: mysql - port: 3306 - clusterIP: None - selector: - app: mysql - diff --git a/docs/tutorials/replicated-stateful-application/mysql-services.yaml b/docs/tutorials/replicated-stateful-application/mysql-services.yaml new file mode 100644 index 0000000000..ef68455396 --- /dev/null +++ b/docs/tutorials/replicated-stateful-application/mysql-services.yaml @@ -0,0 +1,30 @@ +# Headless service for stable DNS entries of Stateful Set members. +apiVersion: v1 +kind: Service +metadata: + name: mysql + labels: + app: mysql +spec: + ports: + - name: mysql + port: 3306 + clusterIP: None + selector: + app: mysql +--- +# Client service for connecting to any MySQL instance for reads. +# For writes, you must instead connect to the master: mysql-0.mysql. +apiVersion: v1 +kind: Service +metadata: + name: mysql-read + labels: + app: mysql +spec: + ports: + - name: mysql + port: 3306 + selector: + app: mysql + From 597758d92f4f5e9f49870c87feb2df4eb022753f Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Fri, 18 Nov 2016 16:59:47 -0800 Subject: [PATCH 05/17] Demonstrate master/slave ConfigMap entries. --- .../replicated-stateful-application/my.cnf | 5 ----- .../mysql-configmap.yaml | 16 ++++++++++++++++ .../mysql-statefulset.yaml | 13 +++++++++---- 3 files changed, 25 insertions(+), 9 deletions(-) delete mode 100644 docs/tutorials/replicated-stateful-application/my.cnf create mode 100644 docs/tutorials/replicated-stateful-application/mysql-configmap.yaml diff --git a/docs/tutorials/replicated-stateful-application/my.cnf b/docs/tutorials/replicated-stateful-application/my.cnf deleted file mode 100644 index c15357b9fd..0000000000 --- a/docs/tutorials/replicated-stateful-application/my.cnf +++ /dev/null @@ -1,5 +0,0 @@ -# Create a ConfigMap resource from this file: -# kubectl create configmap mysql --from-file=./my.cnf -[mysqld] -log-bin - diff --git a/docs/tutorials/replicated-stateful-application/mysql-configmap.yaml b/docs/tutorials/replicated-stateful-application/mysql-configmap.yaml new file mode 100644 index 0000000000..46d34e422c --- /dev/null +++ b/docs/tutorials/replicated-stateful-application/mysql-configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: mysql + labels: + app: mysql +data: + master.cnf: | + # Apply this config only on the master. + [mysqld] + log-bin + slave.cnf: | + # Apply this config only on slaves. + [mysqld] + super-read-only + diff --git a/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml b/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml index accefb2fe2..13ff16c3ef 100644 --- a/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml +++ b/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml @@ -18,13 +18,18 @@ spec: set -ex\n # mysqld --initialize expects an empty data dir.\n rm -rf /mnt/data/lost+found\n - # Copy conf.d from config-map to emptyDir.\n - cp /mnt/config-map/* /mnt/conf.d/\n # Generate mysql server-id from pod ordinal index.\n [[ `hostname` =~ -([0-9]+)$ ]] || exit 1\n + ordinal=${BASH_REMATCH[1]}\n echo [mysqld] > /mnt/conf.d/server-id.cnf\n # Add an offset to avoid reserved server-id=0 value.\n - echo server-id=$((100 + ${BASH_REMATCH[1]})) >> /mnt/conf.d/server-id.cnf\n + echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf\n + # Copy appropriate conf.d files from config-map to emptyDir.\n + if [[ $ordinal -eq 0 ]]; then\n + cp /mnt/config-map/master.cnf /mnt/conf.d/\n + else\n + cp /mnt/config-map/slave.cnf /mnt/conf.d/\n + fi\n "], "volumeMounts": [ {"name": "data", "mountPath": "/mnt/data"}, @@ -102,7 +107,7 @@ spec: # because we're cloning from an existing slave. mv xtrabackup_slave_info change_master_to.sql.in # Ignore xtrabackup_binlog_info in this case (it's useless). - rm xtrabackup_binlog_info + rm -f xtrabackup_binlog_info elif [[ -f xtrabackup_binlog_info ]]; then # We're cloning directly from master. Parse binlog position. [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1 From b28f321c14fe12ef02891410c86d3a08775917fd Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Tue, 22 Nov 2016 12:07:25 -0800 Subject: [PATCH 06/17] Add validation for YAML assets. --- test/examples_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/examples_test.go b/test/examples_test.go index 63fea3c5bc..6aba7203df 100644 --- a/test/examples_test.go +++ b/test/examples_test.go @@ -28,6 +28,8 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/pkg/api/validation" + "k8s.io/kubernetes/pkg/apis/apps" + apps_validation "k8s.io/kubernetes/pkg/apis/apps/validation" "k8s.io/kubernetes/pkg/apis/batch" batch_validation "k8s.io/kubernetes/pkg/apis/batch/validation" "k8s.io/kubernetes/pkg/apis/extensions" @@ -132,6 +134,16 @@ func validateObject(obj runtime.Object) (errors field.ErrorList) { t.Namespace = api.NamespaceDefault } errors = batch_validation.ValidateCronJob(t) + case *api.ConfigMap: + if t.Namespace == "" { + t.Namespace = api.NamespaceDefault + } + errors = validation.ValidateConfigMap(t) + case *apps.StatefulSet: + if t.Namespace == "" { + t.Namespace = api.NamespaceDefault + } + errors = apps_validation.ValidateStatefulSet(t) default: errors = field.ErrorList{} errors = append(errors, field.InternalError(field.NewPath(""), fmt.Errorf("no validation defined for %#v", obj))) @@ -287,6 +299,11 @@ func TestExampleObjectSchemas(t *testing.T) { "secret": &api.Secret{}, "secret-env-pod": &api.Pod{}, }, + "../docs/tutorials/replicated-stateful-application": { + "mysql-services": &api.Service{}, + "mysql-configmap": &api.ConfigMap{}, + "mysql-statefulset": &apps.StatefulSet{}, + }, } capabilities.SetForTests(capabilities.Capabilities{ From 3d08fd0fa22900344574207b12ca359be4fc7282 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Tue, 22 Nov 2016 13:17:03 -0800 Subject: [PATCH 07/17] examples_test: Validate all parts in a multi-doc YAML. --- docs/user-guide/petset.yaml | 4 +- test/examples_test.go | 242 ++++++++++++++++++++---------------- 2 files changed, 136 insertions(+), 110 deletions(-) diff --git a/docs/user-guide/petset.yaml b/docs/user-guide/petset.yaml index 5c29237c48..add3530077 100644 --- a/docs/user-guide/petset.yaml +++ b/docs/user-guide/petset.yaml @@ -14,8 +14,8 @@ spec: selector: app: nginx --- -apiVersion: apps/v1alpha1 -kind: PetSet +apiVersion: apps/v1beta1 +kind: StatefulSet metadata: name: web spec: diff --git a/test/examples_test.go b/test/examples_test.go index 6aba7203df..e41d97467c 100644 --- a/test/examples_test.go +++ b/test/examples_test.go @@ -17,7 +17,10 @@ limitations under the License. package examples_test import ( + "bufio" + "bytes" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -153,7 +156,7 @@ func validateObject(obj runtime.Object) (errors field.ErrorList) { // Walks inDir for any json/yaml files. Converts yaml to json, and calls fn for // each file found with the contents in data. -func walkConfigFiles(inDir string, fn func(name, path string, data []byte)) error { +func walkConfigFiles(inDir string, fn func(name, path string, data [][]byte)) error { return filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -172,137 +175,151 @@ func walkConfigFiles(inDir string, fn func(name, path string, data []byte)) erro } name := strings.TrimSuffix(file, ext) + var docs [][]byte if ext == ".yaml" { - out, err := yaml.ToJSON(data) - if err != nil { - return fmt.Errorf("%s: %v", path, err) + // YAML can contain multiple documents. + splitter := yaml.NewYAMLReader(bufio.NewReader(bytes.NewBuffer(data))) + for { + doc, err := splitter.Read() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("%s: %v", path, err) + } + out, err := yaml.ToJSON(doc) + if err != nil { + return fmt.Errorf("%s: %v", path, err) + } + docs = append(docs, out) } - data = out + } else { + docs = append(docs, data) } - fn(name, path, data) + fn(name, path, docs) } return nil }) } func TestExampleObjectSchemas(t *testing.T) { - cases := map[string]map[string]runtime.Object{ + cases := map[string]map[string][]runtime.Object{ "../docs/user-guide/walkthrough": { - "deployment": &extensions.Deployment{}, - "deployment-update": &extensions.Deployment{}, - "pod-nginx": &api.Pod{}, - "pod-nginx-with-label": &api.Pod{}, - "pod-redis": &api.Pod{}, - "pod-with-http-healthcheck": &api.Pod{}, - "podtemplate": &api.PodTemplate{}, - "service": &api.Service{}, + "deployment": {&extensions.Deployment{}}, + "deployment-update": {&extensions.Deployment{}}, + "pod-nginx": {&api.Pod{}}, + "pod-nginx-with-label": {&api.Pod{}}, + "pod-redis": {&api.Pod{}}, + "pod-with-http-healthcheck": {&api.Pod{}}, + "podtemplate": {&api.PodTemplate{}}, + "service": {&api.Service{}}, }, "../docs/user-guide/update-demo": { - "kitten-rc": &api.ReplicationController{}, - "nautilus-rc": &api.ReplicationController{}, + "kitten-rc": {&api.ReplicationController{}}, + "nautilus-rc": {&api.ReplicationController{}}, }, "../docs/user-guide/persistent-volumes/volumes": { - "local-01": &api.PersistentVolume{}, - "local-02": &api.PersistentVolume{}, - "gce": &api.PersistentVolume{}, - "nfs": &api.PersistentVolume{}, + "local-01": {&api.PersistentVolume{}}, + "local-02": {&api.PersistentVolume{}}, + "gce": {&api.PersistentVolume{}}, + "nfs": {&api.PersistentVolume{}}, }, "../docs/user-guide/persistent-volumes/claims": { - "claim-01": &api.PersistentVolumeClaim{}, - "claim-02": &api.PersistentVolumeClaim{}, - "claim-03": &api.PersistentVolumeClaim{}, + "claim-01": {&api.PersistentVolumeClaim{}}, + "claim-02": {&api.PersistentVolumeClaim{}}, + "claim-03": {&api.PersistentVolumeClaim{}}, }, "../docs/user-guide/persistent-volumes/simpletest": { - "namespace": &api.Namespace{}, - "pod": &api.Pod{}, - "service": &api.Service{}, + "namespace": {&api.Namespace{}}, + "pod": {&api.Pod{}}, + "service": {&api.Service{}}, }, "../docs/user-guide/liveness": { - "exec-liveness": &api.Pod{}, - "http-liveness": &api.Pod{}, - "http-liveness-named-port": &api.Pod{}, + "exec-liveness": {&api.Pod{}}, + "http-liveness": {&api.Pod{}}, + "http-liveness-named-port": {&api.Pod{}}, }, "../docs/user-guide/jobs/work-queue-1": { - "job": &batch.Job{}, + "job": {&batch.Job{}}, }, "../docs/user-guide/jobs/work-queue-2": { - "job": &batch.Job{}, - "redis-pod": &api.Pod{}, - "redis-service": &api.Service{}, + "job": {&batch.Job{}}, + "redis-pod": {&api.Pod{}}, + "redis-service": {&api.Service{}}, }, "../docs/user-guide": { - "bad-nginx-deployment": &extensions.Deployment{}, - "counter-pod": &api.Pod{}, - "curlpod": &extensions.Deployment{}, - "deployment": &extensions.Deployment{}, - "ingress": &extensions.Ingress{}, - "job": &batch.Job{}, - "multi-pod": &api.Pod{}, - "new-nginx-deployment": &extensions.Deployment{}, - "nginx-app": &api.Service{}, - "nginx-deployment": &extensions.Deployment{}, - "nginx-init-containers": &api.Pod{}, - "nginx-lifecycle-deployment": &extensions.Deployment{}, - "nginx-probe-deployment": &extensions.Deployment{}, - "nginx-secure-app": &api.Service{}, - "nginx-svc": &api.Service{}, - "petset": &api.Service{}, - "pod": &api.Pod{}, - "pod-w-message": &api.Pod{}, - "redis-deployment": &extensions.Deployment{}, - "redis-resource-deployment": &extensions.Deployment{}, - "redis-secret-deployment": &extensions.Deployment{}, - "run-my-nginx": &extensions.Deployment{}, - "cronjob": &batch.CronJob{}, + "bad-nginx-deployment": {&extensions.Deployment{}}, + "counter-pod": {&api.Pod{}}, + "curlpod": {&extensions.Deployment{}}, + "deployment": {&extensions.Deployment{}}, + "ingress": {&extensions.Ingress{}}, + "job": {&batch.Job{}}, + "multi-pod": {&api.Pod{}, &api.Pod{}}, + "new-nginx-deployment": {&extensions.Deployment{}}, + "nginx-app": {&api.Service{}, &extensions.Deployment{}}, + "nginx-deployment": {&extensions.Deployment{}}, + "nginx-init-containers": {&api.Pod{}}, + "nginx-lifecycle-deployment": {&extensions.Deployment{}}, + "nginx-probe-deployment": {&extensions.Deployment{}}, + "nginx-secure-app": {&api.Service{}, &extensions.Deployment{}}, + "nginx-svc": {&api.Service{}}, + "petset": {&api.Service{}, &apps.StatefulSet{}}, + "pod": {&api.Pod{}}, + "pod-w-message": {&api.Pod{}}, + "redis-deployment": {&extensions.Deployment{}}, + "redis-resource-deployment": {&extensions.Deployment{}}, + "redis-secret-deployment": {&extensions.Deployment{}}, + "run-my-nginx": {&extensions.Deployment{}}, + "cronjob": {&batch.CronJob{}}, }, "../docs/admin": { - "daemon": &extensions.DaemonSet{}, + "daemon": {&extensions.DaemonSet{}}, }, "../docs/user-guide/downward-api": { - "dapi-pod": &api.Pod{}, - "dapi-container-resources": &api.Pod{}, + "dapi-pod": {&api.Pod{}}, + "dapi-container-resources": {&api.Pod{}}, }, "../docs/user-guide/downward-api/volume/": { - "dapi-volume": &api.Pod{}, - "dapi-volume-resources": &api.Pod{}, + "dapi-volume": {&api.Pod{}}, + "dapi-volume-resources": {&api.Pod{}}, }, "../docs/admin/namespaces": { - "namespace-dev": &api.Namespace{}, - "namespace-prod": &api.Namespace{}, + "namespace-dev": {&api.Namespace{}}, + "namespace-prod": {&api.Namespace{}}, }, "../docs/admin/limitrange": { - "invalid-pod": &api.Pod{}, - "limits": &api.LimitRange{}, - "namespace": &api.Namespace{}, - "valid-pod": &api.Pod{}, + "invalid-pod": {&api.Pod{}}, + "limits": {&api.LimitRange{}}, + "namespace": {&api.Namespace{}}, + "valid-pod": {&api.Pod{}}, }, "../docs/user-guide/logging-demo": { - "synthetic_0_25lps": &api.Pod{}, - "synthetic_10lps": &api.Pod{}, + "synthetic_0_25lps": {&api.Pod{}}, + "synthetic_10lps": {&api.Pod{}}, }, "../docs/user-guide/node-selection": { - "pod": &api.Pod{}, - "pod-with-node-affinity": &api.Pod{}, - "pod-with-pod-affinity": &api.Pod{}, + "pod": {&api.Pod{}}, + "pod-with-node-affinity": {&api.Pod{}}, + "pod-with-pod-affinity": {&api.Pod{}}, }, "../docs/admin/resourcequota": { - "best-effort": &api.ResourceQuota{}, - "compute-resources": &api.ResourceQuota{}, - "limits": &api.LimitRange{}, - "namespace": &api.Namespace{}, - "not-best-effort": &api.ResourceQuota{}, - "object-counts": &api.ResourceQuota{}, + "best-effort": {&api.ResourceQuota{}}, + "compute-resources": {&api.ResourceQuota{}}, + "limits": {&api.LimitRange{}}, + "namespace": {&api.Namespace{}}, + "not-best-effort": {&api.ResourceQuota{}}, + "object-counts": {&api.ResourceQuota{}}, }, "../docs/user-guide/secrets": { - "secret-pod": &api.Pod{}, - "secret": &api.Secret{}, - "secret-env-pod": &api.Pod{}, + "secret-pod": {&api.Pod{}}, + "secret": {&api.Secret{}}, + "secret-env-pod": {&api.Pod{}}, }, "../docs/tutorials/replicated-stateful-application": { - "mysql-services": &api.Service{}, - "mysql-configmap": &api.ConfigMap{}, - "mysql-statefulset": &apps.StatefulSet{}, + "mysql-services": {&api.Service{}, &api.Service{}}, + "mysql-configmap": {&api.ConfigMap{}}, + "mysql-statefulset": {&apps.StatefulSet{}}, }, } @@ -312,43 +329,52 @@ func TestExampleObjectSchemas(t *testing.T) { for path, expected := range cases { tested := 0 - err := walkConfigFiles(path, func(name, path string, data []byte) { - expectedType, found := expected[name] + numExpected := 0 + err := walkConfigFiles(path, func(name, path string, docs [][]byte) { + expectedTypes, found := expected[name] if !found { t.Errorf("%s: %s does not have a test case defined", path, name) return } - tested++ - if expectedType == nil { - t.Logf("skipping : %s/%s\n", path, name) + numExpected += len(expectedTypes) + if len(expectedTypes) != len(docs) { + t.Errorf("%s: number of expected types (%v) doesn't match number of docs in YAML (%v)", path, len(expectedTypes), len(docs)) return } - if strings.Contains(name, "scheduler-policy-config") { - if err := runtime.DecodeInto(schedulerapilatest.Codec, data, expectedType); err != nil { - t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data)) + for i, data := range docs { + expectedType := expectedTypes[i] + tested++ + if expectedType == nil { + t.Logf("skipping : %s/%s\n", path, name) return } - // TODO: Add validate method for - // &schedulerapi.Policy, and remove this - // special case - } else { - codec, err := testapi.GetCodecForObject(expectedType) - if err != nil { - t.Errorf("Could not get codec for %s: %s", expectedType, err) - } - if err := runtime.DecodeInto(codec, data, expectedType); err != nil { - t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data)) - return - } - if errors := validateObject(expectedType); len(errors) > 0 { - t.Errorf("%s did not validate correctly: %v", path, errors) + if strings.Contains(name, "scheduler-policy-config") { + if err := runtime.DecodeInto(schedulerapilatest.Codec, data, expectedType); err != nil { + t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data)) + return + } + // TODO: Add validate method for + // &schedulerapi.Policy, and remove this + // special case + } else { + codec, err := testapi.GetCodecForObject(expectedType) + if err != nil { + t.Errorf("Could not get codec for %s: %s", expectedType, err) + } + if err := runtime.DecodeInto(codec, data, expectedType); err != nil { + t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data)) + return + } + if errors := validateObject(expectedType); len(errors) > 0 { + t.Errorf("%s did not validate correctly: %v", path, errors) + } } } }) if err != nil { t.Errorf("Expected no error, Got %v on Path %v", err, path) } - if tested != len(expected) { + if tested != numExpected { t.Errorf("Directory %v: Expected %d examples, Got %d", path, len(expected), tested) } } From 38edbd87e6fa8f75cb4f396f80bbbc94cb8dfc09 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Wed, 23 Nov 2016 14:19:13 -0800 Subject: [PATCH 08/17] Add tutorial: Running a Replicated Stateful Application --- _data/tutorials.yml | 2 + _includes/default-storage-class-prereqs.md | 4 + docs/tutorials/index.md | 1 + .../run-replicated-stateful-application.md | 522 ++++++++++++++++++ 4 files changed, 529 insertions(+) create mode 100644 _includes/default-storage-class-prereqs.md create mode 100644 docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md diff --git a/_data/tutorials.yml b/_data/tutorials.yml index 61555427d1..5b8bae44cb 100644 --- a/_data/tutorials.yml +++ b/_data/tutorials.yml @@ -55,3 +55,5 @@ toc: section: - title: Running a Single-Instance Stateful Application path: /docs/tutorials/stateful-application/run-stateful-application/ + - title: Running a Replicated Stateful Application + path: /docs/tutorials/replicated-stateful-application/run-replicated-stateful-application/ diff --git a/_includes/default-storage-class-prereqs.md b/_includes/default-storage-class-prereqs.md new file mode 100644 index 0000000000..e9c46a2397 --- /dev/null +++ b/_includes/default-storage-class-prereqs.md @@ -0,0 +1,4 @@ +You need to either have a dynamic Persistent Volume provisioner with a default +[Storage Class](/docs/user-guide/persistent-volumes/#storageclasses), +or [statically provision Persistent Volumes](/docs/user-guide/persistent-volumes/#provisioning) +yourself to satisfy the Persistent Volume Claims used here. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 88c5b75807..7372122896 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -21,6 +21,7 @@ each of which has a sequence of steps. #### Stateful Applications * [Running a Single-Instance Stateful Application](/docs/tutorials/stateful-application/run-stateful-application/) +* [Running a Replicated Stateful Application](/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application/) ### What's next diff --git a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md b/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md new file mode 100644 index 0000000000..f5d2c3aef1 --- /dev/null +++ b/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md @@ -0,0 +1,522 @@ +--- +assignees: +- bprashanth +- enisoc +- erictune +- foxish +- janetkuo +- kow3ns +- smarterclayton + +--- + +{% capture overview %} + +This page shows how to run a replicated stateful application using a +[Stateful Set](/docs/concepts/controllers/statefulsets/) controller. +The example is a MySQL single-master topology with multiple slaves running +asynchronous replication. + +Note that **this is not a production configuration**. +In particular, MySQL settings remain on insecure defaults to keep the focus +on general patterns for running stateful applications in Kubernetes. + +{% endcapture %} + +{% capture prerequisites %} + +* {% include task-tutorial-prereqs.md %} +* {% include default-storage-class-prereqs.md %} +* This tutorial assumes you are familiar with +[Persistent Volumes](/docs/user-guide/persistent-volumes/) +and [Stateful Sets](/docs/concepts/controllers/statefulsets/), +as well as other core concepts like Pods, Services and Config Maps. +* Some familiarity with MySQL will help, but this tutorial aims to present + general patterns that should be useful for other systems. + +{% endcapture %} + +{% capture objectives %} + +* Deploy a replicated MySQL topology with a Stateful Set controller. +* Send MySQL client traffic. +* Observe resistance to downtime. +* Scale the Stateful Set up and down. + +{% endcapture %} + +{% capture lessoncontent %} + +### Deploying MySQL + +The example MySQL deployment consists of a Config Map, two Services, +and a Stateful Set. + +#### Config Map + +Create the Config Map by saving the following manifest to `mysql-configmap.yaml` +and running: + +```shell +kubectl create -f mysql-configmap.yaml +``` + +{% include code.html language="yaml" file="mysql-configmap.yaml" ghlink="/docs/tutorials/replicated-stateful-application/mysql-configmap.yaml" %} + +This Config Map provides `my.cnf` overrides that let you independently control +configuration on the master and the slaves. +In this case, you want the master to be able to serve replication logs to slaves +and you want slaves to reject any writes that don't come via replication. + +There's nothing special about the ConfigMap itself that causes different +portions to apply to different Pods. +Each Pod will decide which portion to look at as it's initializing, +based on information provided by the Stateful Set controller. + +#### Services + +Create the Services by saving the following manifest to `mysql-services.yaml` +and running: + +```shell +kubectl create -f mysql-services.yaml +``` + +{% include code.html language="yaml" file="mysql-services.yaml" ghlink="/docs/tutorials/replicated-stateful-application/mysql-services.yaml" %} + +The Headless Service provides a home for the DNS entries that the Stateful Set +controller will create for each Pod that's part of the set. +Since the Headless Service is named `mysql`, the Pods will be accessible by +resolving `.mysql` from within any other Pod in the same Kubernetes +cluster and namespace. + +The Client Service, called `mysql-read`, is a normal Service with its own +cluster IP that will distribute connections across all MySQL Pods that report +being Ready. The set of endpoints will include the master and all slaves. + +Note that only read queries can use the load-balanced Client Service. +Since there is only one master, clients should connect directly to the master +Pod (through its DNS entry within the Headless Service) to execute writes. + +#### Stateful Set + +Finally, create the Stateful Set by saving the following manifest to +`mysql-statefulset.yaml` and running: + +```shell +kubectl create -f mysql-statefulset.yaml +``` + +{% include code.html language="yaml" file="mysql-statefulset.yaml" ghlink="/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml" %} + +You can watch the startup progress by running: + +```shell +kubectl get pods -l app=mysql --watch +``` + +After a while, you should see all 3 Pods become Running: + +``` +NAME READY STATUS RESTARTS AGE +mysql-0 2/2 Running 0 2m +mysql-1 2/2 Running 0 1m +mysql-2 2/2 Running 0 1m +``` + +Press **Ctrl+C** to cancel the watch. +If you don't see any progress, make sure you have a dynamic Persistent Volume +provisioner enabled as mentioned in the [prerequisites](#before-you-begin). + +This manifest uses a variety of techniques for managing stateful Pods as part of +a Stateful Set. The next section highlights some of these techniques to explain +what happens as the Stateful Set creates Pods. + +### Understanding stateful Pod initialization + +The Stateful Set controller starts Pods one at a time, in order by their +ordinal index. +It waits until each Pod reports being Ready before starting the next one. + +In addition, the controller assigns each Pod a unique, stable name of the form +`-`. +In this case, that results in Pods named `mysql-0`, `mysql-1`, and `mysql-2`. + +The Pod template in the above Stateful Set manifest takes advantage of these +properties to perform orderly startup of MySQL replication. + +#### Generating configuration + +Before starting any of the containers in the Pod spec, the Pod first runs any +[Init Containers](/docs/user-guide/production-pods/#handling-initialization) +in the order defined. +In the Stateful Set manifest, you will find these defined within the +`pod.beta.kubernetes.io/init-containers` annotation. + +The first Init Container, named `init-mysql`, generates special MySQL config +files based on the ordinal index. + +The script determines its own ordinal index by extracting it from the end of +the Pod name, which is returned by the `hostname` command. +Then it saves the ordinal (with a numeric offset to avoid reserved values) +into a file called `server-id.cnf` in the MySQL `conf.d` directory. +This translates the unique, stable identity provided by the Stateful Set +controller into the domain of MySQL server IDs, which require the same +properties. + +The script in the `init-mysql` container also applies either `master.cnf` or +`slave.cnf` from the Config Map by copying the contents into `conf.d`. +Since the example topology consists of a single master and any number of slaves, +the script simply assigns ordinal `0` to be the master, and everyone else to be +slaves. + +#### Cloning existing data + +In general, when a new Pod joins the set as a slave, it must assume the master +may already have data on it. It also must assume that the replication logs may +not go all the way back to the beginning of time. +These conservative assumptions are the key to allowing a running Stateful Set +to scale up and down over time, rather than being fixed at its initial size. + +The second Init Container, named `clone-mysql`, performs a clone operation on +a slave Pod the first time it starts up on an empty Persistent Volume. +That means it copies all existing data from another running Pod, +so its local state is consistent enough to begin replicating from the master. + +MySQL itself does not provide a mechanism to do this, so the example uses a +popular open-source tool called Percona XtraBackup. +During the clone, the source MySQL server may suffer reduced performance. +To minimize impact on the master, the script instructs each Pod to clone from +the Pod whose ordinal index is one lower. +This works because the Stateful Set controller will always ensure Pod `N` is +Ready before starting Pod `N+1`. + +#### Starting replication + +After the Init Containers complete successfully, the regular containers run. +The MySQL Pods consist of a `mysql` container that runs the actual `mysqld` +server, and an `xtrabackup` container that acts as a +[sidecar](http://blog.kubernetes.io/2015/06/the-distributed-system-toolkit-patterns.html). + +The `xtrabackup` sidecar looks at the cloned data files and determines if +it's necessary to initialize MySQL replication on the slave. +If so, it waits for `mysqld` to be ready and then executes the +`CHANGE MASTER TO` and `START SLAVE` commands with replication parameters +extracted from the XtraBackup clone files. + +Once a slave begins replication, by default it will remember its master and +reconnect automatically if the server is restarted or the connection dies. +Also, since slaves look for the master at its stable DNS name (`mysql-0.mysql`), +they will automatically find the master even if it gets a new Pod IP due to +being rescheduled. + +Lastly, after starting replication, the `xtrabackup` container listens for +connections from other Pods requesting a data clone. +This server remains up indefinitely in case the Stateful Set scales up, or in +case the next Pod loses its Persistent Volume Claim and needs to redo the clone. + +### Sending client traffic + +You can send test queries to the master (hostname `mysql-0.mysql`) +by running a temporary container with the `mysql:5.7` image and running the +`mysql` client binary. + +```shell +kubectl run mysql-client --image=mysql:5.7 -i -t --rm --restart=Never --\ + mysql -h mysql-0.mysql <` with the name of the Node you found in the last step. + +This may impact other applications on the Node, so it's best to +**only do this in a test cluster**. + +```shell +kubectl drain --force --delete-local-data --ignore-daemonsets +``` + +Now you can watch as the Pod reschedules on a different Node: + +```shell +kubectl get pod mysql-2 -o wide --watch +``` + +It should look something like this: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +mysql-2 2/2 Terminating 0 15m 10.244.1.56 kubernetes-minion-group-9l2t +[...] +mysql-2 0/2 Pending 0 0s kubernetes-minion-group-fjlm +mysql-2 0/2 Init:0/2 0 0s kubernetes-minion-group-fjlm +mysql-2 0/2 Init:1/2 0 20s 10.244.5.32 kubernetes-minion-group-fjlm +mysql-2 0/2 PodInitializing 0 21s 10.244.5.32 kubernetes-minion-group-fjlm +mysql-2 1/2 Running 0 22s 10.244.5.32 kubernetes-minion-group-fjlm +mysql-2 2/2 Running 0 30s 10.244.5.32 kubernetes-minion-group-fjlm +``` + +And again, you should see server ID `102` disappear from the +`SELECT @@server_id` loop output for a while and then return. + +Now uncordon the Node to return it to a normal state: + +```shell +kubectl uncordon +``` + +### Scaling the number of slaves + +With MySQL replication, you can scale your read query capacity by adding slaves. +With Stateful Set, you can do this with a single command: + +```shell +kubectl scale --replicas=5 statefulset mysql +``` + +Watch the new Pods come up by running: + +```shell +kubectl get pods -l app=mysql --watch +``` + +Once they're up, you should see server IDs `103` and `104` start appearing in +the `SELECT @@server_id` loop output. + +You can also verify that these new servers have the data you added before they +existed: + +```shell +kubectl run mysql-client --image=mysql:5.7 -i -t --rm --restart=Never --\ + mysql -h mysql-3.mysql -e "SELECT * FROM test.messages" +``` + +``` +Waiting for pod default/mysql-client to be running, status is Pending, pod ready: false ++---------+ +| message | ++---------+ +| hello | ++---------+ +pod "mysql-client" deleted +``` + +Scaling back down is also seamless: + +```shell +kubectl scale --replicas=3 statefulset mysql +``` + +Note, however, that while scaling up creates new Persistent Volume Claims +automatically, scaling down does not automatically delete these PVCs. +This gives you the choice to keep those initialized PVCs around to make +scaling back up quicker, or to extract data before deleting them. + +You can see this by running: + +```shell +kubectl get pvc -l app=mysql +``` + +Which will show that all 5 PVCs still exist, despite having scaled the +Stateful Set down to 3: + +``` +NAME STATUS VOLUME CAPACITY ACCESSMODES AGE +data-mysql-0 Bound pvc-8acbf5dc-b103-11e6-93fa-42010a800002 10Gi RWO 20m +data-mysql-1 Bound pvc-8ad39820-b103-11e6-93fa-42010a800002 10Gi RWO 20m +data-mysql-2 Bound pvc-8ad69a6d-b103-11e6-93fa-42010a800002 10Gi RWO 20m +data-mysql-3 Bound pvc-50043c45-b1c5-11e6-93fa-42010a800002 10Gi RWO 2m +data-mysql-4 Bound pvc-500a9957-b1c5-11e6-93fa-42010a800002 10Gi RWO 2m +``` + +If you don't intend to reuse the extra PVCs, you can delete them: + +```shell +kubectl delete pvc data-mysql-3 +kubectl delete pvc data-mysql-4 +``` + +{% endcapture %} + +{% capture cleanup %} + +* Cancel the `SELECT @@server_id` loop by pressing **Ctrl+C** in its terminal, + or running the following from another terminal: + + ```shell + kubectl delete pod mysql-client-loop --now + ``` + +* Delete the Stateful Set. This will also begin terminating the Pods. + + ```shell + kubectl delete statefulset mysql + ``` + +* Verify that the Pods disappear. They may take some time to finish terminating. + + ```shell + kubectl get pods -l app=mysql + ``` + + You'll know the Pods have terminated when the above returns: + + ``` + No resources found. + ``` + +* Delete the ConfigMap, Services, and Persistent Volume Claims. + + ```shell + kubectl delete configmap,service,pvc -l app=mysql + ``` + +{% endcapture %} + +{% capture whatsnext %} + +* Look in the [Helm Charts repository](https://github.com/kubernetes/charts) + for other stateful application examples. + +{% endcapture %} + +{% include templates/tutorial.md %} + From 7b642354994911e1c6d44dd22429e4917906c413 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Wed, 23 Nov 2016 15:44:28 -0800 Subject: [PATCH 09/17] Use gcr.io registry for xtrabackup image. --- .../replicated-stateful-application/mysql-statefulset.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml b/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml index 13ff16c3ef..106f77cd99 100644 --- a/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml +++ b/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml @@ -39,7 +39,7 @@ spec: }, { "name": "clone-mysql", - "image": "enisoc/xtrabackup", + "image": "gcr.io/google-samples/xtrabackup:1.0", "command": ["bash", "-c", " set -ex\n # Skip the clone if data already exists.\n @@ -90,7 +90,7 @@ spec: initialDelaySeconds: 5 timeoutSeconds: 1 - name: xtrabackup - image: enisoc/xtrabackup:latest + image: gcr.io/google-samples/xtrabackup:1.0 ports: - name: xtrabackup containerPort: 3307 From fd7c8eee4f284fdc5ce1378fa5cfc68f8542b103 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Tue, 29 Nov 2016 10:04:17 -0800 Subject: [PATCH 10/17] Use CamelCase for API objects --- _includes/default-storage-class-prereqs.md | 8 +- .../mysql-services.yaml | 2 +- .../run-replicated-stateful-application.md | 74 +++++++++---------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/_includes/default-storage-class-prereqs.md b/_includes/default-storage-class-prereqs.md index e9c46a2397..026d57099f 100644 --- a/_includes/default-storage-class-prereqs.md +++ b/_includes/default-storage-class-prereqs.md @@ -1,4 +1,4 @@ -You need to either have a dynamic Persistent Volume provisioner with a default -[Storage Class](/docs/user-guide/persistent-volumes/#storageclasses), -or [statically provision Persistent Volumes](/docs/user-guide/persistent-volumes/#provisioning) -yourself to satisfy the Persistent Volume Claims used here. +You need to either have a dynamic PersistentVolume provisioner with a default +[StorageClass](/docs/user-guide/persistent-volumes/#storageclasses), +or [statically provision PersistentVolumes](/docs/user-guide/persistent-volumes/#provisioning) +yourself to satisfy the PersistentVolumeClaims used here. diff --git a/docs/tutorials/replicated-stateful-application/mysql-services.yaml b/docs/tutorials/replicated-stateful-application/mysql-services.yaml index ef68455396..f538992566 100644 --- a/docs/tutorials/replicated-stateful-application/mysql-services.yaml +++ b/docs/tutorials/replicated-stateful-application/mysql-services.yaml @@ -1,4 +1,4 @@ -# Headless service for stable DNS entries of Stateful Set members. +# Headless service for stable DNS entries of StatefulSet members. apiVersion: v1 kind: Service metadata: diff --git a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md b/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md index f5d2c3aef1..23a714804b 100644 --- a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md +++ b/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md @@ -13,7 +13,7 @@ assignees: {% capture overview %} This page shows how to run a replicated stateful application using a -[Stateful Set](/docs/concepts/controllers/statefulsets/) controller. +[StatefulSet](/docs/concepts/controllers/statefulsets/) controller. The example is a MySQL single-master topology with multiple slaves running asynchronous replication. @@ -28,9 +28,9 @@ on general patterns for running stateful applications in Kubernetes. * {% include task-tutorial-prereqs.md %} * {% include default-storage-class-prereqs.md %} * This tutorial assumes you are familiar with -[Persistent Volumes](/docs/user-guide/persistent-volumes/) -and [Stateful Sets](/docs/concepts/controllers/statefulsets/), -as well as other core concepts like Pods, Services and Config Maps. +[PersistentVolumes](/docs/user-guide/persistent-volumes/) +and [StatefulSets](/docs/concepts/controllers/statefulsets/), +as well as other core concepts like Pods, Services and ConfigMaps. * Some familiarity with MySQL will help, but this tutorial aims to present general patterns that should be useful for other systems. @@ -38,10 +38,10 @@ as well as other core concepts like Pods, Services and Config Maps. {% capture objectives %} -* Deploy a replicated MySQL topology with a Stateful Set controller. +* Deploy a replicated MySQL topology with a StatefulSet controller. * Send MySQL client traffic. * Observe resistance to downtime. -* Scale the Stateful Set up and down. +* Scale the StatefulSet up and down. {% endcapture %} @@ -49,12 +49,12 @@ as well as other core concepts like Pods, Services and Config Maps. ### Deploying MySQL -The example MySQL deployment consists of a Config Map, two Services, -and a Stateful Set. +The example MySQL deployment consists of a ConfigMap, two Services, +and a StatefulSet. -#### Config Map +#### ConfigMap -Create the Config Map by saving the following manifest to `mysql-configmap.yaml` +Create the ConfigMap by saving the following manifest to `mysql-configmap.yaml` and running: ```shell @@ -63,7 +63,7 @@ kubectl create -f mysql-configmap.yaml {% include code.html language="yaml" file="mysql-configmap.yaml" ghlink="/docs/tutorials/replicated-stateful-application/mysql-configmap.yaml" %} -This Config Map provides `my.cnf` overrides that let you independently control +This ConfigMap provides `my.cnf` overrides that let you independently control configuration on the master and the slaves. In this case, you want the master to be able to serve replication logs to slaves and you want slaves to reject any writes that don't come via replication. @@ -71,7 +71,7 @@ and you want slaves to reject any writes that don't come via replication. There's nothing special about the ConfigMap itself that causes different portions to apply to different Pods. Each Pod will decide which portion to look at as it's initializing, -based on information provided by the Stateful Set controller. +based on information provided by the StatefulSet controller. #### Services @@ -84,7 +84,7 @@ kubectl create -f mysql-services.yaml {% include code.html language="yaml" file="mysql-services.yaml" ghlink="/docs/tutorials/replicated-stateful-application/mysql-services.yaml" %} -The Headless Service provides a home for the DNS entries that the Stateful Set +The Headless Service provides a home for the DNS entries that the StatefulSet controller will create for each Pod that's part of the set. Since the Headless Service is named `mysql`, the Pods will be accessible by resolving `.mysql` from within any other Pod in the same Kubernetes @@ -98,9 +98,9 @@ Note that only read queries can use the load-balanced Client Service. Since there is only one master, clients should connect directly to the master Pod (through its DNS entry within the Headless Service) to execute writes. -#### Stateful Set +#### StatefulSet -Finally, create the Stateful Set by saving the following manifest to +Finally, create the StatefulSet by saving the following manifest to `mysql-statefulset.yaml` and running: ```shell @@ -125,16 +125,16 @@ mysql-2 2/2 Running 0 1m ``` Press **Ctrl+C** to cancel the watch. -If you don't see any progress, make sure you have a dynamic Persistent Volume +If you don't see any progress, make sure you have a dynamic PersistentVolume provisioner enabled as mentioned in the [prerequisites](#before-you-begin). This manifest uses a variety of techniques for managing stateful Pods as part of -a Stateful Set. The next section highlights some of these techniques to explain -what happens as the Stateful Set creates Pods. +a StatefulSet. The next section highlights some of these techniques to explain +what happens as the StatefulSet creates Pods. ### Understanding stateful Pod initialization -The Stateful Set controller starts Pods one at a time, in order by their +The StatefulSet controller starts Pods one at a time, in order by their ordinal index. It waits until each Pod reports being Ready before starting the next one. @@ -142,7 +142,7 @@ In addition, the controller assigns each Pod a unique, stable name of the form `-`. In this case, that results in Pods named `mysql-0`, `mysql-1`, and `mysql-2`. -The Pod template in the above Stateful Set manifest takes advantage of these +The Pod template in the above StatefulSet manifest takes advantage of these properties to perform orderly startup of MySQL replication. #### Generating configuration @@ -150,7 +150,7 @@ properties to perform orderly startup of MySQL replication. Before starting any of the containers in the Pod spec, the Pod first runs any [Init Containers](/docs/user-guide/production-pods/#handling-initialization) in the order defined. -In the Stateful Set manifest, you will find these defined within the +In the StatefulSet manifest, you will find these defined within the `pod.beta.kubernetes.io/init-containers` annotation. The first Init Container, named `init-mysql`, generates special MySQL config @@ -160,12 +160,12 @@ The script determines its own ordinal index by extracting it from the end of the Pod name, which is returned by the `hostname` command. Then it saves the ordinal (with a numeric offset to avoid reserved values) into a file called `server-id.cnf` in the MySQL `conf.d` directory. -This translates the unique, stable identity provided by the Stateful Set +This translates the unique, stable identity provided by the StatefulSet controller into the domain of MySQL server IDs, which require the same properties. The script in the `init-mysql` container also applies either `master.cnf` or -`slave.cnf` from the Config Map by copying the contents into `conf.d`. +`slave.cnf` from the ConfigMap by copying the contents into `conf.d`. Since the example topology consists of a single master and any number of slaves, the script simply assigns ordinal `0` to be the master, and everyone else to be slaves. @@ -175,11 +175,11 @@ slaves. In general, when a new Pod joins the set as a slave, it must assume the master may already have data on it. It also must assume that the replication logs may not go all the way back to the beginning of time. -These conservative assumptions are the key to allowing a running Stateful Set +These conservative assumptions are the key to allowing a running StatefulSet to scale up and down over time, rather than being fixed at its initial size. The second Init Container, named `clone-mysql`, performs a clone operation on -a slave Pod the first time it starts up on an empty Persistent Volume. +a slave Pod the first time it starts up on an empty PersistentVolume. That means it copies all existing data from another running Pod, so its local state is consistent enough to begin replicating from the master. @@ -188,7 +188,7 @@ popular open-source tool called Percona XtraBackup. During the clone, the source MySQL server may suffer reduced performance. To minimize impact on the master, the script instructs each Pod to clone from the Pod whose ordinal index is one lower. -This works because the Stateful Set controller will always ensure Pod `N` is +This works because the StatefulSet controller will always ensure Pod `N` is Ready before starting Pod `N+1`. #### Starting replication @@ -212,8 +212,8 @@ being rescheduled. Lastly, after starting replication, the `xtrabackup` container listens for connections from other Pods requesting a data clone. -This server remains up indefinitely in case the Stateful Set scales up, or in -case the next Pod loses its Persistent Volume Claim and needs to redo the clone. +This server remains up indefinitely in case the StatefulSet scales up, or in +case the next Pod loses its PersistentVolumeClaim and needs to redo the clone. ### Sending client traffic @@ -330,16 +330,16 @@ kubectl exec mysql-2 -c mysql -- mv /usr/bin/mysql.off /usr/bin/mysql #### Delete Pods -The Stateful Set will also recreate Pods if they're deleted, similar to what a -Replica Set does for stateless Pods. +The StatefulSet will also recreate Pods if they're deleted, similar to what a +ReplicaSet does for stateless Pods. ```shell kubectl delete pod mysql-2 ``` -The Stateful Set controller will notice that no `mysql-2` Pod exists anymore, +The StatefulSet controller will notice that no `mysql-2` Pod exists anymore, and will create a new one with the same name and linked to the same -Persistent Volume Claim. +PersistentVolumeClaim. You should see server ID `102` disappear from the loop output for a while and then return on its own. @@ -405,7 +405,7 @@ kubectl uncordon ### Scaling the number of slaves With MySQL replication, you can scale your read query capacity by adding slaves. -With Stateful Set, you can do this with a single command: +With StatefulSet, you can do this with a single command: ```shell kubectl scale --replicas=5 statefulset mysql @@ -444,7 +444,7 @@ Scaling back down is also seamless: kubectl scale --replicas=3 statefulset mysql ``` -Note, however, that while scaling up creates new Persistent Volume Claims +Note, however, that while scaling up creates new PersistentVolumeClaims automatically, scaling down does not automatically delete these PVCs. This gives you the choice to keep those initialized PVCs around to make scaling back up quicker, or to extract data before deleting them. @@ -456,7 +456,7 @@ kubectl get pvc -l app=mysql ``` Which will show that all 5 PVCs still exist, despite having scaled the -Stateful Set down to 3: +StatefulSet down to 3: ``` NAME STATUS VOLUME CAPACITY ACCESSMODES AGE @@ -485,7 +485,7 @@ kubectl delete pvc data-mysql-4 kubectl delete pod mysql-client-loop --now ``` -* Delete the Stateful Set. This will also begin terminating the Pods. +* Delete the StatefulSet. This will also begin terminating the Pods. ```shell kubectl delete statefulset mysql @@ -503,7 +503,7 @@ kubectl delete pvc data-mysql-4 No resources found. ``` -* Delete the ConfigMap, Services, and Persistent Volume Claims. +* Delete the ConfigMap, Services, and PersistentVolumeClaims. ```shell kubectl delete configmap,service,pvc -l app=mysql From 7c800a949abb20ce49154a50697480cbc9dbdb31 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Tue, 29 Nov 2016 10:18:11 -0800 Subject: [PATCH 11/17] Linkify API objects on first mention. --- _includes/default-storage-class-prereqs.md | 4 +++- .../run-replicated-stateful-application.md | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/_includes/default-storage-class-prereqs.md b/_includes/default-storage-class-prereqs.md index 026d57099f..a4747d9032 100644 --- a/_includes/default-storage-class-prereqs.md +++ b/_includes/default-storage-class-prereqs.md @@ -1,4 +1,6 @@ You need to either have a dynamic PersistentVolume provisioner with a default [StorageClass](/docs/user-guide/persistent-volumes/#storageclasses), or [statically provision PersistentVolumes](/docs/user-guide/persistent-volumes/#provisioning) -yourself to satisfy the PersistentVolumeClaims used here. +yourself to satisfy the [PersistentVolumeClaims](/docs/user-guide/persistent-volumes/#persistentvolumeclaims) +used here. + diff --git a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md b/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md index 23a714804b..c4ca73c85d 100644 --- a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md +++ b/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md @@ -28,9 +28,11 @@ on general patterns for running stateful applications in Kubernetes. * {% include task-tutorial-prereqs.md %} * {% include default-storage-class-prereqs.md %} * This tutorial assumes you are familiar with -[PersistentVolumes](/docs/user-guide/persistent-volumes/) -and [StatefulSets](/docs/concepts/controllers/statefulsets/), -as well as other core concepts like Pods, Services and ConfigMaps. + [PersistentVolumes](/docs/user-guide/persistent-volumes/) + and [StatefulSets](/docs/concepts/controllers/statefulsets/), + as well as other core concepts like [Pods](/docs/user-guide/pods/), + [Services](/docs/user-guide/services/), and + [ConfigMaps](/docs/user-guide/configmap/). * Some familiarity with MySQL will help, but this tutorial aims to present general patterns that should be useful for other systems. From 1630ff976ed640e918ddd6cafdcc9ceaa8a87da9 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Tue, 29 Nov 2016 10:25:09 -0800 Subject: [PATCH 12/17] Follow conventions for may/might/since/because --- .../run-replicated-stateful-application.md | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md b/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md index c4ca73c85d..8299c08e57 100644 --- a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md +++ b/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md @@ -88,7 +88,7 @@ kubectl create -f mysql-services.yaml The Headless Service provides a home for the DNS entries that the StatefulSet controller will create for each Pod that's part of the set. -Since the Headless Service is named `mysql`, the Pods will be accessible by +Because the Headless Service is named `mysql`, the Pods will be accessible by resolving `.mysql` from within any other Pod in the same Kubernetes cluster and namespace. @@ -97,7 +97,7 @@ cluster IP that will distribute connections across all MySQL Pods that report being Ready. The set of endpoints will include the master and all slaves. Note that only read queries can use the load-balanced Client Service. -Since there is only one master, clients should connect directly to the master +Because there is only one master, clients should connect directly to the master Pod (through its DNS entry within the Headless Service) to execute writes. #### StatefulSet @@ -168,15 +168,15 @@ properties. The script in the `init-mysql` container also applies either `master.cnf` or `slave.cnf` from the ConfigMap by copying the contents into `conf.d`. -Since the example topology consists of a single master and any number of slaves, -the script simply assigns ordinal `0` to be the master, and everyone else to be -slaves. +Because the example topology consists of a single master and any number of +slaves, the script simply assigns ordinal `0` to be the master, and everyone +else to be slaves. #### Cloning existing data In general, when a new Pod joins the set as a slave, it must assume the master -may already have data on it. It also must assume that the replication logs may -not go all the way back to the beginning of time. +might already have data on it. It also must assume that the replication logs +might not go all the way back to the beginning of time. These conservative assumptions are the key to allowing a running StatefulSet to scale up and down over time, rather than being fixed at its initial size. @@ -187,7 +187,7 @@ so its local state is consistent enough to begin replicating from the master. MySQL itself does not provide a mechanism to do this, so the example uses a popular open-source tool called Percona XtraBackup. -During the clone, the source MySQL server may suffer reduced performance. +During the clone, the source MySQL server might suffer reduced performance. To minimize impact on the master, the script instructs each Pod to clone from the Pod whose ordinal index is one lower. This works because the StatefulSet controller will always ensure Pod `N` is @@ -208,9 +208,9 @@ extracted from the XtraBackup clone files. Once a slave begins replication, by default it will remember its master and reconnect automatically if the server is restarted or the connection dies. -Also, since slaves look for the master at its stable DNS name (`mysql-0.mysql`), -they will automatically find the master even if it gets a new Pod IP due to -being rescheduled. +Also, because slaves look for the master at its stable DNS name +(`mysql-0.mysql`), they will automatically find the master even if it gets a new +Pod IP due to being rescheduled. Lastly, after starting replication, the `xtrabackup` container listens for connections from other Pods requesting a data clone. @@ -260,8 +260,8 @@ kubectl run mysql-client-loop --image=mysql:5.7 -i -t --rm --restart=Never --\ bash -ic "while sleep 1; do mysql -h mysql-read -e 'SELECT @@server_id,NOW()'; done" ``` -You should see the reported `@@server_id` change randomly, since a different -endpoint may be selected upon each connection attempt: +You should see the reported `@@server_id` change randomly, because a different +endpoint might be selected upon each connection attempt: ``` +-------------+---------------------+ @@ -368,7 +368,7 @@ Then drain the Node by running the following command, which will cordon it so no new Pods may schedule there, and then evict any existing Pods. Replace `` with the name of the Node you found in the last step. -This may impact other applications on the Node, so it's best to +This might impact other applications on the Node, so it's best to **only do this in a test cluster**. ```shell @@ -493,7 +493,8 @@ kubectl delete pvc data-mysql-4 kubectl delete statefulset mysql ``` -* Verify that the Pods disappear. They may take some time to finish terminating. +* Verify that the Pods disappear. + They might take some time to finish terminating. ```shell kubectl get pods -l app=mysql From 9a575a6ed20def03cbddb387e17dee09d2e9dcef Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Tue, 29 Nov 2016 10:40:15 -0800 Subject: [PATCH 13/17] Address review comments --- .../run-replicated-stateful-application.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md b/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md index 8299c08e57..299240f80c 100644 --- a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md +++ b/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md @@ -171,6 +171,10 @@ The script in the `init-mysql` container also applies either `master.cnf` or Because the example topology consists of a single master and any number of slaves, the script simply assigns ordinal `0` to be the master, and everyone else to be slaves. +Combined with the StatefulSet controller's +[deployment order guarantee](/docs/concepts/controllers/statefulsets/#deployment-and-scaling-guarantee), +this ensures the master is Ready before creating slaves, so they can begin +replicating. #### Cloning existing data @@ -512,6 +516,11 @@ kubectl delete pvc data-mysql-4 kubectl delete configmap,service,pvc -l app=mysql ``` +* If you manually provisioned PersistentVolumes, you will also need to manually + delete them. If you used a dynamic provisioner, it will automatically delete + the PersistentVolumes when it sees you have deleted the + PersistentVolumeClaims above. + {% endcapture %} {% capture whatsnext %} From c20e8984911fc5ea58e5bc56fbafbcb4afbdcb94 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Fri, 2 Dec 2016 12:52:16 -0800 Subject: [PATCH 14/17] Replicated stateful application doesn't need its own directory. --- _data/tutorials.yml | 2 +- docs/tutorials/index.md | 2 +- .../Dockerfile | 0 .../mysql-configmap.yaml | 0 .../mysql-services.yaml | 0 .../mysql-statefulset.yaml | 0 .../run-replicated-stateful-application.md | 6 +++--- test/examples_test.go | 2 +- 8 files changed, 6 insertions(+), 6 deletions(-) rename docs/tutorials/{replicated-stateful-application => stateful-application}/Dockerfile (100%) rename docs/tutorials/{replicated-stateful-application => stateful-application}/mysql-configmap.yaml (100%) rename docs/tutorials/{replicated-stateful-application => stateful-application}/mysql-services.yaml (100%) rename docs/tutorials/{replicated-stateful-application => stateful-application}/mysql-statefulset.yaml (100%) rename docs/tutorials/{replicated-stateful-application => stateful-application}/run-replicated-stateful-application.md (98%) diff --git a/_data/tutorials.yml b/_data/tutorials.yml index 5b8bae44cb..1af2276a11 100644 --- a/_data/tutorials.yml +++ b/_data/tutorials.yml @@ -56,4 +56,4 @@ toc: - title: Running a Single-Instance Stateful Application path: /docs/tutorials/stateful-application/run-stateful-application/ - title: Running a Replicated Stateful Application - path: /docs/tutorials/replicated-stateful-application/run-replicated-stateful-application/ + path: /docs/tutorials/stateful-application/run-replicated-stateful-application/ diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 7372122896..bcec6590cf 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -21,7 +21,7 @@ each of which has a sequence of steps. #### Stateful Applications * [Running a Single-Instance Stateful Application](/docs/tutorials/stateful-application/run-stateful-application/) -* [Running a Replicated Stateful Application](/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application/) +* [Running a Replicated Stateful Application](/docs/tutorials/stateful-application/run-replicated-stateful-application/) ### What's next diff --git a/docs/tutorials/replicated-stateful-application/Dockerfile b/docs/tutorials/stateful-application/Dockerfile similarity index 100% rename from docs/tutorials/replicated-stateful-application/Dockerfile rename to docs/tutorials/stateful-application/Dockerfile diff --git a/docs/tutorials/replicated-stateful-application/mysql-configmap.yaml b/docs/tutorials/stateful-application/mysql-configmap.yaml similarity index 100% rename from docs/tutorials/replicated-stateful-application/mysql-configmap.yaml rename to docs/tutorials/stateful-application/mysql-configmap.yaml diff --git a/docs/tutorials/replicated-stateful-application/mysql-services.yaml b/docs/tutorials/stateful-application/mysql-services.yaml similarity index 100% rename from docs/tutorials/replicated-stateful-application/mysql-services.yaml rename to docs/tutorials/stateful-application/mysql-services.yaml diff --git a/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml b/docs/tutorials/stateful-application/mysql-statefulset.yaml similarity index 100% rename from docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml rename to docs/tutorials/stateful-application/mysql-statefulset.yaml diff --git a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md b/docs/tutorials/stateful-application/run-replicated-stateful-application.md similarity index 98% rename from docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md rename to docs/tutorials/stateful-application/run-replicated-stateful-application.md index 299240f80c..a3cbe24c72 100644 --- a/docs/tutorials/replicated-stateful-application/run-replicated-stateful-application.md +++ b/docs/tutorials/stateful-application/run-replicated-stateful-application.md @@ -63,7 +63,7 @@ and running: kubectl create -f mysql-configmap.yaml ``` -{% include code.html language="yaml" file="mysql-configmap.yaml" ghlink="/docs/tutorials/replicated-stateful-application/mysql-configmap.yaml" %} +{% include code.html language="yaml" file="mysql-configmap.yaml" ghlink="/docs/tutorials/stateful-application/mysql-configmap.yaml" %} This ConfigMap provides `my.cnf` overrides that let you independently control configuration on the master and the slaves. @@ -84,7 +84,7 @@ and running: kubectl create -f mysql-services.yaml ``` -{% include code.html language="yaml" file="mysql-services.yaml" ghlink="/docs/tutorials/replicated-stateful-application/mysql-services.yaml" %} +{% include code.html language="yaml" file="mysql-services.yaml" ghlink="/docs/tutorials/stateful-application/mysql-services.yaml" %} The Headless Service provides a home for the DNS entries that the StatefulSet controller will create for each Pod that's part of the set. @@ -109,7 +109,7 @@ Finally, create the StatefulSet by saving the following manifest to kubectl create -f mysql-statefulset.yaml ``` -{% include code.html language="yaml" file="mysql-statefulset.yaml" ghlink="/docs/tutorials/replicated-stateful-application/mysql-statefulset.yaml" %} +{% include code.html language="yaml" file="mysql-statefulset.yaml" ghlink="/docs/tutorials/stateful-application/mysql-statefulset.yaml" %} You can watch the startup progress by running: diff --git a/test/examples_test.go b/test/examples_test.go index e41d97467c..3d6d428087 100644 --- a/test/examples_test.go +++ b/test/examples_test.go @@ -316,7 +316,7 @@ func TestExampleObjectSchemas(t *testing.T) { "secret": {&api.Secret{}}, "secret-env-pod": {&api.Pod{}}, }, - "../docs/tutorials/replicated-stateful-application": { + "../docs/tutorials/stateful-application": { "mysql-services": {&api.Service{}, &api.Service{}}, "mysql-configmap": {&api.ConfigMap{}}, "mysql-statefulset": {&apps.StatefulSet{}}, From 8d12e8c888532b1a3056238a02cb18160534838e Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Fri, 2 Dec 2016 13:27:43 -0800 Subject: [PATCH 15/17] Address review comments. --- .../run-replicated-stateful-application.md | 136 +++++++++--------- 1 file changed, 70 insertions(+), 66 deletions(-) diff --git a/docs/tutorials/stateful-application/run-replicated-stateful-application.md b/docs/tutorials/stateful-application/run-replicated-stateful-application.md index a3cbe24c72..e294bb5913 100644 --- a/docs/tutorials/stateful-application/run-replicated-stateful-application.md +++ b/docs/tutorials/stateful-application/run-replicated-stateful-application.md @@ -13,7 +13,7 @@ assignees: {% capture overview %} This page shows how to run a replicated stateful application using a -[StatefulSet](/docs/concepts/controllers/statefulsets/) controller. +[StatefulSet](/docs/concepts/abstractions/controllers/statefulsets/) controller. The example is a MySQL single-master topology with multiple slaves running asynchronous replication. @@ -29,11 +29,11 @@ on general patterns for running stateful applications in Kubernetes. * {% include default-storage-class-prereqs.md %} * This tutorial assumes you are familiar with [PersistentVolumes](/docs/user-guide/persistent-volumes/) - and [StatefulSets](/docs/concepts/controllers/statefulsets/), + and [StatefulSets](/docs/concepts/abstractions/controllers/statefulsets/), as well as other core concepts like [Pods](/docs/user-guide/pods/), [Services](/docs/user-guide/services/), and [ConfigMaps](/docs/user-guide/configmap/). -* Some familiarity with MySQL will help, but this tutorial aims to present +* Some familiarity with MySQL helps, but this tutorial aims to present general patterns that should be useful for other systems. {% endcapture %} @@ -56,57 +56,59 @@ and a StatefulSet. #### ConfigMap -Create the ConfigMap by saving the following manifest to `mysql-configmap.yaml` -and running: +Create the ConfigMap from the following YAML configuration file: ```shell -kubectl create -f mysql-configmap.yaml +export REPO=https://raw.githubusercontent.com/kubernetes/kubernetes.github.io/{{page.docsbranch}} +kubectl create -f $REPO/docs/tutorials/stateful-application/mysql-configmap.yaml ``` {% include code.html language="yaml" file="mysql-configmap.yaml" ghlink="/docs/tutorials/stateful-application/mysql-configmap.yaml" %} This ConfigMap provides `my.cnf` overrides that let you independently control -configuration on the master and the slaves. +configuration on the MySQL master and slaves. In this case, you want the master to be able to serve replication logs to slaves and you want slaves to reject any writes that don't come via replication. There's nothing special about the ConfigMap itself that causes different portions to apply to different Pods. -Each Pod will decide which portion to look at as it's initializing, +Each Pod decides which portion to look at as it's initializing, based on information provided by the StatefulSet controller. #### Services -Create the Services by saving the following manifest to `mysql-services.yaml` -and running: +Create the Services from the following YAML configuration file: ```shell -kubectl create -f mysql-services.yaml +export REPO=https://raw.githubusercontent.com/kubernetes/kubernetes.github.io/{{page.docsbranch}} +kubectl create -f $REPO/docs/tutorials/stateful-application/mysql-services.yaml ``` {% include code.html language="yaml" file="mysql-services.yaml" ghlink="/docs/tutorials/stateful-application/mysql-services.yaml" %} The Headless Service provides a home for the DNS entries that the StatefulSet -controller will create for each Pod that's part of the set. -Because the Headless Service is named `mysql`, the Pods will be accessible by +controller creates for each Pod that's part of the set. +Because the Headless Service is named `mysql`, the Pods are accessible by resolving `.mysql` from within any other Pod in the same Kubernetes cluster and namespace. The Client Service, called `mysql-read`, is a normal Service with its own -cluster IP that will distribute connections across all MySQL Pods that report -being Ready. The set of endpoints will include the master and all slaves. +cluster IP that distributes connections across all MySQL Pods that report +being Ready. The set of potential endpoints includes the MySQL master and all +slaves. Note that only read queries can use the load-balanced Client Service. -Because there is only one master, clients should connect directly to the master -Pod (through its DNS entry within the Headless Service) to execute writes. +Because there is only one MySQL master, clients should connect directly to the +MySQL master Pod (through its DNS entry within the Headless Service) to execute +writes. #### StatefulSet -Finally, create the StatefulSet by saving the following manifest to -`mysql-statefulset.yaml` and running: +Finally, create the StatefulSet from the following YAML configuration file: ```shell -kubectl create -f mysql-statefulset.yaml +export REPO=https://raw.githubusercontent.com/kubernetes/kubernetes.github.io/{{page.docsbranch}} +kubectl create -f $REPO/docs/tutorials/stateful-application/mysql-statefulset.yaml ``` {% include code.html language="yaml" file="mysql-statefulset.yaml" ghlink="/docs/tutorials/stateful-application/mysql-statefulset.yaml" %} @@ -152,7 +154,7 @@ properties to perform orderly startup of MySQL replication. Before starting any of the containers in the Pod spec, the Pod first runs any [Init Containers](/docs/user-guide/production-pods/#handling-initialization) in the order defined. -In the StatefulSet manifest, you will find these defined within the +In the StatefulSet manifest, you can find these defined within the `pod.beta.kubernetes.io/init-containers` annotation. The first Init Container, named `init-mysql`, generates special MySQL config @@ -168,19 +170,19 @@ properties. The script in the `init-mysql` container also applies either `master.cnf` or `slave.cnf` from the ConfigMap by copying the contents into `conf.d`. -Because the example topology consists of a single master and any number of +Because the example topology consists of a single MySQL master and any number of slaves, the script simply assigns ordinal `0` to be the master, and everyone else to be slaves. Combined with the StatefulSet controller's -[deployment order guarantee](/docs/concepts/controllers/statefulsets/#deployment-and-scaling-guarantee), -this ensures the master is Ready before creating slaves, so they can begin +[deployment order guarantee](/docs/concepts/abstractions/controllers/statefulsets/#deployment-and-scaling-guarantee), +this ensures the MySQL master is Ready before creating slaves, so they can begin replicating. #### Cloning existing data -In general, when a new Pod joins the set as a slave, it must assume the master -might already have data on it. It also must assume that the replication logs -might not go all the way back to the beginning of time. +In general, when a new Pod joins the set as a slave, it must assume the MySQL +master might already have data on it. It also must assume that the replication +logs might not go all the way back to the beginning of time. These conservative assumptions are the key to allowing a running StatefulSet to scale up and down over time, rather than being fixed at its initial size. @@ -192,9 +194,9 @@ so its local state is consistent enough to begin replicating from the master. MySQL itself does not provide a mechanism to do this, so the example uses a popular open-source tool called Percona XtraBackup. During the clone, the source MySQL server might suffer reduced performance. -To minimize impact on the master, the script instructs each Pod to clone from -the Pod whose ordinal index is one lower. -This works because the StatefulSet controller will always ensure Pod `N` is +To minimize impact on the MySQL master, the script instructs each Pod to clone +from the Pod whose ordinal index is one lower. +This works because the StatefulSet controller always ensures Pod `N` is Ready before starting Pod `N+1`. #### Starting replication @@ -210,10 +212,10 @@ If so, it waits for `mysqld` to be ready and then executes the `CHANGE MASTER TO` and `START SLAVE` commands with replication parameters extracted from the XtraBackup clone files. -Once a slave begins replication, by default it will remember its master and -reconnect automatically if the server is restarted or the connection dies. +Once a slave begins replication, it remembers its MySQL master and +reconnects automatically if the server restarts or the connection dies. Also, because slaves look for the master at its stable DNS name -(`mysql-0.mysql`), they will automatically find the master even if it gets a new +(`mysql-0.mysql`), they automatically find the master even if it gets a new Pod IP due to being rescheduled. Lastly, after starting replication, the `xtrabackup` container listens for @@ -223,7 +225,7 @@ case the next Pod loses its PersistentVolumeClaim and needs to redo the clone. ### Sending client traffic -You can send test queries to the master (hostname `mysql-0.mysql`) +You can send test queries to the MySQL master (hostname `mysql-0.mysql`) by running a temporary container with the `mysql:5.7` image and running the `mysql` client binary. @@ -336,15 +338,15 @@ kubectl exec mysql-2 -c mysql -- mv /usr/bin/mysql.off /usr/bin/mysql #### Delete Pods -The StatefulSet will also recreate Pods if they're deleted, similar to what a +The StatefulSet also recreates Pods if they're deleted, similar to what a ReplicaSet does for stateless Pods. ```shell kubectl delete pod mysql-2 ``` -The StatefulSet controller will notice that no `mysql-2` Pod exists anymore, -and will create a new one with the same name and linked to the same +The StatefulSet controller notices that no `mysql-2` Pod exists anymore, +and creates a new one with the same name and linked to the same PersistentVolumeClaim. You should see server ID `102` disappear from the loop output for a while and then return on its own. @@ -368,8 +370,8 @@ NAME READY STATUS RESTARTS AGE IP NODE mysql-2 2/2 Running 0 15m 10.244.5.27 kubernetes-minion-group-9l2t ``` -Then drain the Node by running the following command, which will cordon it so -no new Pods may schedule there, and then evict any existing Pods. +Then drain the Node by running the following command, which cordons it so +no new Pods may schedule there, and then evicts any existing Pods. Replace `` with the name of the Node you found in the last step. This might impact other applications on the Node, so it's best to @@ -461,7 +463,7 @@ You can see this by running: kubectl get pvc -l app=mysql ``` -Which will show that all 5 PVCs still exist, despite having scaled the +Which shows that all 5 PVCs still exist, despite having scaled the StatefulSet down to 3: ``` @@ -484,42 +486,44 @@ kubectl delete pvc data-mysql-4 {% capture cleanup %} -* Cancel the `SELECT @@server_id` loop by pressing **Ctrl+C** in its terminal, - or running the following from another terminal: +1. Cancel the `SELECT @@server_id` loop by pressing **Ctrl+C** in its terminal, + or running the following from another terminal: - ```shell - kubectl delete pod mysql-client-loop --now - ``` + ```shell + kubectl delete pod mysql-client-loop --now + ``` -* Delete the StatefulSet. This will also begin terminating the Pods. +1. Delete the StatefulSet. This also begins terminating the Pods. - ```shell - kubectl delete statefulset mysql - ``` + ```shell + kubectl delete statefulset mysql + ``` -* Verify that the Pods disappear. - They might take some time to finish terminating. +1. Verify that the Pods disappear. + They might take some time to finish terminating. - ```shell - kubectl get pods -l app=mysql - ``` + ```shell + kubectl get pods -l app=mysql + ``` - You'll know the Pods have terminated when the above returns: + You'll know the Pods have terminated when the above returns: - ``` - No resources found. - ``` + ``` + No resources found. + ``` -* Delete the ConfigMap, Services, and PersistentVolumeClaims. +1. Delete the ConfigMap, Services, and PersistentVolumeClaims. - ```shell - kubectl delete configmap,service,pvc -l app=mysql - ``` + ```shell + kubectl delete configmap,service,pvc -l app=mysql + ``` -* If you manually provisioned PersistentVolumes, you will also need to manually - delete them. If you used a dynamic provisioner, it will automatically delete - the PersistentVolumes when it sees you have deleted the - PersistentVolumeClaims above. +1. If you manually provisioned PersistentVolumes, you also need to manually + delete them, as well as release the underlying resources. + If you used a dynamic provisioner, it automatically deletes the + PersistentVolumes when it sees that you deleted the PersistentVolumeClaims. + Some dynamic provisioners (such as those for EBS and PD) also release the + underlying resources upon deleting the PersistentVolumes. {% endcapture %} From 06d978995f6bb6b3242be05525f3095163820db6 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Fri, 2 Dec 2016 14:20:40 -0800 Subject: [PATCH 16/17] Link to k8s.io instead of GitHub for manifests. --- .../run-replicated-stateful-application.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/tutorials/stateful-application/run-replicated-stateful-application.md b/docs/tutorials/stateful-application/run-replicated-stateful-application.md index e294bb5913..db4a69b22f 100644 --- a/docs/tutorials/stateful-application/run-replicated-stateful-application.md +++ b/docs/tutorials/stateful-application/run-replicated-stateful-application.md @@ -59,8 +59,7 @@ and a StatefulSet. Create the ConfigMap from the following YAML configuration file: ```shell -export REPO=https://raw.githubusercontent.com/kubernetes/kubernetes.github.io/{{page.docsbranch}} -kubectl create -f $REPO/docs/tutorials/stateful-application/mysql-configmap.yaml +kubectl create -f http://k8s.io/docs/tutorials/stateful-application/mysql-configmap.yaml ``` {% include code.html language="yaml" file="mysql-configmap.yaml" ghlink="/docs/tutorials/stateful-application/mysql-configmap.yaml" %} @@ -80,8 +79,7 @@ based on information provided by the StatefulSet controller. Create the Services from the following YAML configuration file: ```shell -export REPO=https://raw.githubusercontent.com/kubernetes/kubernetes.github.io/{{page.docsbranch}} -kubectl create -f $REPO/docs/tutorials/stateful-application/mysql-services.yaml +kubectl create -f http://k8s.io/docs/tutorials/stateful-application/mysql-services.yaml ``` {% include code.html language="yaml" file="mysql-services.yaml" ghlink="/docs/tutorials/stateful-application/mysql-services.yaml" %} @@ -107,8 +105,7 @@ writes. Finally, create the StatefulSet from the following YAML configuration file: ```shell -export REPO=https://raw.githubusercontent.com/kubernetes/kubernetes.github.io/{{page.docsbranch}} -kubectl create -f $REPO/docs/tutorials/stateful-application/mysql-statefulset.yaml +kubectl create -f http://k8s.io/docs/tutorials/stateful-application/mysql-statefulset.yaml ``` {% include code.html language="yaml" file="mysql-statefulset.yaml" ghlink="/docs/tutorials/stateful-application/mysql-statefulset.yaml" %} From 95d8c39c8cb79c6c191ede6f4a085ec0382b8d6d Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Fri, 2 Dec 2016 15:17:26 -0800 Subject: [PATCH 17/17] Add missing test cases. --- test/examples_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/examples_test.go b/test/examples_test.go index 3d6d428087..1664267d84 100644 --- a/test/examples_test.go +++ b/test/examples_test.go @@ -317,6 +317,8 @@ func TestExampleObjectSchemas(t *testing.T) { "secret-env-pod": {&api.Pod{}}, }, "../docs/tutorials/stateful-application": { + "gce-volume": {&api.PersistentVolume{}}, + "mysql-deployment": {&api.Service{}, &api.PersistentVolumeClaim{}, &extensions.Deployment{}}, "mysql-services": {&api.Service{}, &api.Service{}}, "mysql-configmap": {&api.ConfigMap{}}, "mysql-statefulset": {&apps.StatefulSet{}},