From a6f6fd01cd934dc229fef96f5d47db84b58d08a5 Mon Sep 17 00:00:00 2001 From: John Mulhausen Date: Sun, 6 Mar 2016 17:55:12 +0000 Subject: [PATCH] 1.2 additions for getting-started-guides/ and new non-Markdown files for user-guides --- _data/guides.yml | 6 +- docs/admin/garbage-collection.md | 5 +- docs/getting-started-guides/aws.md | 55 +- .../centos/centos_manual_config.md | 33 +- docs/getting-started-guides/cloudstack.md | 2 +- docs/getting-started-guides/coreos.md | 20 +- .../coreos/azure/addons/skydns-rc.yaml | 15 +- .../coreos/azure/addons/skydns-svc.yaml | 2 +- ...kubernetes-cluster-main-nodes-template.yml | 82 ++- .../coreos/azure/index.md | 19 +- .../coreos/azure/lib/azure_wrapper.js | 31 +- .../coreos/azure/package.json | 2 +- .../coreos/bare_metal_calico.md | 247 +++++--- .../coreos/bare_metal_offline.md | 11 +- .../coreos/cloud-configs/master.yaml | 14 +- .../coreos/cloud-configs/node.yaml | 17 +- .../coreos/coreos_multinode_cluster.md | 13 +- docs/getting-started-guides/dcos.md | 2 +- .../docker-multinode.md | 66 ++- .../docker-multinode/deployDNS.md | 29 +- .../docker-multinode/kube-system.yaml | 4 + .../docker-multinode/master.md | 110 +++- .../docker-multinode/master.sh | 189 ++++-- .../docker-multinode/skydns-svc.yaml.in | 20 - .../{skydns-rc.yaml.in => skydns.yaml.in} | 66 ++- .../docker-multinode/testing.md | 2 +- .../docker-multinode/worker.md | 66 ++- .../docker-multinode/worker.sh | 167 ++++-- docs/getting-started-guides/docker.md | 151 +++-- .../fedora/fedora_ansible_config.md | 3 +- .../fedora/fedora_manual_config.md | 26 +- .../fedora/flannel_multi_node_cluster.md | 11 +- docs/getting-started-guides/gce.md | 2 +- docs/getting-started-guides/index.md | 2 - docs/getting-started-guides/juju.md | 22 +- docs/getting-started-guides/libvirt-coreos.md | 31 +- .../logging-elasticsearch.md | 10 +- docs/getting-started-guides/logging.md | 20 +- docs/getting-started-guides/mesos-docker.md | 135 +++-- docs/getting-started-guides/mesos.md | 12 +- docs/getting-started-guides/mesos/OWNERS | 3 + .../mesos/k8s-firewall.png | Bin 0 -> 88722 bytes .../mesos/k8s-guestbook.png | Bin 0 -> 44000 bytes docs/getting-started-guides/ovirt.md | 2 +- docs/getting-started-guides/rackspace.md | 4 +- docs/getting-started-guides/rkt/index.md | 99 +++- docs/getting-started-guides/rkt/notes.md | 99 ++++ docs/getting-started-guides/scratch.md | 75 ++- docs/getting-started-guides/ubuntu-calico.md | 548 ++++++++++++------ docs/getting-started-guides/ubuntu.md | 170 +++--- docs/getting-started-guides/vagrant.md | 78 ++- docs/getting-started-guides/vsphere.md | 34 +- docs/user-guide/configmap/README.md | 117 ++++ docs/user-guide/configmap/command-pod.yaml | 21 + docs/user-guide/configmap/configmap.yaml | 7 + docs/user-guide/configmap/env-pod.yaml | 21 + .../configmap/kubectl/game.properties | 7 + .../configmap/kubectl/ui.properties | 4 + docs/user-guide/configmap/redis/redis-config | 2 + .../user-guide/configmap/redis/redis-pod.yaml | 30 + docs/user-guide/configmap/volume-pod.yaml | 20 + docs/user-guide/deployment.yaml | 21 + .../hpa-php-apache.yaml | 2 +- .../image/Dockerfile | 14 + docs/user-guide/liveness/http-liveness.yaml | 3 + docs/user-guide/liveness/image/Dockerfile | 14 + docs/user-guide/liveness/image/Makefile | 14 + docs/user-guide/logging-demo/Makefile | 18 +- .../pod-with-node-affinity.yaml | 28 + .../persistent-volumes/simpletest/pod.yaml | 2 +- .../persistent-volumes/volumes/local-01.yaml | 2 +- .../persistent-volumes/volumes/local-02.yaml | 2 +- .../persistent-volumes/volumes/nfs.yaml | 2 +- docs/user-guide/replicaset/frontend.yaml | 42 ++ docs/user-guide/replicaset/redis-slave.yaml | 44 ++ docs/user-guide/secrets/secret-env-pod.yaml | 16 + .../update-demo/images/kitten/Dockerfile | 2 +- .../update-demo/images/nautilus/Dockerfile | 2 +- 78 files changed, 2314 insertions(+), 975 deletions(-) create mode 100644 docs/getting-started-guides/docker-multinode/kube-system.yaml delete mode 100644 docs/getting-started-guides/docker-multinode/skydns-svc.yaml.in rename docs/getting-started-guides/docker-multinode/{skydns-rc.yaml.in => skydns.yaml.in} (57%) create mode 100644 docs/getting-started-guides/mesos/OWNERS create mode 100755 docs/getting-started-guides/mesos/k8s-firewall.png create mode 100755 docs/getting-started-guides/mesos/k8s-guestbook.png create mode 100644 docs/getting-started-guides/rkt/notes.md create mode 100644 docs/user-guide/configmap/README.md create mode 100644 docs/user-guide/configmap/command-pod.yaml create mode 100644 docs/user-guide/configmap/configmap.yaml create mode 100644 docs/user-guide/configmap/env-pod.yaml create mode 100644 docs/user-guide/configmap/kubectl/game.properties create mode 100644 docs/user-guide/configmap/kubectl/ui.properties create mode 100644 docs/user-guide/configmap/redis/redis-config create mode 100644 docs/user-guide/configmap/redis/redis-pod.yaml create mode 100644 docs/user-guide/configmap/volume-pod.yaml create mode 100644 docs/user-guide/deployment.yaml create mode 100644 docs/user-guide/node-selection/pod-with-node-affinity.yaml create mode 100644 docs/user-guide/replicaset/frontend.yaml create mode 100644 docs/user-guide/replicaset/redis-slave.yaml create mode 100644 docs/user-guide/secrets/secret-env-pod.yaml diff --git a/_data/guides.yml b/_data/guides.yml index c83158bbc9..453e84de7b 100644 --- a/_data/guides.yml +++ b/_data/guides.yml @@ -80,7 +80,11 @@ toc: - title: Fedora With Calico Networking path: /docs/getting-started-guides/fedora/fedora-calico/ - title: rkt - path: /docs/getting-started-guides/rkt/ + section: + - title: Running Kubernetes on rat + path: /docs/getting-started-guides/rkt/ + - title: Notes on Different UX with rkt Container Runtime + path: /docs/getting-started-guides/rkt/notes/ - title: Kubernetes on Mesos path: /docs/getting-started-guides/mesos/ - title: Kubernetes on Mesos on Docker diff --git a/docs/admin/garbage-collection.md b/docs/admin/garbage-collection.md index 13f342bb8b..2ccd4bd2e9 100644 --- a/docs/admin/garbage-collection.md +++ b/docs/admin/garbage-collection.md @@ -1,8 +1,9 @@ +--- +--- + * TOC {:toc} -### Introduction - Garbage collection is a helpful function of kubelet that will clean up unreferenced images and unused containers. kubelet will perform garbage collection for containers every minute and garbage collection for images every five minutes. External garbage collection tools are not recommended as these tools can potentially break the behavior of kubelet by removing containers expected to exist. diff --git a/docs/getting-started-guides/aws.md b/docs/getting-started-guides/aws.md index fe055ed1a1..ba576f4f02 100644 --- a/docs/getting-started-guides/aws.md +++ b/docs/getting-started-guides/aws.md @@ -8,7 +8,7 @@ 1. You need an AWS account. Visit [http://aws.amazon.com](http://aws.amazon.com) to get started 2. Install and configure [AWS Command Line Interface](http://aws.amazon.com/cli) -3. You need an AWS [instance profile and role](http://docs.aws.amazon.com/IAM/latest/UserGuide/instance-profiles) with EC2 full access. +3. You need an AWS [instance profile and role](http://docs.aws.amazon.com/IAM/latest/UserGuide/instance-profiles.html) with EC2 full access. NOTE: This script use the 'default' AWS profile by default. You may explicitly set AWS profile to use using the `AWS_DEFAULT_PROFILE` environment variable: @@ -36,28 +36,66 @@ This process takes about 5 to 10 minutes. Once the cluster is up, the IP address as well as information about the default services running in the cluster (monitoring, logging, dns). User credentials and security tokens are written in `~/.kube/config`, they will be necessary to use the CLI or the HTTP Basic Auth. -By default, the script will provision a new VPC and a 4 node k8s cluster in us-west-2a (Oregon) with `t2.micro` instances running on Ubuntu. -You can override the variables defined in [config-default.sh](http://releases.k8s.io/{{page.githubbranch}}/cluster/aws/config-default.sh) to change this behavior as follows: +By default, the script will provision a new VPC and a 4 node k8s cluster in us-west-2a (Oregon) with EC2 instances running on Ubuntu. +You can override the variables defined in [config-default.sh](http://releases.k8s.io/release-1.2/cluster/aws/config-default.sh) to change this behavior as follows: ```shell export KUBE_AWS_ZONE=eu-west-1c -export NUM_MINIONS=2 -export MINION_SIZE=m3.medium +export NUM_NODES=2 +export MASTER_SIZE=m3.medium +export NODE_SIZE=m3.medium export AWS_S3_REGION=eu-west-1 export AWS_S3_BUCKET=mycompany-kubernetes-artifacts export INSTANCE_PREFIX=k8s ... ``` -It will also try to create or reuse a keypair called "kubernetes", and IAM profiles called "kubernetes-master" and "kubernetes-minion". +If you don't specify master and minion sizes, the scripts will attempt to guess +the correct size of the master and worker nodes based on `${NUM_NODES}`. In +version 1.2 these default are: + +* For the master, for clusters of less than 150 nodes it will use an + `m3.medium`, for clusters of greater than 150 nodes it will use an + `m3.large`. + +* For worker nodes, for clusters less than 50 nodes it will use a `t2.micro`, + for clusters between 50 and 150 nodes it will use a `t2.small` and for + clusters with greater than 150 nodes it will use a `t2.medium`. + +WARNING: beware that `t2` instances receive a limited number of CPU credits per hour and might not be suitable for clusters where the CPU is used +consistently. As a rough estimation, consider 15 pods/node the absolute limit a `t2.large` instance can handle before it starts exhausting its CPU credits +steadily, although this number depends heavily on the usage. + +In prior versions of Kubernetes, we defaulted the master node to a t2-class +instance, but found that this sometimes gave hard-to-diagnose problems when the +master ran out of memory or CPU credits. If you are running a test cluster +and want to save money, you can specify `export MASTER_SIZE=t2.micro` but if +your master pauses do check the CPU credits in the AWS console. + +For production usage, we recommend at least `export MASTER_SIZE=m3.medium` and +`export NODE_SIZE=m3.medium`. And once you get above a handful of nodes, be +aware that one m3.large instance has more storage than two m3.medium instances, +for the same price. + +We generally recommend the m3 instances over the m4 instances, because the m3 +instances include local instance storage. Historically local instance storage +has been more reliable than AWS EBS, and performance should be more consistent. +The ephemeral nature of this storage is a match for ephemeral container +workloads also! + +If you use an m4 instance, or another instance type which does not have local +instance storage, you may want to increase the `NODE_ROOT_DISK_SIZE` value, +although the default value of 32 is probably sufficient for the smaller +instance types in the m4 family. + +The script will also try to create or reuse a keypair called "kubernetes", and IAM profiles called "kubernetes-master" and "kubernetes-minion". If these already exist, make sure you want them to be used here. NOTE: If using an existing keypair named "kubernetes" then you must set the `AWS_SSH_KEY` key to point to your private key. ### Alternatives -A contributed [example](/docs/getting-started-guides/coreos/coreos_multinode_cluster) allows you to setup a Kubernetes cluster based on [CoreOS](http://www.coreos.com), using -EC2 with user data (cloud-config). +CoreOS maintains [a CLI tool](https://coreos.com/kubernetes/docs/latest/kubernetes-on-aws.html), `kube-aws` that will create and manage a Kubernetes cluster based on [CoreOS](http://www.coreos.com), using AWS tools: EC2, CloudFormation and Autoscaling. ## Getting started with your cluster @@ -71,6 +109,7 @@ Next, add the appropriate binary folder to your `PATH` to access kubectl: ```shell # OS X export PATH=/platforms/darwin/amd64:$PATH + # Linux export PATH=/platforms/linux/amd64:$PATH ``` diff --git a/docs/getting-started-guides/centos/centos_manual_config.md b/docs/getting-started-guides/centos/centos_manual_config.md index be5892f5a0..553000f43d 100644 --- a/docs/getting-started-guides/centos/centos_manual_config.md +++ b/docs/getting-started-guides/centos/centos_manual_config.md @@ -27,36 +27,19 @@ centos-minion = 192.168.121.65 **Prepare the hosts:** -* Create virt7-testing repo on all hosts - centos-{master,minion} with following information. +* Create a virt7-docker-common-release repo on all hosts - centos-{master,minion} with following information. ```conf -[virt7-testing] -name=virt7-testing -baseurl=http://cbs.centos.org/repos/virt7-testing/x86_64/os/ +[virt7-docker-common-release] +name=virt7-docker-common-release +baseurl=http://cbs.centos.org/repos/virt7-docker-common-release/x86_64/os/ gpgcheck=0 ``` * Install Kubernetes on all hosts - centos-{master,minion}. This will also pull in etcd, docker, and cadvisor. ```shell -yum -y install --enablerepo=virt7-testing kubernetes -``` - -* Note * Using etcd-0.4.6-7 (This is temporary update in documentation) - -If you do not get etcd-0.4.6-7 installed with virt7-testing repo, - -In the current virt7-testing repo, the etcd package is updated which causes service failure. To avoid this, - -```shell -yum erase etcd -``` - -It will uninstall the current available etcd package - -```shell -yum install http://cbs.centos.org/kojifiles/packages/etcd/0.4.6/7.el7.centos/x86_64/etcd-0.4.6-7.el7.centos.x86_64.rpm -yum -y install --enablerepo=virt7-testing kubernetes +yum -y install --enablerepo=virt7-docker-common-release kubernetes ``` * Add master and node to /etc/hosts on all machines (not needed if hostnames already in DNS) @@ -66,11 +49,11 @@ echo "192.168.121.9 centos-master 192.168.121.65 centos-minion" >> /etc/hosts ``` -* Edit `/etc/kubernetes/config` which will be the same on all hosts to contain: +* Edit /etc/kubernetes/config which will be the same on all hosts to contain: ```shell # Comma separated list of nodes in the etcd cluster -KUBE_ETCD_SERVERS="--etcd-servers=http://centos-master:4001" +KUBE_ETCD_SERVERS="--etcd-servers=http://centos-master:2379" # logging to stderr means we get it in the systemd journal KUBE_LOGTOSTDERR="--logtostderr=true" @@ -127,7 +110,7 @@ done ***We need to configure the kubelet and start the kubelet and proxy*** -* Edit `/etc/kubernetes/kubelet` to appear as such: +* Edit /etc/kubernetes/kubelet to appear as such: ```shell # The address for the info server to serve on diff --git a/docs/getting-started-guides/cloudstack.md b/docs/getting-started-guides/cloudstack.md index 5e1b9af4c0..b9aef44c88 100644 --- a/docs/getting-started-guides/cloudstack.md +++ b/docs/getting-started-guides/cloudstack.md @@ -3,7 +3,7 @@ CloudStack is a software to build public and private clouds based on hardware virtualization principles (traditional IaaS). To deploy Kubernetes on CloudStack there are several possibilities depending on the Cloud being used and what images are made available. [Exoscale](http://exoscale.ch) for instance makes a [CoreOS](http://coreos.com) template available, therefore instructions to deploy Kubernetes on coreOS can be used. CloudStack also has a vagrant plugin available, hence Vagrant could be used to deploy Kubernetes either using the existing shell provisioner or using new Salt based recipes. -[CoreOS](http://coreos.com) templates for CloudStack are built [nightly](http://stable.release.core-os.net/amd64-usr/current/). CloudStack operators need to [register](http://docs.cloudstack.apache.org/projects/cloudstack-administration/en/latest/templates) this template in their cloud before proceeding with these Kubernetes deployment instructions. +[CoreOS](http://coreos.com) templates for CloudStack are built [nightly](http://stable.release.core-os.net/amd64-usr/current/). CloudStack operators need to [register](http://docs.cloudstack.apache.org/projects/cloudstack-administration/en/latest/templates.html) this template in their cloud before proceeding with these Kubernetes deployment instructions. This guide uses an [Ansible playbook](https://github.com/runseb/ansible-kubernetes). This is a completely automated, a single playbook deploys Kubernetes based on the coreOS [instructions](/docs/getting-started-guides/coreos/coreos_multinode_cluster). diff --git a/docs/getting-started-guides/coreos.md b/docs/getting-started-guides/coreos.md index 0f3136a85b..68d5a282b9 100644 --- a/docs/getting-started-guides/coreos.md +++ b/docs/getting-started-guides/coreos.md @@ -8,21 +8,27 @@ There are multiple guides on running Kubernetes with [CoreOS](https://coreos.com ### Official CoreOS Guides -These guides are maintained by CoreOS and deploy Kubernetes the "CoreOS Way" with full TLS, the DNS add-on, and more. These guides pass Kubernetes conformance testing and we encourage you to [test this yourself](https://coreos.com/kubernetes/docs/latest/conformance-tests). +These guides are maintained by CoreOS and deploy Kubernetes the "CoreOS Way" with full TLS, the DNS add-on, and more. These guides pass Kubernetes conformance testing and we encourage you to [test this yourself](https://coreos.com/kubernetes/docs/latest/conformance-tests.html). -[**Vagrant Multi-Node**](https://coreos.com/kubernetes/docs/latest/kubernetes-on-vagrant) +[**AWS Multi-Node**](https://coreos.com/kubernetes/docs/latest/kubernetes-on-aws.html) + +Guide and CLI tool for setting up a multi-node cluster on AWS. CloudFormation is used to set up a master and multiple workers in auto-scaling groups. + +
+ +[**Vagrant Multi-Node**](https://coreos.com/kubernetes/docs/latest/kubernetes-on-vagrant.html) Guide to setting up a multi-node cluster on Vagrant. The deployer can independently configure the number of etcd nodes, master nodes, and worker nodes to bring up a fully HA control plane.
-[**Vagrant Single-Node**](https://coreos.com/kubernetes/docs/latest/kubernetes-on-vagrant-single) +[**Vagrant Single-Node**](https://coreos.com/kubernetes/docs/latest/kubernetes-on-vagrant-single.html) The quickest way to set up a Kubernetes development environment locally. As easy as `git clone`, `vagrant up` and configuring `kubectl`.
-[**Full Step by Step Guide**](https://coreos.com/kubernetes/docs/latest/getting-started) +[**Full Step by Step Guide**](https://coreos.com/kubernetes/docs/latest/getting-started.html) A generic guide to setting up an HA cluster on any cloud or bare metal, with full TLS. Repeat the master or worker steps to configure more machines of that role. @@ -54,6 +60,12 @@ Configure a single master, multi-worker cluster locally, running on your choice
+[**Single-node cluster using a small OS X App**](https://github.com/rimusz/kube-solo-osx/blob/master/README.md) + +Guide to running a solo cluster (master + worker) controlled by an OS X menubar application. Uses xhyve + CoreOS under the hood. + +
+ [**Multi-node cluster with Vagrant and fleet units using a small OS X App**](https://github.com/rimusz/coreos-osx-gui-kubernetes-cluster/blob/master/README.md) Guide to running a single master, multi-worker cluster controlled by an OS X menubar application. Uses Vagrant under the hood. diff --git a/docs/getting-started-guides/coreos/azure/addons/skydns-rc.yaml b/docs/getting-started-guides/coreos/azure/addons/skydns-rc.yaml index 00a20f3b4a..ee31c5107c 100644 --- a/docs/getting-started-guides/coreos/azure/addons/skydns-rc.yaml +++ b/docs/getting-started-guides/coreos/azure/addons/skydns-rc.yaml @@ -1,22 +1,22 @@ apiVersion: v1 kind: ReplicationController metadata: - name: kube-dns-v8 + name: kube-dns-v9 namespace: kube-system labels: k8s-app: kube-dns - version: v8 + version: v9 kubernetes.io/cluster-service: "true" spec: replicas: 3 selector: k8s-app: kube-dns - version: v8 + version: v9 template: metadata: labels: k8s-app: kube-dns - version: v8 + version: v9 kubernetes.io/cluster-service: "true" spec: containers: @@ -74,6 +74,13 @@ spec: scheme: HTTP initialDelaySeconds: 30 timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 1 + timeoutSeconds: 5 - name: healthz image: gcr.io/google_containers/exechealthz:1.0 resources: diff --git a/docs/getting-started-guides/coreos/azure/addons/skydns-svc.yaml b/docs/getting-started-guides/coreos/azure/addons/skydns-svc.yaml index a0e979c266..c15822d6bc 100644 --- a/docs/getting-started-guides/coreos/azure/addons/skydns-svc.yaml +++ b/docs/getting-started-guides/coreos/azure/addons/skydns-svc.yaml @@ -10,7 +10,7 @@ metadata: spec: selector: k8s-app: kube-dns - clusterIP: 10.1.0.3 + clusterIP: 10.16.0.3 ports: - name: dns port: 53 diff --git a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml index 340c804139..b3f3331429 100644 --- a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml +++ b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml @@ -25,13 +25,6 @@ coreos: ExecStart=/bin/sh -x -c \ 'until curl --silent --fail https://status.github.com/api/status.json | grep -q \"good\"; do sleep 2; done' - - name: docker.service - drop-ins: - - name: 50-weave-kubernetes.conf - content: | - [Service] - Environment=DOCKER_OPTS='--bridge="weave" -r="false"' - - name: weave-network.target enable: true content: | @@ -92,46 +85,46 @@ coreos: content: | [Unit] After=network-online.target + After=docker.service Before=weave.service - Before=weave-helper.service - Before=docker.service Description=Install Weave Documentation=http://docs.weave.works/ Requires=network-online.target [Service] + EnvironmentFile=-/etc/weave.%H.env + EnvironmentFile=-/etc/weave.env Type=oneshot RemainAfterExit=yes + TimeoutStartSec=0 ExecStartPre=/bin/mkdir -p /opt/bin/ ExecStartPre=/opt/bin/curl-retry.sh \ --silent \ --location \ - https://github.com/weaveworks/weave/releases/download/latest_release/weave \ + git.io/weave \ --output /opt/bin/weave - ExecStartPre=/opt/bin/curl-retry.sh \ - --silent \ - --location \ - https://raw.github.com/errordeveloper/weave-demos/master/poseidon/weave-helper \ - --output /opt/bin/weave-helper ExecStartPre=/usr/bin/chmod +x /opt/bin/weave - ExecStartPre=/usr/bin/chmod +x /opt/bin/weave-helper - ExecStart=/bin/echo Weave Installed + ExecStart=/opt/bin/weave setup [Install] WantedBy=weave-network.target WantedBy=weave.service - - name: weave-helper.service + - name: weaveproxy.service enable: true content: | [Unit] After=install-weave.service After=docker.service - Description=Weave Network Router + Description=Weave proxy for Docker API Documentation=http://docs.weave.works/ Requires=docker.service Requires=install-weave.service [Service] - ExecStart=/opt/bin/weave-helper - Restart=always + EnvironmentFile=-/etc/weave.%H.env + EnvironmentFile=-/etc/weave.env + ExecStartPre=/opt/bin/weave launch-proxy --rewrite-inspect --without-dns + ExecStart=/usr/bin/docker attach weaveproxy + Restart=on-failure + ExecStop=/opt/bin/weave stop-proxy [Install] WantedBy=weave-network.target @@ -147,35 +140,35 @@ coreos: Requires=install-weave.service [Service] TimeoutStartSec=0 - EnvironmentFile=/etc/weave.%H.env - ExecStartPre=/opt/bin/weave setup - ExecStartPre=/opt/bin/weave launch $WEAVE_PEERS + EnvironmentFile=-/etc/weave.%H.env + EnvironmentFile=-/etc/weave.env + ExecStartPre=/opt/bin/weave launch-router $WEAVE_PEERS ExecStart=/usr/bin/docker attach weave Restart=on-failure - Restart=always - ExecStop=/opt/bin/weave stop + ExecStop=/opt/bin/weave stop-router [Install] WantedBy=weave-network.target - - name: weave-create-bridge.service + - name: weave-expose.service enable: true content: | [Unit] - After=network.target After=install-weave.service - Before=weave.service - Before=docker.service - Requires=network.target + After=weave.service + After=docker.service + Documentation=http://docs.weave.works/ + Requires=docker.service Requires=install-weave.service + Requires=weave.service [Service] Type=oneshot - EnvironmentFile=/etc/weave.%H.env - ExecStart=/opt/bin/weave --local create-bridge - ExecStart=/usr/bin/ip addr add dev weave $BRIDGE_ADDRESS_CIDR - ExecStart=/usr/bin/ip route add $BREAKOUT_ROUTE dev weave scope link - ExecStart=/usr/bin/ip route add 224.0.0.0/4 dev weave + RemainAfterExit=yes + TimeoutStartSec=0 + EnvironmentFile=-/etc/weave.%H.env + EnvironmentFile=-/etc/weave.env + ExecStart=/opt/bin/weave expose + ExecStop=/opt/bin/weave hide [Install] - WantedBy=multi-user.target WantedBy=weave-network.target - name: install-kubernetes.service @@ -191,7 +184,7 @@ coreos: Documentation=http://kubernetes.io/ Requires=network-online.target [Service] - Environment=KUBE_RELEASE_TARBALL=https://github.com/GoogleCloudPlatform/kubernetes/releases/download/v1.0.1/kubernetes.tar.gz + Environment=KUBE_RELEASE_TARBALL=https://github.com/kubernetes/kubernetes/releases/download/v1.1.2/kubernetes.tar.gz ExecStartPre=/bin/mkdir -p /opt/ ExecStart=/opt/bin/curl-retry.sh --silent --location $KUBE_RELEASE_TARBALL --output /tmp/kubernetes.tgz ExecStart=/bin/tar xzvf /tmp/kubernetes.tgz -C /tmp/ @@ -222,11 +215,13 @@ coreos: ConditionHost=kube-00 [Service] ExecStart=/opt/kubernetes/server/bin/kube-apiserver \ - --address=0.0.0.0 \ + --insecure-bind-address=0.0.0.0 \ + --advertise-address=$public_ipv4 \ --port=8080 \ $ETCD_SERVERS \ - --service-cluster-ip-range=10.1.0.0/16 \ - --logtostderr=true --v=3 + --service-cluster-ip-range=10.16.0.0/12 \ + --cloud-provider=vagrant \ + --logtostderr=true Restart=always RestartSec=10 [Install] @@ -286,12 +281,13 @@ coreos: [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests/ ExecStart=/opt/kubernetes/server/bin/kubelet \ + --docker-endpoint=unix:/var/run/weave/weave.sock \ --address=0.0.0.0 \ --port=10250 \ --hostname-override=%H \ --api-servers=http://kube-00:8080 \ --logtostderr=true \ - --cluster-dns=10.1.0.3 \ + --cluster-dns=10.16.0.3 \ --cluster-domain=kube.local \ --config=/etc/kubernetes/manifests/ Restart=always @@ -333,7 +329,7 @@ coreos: [Service] Type=oneshot RemainAfterExit=no - ExecStart=/opt/kubernetes/server/bin/kubectl create -f /etc/kubernetes/addons/ + ExecStart=/bin/bash -c 'until /opt/kubernetes/server/bin/kubectl create -f /etc/kubernetes/addons/; do sleep 2; done' SuccessExitStatus=1 [Install] WantedBy=kubernetes-master.target diff --git a/docs/getting-started-guides/coreos/azure/index.md b/docs/getting-started-guides/coreos/azure/index.md index 1750a9f5d2..3134378705 100644 --- a/docs/getting-started-guides/coreos/azure/index.md +++ b/docs/getting-started-guides/coreos/azure/index.md @@ -1,13 +1,11 @@ --- --- - -In this guide I will demonstrate how to deploy a Kubernetes cluster to Azure cloud. You will be using CoreOS with [Weave](http://weave.works), -which implements simple and secure networking, in a transparent, yet robust way. The purpose of this guide is to provide an out-of-the-box -implementation that can ultimately be taken into production with little change. It will demonstrate how to provision a dedicated Kubernetes -master and etcd nodes, and show how to scale the cluster with ease. * TOC {:toc} + + +In this guide I will demonstrate how to deploy a Kubernetes cluster to Azure cloud. You will be using CoreOS with Weave, which implements simple and secure networking, in a transparent, yet robust way. The purpose of this guide is to provide an out-of-the-box implementation that can ultimately be taken into production with little change. It will demonstrate how to provision a dedicated Kubernetes master and etcd nodes, and show how to scale the cluster with ease. ### Prerequisites @@ -37,9 +35,14 @@ Now, all you need to do is: ./create-kubernetes-cluster.js ``` -This script will provision a cluster suitable for production use, where there is a ring of 3 dedicated etcd nodes: 1 kubernetes master and 2 kubernetes nodes. -The `kube-00` VM will be the master, your work loads are only to be deployed on the nodes, `kube-01` and `kube-02`. Initially, all VMs are single-core, to -ensure a user of the free tier can reproduce it without paying extra. I will show how to add more bigger VMs later. +This script will provision a cluster suitable for production use, where there is a ring of 3 dedicated etcd nodes: 1 kubernetes master and 2 kubernetes nodes. The `kube-00` VM will be the master, your work loads are only to be deployed on the nodes, `kube-01` and `kube-02`. Initially, all VMs are single-core, to ensure a user of the free tier can reproduce it without paying extra. I will show how to add more bigger VMs later. +If you need to pass Azure specific options for the creation script you can do this via additional environment variables e.g. + +```shell +AZ_SUBSCRIPTION= AZ_LOCATION="East US" ./create-kubernetes-cluster.js +# or +AZ_VM_COREOS_CHANNEL=beta ./create-kubernetes-cluster.js +``` ![VMs in Azure](/images/docs/initial_cluster.png) diff --git a/docs/getting-started-guides/coreos/azure/lib/azure_wrapper.js b/docs/getting-started-guides/coreos/azure/lib/azure_wrapper.js index d389efbea2..93402c10ed 100644 --- a/docs/getting-started-guides/coreos/azure/lib/azure_wrapper.js +++ b/docs/getting-started-guides/coreos/azure/lib/azure_wrapper.js @@ -13,9 +13,9 @@ var inspect = require('util').inspect; var util = require('./util.js'); var coreos_image_ids = { - 'stable': '2b171e93f07c4903bcad35bda10acf22__CoreOS-Stable-717.3.0', - 'beta': '2b171e93f07c4903bcad35bda10acf22__CoreOS-Beta-723.3.0', // untested - 'alpha': '2b171e93f07c4903bcad35bda10acf22__CoreOS-Alpha-745.1.0' // untested + 'stable': '2b171e93f07c4903bcad35bda10acf22__CoreOS-Stable-835.12.0', // untested + 'beta': '2b171e93f07c4903bcad35bda10acf22__CoreOS-Beta-899.6.0', + 'alpha': '2b171e93f07c4903bcad35bda10acf22__CoreOS-Alpha-942.0.0' // untested }; var conf = {}; @@ -107,8 +107,11 @@ var create_ssh_key = function (prefix) { }; openssl.exec('req', opts, function (err, buffer) { if (err) console.log(clr.red(err)); - fs.chmod(opts.keyout, '0600', function (err) { + openssl.exec('rsa', { in: opts.keyout, out: opts.keyout }, function (err, buffer) { if (err) console.log(clr.red(err)); + fs.chmod(opts.keyout, '0600', function (err) { + if (err) console.log(clr.red(err)); + }); }); }); return { @@ -156,11 +159,18 @@ var get_vm_size = function () { } } +var get_subscription= function () { + if (process.env['AZ_SUBSCRIPTION']) { + return '--subscription=' + process.env['AZ_SUBSCRIPTION']; + } +} + exports.queue_default_network = function () { task_queue.push([ 'network', 'vnet', 'create', get_location(), '--address-space=172.16.0.0', + get_subscription(), conf.resources['vnet'], ]); } @@ -172,11 +182,12 @@ exports.queue_storage_if_needed = function() { 'storage', 'account', 'create', '--type=LRS', get_location(), + get_subscription(), conf.resources['storage_account'], ]); process.env['AZURE_STORAGE_ACCOUNT'] = conf.resources['storage_account']; } else { - // Preserve it for resizing, so we don't create a new one by accedent, + // Preserve it for resizing, so we don't create a new one by accident, // when the environment variable is unset conf.resources['storage_account'] = process.env['AZURE_STORAGE_ACCOUNT']; } @@ -192,6 +203,7 @@ exports.queue_machines = function (name_prefix, coreos_update_channel, cloud_con '--virtual-network-name=' + conf.resources['vnet'], '--no-ssh-password', '--ssh-cert=' + conf.resources['ssh_key']['pem'], + get_subscription(), ]; var cloud_config = cloud_config_creator(x, conf); @@ -216,6 +228,9 @@ exports.queue_machines = function (name_prefix, coreos_update_channel, cloud_con if (conf.resizing && n < conf.old_size) { return []; } else { + if (process.env['AZ_VM_COREOS_CHANNEL']) { + coreos_update_channel = process.env['AZ_VM_COREOS_CHANNEL'] + } return vm_create_base_args.concat(next_host(n), [ coreos_image_ids[coreos_update_channel], 'core', ]); @@ -246,11 +261,11 @@ exports.destroy_cluster = function (state_file) { conf.destroying = true; task_queue = _.map(conf.hosts, function (host) { - return ['vm', 'delete', '--quiet', '--blob-delete', host.name]; + return ['vm', 'delete', '--quiet', '--blob-delete', host.name, get_subscription()]; }); - task_queue.push(['network', 'vnet', 'delete', '--quiet', conf.resources['vnet']]); - task_queue.push(['storage', 'account', 'delete', '--quiet', conf.resources['storage_account']]); + task_queue.push(['network', 'vnet', 'delete', '--quiet', conf.resources['vnet'], get_subscription()]); + task_queue.push(['storage', 'account', 'delete', '--quiet', conf.resources['storage_account'], get_subscription()]); exports.run_task_queue(); }; diff --git a/docs/getting-started-guides/coreos/azure/package.json b/docs/getting-started-guides/coreos/azure/package.json index 3269b418e0..3f7e05f737 100644 --- a/docs/getting-started-guides/coreos/azure/package.json +++ b/docs/getting-started-guides/coreos/azure/package.json @@ -9,7 +9,7 @@ "author": "Ilya Dmitrichenko ", "license": "Apache 2.0", "dependencies": { - "azure-cli": "^0.9.5", + "azure-cli": "^0.9.9", "colors": "^1.0.3", "js-yaml": "^3.2.5", "openssl-wrapper": "^0.2.1", diff --git a/docs/getting-started-guides/coreos/bare_metal_calico.md b/docs/getting-started-guides/coreos/bare_metal_calico.md index eb8e9b642a..fc0d23503a 100644 --- a/docs/getting-started-guides/coreos/bare_metal_calico.md +++ b/docs/getting-started-guides/coreos/bare_metal_calico.md @@ -1,120 +1,197 @@ --- --- + +This document describes how to deploy Kubernetes with Calico networking on _bare metal_ CoreOS. For more information on Project Calico, visit [projectcalico.org](http://projectcalico.org) and the [calico-containers repository](https://github.com/projectcalico/calico-containers). -This guide explains how to deploy a bare-metal Kubernetes cluster on CoreOS using [Calico networking](http://www.projectcalico.org). +To install Calico on an existing Kubernetes cluster, or for more information on deploying Calico with Kubernetes in a number of other environments take a look at our supported [deployment guides](https://github.com/projectcalico/calico-containers/tree/master/docs/cni/kubernetes). Specifically, this guide will have you do the following: + +- Deploy a Kubernetes master node on CoreOS using cloud-config. +- Deploy two Kubernetes compute nodes with Calico Networking using cloud-config. +- Configure `kubectl` to access your cluster. -- Deploy a Kubernetes master node on CoreOS using cloud-config -- Deploy two Kubernetes compute nodes with Calico Networking using cloud-config +The resulting cluster will use SSL between Kubernetes components. It will run the SkyDNS service and kube-ui, and be fully conformant with the Kubernetes v1.1 conformance tests. + +## Prerequisites and Assumptions -## Prerequisites - -1. At least three bare-metal machines (or VMs) to work with. This guide will configure them as follows - - 1 Kubernetes Master - - 2 Kubernetes Nodes -2. Your nodes should have IP connectivity. +- At least three bare-metal machines (or VMs) to work with. This guide will configure them as follows: + - 1 Kubernetes Master + - 2 Kubernetes Nodes +- Your nodes should have IP connectivity to each other and the internet. +- This guide assumes a DHCP server on your network to assign server IPs. +- This guide uses `192.168.0.0/16` as the subnet from which pod IP addresses are assigned. If this overlaps with your host subnet, you will need to configure Calico to use a different [IP pool](https://github.com/projectcalico/calico-containers/blob/master/docs/calicoctl/pool.md#calicoctl-pool-commands). ## Cloud-config This guide will use [cloud-config](https://coreos.com/docs/cluster-management/setup/cloudinit-cloud-config/) to configure each of the nodes in our Kubernetes cluster. -For ease of distribution, the cloud-config files required for this demonstration can be found on [GitHub](https://github.com/projectcalico/calico-kubernetes-coreos-demo). +We'll use two cloud-config files: +- `master-config.yaml`: cloud-config for the Kubernetes master +- `node-config.yaml`: cloud-config for each Kubernetes node -This repo includes two cloud config files: - -- `master-config.yaml`: Cloud-config for the Kubernetes master -- `node-config.yaml`: Cloud-config for each Kubernetes compute host - -In the next few steps you will be asked to configure these files and host them on an HTTP server where your cluster can access them. - -## Building Kubernetes - -To get the Kubernetes source, clone the GitHub repo, and build the binaries. - -```shell -git clone https://github.com/kubernetes/kubernetes.git -cd kubernetes -./build/release.sh -``` +## Download CoreOS -Once the binaries are built, host the entire `/_output/dockerized/bin///` folder on an accessible HTTP server so they can be accessed by the cloud-config. You'll point your cloud-config files at this HTTP server later. - -## Download CoreOS - -Let's download the CoreOS bootable ISO. We'll use this image to boot and install CoreOS on each server. - -```shell -wget http://stable.release.core-os.net/amd64-usr/current/coreos_production_iso_image.iso -``` - -You can also download the ISO from the [CoreOS website](https://coreos.com/docs/running-coreos/platforms/iso/). +Download the stable CoreOS bootable ISO from the [CoreOS website](https://coreos.com/docs/running-coreos/platforms/iso/). ## Configure the Kubernetes Master -Once you've downloaded the image, use it to boot your Kubernetes Master server. Once booted, you should be automatically logged in as the `core` user. +1. Once you've downloaded the ISO image, burn the ISO to a CD/DVD/USB key and boot from it (if using a virtual machine you can boot directly from the ISO). Once booted, you should be automatically logged in as the `core` user at the terminal. At this point CoreOS is running from the ISO and it hasn't been installed yet. -Let's get the master-config.yaml and fill in the necessary variables. Run the following commands on your HTTP server to get the cloud-config files. +2. *On another machine*, download the the [master cloud-config template](https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/cloud-config/master-config-template.yaml) and save it as `master-config.yaml`. +3. Replace the following variables in the `master-config.yaml` file. + + - ``: The public key you will use for SSH access to this server. See [generating ssh keys](https://help.github.com/articles/generating-ssh-keys/) + +4. Copy the edited `master-config.yaml` to your Kubernetes master machine (using a USB stick, for example). + +5. The CoreOS bootable ISO comes with a tool called `coreos-install` which will allow us to install CoreOS and configure the machine using a cloud-config file. The following command will download and install stable CoreOS using the `master-config.yaml` file we just created for configuration. Run this on the Kubernetes master. + + > **Warning:** this is a destructive operation that erases disk `sda` on your server. + + ``` + sudo coreos-install -d /dev/sda -C stable -c master-config.yaml + ``` + +6. Once complete, restart the server and boot from `/dev/sda` (you may need to remove the ISO image). When it comes back up, you should have SSH access as the `core` user using the public key provided in the `master-config.yaml` file. + +### Configure TLS + +The master requires the CA certificate, `ca.pem`; its own certificate, `apiserver.pem` and its private key, `apiserver-key.pem`. This [CoreOS guide](https://coreos.com/kubernetes/docs/latest/openssl.html) explains how to generate these. + +1. Generate the necessary certificates for the master. This [guide for generating Kubernetes TLS Assets](https://coreos.com/kubernetes/docs/latest/openssl.html) explains how to use OpenSSL to generate the required assets. + +2. Send the three files to your master host (using `scp` for example). + +3. Move them to the `/etc/kubernetes/ssl` folder and ensure that only the root user can read the key: + + ```shell + # Move keys + sudo mkdir -p /etc/kubernetes/ssl/ + sudo mv -t /etc/kubernetes/ssl/ ca.pem apiserver.pem apiserver-key.pem + + # Set Permissions + sudo chmod 600 /etc/kubernetes/ssl/apiserver-key.pem + sudo chown root:root /etc/kubernetes/ssl/apiserver-key.pem + ``` + +4. Restart the kubelet to pick up the changes: + + ```shell + sudo systemctl restart kubelet + ``` + +## Configure the compute nodes + +The following steps will set up a single Kubernetes node for use as a compute host. Run these steps to deploy each Kubernetes node in your cluster. + +1. Boot up the node machine using the bootable ISO we downloaded earlier. You should be automatically logged in as the `core` user. + +2. Make a copy of the [node cloud-config template](https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/cloud-config/node-config-template.yaml) for this machine. + +3. Replace the following placeholders in the `node-config.yaml` file to match your deployment. + + - ``: Hostname for this node (e.g. kube-node1, kube-node2) + - ``: The public key you will use for SSH access to this server. + - ``: The IPv4 address of the Kubernetes master. + +4. Replace the following placeholders with the contents of their respective files. + + - ``: Complete contents of `ca.pem` + - ``: Complete contents of `ca-key.pem` + + > **Important:** in a production deployment, embedding the secret key in cloud-config is a bad idea! In production you should use an appropriate secret manager. + + > **Important:** Make sure you indent the entire file to match the indentation of the placeholder. For example: + > + > ``` + > - path: /etc/kubernetes/ssl/ca.pem + > owner: core + > permissions: 0644 + > content: | + > + > ``` + > + > should look like this once the certificate is in place: + > + > ``` + > - path: /etc/kubernetes/ssl/ca.pem + > owner: core + > permissions: 0644 + > content: | + > -----BEGIN CERTIFICATE----- + > MIIC9zCCAd+gAwIBAgIJAJMnVnhVhy5pMA0GCSqGSIb3DQEBCwUAMBIxEDAOBgNV + > ...... + > QHwi1rNc8eBLNrd4BM/A1ZeDVh/Q9KxN+ZG/hHIXhmWKgN5wQx6/81FIFg== + > -----END CERTIFICATE----- + > ``` + +5. Move the modified `node-config.yaml` to your Kubernetes node machine and install and configure CoreOS on the node using the following command. + + > **Warning:** this is a destructive operation that erases disk `sda` on your server. + + ```shell + sudo coreos-install -d /dev/sda -C stable -c node-config.yaml + ``` + +6. Once complete, restart the server and boot into `/dev/sda`. When it comes back up, you should have SSH access as the `core` user using the public key provided in the `node-config.yaml` file. It will take some time for the node to be fully configured. + +## Configure Kubeconfig + +To administer your cluster from a separate host, you will need the client and admin certificates generated earlier (`ca.pem`, `admin.pem`, `admin-key.pem`). With certificates in place, run the following commands with the appropriate filepaths. + ```shell -git clone https://github.com/Metaswitch/calico-kubernetes-demo.git -cd calico-kubernetes-demo/coreos +kubectl config set-cluster calico-cluster --server=https:// --certificate-authority= +kubectl config set-credentials calico-admin --certificate-authority= --client-key= --client-certificate= +kubectl config set-context calico --cluster=calico-cluster --user=calico-admin +kubectl config use-context calico ``` -You'll need to replace the following variables in the `master-config.yaml` file to match your deployment. - -- ``: The public key you will use for SSH access to this server. -- ``: The address used to get the kubernetes binaries over HTTP. - -> **Note:** The config will prepend `"http://"` and append `"/(kubernetes | kubectl | ...)"` to your `KUBERNETES_LOC` variable:, format accordingly - -Host the modified `master-config.yaml` file and pull it on to your Kubernetes Master server. - -The CoreOS bootable ISO comes with a tool called `coreos-install` which will allow us to install CoreOS to disk and configure the install using cloud-config. The following command will download and install stable CoreOS, using the master-config.yaml file for configuration. - +Check your work with `kubectl get nodes`. + +## Install the DNS Addon + +Most Kubernetes deployments will require the DNS addon for service discovery. To install DNS, create the skydns service and replication controller provided. + ```shell -sudo coreos-install -d /dev/sda -C stable -c master-config.yaml +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/master/dns/skydns.yaml ``` -Once complete, eject the bootable ISO and restart the server. When it comes back up, you should have SSH access as the `core` user using the public key provided in the master-config.yaml file. - -## Configure the compute hosts - ->The following steps will set up a Kubernetes node for use as a compute host. This demo uses two compute hosts, so you should run the following steps on each. - -First, boot up your node using the bootable ISO we downloaded earlier. You should be automatically logged in as the `core` user. - -Let's modify the `node-config.yaml` cloud-config file on your HTTP server. Make a copy for this node, and fill in the necessary variables. - -You'll need to replace the following variables in the `node-config.yaml` file to match your deployment. - -- ``: Hostname for this node (e.g. kube-node1, kube-node2) -- ``: The public key you will use for SSH access to this server. -- ``: The IPv4 address of the Kubernetes master. -- ``: The address to use in order to get the kubernetes binaries over HTTP. -- ``: The IP and subnet to use for pods on this node. By default, this should fall within the 192.168.0.0/16 subnet. - -> Note: The DOCKER_BRIDGE_IP is the range used by this Kubernetes node to assign IP addresses to pods on this node. This subnet must not overlap with the subnets assigned to the other Kubernetes nodes in your cluster. Calico expects each DOCKER_BRIDGE_IP subnet to fall within 192.168.0.0/16 by default (e.g. 192.168.1.1/24 for node 1), but if you'd like to use pod IPs within a different subnet, simply run `calicoctl pool add ` and select DOCKER_BRIDGE_IP accordingly. - -Host the modified `node-config.yaml` file and pull it on to your Kubernetes node. - +## Install the Kubernetes UI Addon (Optional) + +The Kubernetes UI can be installed using `kubectl` to run the following manifest file. + ```shell -wget http:///node-config.yaml +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/master/kube-ui/kube-ui.yaml ``` -Install and configure CoreOS on the node using the following command. +## Launch other Services With Calico-Kubernetes + +At this point, you have a fully functioning cluster running on Kubernetes with a master and two nodes networked with Calico. You can now follow any of the [standard documentation](https://github.com/kubernetes/kubernetes/tree/{{page.version}}/examples/) to set up other services on your cluster. + +## Connectivity to outside the cluster +Because containers in this guide have private `192.168.0.0/16` IPs, you will need NAT to allow connectivity between containers and the internet. However, in a production data center deployment, NAT is not always necessary, since Calico can peer with the data center's border routers over BGP. + +### NAT on the nodes + +The simplest method for enabling connectivity from containers to the internet is to use outgoing NAT on your Kubernetes nodes. + +Calico can provide outgoing NAT for containers. To enable it, use the following `calicoctl` command: + ```shell -sudo coreos-install -d /dev/sda -C stable -c node-config.yaml +ETCD_AUTHORITY= calicoctl pool add --nat-outgoing ``` -Once complete, restart the server. When it comes back up, you should have SSH access as the `core` user using the public key provided in the `node-config.yaml` file. It will take some time for the node to be fully configured. Once fully configured, you can check that the node is running with the following command on the Kubernetes master. - +By default, `` will be `192.168.0.0/16`. You can find out which pools have been configured with the following command: + ```shell -/home/core/kubectl get nodes +ETCD_AUTHORITY= calicoctl pool show ``` -## Testing the Cluster - -You should now have a functional bare-metal Kubernetes cluster with one master and two compute hosts. -Try running the [guestbook demo](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/guestbook/) to test out your new cluster! \ No newline at end of file +### NAT at the border router + +In a data center environment, it is recommended to configure Calico to peer with the border routers over BGP. This means that the container IPs will be routable anywhere in the data center, and so NAT is not needed on the nodes (though it may be enabled at the data center edge to allow outbound-only internet connectivity). + +The Calico documentation contains more information on how to configure Calico to [peer with existing infrastructure](https://github.com/projectcalico/calico-containers/blob/master/docs/ExternalConnectivity.md). diff --git a/docs/getting-started-guides/coreos/bare_metal_offline.md b/docs/getting-started-guides/coreos/bare_metal_offline.md index f80f90412c..1a216f7bc9 100644 --- a/docs/getting-started-guides/coreos/bare_metal_offline.md +++ b/docs/getting-started-guides/coreos/bare_metal_offline.md @@ -1,12 +1,12 @@ --- --- -Deploy a CoreOS running Kubernetes environment. This particular guild is made to help those in an OFFLINE system, -whether for testing a POC before the real deal, or you are restricted to be totally offline for your applications. +Deploy a CoreOS running Kubernetes environment. This particular guild is made to help those in an OFFLINE system, wither for testing a POC before the real deal, or you are restricted to be totally offline for your applications. * TOC {:toc} + ## Prerequisites 1. Installed *CentOS 6* for PXE server @@ -31,9 +31,10 @@ whether for testing a POC before the real deal, or you are restricted to be tota | CoreOS Slave 1 | d0:00:67:13:0d:01 | 10.20.30.41 | | CoreOS Slave 2 | d0:00:67:13:0d:02 | 10.20.30.42 | + ## Setup PXELINUX CentOS -To setup CentOS PXELINUX environment there is a complete [guide here](http://docs.fedoraproject.org/en-US/Fedora/7/html/Installation_Guide/ap-pxe-server). This section is the abbreviated version. +To setup CentOS PXELINUX environment there is a complete [guide here](http://docs.fedoraproject.org/en-US/Fedora/7/html/Installation_Guide/ap-pxe-server.html). This section is the abbreviated version. 1. Install packages needed on CentOS @@ -690,5 +691,5 @@ kubectl get nodes Kill all pods: ```shell -for i in `kubectl get pods | awk '{print $1}'`; do kubectl stop pod $i; done -``` \ No newline at end of file +for i in `kubectl get pods | awk '{print $1}'`; do kubectl delete pod $i; done +``` diff --git a/docs/getting-started-guides/coreos/cloud-configs/master.yaml b/docs/getting-started-guides/coreos/cloud-configs/master.yaml index c3b703d1d7..fbacf82ef5 100644 --- a/docs/getting-started-guides/coreos/cloud-configs/master.yaml +++ b/docs/getting-started-guides/coreos/cloud-configs/master.yaml @@ -31,6 +31,8 @@ coreos: fleet: metadata: "role=master" units: + - name: etcd2.service + command: start - name: generate-serviceaccount-key.service command: start content: | @@ -76,14 +78,14 @@ coreos: content: | [Unit] Description=Kubernetes API Server - Documentation=https://github.com/GoogleCloudPlatform/kubernetes + Documentation=https://github.com/kubernetes/kubernetes Requires=setup-network-environment.service etcd2.service generate-serviceaccount-key.service After=setup-network-environment.service etcd2.service generate-serviceaccount-key.service [Service] EnvironmentFile=/etc/network-environment ExecStartPre=-/usr/bin/mkdir -p /opt/bin - ExecStartPre=/usr/bin/curl -L -o /opt/bin/kube-apiserver -z /opt/bin/kube-apiserver https://storage.googleapis.com/kubernetes-release/release/v1.0.3/bin/linux/amd64/kube-apiserver + ExecStartPre=/usr/bin/curl -L -o /opt/bin/kube-apiserver -z /opt/bin/kube-apiserver https://storage.googleapis.com/kubernetes-release/release/v1.1.2/bin/linux/amd64/kube-apiserver ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-apiserver ExecStartPre=/opt/bin/wupiao 127.0.0.1:2379/v2/machines ExecStart=/opt/bin/kube-apiserver \ @@ -107,12 +109,12 @@ coreos: content: | [Unit] Description=Kubernetes Controller Manager - Documentation=https://github.com/GoogleCloudPlatform/kubernetes + Documentation=https://github.com/kubernetes/kubernetes Requires=kube-apiserver.service After=kube-apiserver.service [Service] - ExecStartPre=/usr/bin/curl -L -o /opt/bin/kube-controller-manager -z /opt/bin/kube-controller-manager https://storage.googleapis.com/kubernetes-release/release/v1.0.3/bin/linux/amd64/kube-controller-manager + ExecStartPre=/usr/bin/curl -L -o /opt/bin/kube-controller-manager -z /opt/bin/kube-controller-manager https://storage.googleapis.com/kubernetes-release/release/v1.1.2/bin/linux/amd64/kube-controller-manager ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-controller-manager ExecStart=/opt/bin/kube-controller-manager \ --service-account-private-key-file=/opt/bin/kube-serviceaccount.key \ @@ -125,12 +127,12 @@ coreos: content: | [Unit] Description=Kubernetes Scheduler - Documentation=https://github.com/GoogleCloudPlatform/kubernetes + Documentation=https://github.com/kubernetes/kubernetes Requires=kube-apiserver.service After=kube-apiserver.service [Service] - ExecStartPre=/usr/bin/curl -L -o /opt/bin/kube-scheduler -z /opt/bin/kube-scheduler https://storage.googleapis.com/kubernetes-release/release/v1.0.3/bin/linux/amd64/kube-scheduler + ExecStartPre=/usr/bin/curl -L -o /opt/bin/kube-scheduler -z /opt/bin/kube-scheduler https://storage.googleapis.com/kubernetes-release/release/v1.1.2/bin/linux/amd64/kube-scheduler ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-scheduler ExecStart=/opt/bin/kube-scheduler --master=127.0.0.1:8080 Restart=always diff --git a/docs/getting-started-guides/coreos/cloud-configs/node.yaml b/docs/getting-started-guides/coreos/cloud-configs/node.yaml index b9a4cafcd9..503d67dd55 100644 --- a/docs/getting-started-guides/coreos/cloud-configs/node.yaml +++ b/docs/getting-started-guides/coreos/cloud-configs/node.yaml @@ -18,17 +18,12 @@ coreos: fleet: metadata: "role=node" units: + - name: etcd2.service + command: start - name: fleet.service command: start - name: flanneld.service command: start - drop-ins: - - name: 50-network-config.conf - content: | - [Unit] - Requires=etcd2.service - [Service] - ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{"Network":"10.244.0.0/16", "Backend": {"Type": "vxlan"}}' - name: docker.service command: start - name: setup-network-environment.service @@ -52,12 +47,12 @@ coreos: content: | [Unit] Description=Kubernetes Proxy - Documentation=https://github.com/GoogleCloudPlatform/kubernetes + Documentation=https://github.com/kubernetes/kubernetes Requires=setup-network-environment.service After=setup-network-environment.service [Service] - ExecStartPre=/usr/bin/curl -L -o /opt/bin/kube-proxy -z /opt/bin/kube-proxy https://storage.googleapis.com/kubernetes-release/release/v1.0.3/bin/linux/amd64/kube-proxy + ExecStartPre=/usr/bin/curl -L -o /opt/bin/kube-proxy -z /opt/bin/kube-proxy https://storage.googleapis.com/kubernetes-release/release/v1.1.2/bin/linux/amd64/kube-proxy ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-proxy # wait for kubernetes master to be up and ready ExecStartPre=/opt/bin/wupiao 8080 @@ -71,13 +66,13 @@ coreos: content: | [Unit] Description=Kubernetes Kubelet - Documentation=https://github.com/GoogleCloudPlatform/kubernetes + Documentation=https://github.com/kubernetes/kubernetes Requires=setup-network-environment.service After=setup-network-environment.service [Service] EnvironmentFile=/etc/network-environment - ExecStartPre=/usr/bin/curl -L -o /opt/bin/kubelet -z /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.0.3/bin/linux/amd64/kubelet + ExecStartPre=/usr/bin/curl -L -o /opt/bin/kubelet -z /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.1.2/bin/linux/amd64/kubelet ExecStartPre=/usr/bin/chmod +x /opt/bin/kubelet # wait for kubernetes master to be up and ready ExecStartPre=/opt/bin/wupiao 8080 diff --git a/docs/getting-started-guides/coreos/coreos_multinode_cluster.md b/docs/getting-started-guides/coreos/coreos_multinode_cluster.md index e08345123a..cc786e3ec4 100644 --- a/docs/getting-started-guides/coreos/coreos_multinode_cluster.md +++ b/docs/getting-started-guides/coreos/coreos_multinode_cluster.md @@ -125,7 +125,7 @@ nova list #### Get a Suitable CoreOS Image -You'll need a [suitable version of CoreOS image for OpenStack](https://coreos.com/os/docs/latest/booting-on-openstack) +You'll need a [suitable version of CoreOS image for OpenStack](https://coreos.com/os/docs/latest/booting-on-openstack.html) Once you download that, upload it to glance. An example is shown below: ```shell @@ -159,8 +159,7 @@ kube-master `` is the keypair name that you already generated to access the instance. -`` is the flavor ID you use to size the instance. Run `nova flavor-list` -to get the IDs. 3 on the system this was tested with gives the m1.large size. +`` is the flavor ID you use to size the instance. Run `nova flavor-list` to get the IDs. 3 on the system this was tested with gives the m1.large size. The important part is to ensure you have the files/master.yml as this is what will do all the post boot configuration. This path is relevant so we are assuming in this example that you are running the nova command in a directory where there is a subdirectory called files that has the master.yml file in it. Absolute paths also work. @@ -176,15 +175,11 @@ Get an IP address that's free and run: nova floating-ip-associate kube-master ``` -...where `` is the IP address that was available from the `nova floating-ip-list` -command. +where `` is the IP address that was available from the `nova floating-ip-list` command. #### Provision Worker Nodes -Edit `node.yaml` -and replace all instances of `````` -with the private IP address of the master node. You can get this by running ```nova show kube-master``` -assuming you named your instance kube master. This is not the floating IP address you just assigned it. +Edit `node.yaml` and replace all instances of `` with the private IP address of the master node. You can get this by running `nova show kube-master` assuming you named your instance kube master. This is not the floating IP address you just assigned it. ```shell nova boot \ diff --git a/docs/getting-started-guides/dcos.md b/docs/getting-started-guides/dcos.md index efc64002b7..d46cb1fb65 100644 --- a/docs/getting-started-guides/dcos.md +++ b/docs/getting-started-guides/dcos.md @@ -105,7 +105,7 @@ $ dcos kubectl get pods --namespace=kube-system Names and ages may vary. -Now that Kubernetes is installed on DCOS, you may wish to explore the [Kubernetes Examples](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/) or the [Kubernetes User Guide](/docs/user-guide/). +Now that Kubernetes is installed on DCOS, you may wish to explore the [Kubernetes Examples](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/README.md) or the [Kubernetes User Guide](/docs/user-guide/). ## Uninstall diff --git a/docs/getting-started-guides/docker-multinode.md b/docs/getting-started-guides/docker-multinode.md index d42e1d5970..ded684e109 100644 --- a/docs/getting-started-guides/docker-multinode.md +++ b/docs/getting-started-guides/docker-multinode.md @@ -1,29 +1,27 @@ --- --- +_Note_: +These instructions are somewhat significantly more advanced than the [single node](docker.md) instructions. If you are +interested in just starting to explore Kubernetes, we recommend that you start there. + +* TOC +{:toc} + +## Prerequisites + +The only thing you need is a machine with **Docker 1.7.1 or higher** + +## Overview + This guide will set up a 2-node Kubernetes cluster, consisting of a _master_ node which hosts the API server and orchestrates work -and a _worker_ node which receives work from the master. You can repeat the process of adding worker nodes an arbitrary number of +and a _worker_ node which receives work from the master. You can repeat the process of adding worker nodes an arbitrary number of times to create larger clusters. Here's a diagram of what the final result will look like: ![Kubernetes Single Node on Docker](/images/docs/k8s-docker.png) -_Note_: -These instructions are somewhat significantly more advanced than the [single node](/docs/getting-started-guides/docker) instructions. If you are -interested in just starting to explore Kubernetes, we recommend that you start there. - -_Note_: -There is a [bug](https://github.com/docker/docker/issues/14106) in Docker 1.7.0 that prevents this from working correctly. -Please install Docker 1.6.2 or Docker 1.7.1. - -* TOC -{:toc} - -## Prerequisites - -You need a machine with docker of right version installed. - ### Bootstrap Docker This guide also uses a pattern of running two instances of the Docker daemon @@ -34,10 +32,14 @@ This pattern is necessary because the `flannel` daemon is responsible for settin all of the Docker containers created by Kubernetes. To achieve this, it must run outside of the _main_ Docker daemon. However, it is still useful to use containers for deployment and management, so we create a simpler _bootstrap_ daemon to achieve this. -You can specify k8s version on very node before install: +You can specify the version on every node before install: ```shell -export K8S_VERSION= +export K8S_VERSION= +export ETCD_VERSION= +export FLANNEL_VERSION= +export FLANNEL_IFACE= +export FLANNEL_IPMASQ= ``` Otherwise, we'll use latest `hyperkube` image as default k8s version. @@ -46,14 +48,16 @@ Otherwise, we'll use latest `hyperkube` image as default k8s version. The first step in the process is to initialize the master node. -Clone the Kubernetes repo, and run [master.sh](/docs/getting-started-guides/docker-multinode/master.sh) on the master machine with root: +The MASTER_IP step here is optional, it defaults to the first value of `hostname -I`. +Clone the Kubernetes repo, and run [master.sh](/docs/getting-started-guides/docker-multinode/master.sh) on the master machine _with root_: ```shell -cd kubernetes/docs/getting-started-guides/docker-multinode/ -./master.sh -... +$ export MASTER_IP= +$ cd kubernetes/docs/getting-started-guides/docker-multinode/ +$ ./master.sh +``` + `Master done!` -``` See [here](/docs/getting-started-guides/docker-multinode/master) for detailed instructions explanation. @@ -61,17 +65,17 @@ See [here](/docs/getting-started-guides/docker-multinode/master) for detailed in Once your master is up and running you can add one or more workers on different machines. -Clone the Kubernetes repo, and run [worker.sh](/docs/getting-started-guides/docker-multinode/worker.sh) on the worker machine with root: +Clone the Kubernetes repo, and run [worker.sh](/docs/getting-started-guides/docker-multinode/worker.sh) on the worker machine _with root_: ```shell -export MASTER_IP= -cd kubernetes/docs/getting-started-guides/docker-multinode/ -./worker.sh -... -`Worker done!` -```` +$ export MASTER_IP= +$ cd kubernetes/docs/getting-started-guides/docker-multinode/ +$ ./worker.sh +``` -See [here](/docs/getting-started-guides/docker-multinode/worker) for detailed instructions explanation. +`Worker done!` + +See [here](/docs/getting-started-guides/docker-multinode/worker) for a detailed explanation. ## Deploy a DNS diff --git a/docs/getting-started-guides/docker-multinode/deployDNS.md b/docs/getting-started-guides/docker-multinode/deployDNS.md index 9bd738ea13..7908e05b0c 100644 --- a/docs/getting-started-guides/docker-multinode/deployDNS.md +++ b/docs/getting-started-guides/docker-multinode/deployDNS.md @@ -3,15 +3,13 @@ ### Get the template file -First of all, download the template dns rc and svc file from +First of all, download the dns template -[skydns-rc template](/docs/getting-started-guides/docker-multinode/skydns-rc.yaml.in) +[skydns template](/docs/getting-started-guides/docker-multinode/skydns.yaml.in) -[skydns-svc template](/docs/getting-started-guides/docker-multinode/skydns-svc.yaml.in) +### Set environment variables -### Set env - -Then you need to set `DNS_REPLICAS` , `DNS_DOMAIN` , `DNS_SERVER_IP` , `KUBE_SERVER` ENV. +Then you need to set `DNS_REPLICAS`, `DNS_DOMAIN` and `DNS_SERVER_IP` envs ```shell $ export DNS_REPLICAS=1 @@ -19,25 +17,18 @@ $ export DNS_REPLICAS=1 $ export DNS_DOMAIN=cluster.local # specify in startup parameter `--cluster-domain` for containerized kubelet $ export DNS_SERVER_IP=10.0.0.10 # specify in startup parameter `--cluster-dns` for containerized kubelet - -$ export KUBE_SERVER=10.10.103.250 # your master server ip, you may change it ``` -### Replace the corresponding value in the template. +### Replace the corresponding value in the template and create the pod ```shell -$ sed -e "s/{{ pillar\['dns_replicas'\] }}/${DNS_REPLICAS}/g;s/{{ pillar\['dns_domain'\] }}/${DNS_DOMAIN}/g;s/{kube_server_url}/${KUBE_SERVER}/g;" skydns-rc.yaml.in > ./skydns-rc.yaml +$ sed -e "s/{{ pillar\['dns_replicas'\] }}/${DNS_REPLICAS}/g;s/{{ pillar\['dns_domain'\] }}/${DNS_DOMAIN}/g;s/{{ pillar\['dns_server'\] }}/${DNS_SERVER_IP}/g" skydns.yaml.in > ./skydns.yaml -$ sed -e "s/{{ pillar\['dns_server'\] }}/${DNS_SERVER_IP}/g" skydns-svc.yaml.in > ./skydns-svc.yaml -``` +# If the kube-system namespace isn't already created, create it +$ kubectl get ns +$ kubectl create -f ./kube-system.yaml -### Use `kubectl` to create skydns rc and service - - -```shell -$ kubectl -s "$KUBE_SERVER:8080" --namespace=kube-system create -f ./skydns-rc.yaml - -$ kubectl -s "$KUBE_SERVER:8080" --namespace=kube-system create -f ./skydns-svc.yaml +$ kubectl create -f ./skydns.yaml ``` ### Test if DNS works diff --git a/docs/getting-started-guides/docker-multinode/kube-system.yaml b/docs/getting-started-guides/docker-multinode/kube-system.yaml new file mode 100644 index 0000000000..986f4b4822 --- /dev/null +++ b/docs/getting-started-guides/docker-multinode/kube-system.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-system diff --git a/docs/getting-started-guides/docker-multinode/master.md b/docs/getting-started-guides/docker-multinode/master.md index 5867a0de01..0677ab1165 100644 --- a/docs/getting-started-guides/docker-multinode/master.md +++ b/docs/getting-started-guides/docker-multinode/master.md @@ -1,9 +1,23 @@ --- --- -We'll begin by setting up the master node. For the purposes of illustration, we'll assume that the IP of this machine is `${MASTER_IP}` +We'll begin by setting up the master node. For the purposes of illustration, we'll assume that the IP of this machine +is `${MASTER_IP}`. We'll need to run several versioned Kubernetes components, so we'll assume that the version we want +to run is `${K8S_VERSION}`, which should hold a released version of Kubernetes >= "1.2.0-alpha.7" + +Enviroinment variables used: + +```shell +export MASTER_IP= +export K8S_VERSION= +export ETCD_VERSION= +export FLANNEL_VERSION= +export FLANNEL_IFACE= +export FLANNEL_IPMASQ= +``` There are two main phases to installing the master: + * [Setting up `flanneld` and `etcd`](#setting-up-flanneld-and-etcd) * [Starting the Kubernetes master components](#starting-the-kubernetes-master) @@ -11,10 +25,9 @@ There are two main phases to installing the master: ## Setting up flanneld and etcd _Note_: -There is a [bug](https://github.com/docker/docker/issues/14106) in Docker 1.7.0 that prevents this from working correctly. -Please install Docker 1.6.2 or Docker 1.7.1. +This guide expects **Docker 1.7.1 or higher**. -### Setup Docker-Bootstrap +### Setup Docker Bootstrap We're going to use `flannel` to set up networking between Docker daemons. Flannel itself (and etcd on which it relies) will run inside of Docker containers themselves. To achieve this, we need a separate "bootstrap" instance of the Docker daemon. This daemon will be started with @@ -26,6 +39,12 @@ Run: sudo sh -c 'docker -d -H unix:///var/run/docker-bootstrap.sock -p /var/run/docker-bootstrap.pid --iptables=false --ip-masq=false --bridge=none --graph=/var/lib/docker-bootstrap 2> /var/log/docker-bootstrap.log 1> /dev/null &' ``` +_If you have Docker 1.8.0 or higher run this instead_ + +```shell +sudo sh -c 'docker daemon -H unix:///var/run/docker-bootstrap.sock -p /var/run/docker-bootstrap.pid --iptables=false --ip-masq=false --bridge=none --graph=/var/lib/docker-bootstrap 2> /var/log/docker-bootstrap.log 1> /dev/null &' +``` + _Important Note_: If you are running this on a long running system, rather than experimenting, you should run the bootstrap Docker instance under something like SysV init, upstart or systemd so that it is restarted across reboots and failures. @@ -36,15 +55,25 @@ across reboots and failures. Run: ```shell -sudo docker -H unix:///var/run/docker-bootstrap.sock run --net=host -d gcr.io/google_containers/etcd:2.0.12 /usr/local/bin/etcd --addr=127.0.0.1:4001 --bind-addr=0.0.0.0:4001 --data-dir=/var/etcd/data +sudo docker -H unix:///var/run/docker-bootstrap.sock run -d \ + --net=host \ + gcr.io/google_containers/etcd-amd64:${ETCD_VERSION} \ + /usr/local/bin/etcd \ + --listen-client-urls=http://127.0.0.1:4001,http://${MASTER_IP}:4001 \ + --advertise-client-urls=http://${MASTER_IP}:4001 \ + --data-dir=/var/etcd/data ``` Next, you need to set a CIDR range for flannel. This CIDR should be chosen to be non-overlapping with any existing network you are using: ```shell -sudo docker -H unix:///var/run/docker-bootstrap.sock run --net=host gcr.io/google_containers/etcd:2.0.12 etcdctl set /coreos.com/network/config '{ "Network": "10.1.0.0/16" }' +sudo docker -H unix:///var/run/docker-bootstrap.sock run \ + --net=host \ + gcr.io/google_containers/etcd-amd64:${ETCD_VERSION} \ + etcdctl set /coreos.com/network/config '{ "Network": "10.1.0.0/16" }' ``` + ### Set up Flannel on the master node Flannel is a network abstraction layer build by CoreOS, we will use it to provide simplified networking between our Pods of containers. @@ -67,6 +96,12 @@ or sudo systemctl stop docker ``` +or + +```shell +sudo service docker stop +``` + or it may be something else. #### Run flannel @@ -74,10 +109,16 @@ or it may be something else. Now run flanneld itself: ```shell -sudo docker -H unix:///var/run/docker-bootstrap.sock run -d --net=host --privileged -v /dev/net:/dev/net quay.io/coreos/flannel:0.5.0 +sudo docker -H unix:///var/run/docker-bootstrap.sock run -d \ + --net=host \ + --privileged \ + -v /dev/net:/dev/net \ + quay.io/coreos/flannel:${FLANNEL_VERSION} \ + --ip-masq=${FLANNEL_IPMASQ} \ + --iface=${FLANNEL_IFACE} ``` -The previous command should have printed a really long hash, copy this hash. +The previous command should have printed a really long hash, the container id, copy this hash. Now get the subnet settings from flannel: @@ -130,40 +171,63 @@ Ok, now that your networking is set up, you can startup Kubernetes, this is the sudo docker run \ --volume=/:/rootfs:ro \ --volume=/sys:/sys:ro \ - --volume=/dev:/dev \ --volume=/var/lib/docker/:/var/lib/docker:rw \ --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \ --volume=/var/run:/var/run:rw \ --net=host \ --privileged=true \ - --pid=host \ + --pid=host \ -d \ - gcr.io/google_containers/hyperkube:v1.0.1 /hyperkube kubelet --api-servers=http://localhost:8080 --v=2 --address=0.0.0.0 --enable-server --hostname-override=127.0.0.1 --config=/etc/kubernetes/manifests-multi --cluster-dns=10.0.0.10 --cluster-domain=cluster.local + gcr.io/google_containers/hyperkube-amd64:v${K8S_VERSION} \ + /hyperkube kubelet \ + --allow-privileged=true \ + --api-servers=http://localhost:8080 \ + --v=2 \ + --address=0.0.0.0 \ + --enable-server \ + --hostname-override=127.0.0.1 \ + --config=/etc/kubernetes/manifests-multi \ + --containerized \ + --cluster-dns=10.0.0.10 \ + --cluster-domain=cluster.local ``` > Note that `--cluster-dns` and `--cluster-domain` is used to deploy dns, feel free to discard them if dns is not needed. -### Also run the service proxy - -```shell -sudo docker run -d --net=host --privileged gcr.io/google_containers/hyperkube:v1.0.1 /hyperkube proxy --master=http://127.0.0.1:8080 --v=2 -``` - ### Test it out At this point, you should have a functioning 1-node cluster. Let's test it out! -Download the kubectl binary and make it available by editing your PATH ENV. -([OS X](http://storage.googleapis.com/kubernetes-release/release/v1.0.1/bin/darwin/amd64/kubectl)) -([linux](http://storage.googleapis.com/kubernetes-release/release/v1.0.1/bin/linux/amd64/kubectl)) +Download the kubectl binary for `${K8S_VERSION}` ({{page.version}}) and make it available by editing your PATH environment variable. +([OS X/amd64](http://storage.googleapis.com/kubernetes-release/release/{{page.version}}.0-alpha.7/bin/darwin/amd64/kubectl)) +([OS X/386](http://storage.googleapis.com/kubernetes-release/release/{{page.version}}.0-alpha.7/bin/darwin/386/kubectl)) +([linux/amd64](http://storage.googleapis.com/kubernetes-release/release/{{page.version}}.0-alpha.7/bin/linux/amd64/kubectl)) +([linux/386](http://storage.googleapis.com/kubernetes-release/release/{{page.version}}.0-alpha.7/bin/linux/386/kubectl)) +([linux/arm](http://storage.googleapis.com/kubernetes-release/release/{{page.version}}.0-alpha.7/bin/linux/arm/kubectl)) -List the nodes +For example, OS X: + +```console +$ wget http://storage.googleapis.com/kubernetes-release/release/v${K8S_VERSION}/bin/darwin/amd64/kubectl +$ chmod 755 kubectl +$ PATH=$PATH:`pwd` +``` + +Linux: + +```console +$ wget http://storage.googleapis.com/kubernetes-release/release/v${K8S_VERSION}/bin/linux/amd64/kubectl +$ chmod 755 kubectl +$ PATH=$PATH:`pwd` +``` + +Now you can list the nodes: ```shell kubectl get nodes ``` -This should print: +This should print something like: ```shell NAME LABELS STATUS @@ -176,4 +240,4 @@ If all else fails, ask questions on [Slack](/docs/troubleshooting/#slack). ### Next steps -Move on to [adding one or more workers](/docs/getting-started-guides/docker-multinode/worker) or [deploy a dns](/docs/getting-started-guides/docker-multinode/deployDNS) \ No newline at end of file +Move on to [adding one or more workers](/docs/getting-started-guides/docker-multinode/worker/) or [deploy a dns](/docs/getting-started-guides/docker-multinode/deployDNS/) diff --git a/docs/getting-started-guides/docker-multinode/master.sh b/docs/getting-started-guides/docker-multinode/master.sh index 0a247804ec..d7f59224c5 100755 --- a/docs/getting-started-guides/docker-multinode/master.sh +++ b/docs/getting-started-guides/docker-multinode/master.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# A scripts to install k8s worker node. -# Author @wizard_cxy @reouser +# A script to setup the k8s master in docker containers. +# Authors @wizard_cxy @resouer set -e @@ -26,13 +26,12 @@ if ( ! ps -ef | grep "/usr/bin/docker" | grep -v 'grep' &> /dev/null ); then fi # Make sure k8s version env is properly set -if [ -z ${K8S_VERSION} ]; then - K8S_VERSION="1.0.3" - echo "K8S_VERSION is not set, using default: ${K8S_VERSION}" -else - echo "k8s version is set to: ${K8S_VERSION}" -fi - +K8S_VERSION=${K8S_VERSION:-"1.2.0-alpha.7"} +ETCD_VERSION=${ETCD_VERSION:-"2.2.1"} +FLANNEL_VERSION=${FLANNEL_VERSION:-"0.5.5"} +FLANNEL_IPMASQ=${FLANNEL_IPMASQ:-"true"} +FLANNEL_IFACE=${FLANNEL_IFACE:-"eth0"} +ARCH=${ARCH:-"amd64"} # Run as root if [ "$(id -u)" != "0" ]; then @@ -40,6 +39,19 @@ if [ "$(id -u)" != "0" ]; then exit 1 fi +# Make sure master ip is properly set +if [ -z ${MASTER_IP} ]; then + MASTER_IP=$(hostname -I | awk '{print $1}') +fi + +echo "K8S_VERSION is set to: ${K8S_VERSION}" +echo "ETCD_VERSION is set to: ${ETCD_VERSION}" +echo "FLANNEL_VERSION is set to: ${FLANNEL_VERSION}" +echo "FLANNEL_IFACE is set to: ${FLANNEL_IFACE}" +echo "FLANNEL_IPMASQ is set to: ${FLANNEL_IPMASQ}" +echo "MASTER_IP is set to: ${MASTER_IP}" +echo "ARCH is set to: ${ARCH}" + # Check if a command is valid command_exists() { command -v "$@" > /dev/null 2>&1 @@ -49,13 +61,14 @@ lsb_dist="" # Detect the OS distro, we support ubuntu, debian, mint, centos, fedora dist detect_lsb() { + # TODO: remove this when ARM support is fully merged case "$(uname -m)" in - *64) - ;; - *) - echo "Error: We currently only support 64-bit platforms." - exit 1 - ;; + *64) + ;; + *) + echo "Error: We currently only support 64-bit platforms." + exit 1 + ;; esac if command_exists lsb_release; then @@ -75,13 +88,39 @@ detect_lsb() { fi lsb_dist="$(echo ${lsb_dist} | tr '[:upper:]' '[:lower:]')" + + case "${lsb_dist}" in + amzn|centos|debian|ubuntu) + ;; + *) + echo "Error: We currently only support ubuntu|debian|amzn|centos." + exit 1 + ;; + esac } # Start the bootstrap daemon +# TODO: do not start docker-bootstrap if it's already running bootstrap_daemon() { - sudo -b docker -d -H unix:///var/run/docker-bootstrap.sock -p /var/run/docker-bootstrap.pid --iptables=false --ip-masq=false --bridge=none --graph=/var/lib/docker-bootstrap 2> /var/log/docker-bootstrap.log 1> /dev/null - + # Detecting docker version so we could run proper docker_daemon command + [[ $(eval "docker --version") =~ ([0-9][.][0-9][.][0-9]*) ]] && version="${BASH_REMATCH[1]}" + local got=$(echo -e "${version}\n1.8.0" | sed '/^$/d' | sort -nr | head -1) + if [[ "${got}" = "${version}" ]]; then + docker_daemon="docker -d" + else + docker_daemon="docker daemon" + fi + ${docker_daemon} \ + -H unix:///var/run/docker-bootstrap.sock \ + -p /var/run/docker-bootstrap.pid \ + --iptables=false \ + --ip-masq=false \ + --bridge=none \ + --graph=/var/lib/docker-bootstrap \ + 2> /var/log/docker-bootstrap.log \ + 1> /dev/null & + sleep 5 } @@ -89,79 +128,107 @@ bootstrap_daemon() { DOCKER_CONF="" start_k8s(){ - # Start etcd - docker -H unix:///var/run/docker-bootstrap.sock run --restart=always --net=host -d gcr.io/google_containers/etcd:2.0.12 /usr/local/bin/etcd --addr=127.0.0.1:4001 --bind-addr=0.0.0.0:4001 --data-dir=/var/etcd/data + # Start etcd + docker -H unix:///var/run/docker-bootstrap.sock run \ + --restart=on-failure \ + --net=host \ + -d \ + gcr.io/google_containers/etcd-${ARCH}:${ETCD_VERSION} \ + /usr/local/bin/etcd \ + --listen-client-urls=http://127.0.0.1:4001,http://${MASTER_IP}:4001 \ + --advertise-client-urls=http://${MASTER_IP}:4001 \ + --data-dir=/var/etcd/data sleep 5 # Set flannel net config - docker -H unix:///var/run/docker-bootstrap.sock run --net=host gcr.io/google_containers/etcd:2.0.12 etcdctl set /coreos.com/network/config '{ "Network": "10.1.0.0/16", "Backend": {"Type": "vxlan"}}' + docker -H unix:///var/run/docker-bootstrap.sock run \ + --net=host gcr.io/google_containers/etcd:${ETCD_VERSION} \ + etcdctl \ + set /coreos.com/network/config \ + '{ "Network": "10.1.0.0/16", "Backend": {"Type": "vxlan"}}' # iface may change to a private network interface, eth0 is for default - flannelCID=$(docker -H unix:///var/run/docker-bootstrap.sock run --restart=always -d --net=host --privileged -v /dev/net:/dev/net quay.io/coreos/flannel:0.5.0 /opt/bin/flanneld -iface="eth0") + flannelCID=$(docker -H unix:///var/run/docker-bootstrap.sock run \ + --restart=on-failure \ + -d \ + --net=host \ + --privileged \ + -v /dev/net:/dev/net \ + quay.io/coreos/flannel:${FLANNEL_VERSION} \ + /opt/bin/flanneld \ + --ip-masq="${FLANNEL_IPMASQ}" \ + --iface="${FLANNEL_IFACE}") sleep 8 # Copy flannel env out and source it on the host - docker -H unix:///var/run/docker-bootstrap.sock cp ${flannelCID}:/run/flannel/subnet.env . + docker -H unix:///var/run/docker-bootstrap.sock \ + cp ${flannelCID}:/run/flannel/subnet.env . source subnet.env # Configure docker net settings, then restart it - case "$lsb_dist" in - fedora|centos|amzn) + case "${lsb_dist}" in + amzn) DOCKER_CONF="/etc/sysconfig/docker" - ;; - ubuntu|debian|linuxmint) + echo "OPTIONS=\"\$OPTIONS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" | tee -a ${DOCKER_CONF} + ifconfig docker0 down + yum -y -q install bridge-utils && brctl delbr docker0 && service docker restart + ;; + centos) + DOCKER_CONF="/etc/sysconfig/docker" + echo "OPTIONS=\"\$OPTIONS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" | tee -a ${DOCKER_CONF} + if ! command_exists ifconfig; then + yum -y -q install net-tools + fi + ifconfig docker0 down + yum -y -q install bridge-utils && brctl delbr docker0 && systemctl restart docker + ;; + ubuntu|debian) DOCKER_CONF="/etc/default/docker" - ;; - esac - - # Append the docker opts - echo "DOCKER_OPTS=\"\$DOCKER_OPTS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" | sudo tee -a ${DOCKER_CONF} - - - # sleep a little bit - ifconfig docker0 down - - case "$lsb_dist" in - fedora|centos|amzn) - yum install bridge-utils && brctl delbr docker0 && systemctl restart docker - ;; - ubuntu|debian|linuxmint) - apt-get install bridge-utils && brctl delbr docker0 && service docker restart - ;; + echo "DOCKER_OPTS=\"\$DOCKER_OPTS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" | tee -a ${DOCKER_CONF} + ifconfig docker0 down + apt-get install bridge-utils + brctl delbr docker0 + service docker stop + while [ `ps aux | grep /usr/bin/docker | grep -v grep | wc -l` -gt 0 ]; do + echo "Waiting for docker to terminate" + sleep 1 + done + service docker start + ;; + *) + echo "Unsupported operations system ${lsb_dist}" + exit 1 + ;; esac # sleep a little bit sleep 5 - # Start kubelet & proxy, then start master components as pods + # Start kubelet and then start master components as pods docker run \ --net=host \ --pid=host \ --privileged \ - --restart=always \ + --restart=on-failure \ -d \ -v /sys:/sys:ro \ -v /var/run:/var/run:rw \ -v /:/rootfs:ro \ - -v /dev:/dev \ - -v /var/lib/docker/:/var/lib/docker:ro \ + -v /var/lib/docker/:/var/lib/docker:rw \ -v /var/lib/kubelet/:/var/lib/kubelet:rw \ - gcr.io/google_containers/hyperkube:v${K8S_VERSION} \ + gcr.io/google_containers/hyperkube-${ARCH}:v${K8S_VERSION} \ /hyperkube kubelet \ - --api-servers=http://localhost:8080 \ - --v=2 --address=0.0.0.0 --enable-server \ - --hostname-override=127.0.0.1 \ - --config=/etc/kubernetes/manifests-multi \ - --cluster-dns=10.0.0.10 \ - --cluster-domain=cluster.local - - docker run \ - -d \ - --net=host \ - --privileged \ - gcr.io/google_containers/hyperkube:v${K8S_VERSION} \ - /hyperkube proxy --master=http://127.0.0.1:8080 --v=2 + --address=0.0.0.0 \ + --allow-privileged=true \ + --enable-server \ + --api-servers=http://localhost:8080 \ + --config=/etc/kubernetes/manifests-multi \ + --cluster-dns=10.0.0.10 \ + --cluster-domain=cluster.local \ + --containerized \ + --v=2 + } echo "Detecting your OS distro ..." diff --git a/docs/getting-started-guides/docker-multinode/skydns-svc.yaml.in b/docs/getting-started-guides/docker-multinode/skydns-svc.yaml.in deleted file mode 100644 index 242c8871ee..0000000000 --- a/docs/getting-started-guides/docker-multinode/skydns-svc.yaml.in +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kube-dns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "KubeDNS" -spec: - selector: - k8s-app: kube-dns - clusterIP: {{ pillar['dns_server'] }} - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP diff --git a/docs/getting-started-guides/docker-multinode/skydns-rc.yaml.in b/docs/getting-started-guides/docker-multinode/skydns.yaml.in similarity index 57% rename from docs/getting-started-guides/docker-multinode/skydns-rc.yaml.in rename to docs/getting-started-guides/docker-multinode/skydns.yaml.in index 845af9bf94..098b85ddf3 100644 --- a/docs/getting-started-guides/docker-multinode/skydns-rc.yaml.in +++ b/docs/getting-started-guides/docker-multinode/skydns.yaml.in @@ -1,31 +1,35 @@ apiVersion: v1 kind: ReplicationController metadata: - name: kube-dns-v8 + name: kube-dns-v10 namespace: kube-system labels: k8s-app: kube-dns - version: v8 + version: v10 kubernetes.io/cluster-service: "true" spec: replicas: {{ pillar['dns_replicas'] }} selector: k8s-app: kube-dns - version: v8 + version: v10 template: metadata: labels: k8s-app: kube-dns - version: v8 + version: v10 kubernetes.io/cluster-service: "true" spec: containers: - name: etcd - image: gcr.io/google_containers/etcd:2.0.9 + image: gcr.io/google_containers/etcd-amd64:2.2.1 resources: + # keep request = limit to keep this container in guaranteed class limits: cpu: 100m memory: 50Mi + requests: + cpu: 100m + memory: 50Mi command: - /usr/local/bin/etcd - -data-dir @@ -40,25 +44,33 @@ spec: - name: etcd-storage mountPath: /var/etcd/data - name: kube2sky - image: gcr.io/google_containers/kube2sky:1.11 + image: gcr.io/google_containers/kube2sky:1.12 resources: + # keep request = limit to keep this container in guaranteed class limits: cpu: 100m memory: 50Mi + requests: + cpu: 100m + memory: 50Mi args: # command = "/kube2sky" - - -domain={{ pillar['dns_domain'] }} - - -kube_master_url=http://{kube_server_url}:8080 + - --domain={{ pillar['dns_domain'] }} - name: skydns - image: gcr.io/google_containers/skydns:2015-03-11-001 + image: gcr.io/google_containers/skydns:2015-10-13-8c72f8c resources: + # keep request = limit to keep this container in guaranteed class limits: cpu: 100m memory: 50Mi + requests: + cpu: 100m + memory: 50Mi args: # command = "/skydns" - - -machines=http://localhost:4001 + - -machines=http://127.0.0.1:4001 - -addr=0.0.0.0:53 + - -ns-rotate=false - -domain={{ pillar['dns_domain'] }}. ports: - containerPort: 53 @@ -74,14 +86,25 @@ spec: scheme: HTTP initialDelaySeconds: 30 timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 1 + timeoutSeconds: 5 - name: healthz image: gcr.io/google_containers/exechealthz:1.0 resources: + # keep request = limit to keep this container in guaranteed class limits: cpu: 10m memory: 20Mi + requests: + cpu: 10m + memory: 20Mi args: - - -cmd=nslookup kubernetes.default.svc.{{ pillar['dns_domain'] }} localhost >/dev/null + - -cmd=nslookup kubernetes.default.svc.{{ pillar['dns_domain'] }} 127.0.0.1 >/dev/null - -port=8080 ports: - containerPort: 8080 @@ -90,3 +113,24 @@ spec: - name: etcd-storage emptyDir: {} dnsPolicy: Default # Don't use cluster DNS. +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: {{ pillar['dns_server'] }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/docs/getting-started-guides/docker-multinode/testing.md b/docs/getting-started-guides/docker-multinode/testing.md index 708ecea7a5..196d15f19e 100644 --- a/docs/getting-started-guides/docker-multinode/testing.md +++ b/docs/getting-started-guides/docker-multinode/testing.md @@ -31,7 +31,7 @@ now run `docker ps` you should see nginx running. You may need to wait a few mi kubectl expose rc nginx --port=80 ``` -Run the following command to obtain the IP of this service we just created. There are two IPs, the first one is internal (`CLUSTER_IP`), and the second one is the external load-balanced IP. +Run the following command to obtain the IP of this service we just created. There are two IPs, the first one is internal (CLUSTER_IP), and the second one is the external load-balanced IP. ```shell kubectl get svc nginx diff --git a/docs/getting-started-guides/docker-multinode/worker.md b/docs/getting-started-guides/docker-multinode/worker.md index 4a0acd58f6..c2c3923f43 100644 --- a/docs/getting-started-guides/docker-multinode/worker.md +++ b/docs/getting-started-guides/docker-multinode/worker.md @@ -3,21 +3,31 @@ These instructions are very similar to the master set-up above, but they are duplicated for clarity. You need to repeat these instructions for each node you want to join the cluster. -We will assume that the IP address of this node is `${NODE_IP}` and you have the IP address of the master in `${MASTER_IP}` that you created in the [master instructions](/docs/getting-started-guides/docker-multinode/master). +We will assume that you have the IP address of the master in `${MASTER_IP}` that you created in the [master instructions](/docs/getting-started-guides/docker-multinode/master/). We'll need to run several versioned Kubernetes components, so we'll assume that the version we want +to run is `${K8S_VERSION}`, which should hold a released version of Kubernetes >= "1.2.0-alpha.6" + +Enviroinment variables used: + +```sh +export MASTER_IP= +export K8S_VERSION= +export FLANNEL_VERSION= +export FLANNEL_IFACE= +export FLANNEL_IPMASQ= +``` For each worker node, there are three steps: -* [Set up `flanneld` on the worker node](#set-up-flanneld-on-the-worker-node) -* [Start Kubernetes on the worker node](#start-kubernetes-on-the-worker-node) -* [Add the worker to the cluster](#add-the-node-to-the-cluster) + * [Set up `flanneld` on the worker node](#set-up-flanneld-on-the-worker-node) + * [Start Kubernetes on the worker node](#start-kubernetes-on-the-worker-node) + * [Add the worker to the cluster](#add-the-node-to-the-cluster) ### Set up Flanneld on the worker node As before, the Flannel daemon is going to provide network connectivity. _Note_: -There is a [bug](https://github.com/docker/docker/issues/14106) in Docker 1.7.0 that prevents this from working correctly. -Please install Docker 1.6.2 or wait for Docker 1.7.1. +This guide expects **Docker 1.7.1 or higher**. #### Set up a bootstrap docker @@ -30,6 +40,12 @@ Run: sudo sh -c 'docker -d -H unix:///var/run/docker-bootstrap.sock -p /var/run/docker-bootstrap.pid --iptables=false --ip-masq=false --bridge=none --graph=/var/lib/docker-bootstrap 2> /var/log/docker-bootstrap.log 1> /dev/null &' ``` +_If you have Docker 1.8.0 or higher run this instead_ + +```sh +sudo sh -c 'docker daemon -H unix:///var/run/docker-bootstrap.sock -p /var/run/docker-bootstrap.pid --iptables=false --ip-masq=false --bridge=none --graph=/var/lib/docker-bootstrap 2> /var/log/docker-bootstrap.log 1> /dev/null &' +``` + _Important Note_: If you are running this on a long running system, rather than experimenting, you should run the bootstrap Docker instance under something like SysV init, upstart or systemd so that it is restarted across reboots and failures. @@ -57,10 +73,18 @@ or it may be something else. Now run flanneld itself, this call is slightly different from the above, since we point it at the etcd instance on the master. ```shell -sudo docker -H unix:///var/run/docker-bootstrap.sock run -d --net=host --privileged -v /dev/net:/dev/net quay.io/coreos/flannel:0.5.0 /opt/bin/flanneld --etcd-endpoints=http://${MASTER_IP}:4001 +sudo docker -H unix:///var/run/docker-bootstrap.sock run -d \ + --net=host \ + --privileged \ + -v /dev/net:/dev/net \ + quay.io/coreos/flannel:${FLANNEL_VERSION} \ + /opt/bin/flanneld \ + --ip-masq=${FLANNEL_IPMASQ} \ + --etcd-endpoints=http://${MASTER_IP}:4001 \ + --iface=${FLANNEL_IFACE} ``` -The previous command should have printed a really long hash, copy this hash. +The previous command should have printed a really long hash, the container id, copy this hash. Now get the subnet settings from flannel: @@ -68,6 +92,7 @@ Now get the subnet settings from flannel: sudo docker -H unix:///var/run/docker-bootstrap.sock exec cat /run/flannel/subnet.env ``` + #### Edit the docker configuration You now need to edit the docker configuration to activate new flags. Again, this is system specific. @@ -99,7 +124,7 @@ Again this is system dependent, it may be: sudo /etc/init.d/docker start ``` -it may be: +or it may be: ```shell systemctl start docker @@ -121,9 +146,18 @@ sudo docker run \ --volume=/var/run:/var/run:rw \ --net=host \ --privileged=true \ - --pid=host \ + --pid=host \ -d \ - gcr.io/google_containers/hyperkube:v1.0.1 /hyperkube kubelet --api-servers=http://${MASTER_IP}:8080 --v=2 --address=0.0.0.0 --enable-server --hostname-override=$(hostname -i) --cluster-dns=10.0.0.10 --cluster-domain=cluster.local + gcr.io/google_containers/hyperkube-amd64:v${K8S_VERSION} \ + /hyperkube kubelet \ + --allow-privileged=true \ + --api-servers=http://${MASTER_IP}:8080 \ + --v=2 \ + --address=0.0.0.0 \ + --enable-server \ + --containerized \ + --cluster-dns=10.0.0.10 \ + --cluster-domain=cluster.local ``` #### Run the service proxy @@ -131,9 +165,15 @@ sudo docker run \ The service proxy provides load-balancing between groups of containers defined by Kubernetes `Services` ```shell -sudo docker run -d --net=host --privileged gcr.io/google_containers/hyperkube:v1.0.1 /hyperkube proxy --master=http://${MASTER_IP}:8080 --v=2 +sudo docker run -d \ + --net=host \ + --privileged \ + gcr.io/google_containers/hyperkube-amd64:v${K8S_VERSION} \ + /hyperkube proxy \ + --master=http://${MASTER_IP}:8080 \ + --v=2 ``` ### Next steps -Move on to [testing your cluster](/docs/getting-started-guides/docker-multinode/testing) or add another node](#). \ No newline at end of file +Move on to [testing your cluster](/docs/getting-started-guides/docker-multinode/testing/) or [add another node](#adding-a-kubernetes-worker-node-via-docker) diff --git a/docs/getting-started-guides/docker-multinode/worker.sh b/docs/getting-started-guides/docker-multinode/worker.sh index dda80b553c..6404884486 100755 --- a/docs/getting-started-guides/docker-multinode/worker.sh +++ b/docs/getting-started-guides/docker-multinode/worker.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# A scripts to install k8s worker node. -# Author @wizard_cxy @reouser +# A script to the k8s worker in docker containers. +# Authors @wizard_cxy @resouer set -e @@ -26,14 +26,11 @@ if ( ! ps -ef | grep "/usr/bin/docker" | grep -v 'grep' &> /dev/null ); then fi # Make sure k8s version env is properly set -if [ -z ${K8S_VERSION} ]; then - K8S_VERSION="1.0.3" - echo "K8S_VERSION is not set, using default: ${K8S_VERSION}" -else - echo "k8s version is set to: ${K8S_VERSION}" -fi - - +K8S_VERSION=${K8S_VERSION:-"1.2.0-alpha.7"} +FLANNEL_VERSION=${FLANNEL_VERSION:-"0.5.5"} +FLANNEL_IFACE=${FLANNEL_IFACE:-"eth0"} +FLANNEL_IPMASQ=${FLANNEL_IPMASQ:-"true"} +ARCH=${ARCH:-"amd64"} # Run as root if [ "$(id -u)" != "0" ]; then @@ -45,10 +42,15 @@ fi if [ -z ${MASTER_IP} ]; then echo "Please export MASTER_IP in your env" exit 1 -else - echo "k8s master is set to: ${MASTER_IP}" fi +echo "K8S_VERSION is set to: ${K8S_VERSION}" +echo "FLANNEL_VERSION is set to: ${FLANNEL_VERSION}" +echo "FLANNEL_IFACE is set to: ${FLANNEL_IFACE}" +echo "FLANNEL_IPMASQ is set to: ${FLANNEL_IPMASQ}" +echo "MASTER_IP is set to: ${MASTER_IP}" +echo "ARCH is set to: ${ARCH}" + # Check if a command is valid command_exists() { command -v "$@" > /dev/null 2>&1 @@ -59,12 +61,12 @@ lsb_dist="" # Detect the OS distro, we support ubuntu, debian, mint, centos, fedora dist detect_lsb() { case "$(uname -m)" in - *64) - ;; - *) - echo "Error: We currently only support 64-bit platforms." - exit 1 - ;; + *64) + ;; + *) + echo "Error: We currently only support 64-bit platforms." + exit 1 + ;; esac if command_exists lsb_release; then @@ -84,12 +86,37 @@ detect_lsb() { fi lsb_dist="$(echo ${lsb_dist} | tr '[:upper:]' '[:lower:]')" + + case "${lsb_dist}" in + amzn|centos|debian|ubuntu) + ;; + *) + echo "Error: We currently only support ubuntu|debian|amzn|centos." + exit 1 + ;; + esac } # Start the bootstrap daemon bootstrap_daemon() { - sudo -b docker -d -H unix:///var/run/docker-bootstrap.sock -p /var/run/docker-bootstrap.pid --iptables=false --ip-masq=false --bridge=none --graph=/var/lib/docker-bootstrap 2> /var/log/docker-bootstrap.log 1> /dev/null + # Detecting docker version so we could run proper docker_daemon command + [[ $(eval "docker --version") =~ ([0-9][.][0-9][.][0-9]*) ]] && version="${BASH_REMATCH[1]}" + local got=$(echo -e "${version}\n1.8.0" | sed '/^$/d' | sort -nr | head -1) + if [[ "${got}" = "${version}" ]]; then + docker_daemon="docker -d" + else + docker_daemon="docker daemon" + fi + ${docker_daemon} \ + -H unix:///var/run/docker-bootstrap.sock \ + -p /var/run/docker-bootstrap.pid \ + --iptables=false \ + --ip-masq=false \ + --bridge=none \ + --graph=/var/lib/docker-bootstrap \ + 2> /var/log/docker-bootstrap.log \ + 1> /dev/null & sleep 5 } @@ -99,67 +126,97 @@ DOCKER_CONF="" # Start k8s components in containers start_k8s() { # Start flannel - flannelCID=$(sudo docker -H unix:///var/run/docker-bootstrap.sock run -d --restart=always --net=host --privileged -v /dev/net:/dev/net quay.io/coreos/flannel:0.5.0 /opt/bin/flanneld --etcd-endpoints=http://${MASTER_IP}:4001 -iface="eth0") + flannelCID=$(docker -H unix:///var/run/docker-bootstrap.sock run \ + -d \ + --restart=on-failure \ + --net=host \ + --privileged \ + -v /dev/net:/dev/net \ + quay.io/coreos/flannel:${FLANNEL_VERSION} \ + /opt/bin/flanneld \ + --ip-masq="${FLANNEL_IPMASQ}" \ + --etcd-endpoints=http://${MASTER_IP}:4001 \ + --iface="${FLANNEL_IFACE}") - sleep 8 + sleep 10 # Copy flannel env out and source it on the host - sudo docker -H unix:///var/run/docker-bootstrap.sock cp ${flannelCID}:/run/flannel/subnet.env . + docker -H unix:///var/run/docker-bootstrap.sock \ + cp ${flannelCID}:/run/flannel/subnet.env . source subnet.env # Configure docker net settings, then restart it - case "$lsb_dist" in - fedora|centos|amzn) + case "${lsb_dist}" in + centos) DOCKER_CONF="/etc/sysconfig/docker" - ;; - ubuntu|debian|linuxmint) + echo "OPTIONS=\"\$OPTIONS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" | tee -a ${DOCKER_CONF} + if ! command_exists ifconfig; then + yum -y -q install net-tools + fi + ifconfig docker0 down + yum -y -q install bridge-utils && brctl delbr docker0 && systemctl restart docker + ;; + amzn) + DOCKER_CONF="/etc/sysconfig/docker" + echo "OPTIONS=\"\$OPTIONS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" | tee -a ${DOCKER_CONF} + ifconfig docker0 down + yum -y -q install bridge-utils && brctl delbr docker0 && service docker restart + ;; + ubuntu|debian) # TODO: today ubuntu uses systemd. Handle that too DOCKER_CONF="/etc/default/docker" - ;; - esac - - echo "DOCKER_OPTS=\"\$DOCKER_OPTS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" | sudo tee -a ${DOCKER_CONF} - - ifconfig docker0 down - - case "$lsb_dist" in - fedora|centos) - yum install bridge-utils && brctl delbr docker0 && systemctl restart docker - ;; - ubuntu|debian|linuxmint) - apt-get install bridge-utils && brctl delbr docker0 && service docker restart - ;; + echo "DOCKER_OPTS=\"\$DOCKER_OPTS --mtu=${FLANNEL_MTU} --bip=${FLANNEL_SUBNET}\"" | tee -a ${DOCKER_CONF} + ifconfig docker0 down + apt-get install bridge-utils + brctl delbr docker0 + service docker stop + while [ `ps aux | grep /usr/bin/docker | grep -v grep | wc -l` -gt 0 ]; do + echo "Waiting for docker to terminate" + sleep 1 + done + service docker start + ;; + *) + echo "Unsupported operations system ${lsb_dist}" + exit 1 + ;; esac # sleep a little bit sleep 5 - + # Start kubelet & proxy in container + # TODO: Use secure port for communication docker run \ --net=host \ --pid=host \ --privileged \ - --restart=always \ + --restart=on-failure \ -d \ -v /sys:/sys:ro \ -v /var/run:/var/run:rw \ - -v /dev:/dev \ - -v /var/lib/docker/:/var/lib/docker:ro \ + -v /:/rootfs:ro \ + -v /var/lib/docker/:/var/lib/docker:rw \ -v /var/lib/kubelet/:/var/lib/kubelet:rw \ - gcr.io/google_containers/hyperkube:v${K8S_VERSION} \ - /hyperkube kubelet --api-servers=http://${MASTER_IP}:8080 \ - --v=2 --address=0.0.0.0 --enable-server \ - --hostname-override=$(hostname -i) \ - --cluster-dns=10.0.0.10 \ - --cluster-domain=cluster.local - + gcr.io/google_containers/hyperkube-${ARCH}:v${K8S_VERSION} \ + /hyperkube kubelet \ + --allow-privileged=true \ + --api-servers=http://${MASTER_IP}:8080 \ + --address=0.0.0.0 \ + --enable-server \ + --cluster-dns=10.0.0.10 \ + --cluster-domain=cluster.local \ + --containerized \ + --v=2 + docker run \ -d \ --net=host \ --privileged \ - --restart=always \ - gcr.io/google_containers/hyperkube:v${K8S_VERSION} \ - /hyperkube proxy --master=http://${MASTER_IP}:8080 \ - --v=2 + --restart=on-failure \ + gcr.io/google_containers/hyperkube-${ARCH}:v${K8S_VERSION} \ + /hyperkube proxy \ + --master=http://${MASTER_IP}:8080 \ + --v=2 } echo "Detecting your OS distro ..." diff --git a/docs/getting-started-guides/docker.md b/docs/getting-started-guides/docker.md index 42468a3dfc..e5cb648a34 100644 --- a/docs/getting-started-guides/docker.md +++ b/docs/getting-started-guides/docker.md @@ -13,80 +13,82 @@ Here's a diagram of what the final result will look like: ## Prerequisites 1. You need to have docker installed on one machine. -2. Your kernel should support memory and swap accounting. Ensure that the -following configs are turned on in your linux kernel: +2. Decide what Kubernetes version to use. Set the `${K8S_VERSION}` variable to + a released version of Kubernetes >= "1.2.0-alpha.7" -```shell -CONFIG_RESOURCE_COUNTERS=y -CONFIG_MEMCG=y -CONFIG_MEMCG_SWAP=y -CONFIG_MEMCG_SWAP_ENABLED=y -CONFIG_MEMCG_KMEM=y -``` - -3. Enable the memory and swap accounting in the kernel, at boot, as command line -parameters as follows: - -```shell -GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" -``` - -NOTE: The above is specifically for GRUB2. - You can check the command line parameters passed to your kernel by looking at the - output of /proc/cmdline: - -```shell -$cat /proc/cmdline - BOOT_IMAGE=/boot/vmlinuz-3.18.4-aufs root=/dev/sda5 ro cgroup_enable=memory - swapaccount=1 -``` - -### Step One: Run etcd - -```shell -docker run --net=host -d gcr.io/google_containers/etcd:2.0.12 /usr/local/bin/etcd --addr=127.0.0.1:4001 --bind-addr=0.0.0.0:4001 --data-dir=/var/etcd/data -``` - -### Step Two: Run the master +### Run it ```shell docker run \ --volume=/:/rootfs:ro \ --volume=/sys:/sys:ro \ - --volume=/dev:/dev \ - --volume=/var/lib/docker/:/var/lib/docker:ro \ + --volume=/var/lib/docker/:/var/lib/docker:rw \ --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \ --volume=/var/run:/var/run:rw \ --net=host \ --pid=host \ --privileged=true \ -d \ - gcr.io/google_containers/hyperkube:v1.0.1 \ - /hyperkube kubelet --containerized --hostname-override="127.0.0.1" --address="0.0.0.0" --api-servers=http://localhost:8080 --config=/etc/kubernetes/manifests + gcr.io/google_containers/hyperkube-amd64:v${K8S_VERSION} \ + /hyperkube kubelet \ + --containerized \ + --hostname-override="127.0.0.1" \ + --address="0.0.0.0" \ + --api-servers=http://localhost:8080 \ + --config=/etc/kubernetes/manifests \ + --cluster-dns=10.0.0.10 \ + --cluster-domain=cluster.local \ + --allow-privileged=true --v=2 ``` -This actually runs the kubelet, which in turn runs a [pod](/docs/user-guide/pods) that contains the other master components. +> Note that `--cluster-dns` and `--cluster-domain` is used to deploy dns, feel free to discard them if dns is not needed. + +> If you would like to mount an external device as a volume, add `--volume=/dev:/dev` to the command above. It may however, cause some problems described in [#18230](https://github.com/kubernetes/kubernetes/issues/18230) + +This actually runs the kubelet, which in turn runs a [pod](/docs/user-guide/pods/) that contains the other master components. -### Step Three: Run the service proxy +### Download `kubectl` + +At this point you should have a running Kubernetes cluster. You can test this +by downloading the kubectl binary for `${K8S_VERSION}` (look at the URL in the +following links) and make it available by editing your PATH environment +variable. +([OS X/amd64](http://storage.googleapis.com/kubernetes-release/release/{{page.version}}.0-alpha.7/bin/darwin/amd64/kubectl)) +([OS X/386](http://storage.googleapis.com/kubernetes-release/release/{{page.version}}.0-alpha.7/bin/darwin/386/kubectl)) +([linux/amd64](http://storage.googleapis.com/kubernetes-release/release/{{page.version}}.0-alpha.7/bin/linux/amd64/kubectl)) +([linux/386](http://storage.googleapis.com/kubernetes-release/release/{{page.version}}.0-alpha.7/bin/linux/386/kubectl)) +([linux/arm](http://storage.googleapis.com/kubernetes-release/release/{{page.version}}.0-alpha.7/bin/linux/arm/kubectl)) + +For example, OS X: + +```shell +$ wget http://storage.googleapis.com/kubernetes-release/release/v${K8S_VERSION}/bin/darwin/amd64/kubectl +$ chmod 755 kubectl +$ PATH=$PATH:`pwd` +``` + +Linux: ```shell -docker run -d --net=host --privileged gcr.io/google_containers/hyperkube:v1.0.1 /hyperkube proxy --master=http://127.0.0.1:8080 --v=2 +$ wget http://storage.googleapis.com/kubernetes-release/release/v${K8S_VERSION}/bin/linux/amd64/kubectl +$ chmod 755 kubectl +$ PATH=$PATH:`pwd` ``` -### Test it out - -At this point you should have a running Kubernetes cluster. You can test this by downloading the kubectl -binary -([OS X](https://storage.googleapis.com/kubernetes-release/release/v1.0.1/bin/darwin/amd64/kubectl)) -([linux](https://storage.googleapis.com/kubernetes-release/release/v1.0.1/bin/linux/amd64/kubectl)) - -*Note:* -On OS/X you will need to set up port forwarding via ssh: +Create configuration: ```shell -boot2docker ssh -L8080:localhost:8080 +$ kubectl config set-cluster test-doc --server=http://localhost:8080 +$ kubectl config set-context test-doc --cluster=test-doc +$ kubectl config use-context test-doc ``` +For Max OS X users instead of `localhost` you will have to use IP address of your docker machine, +which you can find by running `docker-machine env ` (see [documentation](https://docs.docker.com/machine/reference/env/) +for details). + +### Test it out + List the nodes in your cluster by running: ```shell @@ -96,16 +98,14 @@ kubectl get nodes This should print: ```shell -NAME LABELS STATUS -127.0.0.1 Ready +NAME LABELS STATUS +127.0.0.1 kubernetes.io/hostname=127.0.0.1 Ready ``` -If you are running different Kubernetes clusters, you may need to specify `-s http://localhost:8080` to select the local cluster. - ### Run an application ```shell -kubectl -s http://localhost:8080 run nginx --image=nginx --port=80 +kubectl run nginx --image=nginx --port=80 ``` Now run `docker ps` you should see nginx running. You may need to wait a few minutes for the image to get pulled. @@ -116,7 +116,7 @@ Now run `docker ps` you should see nginx running. You may need to wait a few mi kubectl expose rc nginx --port=80 ``` -Run the following command to obtain the IP of this service we just created. There are two IPs, the first one is internal (CLUSTER_IP), and the second one is the external load-balanced IP. +Run the following command to obtain the IP of this service we just created. There are two IPs, the first one is internal (CLUSTER_IP), and the second one is the external load-balanced IP (if a LoadBalancer is configured) ```shell kubectl get svc nginx @@ -136,9 +136,46 @@ curl Note that you will need run this curl command on your boot2docker VM if you are running on OS X. +## Deploy a DNS + +See [here](/docs/getting-started-guides/docker-multinode/deployDNS/) for instructions. + ### A note on turning down your cluster Many of these containers run under the management of the `kubelet` binary, which attempts to keep containers running, even if they fail. So, in order to turn down the cluster, you need to first kill the kubelet container, and then any other containers. You may use `docker kill $(docker ps -aq)`, note this removes _all_ containers running under Docker, so use with caution. + +### Troubleshooting + +#### Node is in `NotReady` state + +If you see your node as `NotReady` it's possible that your OS does not have memcg and swap enabled. + +1. Your kernel should support memory and swap accounting. Ensure that the +following configs are turned on in your linux kernel: + + ```shell + CONFIG_RESOURCE_COUNTERS=y + CONFIG_MEMCG=y + CONFIG_MEMCG_SWAP=y + CONFIG_MEMCG_SWAP_ENABLED=y + CONFIG_MEMCG_KMEM=y + ``` + +2. Enable the memory and swap accounting in the kernel, at boot, as command line +parameters as follows: + + ```shell + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + ``` + + NOTE: The above is specifically for GRUB2. + You can check the command line parameters passed to your kernel by looking at the + output of /proc/cmdline: + + ```shell + $ cat /proc/cmdline + BOOT_IMAGE=/boot/vmlinuz-3.18.4-aufs root=/dev/sda5 ro cgroup_enable=memory swapaccount=1 + ``` diff --git a/docs/getting-started-guides/fedora/fedora_ansible_config.md b/docs/getting-started-guides/fedora/fedora_ansible_config.md index 4956f48ec8..b82242b92c 100644 --- a/docs/getting-started-guides/fedora/fedora_ansible_config.md +++ b/docs/getting-started-guides/fedora/fedora_ansible_config.md @@ -61,7 +61,7 @@ kube-node-02.example.com ## Setting up ansible access to your nodes -If you already are running on a machine which has passwordless ssh access to the kube-master and kube-node-{01,02} nodes, and 'sudo' privileges, simply set the value of `ansible_ssh_user` in `~/contrib/ansible/group_vars/all.yaml` to the username which you use to ssh to the nodes (i.e. `fedora`), and proceed to the next step... +If you already are running on a machine which has passwordless ssh access to the kube-master and kube-node-{01,02} nodes, and 'sudo' privileges, simply set the value of `ansible_ssh_user` in `~/contrib/ansible/group_vars/all.yml` to the username which you use to ssh to the nodes (i.e. `fedora`), and proceed to the next step... *Otherwise* setup ssh on the machines like so (you will need to know the root password to all machines in the cluster). @@ -171,6 +171,7 @@ systemctl | grep -i kube ```shell iptables -nvL ``` + **Create /tmp/apache.json on the master with the following contents and deploy pod** ```json diff --git a/docs/getting-started-guides/fedora/fedora_manual_config.md b/docs/getting-started-guides/fedora/fedora_manual_config.md index 9b638163a6..224e10fce9 100644 --- a/docs/getting-started-guides/fedora/fedora_manual_config.md +++ b/docs/getting-started-guides/fedora/fedora_manual_config.md @@ -10,16 +10,11 @@ ## Instructions -This is a getting started guide for [Fedora](http://fedoraproject.org). It is a manual configuration so you understand all the underlying packages / services / ports, etc... +This is a getting started guide for Fedora. It is a manual configuration so you understand all the underlying packages / services / ports, etc... -This guide will only get ONE node (previously minion) working. Multiple nodes require a functional [networking configuration](/docs/admin/networking) -done outside of Kubernetes. Although the additional Kubernetes configuration requirements should be obvious. +This guide will only get ONE node (previously minion) working. Multiple nodes require a functional [networking configuration](/docs/admin/networking/) done outside of Kubernetes. Although the additional Kubernetes configuration requirements should be obvious. -The Kubernetes package provides a few services: kube-apiserver, kube-scheduler, kube-controller-manager, kubelet, kube-proxy. These -services are managed by systemd and the configuration resides in a central location: /etc/kubernetes. We will break the services up -between the hosts. The first host, fed-master, will be the Kubernetes master. This host will run the kube-apiserver, kube-controller-manager, -and kube-scheduler. In addition, the master will also run _etcd_ (not needed if _etcd_ runs on a different host but this guide assumes -that _etcd_ and Kubernetes master run on the same host). The remaining host, fed-node will be the node and run kubelet, proxy and docker. +The Kubernetes package provides a few services: kube-apiserver, kube-scheduler, kube-controller-manager, kubelet, kube-proxy. These services are managed by systemd and the configuration resides in a central location: /etc/kubernetes. We will break the services up between the hosts. The first host, fed-master, will be the Kubernetes master. This host will run the kube-apiserver, kube-controller-manager, and kube-scheduler. In addition, the master will also run _etcd_ (not needed if _etcd_ runs on a different host but this guide assumes that _etcd_ and Kubernetes master run on the same host). The remaining host, fed-node will be the node and run kubelet, proxy and docker. **System Information:** @@ -32,15 +27,9 @@ fed-node = 192.168.121.65 **Prepare the hosts:** -* Install Kubernetes on all hosts - fed-{master,node}. This will also pull in docker. Also install etcd on fed-master. - This guide has been tested with kubernetes-0.18 and beyond. -* The [--enablerepo=updates-testing](https://fedoraproject.org/wiki/QA:Updates_Testing) directive in the yum - command below will ensure that the most recent Kubernetes version that is scheduled for pre-release will - be installed. This should be a more recent version than the Fedora "stable" release for Kubernetes that you - would get without adding the directive. -* If you want the very latest Kubernetes release [you can download and yum install the RPM directly from - Fedora Koji](http://koji.fedoraproject.org/koji/packageinfo?packageID=19202) instead of using the yum - install command below. +* Install Kubernetes on all hosts - fed-{master,node}. This will also pull in docker. Also install etcd on fed-master. This guide has been tested with kubernetes-0.18 and beyond. +* The [--enablerepo=updates-testing](https://fedoraproject.org/wiki/QA:Updates_Testing) directive in the yum command below will ensure that the most recent Kubernetes version that is scheduled for pre-release will be installed. This should be a more recent version than the Fedora "stable" release for Kubernetes that you would get without adding the directive. +* If you want the very latest Kubernetes release [you can download and yum install the RPM directly from Fedora Koji](http://koji.fedoraproject.org/koji/packageinfo?packageID=19202) instead of using the yum install command below. ```shell yum -y install --enablerepo=updates-testing kubernetes @@ -84,8 +73,7 @@ systemctl stop iptables-services firewalld **Configure the Kubernetes services on the master.** -* Edit /etc/kubernetes/apiserver to appear as such. The service-cluster-ip-range IP addresses must be an unused block of addresses, not used anywhere else. -They do not need to be routed or assigned to anything. +* Edit /etc/kubernetes/apiserver to appear as such. The service-cluster-ip-range IP addresses must be an unused block of addresses, not used anywhere else. They do not need to be routed or assigned to anything. ```shell # The address on the local server to listen to. diff --git a/docs/getting-started-guides/fedora/flannel_multi_node_cluster.md b/docs/getting-started-guides/fedora/flannel_multi_node_cluster.md index d10eb42fb6..8d06892df7 100644 --- a/docs/getting-started-guides/fedora/flannel_multi_node_cluster.md +++ b/docs/getting-started-guides/fedora/flannel_multi_node_cluster.md @@ -1,10 +1,7 @@ ---- ---- - -This document describes how to deploy Kubernetes on multiple hosts to set up a multi-node cluster and networking with flannel. Follow fedora [getting started guide](/docs/getting-started-guides/fedora/fedora_manual_config) to setup 1 master (fed-master) and 2 or more nodes. Make sure that all nodes have different names (fed-node1, fed-node2 and so on) and labels (fed-node1-label, fed-node2-label, and so on) to avoid any conflict. Also make sure that the Kubernetes master host is running etcd, kube-controller-manager, kube-scheduler, and kube-apiserver services, and the nodes are running docker, kube-proxy and kubelet services. Now install flannel on Kubernetes nodes. flannel on each node configures an overlay network that docker uses. flannel runs on each node to setup a unique class-C container network. - * TOC -{:toc} +{:toc} + +This document describes how to deploy Kubernetes on multiple hosts to set up a multi-node cluster and networking with flannel. Follow fedora [getting started guide](/docs/getting-started-guides/fedora/fedora_manual_config/) to setup 1 master (fed-master) and 2 or more nodes. Make sure that all nodes have different names (fed-node1, fed-node2 and so on) and labels (fed-node1-label, fed-node2-label, and so on) to avoid any conflict. Also make sure that the Kubernetes master host is running etcd, kube-controller-manager, kube-scheduler, and kube-apiserver services, and the nodes are running docker, kube-proxy and kubelet services. Now install flannel on Kubernetes nodes. flannel on each node configures an overlay network that docker uses. flannel runs on each node to setup a unique class-C container network. ## Prerequisites @@ -14,7 +11,7 @@ You need 2 or more machines with Fedora installed. **Perform following commands on the Kubernetes master** -Configure flannel by creating a `flannel-config.json` in your current directory on fed-master. flannel provides udp and vxlan among other overlay networking backend options. In this guide, we choose kernel based vxlan backend. The contents of the json are: +* Configure flannel by creating a `flannel-config.json` in your current directory on fed-master. flannel provides udp and vxlan among other overlay networking backend options. In this guide, we choose kernel based vxlan backend. The contents of the json are: ```json { diff --git a/docs/getting-started-guides/gce.md b/docs/getting-started-guides/gce.md index d987ee66ff..ffead1c97f 100644 --- a/docs/getting-started-guides/gce.md +++ b/docs/getting-started-guides/gce.md @@ -17,7 +17,7 @@ If you want to use custom binaries or pure open source Kubernetes, please contin 1. You need a Google Cloud Platform account with billing enabled. Visit the [Google Developers Console](http://cloud.google.com/console) for more details. 1. Install `gcloud` as necessary. `gcloud` can be installed as a part of the [Google Cloud SDK](https://cloud.google.com/sdk/). -1. Then, make sure you have the `gcloud preview` command line component installed. Run `gcloud preview` at the command line - if it asks to install any components, go ahead and install them. If it simply shows help text, you're good to go. This is required as the cluster setup script uses GCE [Instance Groups](https://cloud.google.com/compute/docs/instance-groups/), which are in the gcloud preview namespace. You will also need to **enable [`Compute Engine Instance Group Manager API`](https://developers.google.com/console/help/new/#activatingapis)** in the developers console. +1. Enable the [Compute Engine Instance Group Manager API](https://developers.google.com/console/help/new/#activatingapis) in the [Google Cloud developers console](https://console.developers.google.com). 1. Make sure that gcloud is set to use the Google Cloud Platform project you want. You can check the current project using `gcloud config list project` and change it via `gcloud config set project `. 1. Make sure you have credentials for GCloud by running ` gcloud auth login`. 1. Make sure you can start up a GCE VM from the command line. At least make sure you can do the [Create an instance](https://cloud.google.com/compute/docs/instances/#startinstancegcloud) part of the GCE Quickstart. diff --git a/docs/getting-started-guides/index.md b/docs/getting-started-guides/index.md index f3a42c476d..c733c52393 100644 --- a/docs/getting-started-guides/index.md +++ b/docs/getting-started-guides/index.md @@ -118,7 +118,6 @@ Azure | CoreOS | CoreOS | Weave | [docs](/docs/gettin Docker Single Node | custom | N/A | local | [docs](/docs/getting-started-guides/docker) | | Project ([@brendandburns](https://github.com/brendandburns)) Docker Multi Node | Flannel | N/A | local | [docs](/docs/getting-started-guides/docker-multinode) | | Project ([@brendandburns](https://github.com/brendandburns)) Bare-metal | Ansible | Fedora | flannel | [docs](/docs/getting-started-guides/fedora/fedora_ansible_config) | | Project -Digital Ocean | custom | Fedora | Calico | [docs](/docs/getting-started-guides/fedora/fedora-calico) | | Community (@djosborne) Bare-metal | custom | Fedora | _none_ | [docs](/docs/getting-started-guides/fedora/fedora_manual_config) | | Project Bare-metal | custom | Fedora | flannel | [docs](/docs/getting-started-guides/fedora/flannel_multi_node_cluster) | | Community ([@aveshagarwal](https://github.com/aveshagarwal)) libvirt | custom | Fedora | flannel | [docs](/docs/getting-started-guides/fedora/flannel_multi_node_cluster) | | Community ([@aveshagarwal](https://github.com/aveshagarwal)) @@ -140,7 +139,6 @@ Joyent | Juju | Ubuntu | flannel | [docs](/docs/gettin AWS | Saltstack | Ubuntu | OVS | [docs](/docs/getting-started-guides/aws) | | Community ([@justinsb](https://github.com/justinsb)) Bare-metal | custom | Ubuntu | Calico | [docs](/docs/getting-started-guides/ubuntu-calico) | | Community ([@djosborne](https://github.com/djosborne)) Bare-metal | custom | Ubuntu | flannel | [docs](/docs/getting-started-guides/ubuntu) | | Community ([@resouer](https://github.com/resouer), [@WIZARD-CXY](https://github.com/WIZARD-CXY)) -Local | | | _none_ | [docs](/docs/getting-started-guides/locally) | | Community ([@preillyme](https://github.com/preillyme)) libvirt/KVM | CoreOS | CoreOS | libvirt/KVM | [docs](/docs/getting-started-guides/libvirt-coreos) | | Community ([@lhuard1A](https://github.com/lhuard1A)) oVirt | | | | [docs](/docs/getting-started-guides/ovirt) | | Community ([@simon3z](https://github.com/simon3z)) Rackspace | CoreOS | CoreOS | flannel | [docs](/docs/getting-started-guides/rackspace) | | Community ([@doublerr](https://github.com/doublerr)) diff --git a/docs/getting-started-guides/juju.md b/docs/getting-started-guides/juju.md index c5de10069a..aa6a6efa32 100644 --- a/docs/getting-started-guides/juju.md +++ b/docs/getting-started-guides/juju.md @@ -230,18 +230,16 @@ github.com: ### Cloud compatibility Juju runs natively against a variety of public cloud providers. Juju currently -works with: - -- [Amazon Web Service](https://jujucharms.com/docs/stable/config-aws) -- [Windows Azure](https://jujucharms.com/docs/stable/config-azure) -- [DigitalOcean](https://jujucharms.com/docs/stable/config-digitalocean) -- [Google Compute Engine](https://jujucharms.com/docs/stable/config-gce) -- [HP Public Cloud](https://jujucharms.com/docs/stable/config-hpcloud) -- [Joyent](https://jujucharms.com/docs/stable/config-joyent) -- [LXC](https://jujucharms.com/docs/stable/config-LXC) -- Any [OpenStack](https://jujucharms.com/docs/stable/config-openstack) deployment -- [Vagrant](https://jujucharms.com/docs/stable/config-vagrant) -- [Vmware vSphere](https://jujucharms.com/docs/stable/config-vmware) +works with [Amazon Web Service](https://jujucharms.com/docs/stable/config-aws), +[Windows Azure](https://jujucharms.com/docs/stable/config-azure), +[DigitalOcean](https://jujucharms.com/docs/stable/config-digitalocean), +[Google Compute Engine](https://jujucharms.com/docs/stable/config-gce), +[HP Public Cloud](https://jujucharms.com/docs/stable/config-hpcloud), +[Joyent](https://jujucharms.com/docs/stable/config-joyent), +[LXC](https://jujucharms.com/docs/stable/config-LXC), any +[OpenStack](https://jujucharms.com/docs/stable/config-openstack) deployment, +[Vagrant](https://jujucharms.com/docs/stable/config-vagrant), and +[Vmware vSphere](https://jujucharms.com/docs/stable/config-vmware). If you do not see your favorite cloud provider listed many clouds can be configured for [manual provisioning](https://jujucharms.com/docs/stable/config-manual). diff --git a/docs/getting-started-guides/libvirt-coreos.md b/docs/getting-started-guides/libvirt-coreos.md index b52a6bc717..8d8a447209 100644 --- a/docs/getting-started-guides/libvirt-coreos.md +++ b/docs/getting-started-guides/libvirt-coreos.md @@ -14,7 +14,7 @@ The primary goal of the `libvirt-coreos` cluster provider is to deploy a multi-node Kubernetes cluster on local VMs as fast as possible and to be as light as possible in term of resources used. -In order to achieve that goal, its deployment is very different from the 'standard production deployment'? method used on other providers. This was done on purpose in order to implement some optimizations made possible by the fact that we know that all VMs will be running on the same physical machine. +In order to achieve that goal, its deployment is very different from the "standard production deployment" method used on other providers. This was done on purpose in order to implement some optimizations made possible by the fact that we know that all VMs will be running on the same physical machine. The `libvirt-coreos` cluster provider doesn't aim at being production look-alike. @@ -36,15 +36,16 @@ On the other hand, `libvirt-coreos` might be useful for people investigating low ### Prerequisites -1. Install [dnsmasq](http://www.thekelleys.org.uk/dnsmasq/doc) +1. Install [dnsmasq](http://www.thekelleys.org.uk/dnsmasq/doc.html) 2. Install [ebtables](http://ebtables.netfilter.org/) 3. Install [qemu](http://wiki.qemu.org/Main_Page) 4. Install [libvirt](http://libvirt.org/) -5. Enable and start the libvirt daemon, e.g: - * ``systemctl enable libvirtd`` - * ``systemctl start libvirtd`` -6. [Grant libvirt access to your user¹](https://libvirt.org/aclpolkit) -7. Check that your $HOME is accessible to the qemu user² +5. Install [openssl](http://openssl.org/) +6. Enable and start the libvirt daemon, e.g: + * ``systemctl enable libvirtd && systemctl start libvirtd`` # for systemd-based systems + * ``/etc/init.d/libvirt-bin start`` # for init.d-based systems +7. [Grant libvirt access to your user¹](https://libvirt.org/aclpolkit.html) +8. Check that your $HOME is accessible to the qemu user² #### ¹ Depending on your distribution, libvirt access may be denied by default or may require a password at each access. @@ -127,11 +128,11 @@ cluster/kube-up.sh The `KUBERNETES_PROVIDER` environment variable tells all of the various cluster management scripts which variant to use. If you forget to set this, the assumption is you are running on Google Compute Engine. -The `NUM_MINIONS` environment variable may be set to specify the number of nodes to start. If it is not set, the number of nodes defaults to 3. +The `NUM_NODES` environment variable may be set to specify the number of nodes to start. If it is not set, the number of nodes defaults to 3. The `KUBE_PUSH` environment variable may be set to specify which Kubernetes binaries must be deployed on the cluster. Its possible values are: -* `release` (default if `KUBE_PUSH` is not set) will deploy the binaries of `_output/release-tars/kubernetes-server-'|.tar.gz`. This is built with `make release` or `make release-skip-tests`. +* `release` (default if `KUBE_PUSH` is not set) will deploy the binaries of `_output/release-tars/kubernetes-server-….tar.gz`. This is built with `make release` or `make release-skip-tests`. * `local` will deploy the binaries of `_output/local/go/bin`. These are built with `make`. You can check that your machines are there and running with: @@ -141,10 +142,10 @@ $ virsh -c qemu:///system list Id Name State ---------------------------------------------------- 15 kubernetes_master running - 16 kubernetes_minion-01 running - 17 kubernetes_minion-02 running - 18 kubernetes_minion-03 running -``` + 16 kubernetes_node-01 running + 17 kubernetes_node-02 running + 18 kubernetes_node-03 running + ``` You can check that the Kubernetes cluster is working with: @@ -168,7 +169,7 @@ Connect to `kubernetes_master`: ssh core@192.168.10.1 ``` -Connect to `kubernetes_minion-01`: +Connect to `kubernetes_node-01`: ```shell ssh core@192.168.10.2 @@ -185,7 +186,7 @@ export KUBERNETES_PROVIDER=libvirt-coreos Bring up a libvirt-CoreOS cluster of 5 nodes ```shell -NUM_MINIONS=5 cluster/kube-up.sh +NUM_NODES=5 cluster/kube-up.sh ``` Destroy the libvirt-CoreOS cluster diff --git a/docs/getting-started-guides/logging-elasticsearch.md b/docs/getting-started-guides/logging-elasticsearch.md index 9fe352eb47..00202e964a 100644 --- a/docs/getting-started-guides/logging-elasticsearch.md +++ b/docs/getting-started-guides/logging-elasticsearch.md @@ -45,10 +45,10 @@ $ kubectl get pods --namespace=kube-system NAME READY REASON RESTARTS AGE elasticsearch-logging-v1-78nog 1/1 Running 0 2h elasticsearch-logging-v1-nj2nb 1/1 Running 0 2h -fluentd-elasticsearch-kubernetes-minion-5oq0 1/1 Running 0 2h -fluentd-elasticsearch-kubernetes-minion-6896 1/1 Running 0 2h -fluentd-elasticsearch-kubernetes-minion-l1ds 1/1 Running 0 2h -fluentd-elasticsearch-kubernetes-minion-lz9j 1/1 Running 0 2h +fluentd-elasticsearch-kubernetes-node-5oq0 1/1 Running 0 2h +fluentd-elasticsearch-kubernetes-node-6896 1/1 Running 0 2h +fluentd-elasticsearch-kubernetes-node-l1ds 1/1 Running 0 2h +fluentd-elasticsearch-kubernetes-node-lz9j 1/1 Running 0 2h kibana-logging-v1-bhpo8 1/1 Running 0 2h kube-dns-v3-7r1l9 3/3 Running 0 2h monitoring-heapster-v4-yl332 1/1 Running 1 2h @@ -219,7 +219,7 @@ $ curl --header "Authorization: Bearer JsUe2Z3cXqa17UQqQ8qWGGf4nOSLwSnp" --insec } ``` -The Elasticsearch website contains information about [URI search queries](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-uri-request) which can be used to extract the required logs. +The Elasticsearch website contains information about [URI search queries](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-uri-request.html) which can be used to extract the required logs. Alternatively you can view the ingested logs using Kibana. The first time you visit the Kibana URL you will be presented with a page that asks you to configure your view of the ingested logs. Select the option for diff --git a/docs/getting-started-guides/logging.md b/docs/getting-started-guides/logging.md index d0dce5ef64..a679955aed 100644 --- a/docs/getting-started-guides/logging.md +++ b/docs/getting-started-guides/logging.md @@ -9,17 +9,17 @@ logging and DNS resolution for names of Kubernetes services: ```shell $ kubectl get pods --namespace=kube-system NAME READY REASON RESTARTS AGE -fluentd-cloud-logging-kubernetes-minion-0f64 1/1 Running 0 32m -fluentd-cloud-logging-kubernetes-minion-27gf 1/1 Running 0 32m -fluentd-cloud-logging-kubernetes-minion-pk22 1/1 Running 0 31m -fluentd-cloud-logging-kubernetes-minion-20ej 1/1 Running 0 31m +fluentd-cloud-logging-kubernetes-node-0f64 1/1 Running 0 32m +fluentd-cloud-logging-kubernetes-node-27gf 1/1 Running 0 32m +fluentd-cloud-logging-kubernetes-node-pk22 1/1 Running 0 31m +fluentd-cloud-logging-kubernetes-node-20ej 1/1 Running 0 31m kube-dns-v3-pk22 3/3 Running 0 32m monitoring-heapster-v1-20ej 0/1 Running 9 32m ``` Here is the same information in a picture which shows how the pods might be placed on specific nodes. -![Cluster](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/blog-logging/diagrams/cloud-logging.png) +![Cluster](https://raw.githubusercontent.com/kubernetes/kubernetes/{{page.githubbranch}}/examples/blog-logging/diagrams/cloud-logging.png) This diagram shows four nodes created on a Google Compute Engine cluster with the name of each VM node on a purple background. The internal and public IPs of each node are shown on gray boxes and the pods running in each node are shown in green boxes. Each pod box shows the name of the pod and the namespace it runs in, the IP address of the pod and the images which are run as part of the pod's execution. Here we see that every node is running a fluentd-cloud-logging pod which is collecting the log output of the containers running on the same node and sending them to Google Cloud Logging. A pod which provides the [cluster DNS service](/docs/admin/dns) runs on one of the nodes and a pod which provides monitoring support runs on another node. @@ -48,7 +48,7 @@ This step may take a few minutes to download the ubuntu:14.04 image during which One of the nodes is now running the counter pod: -![Counter Pod](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/blog-logging/diagrams/27gf-counter.png) +![Counter Pod](https://raw.githubusercontent.com/kubernetes/kubernetes/{{page.githubbranch}}/examples/blog-logging/diagrams/27gf-counter.png) When the pod status changes to `Running` we can use the kubectl logs command to view the output of this counter pod. @@ -75,10 +75,10 @@ root 479 0.0 0.0 4348 812 ? S 00:05 0:00 sleep 1 root 480 0.0 0.0 15572 2212 ? R 00:05 0:00 ps aux ``` -What happens if for any reason the image in this pod is killed off and then restarted by Kubernetes? Will we still see the log lines from the previous invocation of the container followed by the log lines for the started container? Or will we lose the log lines from the original container's execution and only see the log lines for the new container? Let's find out. First let's stop the currently running counter. +What happens if for any reason the image in this pod is killed off and then restarted by Kubernetes? Will we still see the log lines from the previous invocation of the container followed by the log lines for the started container? Or will we lose the log lines from the original container's execution and only see the log lines for the new container? Let’s find out. First let's delete the currently running counter. ```shell -$ kubectl stop pod counter +$ kubectl delete pod counter pods/counter ``` @@ -131,8 +131,8 @@ Note the first container counted to 108 and then it was terminated. When the nex ```shell SELECT metadata.timestamp, structPayload.log - FROM [mylogs.kubernetes_counter_default_count_20150611] - ORDER BY metadata.timestamp DESC +FROM [mylogs.kubernetes_counter_default_count_20150611] +ORDER BY metadata.timestamp DESC ``` Here is some sample output: diff --git a/docs/getting-started-guides/mesos-docker.md b/docs/getting-started-guides/mesos-docker.md index 5578025751..d2b8e95f87 100644 --- a/docs/getting-started-guides/mesos-docker.md +++ b/docs/getting-started-guides/mesos-docker.md @@ -42,7 +42,7 @@ The cluster consists of several docker containers linked together by docker-mana | Mesos Master | mesosmaster1 | REST endpoint for interacting with Mesos | | Mesos Slave (x2) | mesosslave1, mesosslave2 | Mesos agents that offer resources and run framework executors (e.g. Kubernetes Kublets) | | Kubernetes API Server | apiserver | REST endpoint for interacting with Kubernetes | -| Kubernetes Controller Manager | controller | | +| Kubernetes Controller Manager | controller | | | Kubernetes Scheduler | scheduler | Schedules container deployment by accepting Mesos offers | ## Prerequisites @@ -52,13 +52,13 @@ Required: - [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - version control system - [Docker CLI](https://docs.docker.com/) - container management command line client - [Docker Engine](https://docs.docker.com/) - container management daemon - - On Mac, use [Boot2Docker](http://boot2docker.io/) or [Docker Machine](https://docs.docker.com/machine/install-machine/) + - On Mac, use [Docker Machine](https://docs.docker.com/machine/install-machine/) - [Docker Compose](https://docs.docker.com/compose/install/) - multi-container application orchestration Optional: -- [Virtual Box](https://www.virtualbox.org/wiki/Downloads) - x86 hardware virtualizer - - Required by Boot2Docker and Docker Machine +- [Virtual Box](https://www.virtualbox.org/wiki/Downloads) + - Free x86 virtualization engine with a Docker Machine driver - [Golang](https://golang.org/doc/install) - Go programming language - Required to build Kubernetes locally - [Make](https://en.wikipedia.org/wiki/Make_(software)) - Utility for building executables from source @@ -70,14 +70,14 @@ It's possible to install all of the above via [Homebrew](http://brew.sh/) on a M Some steps print instructions for configuring or launching. Make sure each is properly set up before continuing to the next step. - brew install git - brew install caskroom/cask/brew-cask - brew cask install virtualbox - brew install docker - brew install boot2docker - boot2docker init - boot2docker up - brew install docker-compose +```shell +brew install git +brew install caskroom/cask/brew-cask +brew cask install virtualbox +brew install docker +brew install docker-machine +brew install docker-compose +``` ### Install on Linux @@ -91,27 +91,42 @@ In order to build Kubernetes, the current user must be in a docker group with su See the docker docs for [instructions](https://docs.docker.com/installation/ubuntulinux/#create-a-docker-group). -### Boot2Docker Config (Mac) +#### Docker Machine Config (Mac) -If on a mac using boot2docker, the following steps will make the docker IPs (in the virtualbox VM) reachable from the -host machine (mac). +If on a Mac using docker-machine, the following steps will make the docker IPs (in the virtualbox VM) reachable from the +host machine (Mac). -1. Set the VM's host-only network to "promiscuous mode": +1. Create VM - boot2docker stop - VBoxManage modifyvm boot2docker-vm --nicpromisc2 allow-all - boot2docker start + oracle-virtualbox + + ```shell + docker-machine create --driver virtualbox kube-dev + eval "$(docker-machine env kube-dev)" + ``` + +2. Set the VM's host-only network to "promiscuous mode": + + oracle-virtualbox + + ```conf + docker-machine stop kube-dev + VBoxManage modifyvm kube-dev --nicpromisc2 allow-all + docker-machine start kube-dev + ``` This allows the VM to accept packets that were sent to a different IP. Since the host-only network routes traffic between VMs and the host, other VMs will also be able to access the docker IPs, if they have the following route. -1. Route traffic to docker through the boot2docker IP: +1. Route traffic to docker through the docker-machine IP: - sudo route -n add -net 172.17.0.0 $(boot2docker ip) +```shell +sudo route -n add -net 172.17.0.0 $(docker-machine ip kube-dev) +``` - Since the boot2docker IP can change when the VM is restarted, this route may need to be updated over time. + Since the docker-machine IP can change when the VM is restarted, this route may need to be updated over time. To delete the route later: `sudo route delete 172.17.0.0` @@ -119,8 +134,10 @@ host machine (mac). 1. Checkout source - git clone https://github.com/kubernetes/kubernetes - cd kubernetes + ```shell + git clone https://github.com/kubernetes/kubernetes + cd kubernetes + ``` By default, that will get you the bleeding edge of master branch. You may want a [release branch](https://github.com/kubernetes/kubernetes/releases) instead, @@ -132,7 +149,9 @@ host machine (mac). Building a new release covers both cases: - KUBERNETES_CONTRIB=mesos build/release.sh + ```shell + KUBERNETES_CONTRIB=mesos build/release.sh + ``` For developers, it may be faster to [build locally](#build-locally). @@ -142,13 +161,17 @@ host machine (mac). 1. Test image includes all the dependencies required for running e2e tests. - ./cluster/mesos/docker/test/build.sh + ```shell + ./cluster/mesos/docker/test/build.sh + ``` In the future, this image may be available to download. It doesn't contain anything specific to the current release, except its build dependencies. 1. Kubernetes-Mesos image includes the compiled linux binaries. - ./cluster/mesos/docker/km/build.sh + ```shell + ./cluster/mesos/docker/km/build.sh + ``` This image needs to be built every time you recompile the server binaries. @@ -159,23 +182,29 @@ host machine (mac). If you delete the `MESOS_RESOURCES` environment variables, the resource amounts will be auto-detected based on the host resources, which will over-provision by > 2x. If the configured resources are not available on the host, you may want to increase the resources available to Docker Engine. - You may have to increase you VM disk, memory, or cpu allocation in VirtualBox, - [Docker Machine](https://docs.docker.com/machine/#oracle-virtualbox), or - [Boot2Docker](https://ryanfb.github.io/etc/2015/01/28/increasing_boot2docker_allocations_on_os_x). + You may have to increase you VM disk, memory, or cpu allocation. See the Docker Machine docs for details + ([Virtualbox](https://docs.docker.com/machine/drivers/virtualbox)) + 1. Configure provider - export KUBERNETES_PROVIDER=mesos/docker + ```shell + export KUBERNETES_PROVIDER=mesos/docker + ``` This tells cluster scripts to use the code within `cluster/mesos/docker`. 1. Create cluster - ./cluster/kube-up.sh + ```shell + ./cluster/kube-up.sh + ``` If you manually built all the above docker images, you can skip that step during kube-up: - MESOS_DOCKER_SKIP_BUILD=true ./cluster/kube-up.sh + ```shell + MESOS_DOCKER_SKIP_BUILD=true ./cluster/kube-up.sh + ``` After deploying the cluster, `~/.kube/config` will be created or updated to configure kubectl to target the new cluster. @@ -188,7 +217,9 @@ host machine (mac). 1. Destroy cluster - ./cluster/kube-down.sh + ```shell + ./cluster/kube-down.sh + ``` ## Addons @@ -196,7 +227,9 @@ The `kube-up` for the mesos/docker provider will automatically deploy KubeDNS an Check their status with: - ./cluster/kubectl.sh get pods --namespace=kube-system +```shell +./cluster/kubectl.sh get pods --namespace=kube-system +``` ### KubeUI @@ -213,7 +246,9 @@ Warning: e2e tests can take a long time to run. You may not want to run them imm While your cluster is up, you can run the end-to-end tests: - ./cluster/test-e2e.sh +```shell +./cluster/test-e2e.sh +``` Notable parameters: - Increase the logging verbosity: `-v=2` @@ -221,35 +256,45 @@ Notable parameters: To build, deploy, test, and destroy, all in one command (plus unit & integration tests): - make test_e2e +```shell +make test_e2e +``` ## Kubernetes CLI When compiling from source, it's simplest to use the `./cluster/kubectl.sh` script, which detects your platform & architecture and proxies commands to the appropriate `kubectl` binary. - `./cluster/kubectl.sh get pods` +ex: `./cluster/kubectl.sh get pods` ## Helpful scripts -Kill all docker containers - - docker ps -q -a | xargs docker rm -f +- Kill all docker containers -Clean up unused docker volumes - + ```shell + docker ps -q -a | xargs docker rm -f + ``` + +- Clean up unused docker volumes + + ```shell docker run -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker:/var/lib/docker --rm martin/docker-cleanup-volumes + ``` ## Build Locally The steps above tell you how to build in a container, for minimal local dependencies. But if you have Go and Make installed you can build locally much faster: - KUBERNETES_CONTRIB=mesos make +```shell +KUBERNETES_CONTRIB=mesos make +``` However, if you're not on linux, you'll still need to compile the linux/amd64 server binaries: - KUBERNETES_CONTRIB=mesos build/run.sh hack/build-go.sh +```shell +KUBERNETES_CONTRIB=mesos build/run.sh hack/build-go.sh +``` The above two steps should be significantly faster than cross-compiling a whole new release for every supported platform (which is what `./build/release.sh` does). diff --git a/docs/getting-started-guides/mesos.md b/docs/getting-started-guides/mesos.md index 2b4068ad80..2ac895aaab 100644 --- a/docs/getting-started-guides/mesos.md +++ b/docs/getting-started-guides/mesos.md @@ -27,7 +27,7 @@ Further information is available in the Kubernetes on Mesos [contrib directory][ - A running [Mesos cluster on Google Compute Engine][5] - A [VPN connection][10] to the cluster - A machine in the cluster which should become the Kubernetes *master node* with: - - GoLang > 1.2 + - Go (see [here](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/devel/development.md#go-versions) for required versions) - make (i.e. build-essential) - Docker @@ -66,7 +66,7 @@ Start etcd and verify that it is running: ```shell sudo docker run -d --hostname $(uname -n) --name etcd \ - -p 4001:4001 -p 7001:7001 quay.io/coreos/etcd:v2.0.12 \ + -p 4001:4001 -p 7001:7001 quay.io/coreos/etcd:v2.2.1 \ --listen-client-urls http://0.0.0.0:4001 \ --advertise-client-urls http://${KUBERNETES_MASTER_IP}:4001 ``` @@ -74,7 +74,7 @@ sudo docker run -d --hostname $(uname -n) --name etcd \ ```shell $ sudo docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -fd7bac9e2301 quay.io/coreos/etcd:v2.0.12 "/etcd" 5s ago Up 3s 2379/tcp, 2380/... etcd +fd7bac9e2301 quay.io/coreos/etcd:v2.2.1 "/etcd" 5s ago Up 3s 2379/tcp, 2380/... etcd ``` It's also a good idea to ensure your etcd instance is reachable by testing it @@ -147,12 +147,6 @@ disown -a #### Validate KM Services -Add the appropriate binary folder to your `PATH` to access kubectl: - -```shell -export PATH=/platforms/linux/amd64:$PATH -``` - Interact with the kubernetes-mesos framework via `kubectl`: ```shell diff --git a/docs/getting-started-guides/mesos/OWNERS b/docs/getting-started-guides/mesos/OWNERS new file mode 100644 index 0000000000..7cdb4cdb49 --- /dev/null +++ b/docs/getting-started-guides/mesos/OWNERS @@ -0,0 +1,3 @@ +assignees: + - jdef + - karlkfi diff --git a/docs/getting-started-guides/mesos/k8s-firewall.png b/docs/getting-started-guides/mesos/k8s-firewall.png new file mode 100755 index 0000000000000000000000000000000000000000..ed1c57ca7d0980056b9d088b4d3822c4dc0fd223 GIT binary patch literal 88722 zcmXtA2Ou16v|V*|t9Mo}(WBQ5qD2rywCF)d^oVW|L4+lUXh9G?T101cBBDj~PW0aU z`~2^{Yj<{L?97_k@4NThbI%p2tF1;%Ku-XHK!`QeRrDYb%tG+Yz{3Q4cy$BVArLr3 zL*<@3fkUQ;EjhdStR#i|>&G>GVNH&>Rl$8E`vQcz2a_)EqHDZGVagf*6G}HiVXlSUb zs{>15H8q)AMMXsi2ObkmH8oO-ip}6xO-+HG0l=y+*7&bMuBIHheZEG!%z z9-f|_zArhbtgMWRB1}8j+$0jZeS;c`S>Eq>GaIFFe!7TKygpr|5iH9vqg-5Qrsk<+d9_Vz~o zXJJaL8T~W_VP*=7vXa1(dY4@@dY8-Wot>TS?CgBO)6UM(*Vo@_OQ`0CXw!IoukTy@vt!P)M&wNr#JEm4>&VHo(MkY49l!3g1q>P^quG(nng z18eRh;1Ck%1-2YBYBUOK>N0d6irJ!zw}KzmmZi%BBU-Sr_iLSZh&rP0`qSAC3R{bLc@`^JkMB`!8d~$z*VhMw z&BKH6`nBGFTOJ-BPEL2WP0yNj{rvpY)S%ait6g4~em|;hvMVboWMyS!WMpOQ>*~M@ zm~CJk*yO~xqK%rGnjbXC8tUh4>Sqcj!;>fWuV25`(fMjob~`lQzQx7Gy?qG}A3vT0 z1}CAVMUSdkSXj6O1+jP??~e=*huZ9_FpdHPt7_%GetuAJ3~u-vCre99Z*TASrIC@5 zn?@3;ecD>L_8yG9_wVOEfOA5jP&X^@R}Gq6@oV6IJ?R#-v9;w$Woq;3?CRp?>g49) z`U)oIf96p;wYkaPWSEzCM#DS%IXSsS(p4d&tJ*C~*9e%2iHWhXv4zF#*x1<2#Ke*l zxDntuKPRVfq0JWM$;nAs@H07?nwko(9f{9aSzTS-T>aYB75*j*%-P{eeI;}dKEdRR zAKd=Htq?H)@8WxTx(B|wws6ut&%x23CYZtK@$vDNmKHG6w*RyF`T4;?GSW1_j-KIlQ#A1pL_LQYkYh> z@Q7g9mYt1N+|~xx78e(5jj^e}?hTe&yd+dpkSeAOHPBp4Xb)`Yqs2{H?7$ zJw06=^1;E?r9$y*Ypbi9tE=rn-nVz#&kqbmVEX?pA2{7_fa<`@bh5)3;93pgz{ z4AXA&)hm*4EZRj^nU@m4t&|pdjJ=SE8gcwqAd>P6H+#@Qbd4A{yT`z$+@#nryT?{I zAz_8$tW3*wBUJ;dMu3oMlPoiG37F*)GT)lD?w)oy$(PSgG0&ous zi;6m|<@geNazvAr3KDvjH4`+ZW8gY|nT<)`H;Ako;D)7wB&E@3e1WRCGbD7FJ{>9@b zQJm!X+PsP2;tR|KpYZJ&7ZT%!_SlN1_KAy&bHx{Y{n}|=Zd77;*Mcivqjc1%$^^U< z1OxhL&5{qj%c99zS-M~goIsby)$T?2vU{RtG+5sS9*)=?Tz+8!U$jZ+iuJr@y0*O- zeV<+Bh$9$%+rwdRLba%m6e)abY@$=7J_y-0y>4K1q!)VAu8M`RJ z&(AL;Q2OoLx8maBZ{OHXOwG;BfoBB%5jaP4^UiDF0FI6pIy*ao2TKZR^=-VKZwqgx z2)nO%IZvvit?l*tb=1cFHxj^MbR4~cvsr%x&J4Jf_;?C4$N-biE}7pd?U{)1>FVyS zXWN~Fs4eih)ovoQZ>=r6?dmF^;f|H_!?Nfi4ymr9VLt^aU@C{6?p|I$g%*JeWSqxY z4d>SREa!K8IGFA9?k!ANuEE#a$H&Ll*Vohkdv$gAnB{91)E5kFw+`#LK%J-u4<7XP z0_RjzRCMP~5n)u+!G8vJpO&ob`>4d!`Ue8&1W*-#RKWhjQ+8g?g5X;x4D-)IHW&63 z6aW{!2O?2qWWn0=DLW%Enb0{)TTf3**UY>;?)V;0dr|quMgXs-leIup1Oe91Pd2ux zwpMz4Y%Eq?OiZkOVecHECDBMzIc7Ds8L*Fo97G37j5q9Yt{jZf_p`x3W=5&74;m5S ze^G?(o;NU;LZ274t$Eg@30C(md+oWlSXEAzR8?_R(#;n9OQ8pTN@DbrlcOX5NBXD^ zOEJ-$Cp=38IN>}6W?67C&QIKU33E{vS&*_h^DKkHKaN$Z*?%3xOiJ`}Tk$){@#h(l zFsUvjf=KL7P^lbY%&xgyjBtn~qq4V9eD5Bmj5JARcVTQ7tBM6#-LQ6mjNGv8CAIM*gzn$ z`gHEYF7?l!KlzFZVfnh2@+4?-BI(pLImo>}zk9d8 zzh6;N0haswx9i~0pFcGxz}Woujha0IbG_HrJy)r~>JjR zA-htnrEtXPyAEq_>36umdvcItd>a1 zAKCiW)z{xmCb}q35}|7hXv};V{`PHz784y^%keF|;D{pU;s$Vu>+9FAXhBMXFn_^} z)LWm|5!L|$!ARl=cqA?-CO%VTe&PIrI;z9E94}4a8%; z0KD91zPUUb&eJJiM7~3}ZG@v(6a!B|q83hi@0o5sChq5-7)ePdNo`tDt6>Eu4UE|A-VhpVZo;^IOt&VgNAoL_(w)z{VU zLEjlLS8Gqbe)Y=h)ho~2`<=OMYGP_?^sT?<+*5`aS0Pb5Pp{yu3g(SFnBap({4ry4 zahYWhQb`W-y>nj`tYFK5tvxA55SBm^^Hi|9ZSDKtzW{es)^^%Tq-xn^588^Q&@QDq zY4Ja-Fe)~T8u=;OG+}a1=8x1a&Esc0gy?xi8;0^T(&E0y7{l_7>t<6peOU%=ZA`FQ zRJSr%tA2z~U2 zY|iCv$a70+2ckDpcn+|XIb$i5c4^mh-q5`KumPqFh@Ow_fj7+m}8K#sA(hV7i z03ojdUx^_h2ku|dP07_Nne;)2KWmsSrFuQ9Yx?8~^lQtS=il*-2tPyuax=~S905maLSv=Kquaq8hj_RWz>61u5CK( zgQSR7lJLc09mT739{##l8-GFO;YjuV%G}iS>C>ksCYZdztw-m+ZUK!Ey3YUrcLGVF zlhbQ^n(lY`*tG9sm*(b>wC)YczAkwF`K}>tfW6QamzvwzJw5%jAGn#LBrAbv9hUGW zmHCG(O;Y%In?dyTfs4Krb=8m*Pc0FUtE;U7$wOnuPf6JhYcl>m;L;a{hAvx`nsh&< zr>3T*S(D8zEwzJe=TD!t;zJ#shv4US8LjgkpbKp+Z99R#Bm6r-m^9=y1fN-Bo{al7oaA z#A72lW&ra*GEJ@KzH@XY2bhSRJaX&G6fErW)dO>l zj1GRUl+w1E@-dzb4jXr@mHr1}80TA-aI$oS7%mfT`+cd9sFp;EIHZn7uHhX6P0g!B zzC@GkN6-I^am_FzG5bOt)JX7!Yih z78g@Sb?`tVI;s80F`~KEv!|du%mx$)$)-M>meR;_3DAXu(V3W*7@EE;0oM;^-t~}OC7so~7E5gWEW>iVw;8h$ z>R*2R_-kDbhG%NuTL;JrRN1m}yx2e4$r1LJ_PuK3y;8}!lvk?pgJw2AjQCaKBQ+VM zuuic^SMc8~5EbIUxm=Zq*=umICG&YtuL9Xa;W!B}D-G zBxM0l7cK?y6cWAY=l~&;bFOuNXb6n&{h|H!b$U@{y#TVIN?pswh;$Bk=H1h*~r!}rq#|MHWq_)r}*0(ryEKca0_w7J1=(N(yHV{R9ILOR{@rHW7FIn&_8xl zV(S;xmbPhVXh>NU03oV!j8eH-SO{VA@_I)L^--jvL7cKLD=H{_kwm?N%n0jdut7Ms zRdV22dFu)!nf*8{eVv_LGeUuY@OWx!Y60f{zuW)=Y~4*LMOi~dMOlqhb6M<4NYi+2 z?Rc#;p2pWa`>!p#IuC(GqfqQp08uc}(=*f2-QUFTA<91Y^OdQuGkTh#Z#11ubPT@; zH_9(5Am#&0CXekFZg@$IJkxY=Zz z`EdQDM<#iH^B+5YT-0OC99aYD_l%{klUS>#h1%>CSA5J({>a*!TG3WK$Op3)q`uKJ znu`LzbmTxYgb8j$FkfAB+YH}2>1Tr@kP(Ne4aTg%*U!1HW)d{)@Tggn2oQ3 zb0E3Ye#0Ik6Q2G15eSV-b|z0Eh+cRZM&?(Im^i(xl1Tdrrd5`l>n(+egB=K*<4OGX zhxSk3=TD%BYqji0Y|J74!5`uL&@8;C9OS+*NaluVN~)=eNy^aL$B!R>{E&D%c{_3* zzQkk|IYv-G{$-Yz$9ZDydCLtBAQ4%B$xOGyRwnGgP9s{~-d6qAFCKXXLc|^3Pe zF^Y&3FElBQ}vuU}gA)xbJu8?L97g;kP1wqEu6P6w=EMR`e<%#J+ zv1HrszUjTFP1qx|jsP`@2p$c5^-cNT{;sZN)y+CWjU9iFW!OZoA1Ec1)z^o|6330M zuk+zcBKBdESb7d1<+QdAiNm0vuo>qciM}OD0ptV(EHwNait7u2XXeqyTF>EO=Eqq; z-i^*G3QI*O{sI`G>H7LQ7(9YMfBrnv)bx1Gu7s@ za?%>{CLy3Eu5Oo82~z|r6}qoUZ9n(Yn)^csDiE)^EK2t|M#X}k`a!#OtKSgSPg~JH3h75;9YkgdI+c6>_VrGVxV)Vc4?s~2JG=}065~*en=8tYXQV`y=Q8&mRUZYL2 z6rN(V6E$|^hvP@YBaAG|O<>fJPZ|WeEBPKW=@O~LJTYdB_psbf;pIXF#k8ePB z1mNlHY(|!l4$qC6+8{q<{?EX!@H1n;(KztX-nw=s1|~YNeD>-p?h!)4iLN_F|4|TN zMVy^umai^TS(q@+l$h^hP~ye@>-+sXs~IAKv5q4JQ4+DeL0$|_y118*HiD@*5bgHw8)62lX_oHRJBXK3I_JvsUOJc$H} znqi|L5tAvQZf{C<+6fEs4{gaD1~hfLc^(#;UAN{rg>Lx)QaFFU47hU#E7;#1x|hH8 z9e@7bV+qxgf#JbZyF`EwrVRuH`yyda-Lzfx_zSO8rhHONNOI%w8g}r`TLu|OHlrnTFIbUU~&wfE0q6AH3l60%Md|z?!AidDSPXo#3S7kvi$W|6dC*5=r|bFE6jO6m=g0?i@Qi z3`#^oOiV;_OiEkC-ux370!dwPA*{-~Wt5s7`B6};NOE5jV!$u`Kt(~y+%$GAx5 zd^-5bsKJjxe%T?twEPcZhV-EYm?dI#77JS%`9iih-qB{7LvjeI76~h=p0%~I!a_$r zOMMaMf%_(}`Ji~OLOj{o8oyMK7b7&D^CsS`iSi`Um{T>1&5Jl5ApKca_4tHd5NB2^ z=is=zt#`41Z8U*9d#};2GVg?h%Z{89MWW}JpwCpkWKKa0vBHCw5IvdS8{QYPTHi}% zDeb^cq@qepOoVT-UPE!SQObZi_)fpDMMfB#g0jY}GI^*@D%DkPQbO%1pCzhsrvAc3 zH2E;Wks(?uK7Y{Rl}Y10l>8rGHAhJ7fkCSMGUV&J4L57>=N^Zljl-Nv?iTpZEt96$ zOL&%o>c6wc*tvRg;{gAp%3u@`B05fGcVPv-BbSgJn?b^4yusIkpnd zBo)+lCnjB{NG2vHt-KM6A+%@|$O}MX6A;j-iy0FW&ALXry1{FyI(M2sg3 z3G{Bxh<8lFfp;Pr0<_vfHeF@Yq zHm|~fi4jtb3IdC%gNPnYPd`-=0b&g>1z-eaN%gn7l7;GCzyyNw2pXlK>2{^G%1ND=_5fH=K7VC<#Qh&P(HJA zV@InK6&y4EY9nU{Fw=uPeakVh2?rfc;;*(2^9->cZ^^R17&pKm`1OOhNp7F{*9l z$h*-QHMw$fb`>a9g1AAMAR=Ucy)nu~CXcz^rL!}$DJ`n}J5gL!r5aDyiU62Smxszs zt#dBtOGR;U8m-@#_D8;5#KLb?5BA>yT=e#@(noWiflxSvhi7v``7!ea{^`9z3Qj@) zgx^{O_xLy=o(0mn>J$1c%$xGlgwYspP9M@MFL1y4pwZ1fz=mP)XzxAaLyG(y-5r)N zCXNS`Kk0dXvL2xyBn3a=b`1XA^ZS=UpmoA}Si8ak^~VIPqY+2Gy6gcGGZj9R;luS( zEHAxfbKrfKMRWBcy|kJ72Yav}wvcVIu(@1TZjQK2Ey0zNI6eY?yiNQZUP^c*!~u(d z9%c*ija$|y{!D+czP`SBEK@(NO@^zyJz-KIX}J|_S2^xKOwk<~0BtGkU%~P~1?gu% z-dibC;o&-0Y$1>&h!0G7*sb#n9vszO(HjlV`IlsxT%{P1ksn7N>?vwO0D}+1`;vmv zV!9vj9T)FmECL_!`IqXilna^W)cGhNYx@460$TN>DsG%iBC~^il>Ibax0-p|*vaCx zWaO7vQQ(AIj`NzESw37n(!L-iCLtjqIzBl$IX*lDXzW#>#^T6JY``Eq*`5V8M-S1Xb-o<>xL-L!308bzbg8b_Ok|R*N zoL>RF-rO9_@e&Iwv@B)g~O|^g}t{b%3^N?Z{Qxqi#*yKii``NNkH2;$B~>O)PN1|r{>%G$=)LSgq$#IOhGTh6VPPGJM~|ObVa?Fk z9V-z8|K8P$W01_kW65JPVaWAGGuLis8lWaFLqCvf_K^|eQMg$#G`f<~lVsgMi*)QW zwS7O(%gt5bHV8f^dd@&69YVgq`bOzK-%1oGSnDQ7I^dJr>Gk8x*Ofg(s7seAL4pX# z7IW6o5-9zB23L4i1?D*p-ew<{$HRxj5&S<(%3nyx796Y7qTpyycOT#S zF=4W_r!>w)2A9I4LUffCiaoH-<%3ol5DIRn$c|7p%S9?C^O4Y|JcO zVNmH}Cv!N;stu*m0dgsKr~9j?n8C=5E04;VR!jKY;zP`VlX7m=?8vvcq9Q5_pS8I3 zqoaTIgrA(=`-N&?p+Oop0$T0-?QONiJAg4Qh%a5~8_JV~VJS^L9j6Q%tuY6G(~8lE z?FrG?J_xOhOj4_GaML34hfRG+vEul)m|Z3QZzc4Fmp#q;DT9vmw|^HEx?`5Ya=wSFoRoo! z(UO6CQ5OE^Yk7({XZ?y-hqvsx!EIm)y$blM%}-q|A^Fxa}D@lgx-G zsvAMOSL&_~dV3#uFS8xOd1ueg&cIv%${l9TM92t-da;`Pn~M-Zg_?6FW)RR=Hn%rF zP++>@|KeI(T8i+y+}g_8$~dCTivVYdl*?N+x z2&?7)O*Ed^+HDn zOujosg`dVyk{~1y&}p+#KgrV6dxXKw^8G*~hn!;;Fu31hdzXm1gIhRA;F8P{7~}VQ zazF`(FJ3c;eCVc^%Dq5~FHU`lpjIl(>-N)@DF_(KT4j5#@hSJ&$M(MZyaLw z!sVYWfPyB!?y0^OVqEbZPKZJvr~`jD;z!%nPyYkqKpT1hUFw~8{o}U9jD740QT83U zmiv(JA+;MDc0)RUIc7!lza(oe>pm;zEDXS#jA@)H_ zV{x&z%YVU+JjDx{pASBkirZqEi@+0wNt;b_jM0+iwSH_G7F4b zZW}&XUnlU#p{GJv#6pp-B71Nl_qi%pIU!;*OVS1Wgfrt}SHK3ix5{E3dwIRimnJ70 zKmIMzqBwCyM^W}2yH_#)hK8EPqgIamN@M0q5e$2f*8tF}pzyiRZvObLg2M0jRe(!V zQw6*l@(@HS-ESb~ei_X8DUUw#@|IaI#7ia4--~ziPC+ilQH?gnIM*jz?9rv)BtXsfsInq zM9{itv9oB%j0NT_CLG46lZ0wjj(0QphfVmw<*TmaBT)N(htMV!TA*|gn2lu59qKnG zT#TH$HYy$6&~=6#4x1v!K_D*UxC^PkhK!U-(&}b$Q!#t&6%|-5Km-Ke)e;sx<||Rn z9$x!SFK~hQLoaA%N%nTovONlD=jzi7PB41@1nqeFF$xcONz=1ZM97~`(TeP)BJ?wosrp+)vTl;+j>>7!J?v}O@)E-SG@?xtR& z#m?N;582xk_Lp;oc-)t}(p;mm z>}7jgWAQ12`*+Z*#Hl&MzurT5^VjDS^mH?ux5CU#a)#*hD@Cj@P<`yj_LAivTp4Nx zgwb&2a1^iyhIfJv`Xpz)7o)-SXxRJ5;L>!)6&h60bWdBW%8M&$(NVdui%{qpn4LiY zcDX!pxDycqrhXv;Q4_Io+R7^{cB%$bPW!ya(nB13pL~k;(*oS^&x$b0?;qaxHqWpc zwE@@e@RL|)YX&ZSQ`4occ1MZ)!=S*v{{EGHAQO{TzvRkuesd+}daI|jv@B~50y-pU zq5S0o-5chOsg2vKcpK^>D|y|fyv4M&d+Ox;Ty>;bBr&8-N0}`+W(;`uNLBXJAd>Lp z{ft@og?)4b<9=wCB?mnSxGII={DV`hmLPZ2(^b|}y>g@`u7OZyPR4f}12x$3#XN-VEN*qt_l>xp zm2@a&Nc#cd15yKe?}EFr3M?`L-&itOnaRx+6A}{}=&~3)WiLnD`{Iq5QTMSG=1-z& zW5qBC!c^FTZ7DRG0nR9R>=>Q$$HSHZF&zqM3qhMayy3^BW7n5G>n4KHOFKn7H+?f*r$0Bm=3aH>RH>G=k7DM8NPhnxE6E5>iXN zFwt#E>yELC5qy|`uCJ7_w!OWvrCdA!X2AU0rW{rWN8f_bxz1IZ0E_`*1SEo9QTjB* zhiP6SC}$^oo|f2Eg%vQjV5(D4LKLCK>yd#mm19;OO^@+>qj^M``PJHXtKItyIwouV)#CA-oaT zl-?I(-_`ZpcLY$^27$MNJA3Y&I}YbNxvYjE*MWAp1EbtTOgtgCvLh#u^gM4ViK7{c zU2VH$A%J|kR<#k)J77KHo(#&zaQIFt%4#=fp^rsxLHJ6{e(|wml~T))ZrK3L43r?5 z>|Z=LBVH%O%^v)v`AwXw(%41J@kl07HPlzp`1j*T7y7B@pKCXq((=za!hElj>{te# z!MjG?M_4x?OV4FYu5%f4pSepv7~&)!JH^}B27`|^X;>FxurN3-MCFa2DHk-1gT}{= z*oiHOHzGBV>p4lb?Qd_m98ky{UL(SlU-kWYgfT-jmwc$p>&ebFqy$952?SIp%y!o| zGJlQK+5BrZq!>w(Chvc{mS7m(1?BhD%qr9;d^sit%hTZP;x{{_u~O*8z0*sD(j7Y) z6qJ~3ZA9a+mpo2Bi~U0Gskmd>S>1a; z*r#MGhH$!*>fPEt_yWcjuWetafsLLhIKQwel93XN=xi>3L49;hHr2~Ei#2v-`z<xmzS1M2J6Depl3qA{BZZs?Foxn+!tRAn(W-4$$2X8#{)fEJ zf~Gu;K`A063cD*zEsU(_R%$MP`lQt34N5 zWrhe;O!@3woy_{}H2R%vH(ni2x-=gBO;tP}1W&S5ysLkyE)8e@Hb4XlT=|_YIa71c z$xu35djME8@ZxcC3|^bB&(meAErPC&H%bA?wl%slb=G6*GTnTGSWVLxpnekNlJYZ_ zr#Yea-5Hv}a|~AMt=qD83B4DjYifLAu&;&(kG%hIszWp$9#h8Xb}6#5 zNwV2~0u)e)DXSIS08>v-aMKLtxD!03)PYsNR3Kaay_o17b_6+`JLfO{2llzw?`#}W zRdDAX@UXG!!_TqwWA}t9R_^dVv{x^%Y;(_dvgn5CYJ7eZtQD)Xgt4<3z8X(}(+yvCE#HMZBkaCGYV z60UJmQ_=ss8QYt^>EY>1U8ka|$??~ioEx^xmerG4l$nD;Vy9g@2Q9v-4Sv(kk}mBY zAHMJY3v+ZTP}K#c@HjC2JnNhpG1Ig=Im@)0`$-+Rih};~!dLmxRXzc~9PnNSFA@0P zHRqv^zHbw!j@_0tu7dQW*k!)^Aod$vbamT~KG5Jc8JqMbM{z+ir7D69Fn~yq%-xb( zETFMb>(A^ou_{s^;U;o^XJ?0Hl?Aa*n$ZNbU$GT%NX2+LP#PCo2G@rX!CKKkh%uf# zZYC&6nsL+WuaKk`8FcH!12MJ(ElZFmUVHjZhz!Uc)==vbzW`U*|-m}1+ z;lIDv#nSA(w*Y!iz(&vQ9`I|%3&pdgc+gunH;6hrUtL6vb>8;Y4XV*VBYq~9~ttG_Fg8kCm3~x*%REM${U)U*|BmbK0dTe{oeg}`I!a0D^_Q}OJ;_Pi;G`1 zu2AaOV!QINNZe2N8>>Lc(RhiZ-@(jGh9wlP|f`F?&0GWzFNeQR|C;z4t}QKv_NQbpGn1 zMolt%;5@dBMaJ(}E0&WWnfLWOPV#aiElV8%p9TXVs*jKy&`ZIUN3H0J^&S3m__oi) zByac;w0Y3hmUPe0e9it`A>F(y<$7F-#hD83fY2=<6D;Ef^qZ?t0Ab$h!zpIu-ocAj z*d1(#nrJwosCP5g0+AgLhc0|b7-0bA`3xh$t}vR*>@y-xAbBII?065mzrQbNU7c;t z{q9u|k`!C%t2Rl!ZAshV>oqP)Xx#AKmrVTw3wu(RmzI;Sho@tBxSLTS zBz%yCdf+Y0cU8=f>27}8`tzv4o;e6KY_dWFi?G=*ys=%g_v1Bl`KWTP()gES%WS;do*Ts5Ocs-$I307DxK(K7;i8Fa+RVOZiw7QcywPf@wX-aDF&$e5 zkkl0?J+`7)S6EZ%ARbwXx1x7i&c;@G+|LIGAH6_jZMk@g$x29k?u)VMAERYX0hN%? zpMQfw`L3&#qdU1crU{sNXJ*d-0X&O2#%uqqGC(5#ez*XnrQItmMp$`L6>(3^%v73b zzd&j6u9X2O4LNslNxC;VI8Ww__xiP09NzZUwumuv1|G6iaislGQZwiU-ap01g;Yca zp&txja=_&jZBpt(udcuYDC`XQUIhe1TP_vFDOyBdp0&m)lH_^VdLE99nqTZ*2e5X+ zX7nGVYLWOS?!If%TFq5{_QI-}MfXe<_1m5>PJ{%Vx=y0R4j+QyJv_zda^o18d(8OL z0Ud0X70bH>)S%xC1~3df^OWAI2%G9$`|Pm0^JeC&AG^r^pgm&8+jmL1yT^)%Z7I5z z`hO>mE@!ge1Qw!R4&qEsCaf*BRrX*H?l0$jHe1oXrCIO$ zoou^timHH}lAw!L-CXfF`y{9XBq-jI(aqo9g+Sg(6E&!1cD9cBg?L%T2kDuaCAZ^sq7h;A~)_ zs`_puBysoA45yAoh(9{e=>eh4p@SC-8)Ox<$Ppcp35Qv7VeMJ!d<6xOAX{(jSkl9A zc0$fWP&IS*^7@E6>0ADnA9?dtySWT~e&N%YL1_c3b3mJV=gu9FbsrECnlZ+5|3<#W zK|g?mARjXmXUZS~E`_poJI-*h_d$)2yQ?eqIvtOCbe<`8B0bF`YbTV3q;4HjS~eVT zx|_B7x5SiYW#zs9%rpAat&*+u^ru!|&H1SZ348(ruSdEwN>#5f7ukdA>guaxwx;V} z8$bBG)Olqt6`+=n6^rin`ohIh_Mh!Hz7|I?O(EPC-B0K zaO?OQWY1yUkWbPa87IpZu_x@Ju7uQDxJ*<21M`5eTsITs zeuVi*wNuUNh6e9cV;$Ah1UiwUy2>6xbfO_Vj0n82HpF7~4ohv|`%jRz-2c}CR4-O~ zvx0=-lQ=AjgYErso+$Q`@xzBe-d*yt1kh(MnN}^9n9mnW@X6=urUTlw=oHVuEMCD^&`cQJ+ux#c~fYx*j z2@NPBQh|O|eKwpjg?_Fd*zf+VbQ-``KH!l@w*ksrs`m`XsYX#s%E8G=dDHcfl!#$$ zMs`}dVQNBga!Pi3Ox}1}&S+A`Wb)E+sXwSDPSL!yw6bUe0O=8Hk>wV^`|-yCy=eGO z!^O7jPSeHj`km><*E?I)vrT?GGk!bGK_@ef^#H6NY}dQYHeMW0*Sj)v+(zcq40+ zhAj&9W6Sn9?_-Le;uFW@$G|t;bJCT5BiAW8?zcz(f#kSkDq-P4+NqbWqY9g2BZ2Ho zN`Qi8)pK=CGS%r5T|!Sg$BCrP-XXpAr#oj#>!hs+sZ!J-M)w~Jz7@jgD|{jR4?ycL zV0h>Gl^DH%8^lDw!KVszuxjRJCcOzG5@XbSCM+7;!uP22aWuR<;k0u8D8PK-iIbQK zlOHcrhlqoGW?*wlN{Y#yyVnQ1Q|aP6dwYszH7onR@|PJssK5Zx9l~9!fbQCsM#AWG zUU(*k5_5Ie%F&}k&@{z1?+X4;fX8zW<$!=MS**-apgkV%$qL4AY5&R?Spy1CMH=1M zJ98GqXH^oW#pXtMHAmWpXkC5iu~q((m6e5IzNqU6TJXH(J#gNBMx0927D@RXAr-;D zAb9@MRb|%%PKCaM*5Cnm1>hMQ_OB4gDC?e0|Ve3Fx2eg=N2pk65()jCA0y*SMAF z$Nybl&$wjof(CCO!CW5U{9rm|v5SKj1h9s(!+9x-xqO3?_51KlnYO941CM`&U3J)` zqyT~_jGmBA_=EnbZ>1Dj(YCMc?6?v&f%xJ|xeb>*dqpw&a+)+P$fNhTcV{fwG=Jf`{T`O?mZY?|93hLYH~_qJV{k=+5BXV!BQwzxo`fM#cB2X3?X&K=hWms>b` zIFqr)X!Ga6JHM6>fWGhX$jIxR>7xANV)7?gx?$71LVLt2sCaSaxLwI4?$xX8KN*&f z5s$hlGpeWZax#4J@SE{lBmKAo*tUDVefr6mkgorobyEeNXfZ<5r$zqbY#uVkobbfp zgK98kNDlOk!S40d#S3a>A0K05x>6;B>{1jyWER73COjp$LZD={|6rtlK+B7jsKwD65P#--owl9yz^xKdzwycgG! zt(gDUgs_|y6VDL`y9F%}x+&(2-trRBoZZ;-1#$}Lx? z$~nYKh03V=v;_n1)P-xSq_TLoS#jn+G|G5M3neRvGTfFL1oXVBdv zDe(q<&rYFxQ3Jm_LvM6L z_r9*r)!=h?xI{_tNS})yK7Fp%x#)C1hfFU|gP-8-PIe8%M z9aqSCt(|yr&t#V2&y<{~=EdQoTcElEqSXKbCPS-UZx{@h6E8uY)>eQma(aO_enC6$H| zvQX8gu1~^*H0Nz#T)W{JnH>NZSK>QG;srNv*~AJ;CDh~v;GAg0L8k=U?t{D$1q&y z6q#x|K}s`YI!HOhK3188=GhFF}{NzCJSSe0&!nJV)YHz@Q6HMU%&o*@IgWV@W`( zQQh4?5Mhq~`Ga1SJvBwq#PRFD|LR|Uv_*LuXVVUibHOm0mwxi(1q(FtYGZ?v|B5x6 z&lx`b`#1XQbMz}saej+)@or`(g(a%R7lLlrF2SAR2q00?jOYN(;dk%8Cui3{Xak1V zUfWpVTIM%gsRgvjK^Kq4Dl7EjRFKO05r?a6{WD3S%8`t~2)0j}+_i|(zh?rpKy)otL#?fbpq*t4F%^~@Ex&8lYmkz~B-U4i|Q8tv7@ z8_sLB>DvSPE1{yBUz=__hI8|LE~}c0S(|)hkCE2Q@ci-!h1Lt-){C1~jg1L}ODLiaq`im;!jJe;){J{sNJ&Wn z?ShoFnq5LlNE zi@YAs5}F{DD8ET{=!jHN`s(6V<3v+Rv|id~|0Ry-%QsE`|C?^e3j+&@y8Sa z0m6#a;-Db9yXQ$K2GMGuI$LmX94OK`RwrW-M6xqPMHGWKvdPeTiJy*w7~g(LUdOY9 zSXZT42D99-XA_>@8J1C4|0b-3qyx~7L_H6Y9GcoE^kGCkLl;U6?q0a}z`YvXt%hAo zA*_v4AD|LM{?W&#aA5_sN>kwlzK-o#>9e%JN&H}Fus9i&A2g^pzU3}BizS7at)1*q zQE%f2)E6cr_}7a&iUJ!YH-9SgjJ~{Qkl_cu!Vk@%fX&n30lu{B_R;Tod8Ci;IW|s? zZ#jxC!Q`Q(rG?&cKS~nlcmEHg6dV*v46jqFggZDmJR4}ZVPj_-`z=N8Gz!jg)5}+HoQtX5X%^Oz!Q>TQ_ra z*Ff{p-;X0z7f#3!uv8J)-tnjQJ@O#i5PtV>&O~EX!$Shv@{c>ORC;9m zf&Bu-(Njl@XojAfSiqTRjlxS6%*PdMDo{|^;hf3mkwUPJj0;RI@syZuXfz4v)_f<^ zTZksK{^V^?rp1smy;Ju%Hto`{YBu`w)7$tjL)L09KK<^e^4QMW(D0>z3I6m>!n5O{ z$Ul>Lmok9P!=5N*mLt;=$nj`&5_Jm4hJUf9DE>=ZDw}oOY|O#|>H35va1MI?Bz?Z27n|EGCZ_ft=lp5#d@=SO68Rg5pC=)4CQOvFQT`vHsMj7%8!ag}&G7q4K@-HmxtA~kc3lqTn z**PCPsIY}@V9dF14{Q$%goY68rUtfy$6voP(%dPLwD$F>uJN$WLV19EL!>c0g8V3z=M^x1#^0RQMMc;I2c*Ln{3KbA(WUUp_@ zW&(>*rm@k_{m!08fearNeg?7wEnw&x!KyDm+cW8dA5ALVrHvya_4qP|zlZKW60GOB zySn~d^a(w=vOD=;K+{maxU0wY;6fsu!7D)Z4rPL05 z+}x^)by!G}zy10JME7$B%l`MLPLIMqe@>xXIfznN5W%Y=@sMJuXD9iGKdAoTDZ4p1 z4o0SP2?629{#Dv0-|G9>-L8;}vh*lqG3e<4&69J*K5%2i?w&K8V9r-qg6WXK13Bmv zKI9p2CtCz)D`i8@hNou<5l0?id(iOS#(rXphTbv?;Fq+^0V_R8d~%MPX4;T&fMKIaUZJ=_yVoexzxfe?V>}zd7u4 zko%20ml>E65Qi8@+kMNyCy{}DG2=l$;DenoB;CfRB+`|SQm3SCJO5@lf29dSFvvj# zm&!{@MJIclx{panWsGGGvD=noSVV;(b7%nB zGbm#577n|Tc-lK$t{JzM`0K$H!JgP|Z2|<_(t_Vw?ps>G72Pu50xI_ZCMFi9rAMXt zg@w^3vonY`%d2MlutiQ%wotpB!tUC5BK(Z3-=)>1bE0~wBf4U-+XSAN`p^hK-xO(! zA_2stQvdHVvtL^|swYV^7##ZN zvAkkpx8KXlh%tVYmxIrJo7e*L6k7OLMW7P*=T_=_>G>xGErut3$NksVE@z7bL zRt~@00gFS~dq=S3(%xT|^WE96@NMor& zuHN4Pz%-;xNg3GJ19zsZ2Xqd7rSd820*xJ#I}2Qif9QaSzUSlz_;SC7&J&HN+|P6S zhe1IVqdIm$TOa((HNGepKl=G2-)K+ueFj?CKV+7_BV`g~l6<0OfjE+0m|}W55v(K( z!ZJI0$Ux;&**hTJk68LI*J9_dZ^({l(N5kh^?${3T~2-mC`te(rhSGVzd7_p{ab0{IzG6XeJ1|S&d!gG*|so~ z+}XbQUMgWx&rI@904O#+YHA)iMJd#N+8cU&t@oTMezRQnf8xjLbzSh73?IMJ>)E>3 z#?8fUybakb9B_I1;4qTA6weH;n;s@&UV@Jb9mIFR<^Fq*3q;AV!#m8v*V_%9IQIa3 z?46J*3j!|M!e_)AK7q~pL5w86-x7XZ2*1{YpXo_oE)hOFIlu)x>mULl;ot4%QsSbc zrD*`uTj~2gO=Y9=ecywvv+Kx6>``wE$Z@3r;;F?w;Ms8!5JkAxHKep7H)r&vvcm&j zh|Q;5V+T$pj1bbZ=zQDvTkiht_w2{$tB8Hu&xHTlsDTt+8HjTcKG4foToGPGBFnf? zdU-Du#lwWQIK23xP@oa}s1ZUfy@m58r!5)(`)|lEOut~O^S^(>Aqm}(+=J}zcQTV; zf}WC;1jIJU-~rf>J$#&^r=z2zt1J6lAJn4S+S+g0IsDTAWux8Tlj4Q9i80ac(-z1! z%aFXEDL+-T-zkCjO>lL2aWQm+4~xKE43!qcW$qFF*V$+T$O|Oux1b;FsHQ*nPzxZx z5M$UG9*{)1(3`q2a$=rrhOMZSw!S0gceok!v6M0a%;c;29iOmaq7;SeGV@TX2SnmC zD^P!~e(%}+GhsVZ1pNI9g6*ZD08z;zlX-fc%0Q){*D>^6?%OjN4!KCFX;hzPBSPh8 zXz3hO1Yge}M^L{&z}P}~+uVbuQb{E-2{h>_%^jhHDvXO@BLIw-U$wP0o~HLE614YU zv%(r>l$0b4pf_l15KB4>3yWJ{E5C#ko4|dvGhlc#H>WLt+MpZ<2}sMh;GN3{r=3ht&}vtS7_KVm;L3lUrRF8>_6YK0Um?#tp~!6S;@qTklAhRo`!P z2M%SHErV5K0eQE83@ZnQwB&!yLorhmPRPEIf&I+;)j#RhT~Sf$HOi;{BhNf9NWZV3 zPjlz$8L0td3^*t$Q&LJ>7$O1|O-xSiWKQLg80AMMCX`^U@L#BMjW1SY3G_Ux&_n51 z{9dYhn)1WXFm%(`qaM{ddh`t^iEcG>YDZ6Vy|`^X>vfC=PCsNHGp>b?*IGcRn#7;iU9YI$d~@Haokuwbj?h zx$riA^9Q^GYzfnU=Io~DN|TBb;GLeI0pyXSB);wfk$+xEWo`L+54hRycs1L>bno8f zI_W$)pSo8*{Zw+g2s59XV~A96G7zF3NY6zY&>!qS7)G{nQvBY6+rP7~#5Lpyo~6w8 zKf|=*$PJu{q`5?uE&c*DBZ-{KT_R0x=3>|OAI4>v@?TAR^Gdx+`xl)X?1b0WKvQ2> zYX60GReMCcbc6mg-o$~{t0AX=lk|n8KFcv1GRK-|@+V(Rb#ak%I#2iRku$zkIlq1i zzja>-kMU__U(N~-ct$9?oK-mC(vaH-yvp3AKQn`m@Gydv2A|E%%wSlwrFv(*6QT+J z{VZ5UPDpN02n(Aiv0X!=ic5&wzV}D}s+6Do-5jD#qfm(ZhK!p`ACLfi_nNKlCNAJ6 zucyS{y&D~3q6cP_WY;f`C+~@;2G;z^HuGGRh8eV8b&;w|;kR6CYo>UfzP! zJW<{PF-~3qkzL@tl$1@ol6JN`%93V;FJ{~lVQd7^#b{n&++{>uKq0+c(v`|WBG^)XBv3cyv}Jwju~4$&WdAVt!Opv=Upy}`!DKEgh#bg9IS#OA-nj^qiC zC83HxXJ=v2r^A}Zp`xa~T9g&OqBXbslb@g8)x|Qo`^F|K89RkZTR}Q5DG9BQ@aI0- za;J;gO0f$sA0MIZhs8tBC-}K$BPAXQ=_x?xoRXs1d~ZH)5_HF$t@*Td${0SDxD0qz zhdqb4cy&G1dNvPBByEJHkqu_*?Z=NF`}>u;PQ!1J0Nu&ePpt>T@FNp2AVMju9sr-Q zt3x5;#6=Z`7(<28KE9B;zR>b2_#hRBbR&@AbU|{8Uos>cO&t#fdo>#qW{Ku@$~MI) zNFJ>U1}vn#v71ujc|nI@p^k>GsJ+(bK?W;7s&Np=Vi*&9vPYUj7$^|p99YBt1vcZ@ zQD4<94MTqV`_!C}ANvtrSWplO|J^Lvqs!a_xMS?yPIlwHnG4LqdGMK#Q1kSy=A_h{ z>%)y!{9~hrRhgK+>uaBr_p)M{k{E{_UOyy9Ov9@s-#0U3)uw-?j&Oax@m3bg)*gK9 zY2X~z#yd4n6atHsy32QF((T#AdRNF`LKUrAHp1KM5zIlpSpXxs>V084| z%tp_1?nj&_q*BouXB24JBt=-;?ss-IPeNwDYDo{fqq+RkyW=pChGm6_5TgB04(q z)7kO4Q~+>Jt@Hp9vxT|E8wi~brMo2KS5MA7dSdF?xv3AQo7!OPK&YviYB*i0=XP=o z3slxcIL4fWV!o^ncwIpJs**F`uj|PsO1e%vsCtGdFRuV+yzs@cLi|%(XoN4i#U;d(Nc#xj z)VhQ7Qo>izXnL*dKZJCMb9hh!;3x$J@eaZLvAd6~cBOJ0-unx2K%EgsM@Kis&Cz<4 z15Mb5{)yGS<`WWnoJ`K?)gm39`el?j>?y>UIj>*d+y_a@&v)-fcfNL zr37?jx^{>tVLQfew6UETO#Ro^z9;7Z22w&`*K&^nl8d{T7$T7v%E!n2bqU7{6^|7b zKM5tB8HGgZe7rRL&mb_R1F>|ly`P|3koorQcou!aRr^R3;J(@mw_WBLXnyx=xDTi} z?7)_xl!3!r70cH7A!oc#TN#HTxYCfym0dhCgLReh&mS+k-fPlcK{}%3{RCu(e{5ze z`>DX%XhfsF9CP493{{L8WCj>T+j8Ns`>x6Zpp&_rq4Fyj8Y;Yr#OK+uXo<%~&OUBR zx`#7&1~Xf{Qoe`ZolQ@zwwA8+mm|G*r0=b2zPSbmUu8Q!qZ<+j7vLiWBN=PAw{b9| z^0d-=UMyJ>FJ6R{Y7p9z^;`K0*k3!#42;rJp!3>e405iu_;vW*`IpW1tWX=4v3n?X z$G`KkvQJYI9cf0k=eA2a-{u`eczLLvqo`K7xfR7n#||v-h2*(3uaHKMI$K&=8c{g0 zV+HCa`eT0p7cxVk3#_4Yta83CPflj-Qo#ygngV;oz(DsXu{(gp?=t6Y`LtaPj`7o% z>kYU_VhVp+g5-UQi(#R$RZnle-jMpw%e2+8oP! zpg?|f#$4JkrnV{o-HiWXYM=l9%}FmEOn4}CVd;FivAVuaNk$fyK@yT(&q}_&wvpGO zr(=Ytiol#l-a6e^UZJDUWPxRyY=4_H`Ac%<>Ecw`_ubuHhbdO9ENvYHH&?gJw?;9v z;z0|u%t|q0PoIAKR`8p(tB0KhwjUc2jfdN+$msm$;jM;|EpSDG_6r& zJ^<$CSG|^ksV7n~NG0>u(#A$0T}YG$+{FSND3L5$wZVHGclX5_d$P<|2iamr1UJ{` zj3}P+vi6rIC```w?1l(kc1|&>%xhUuN2AsNNuoj3XNmBW^`V{Va+B78I}H6`$Axdl z;(UYQ;lU*-)4Sr?rZdfl&RiUSt;Ul)M;9r5L;+^>6b!39scn$-O_(L=A9s9(oCQYF{l5Sf2C&Th>C>IF9sek*sMy3u(fm-zq^rUE7l0B$=F1nX|5uGdajUe~w;{n{-tU{;bM5cdrb4 zLeYxj2{VwO#Fc!>$x7tw;{#6Ozo;j%a(U5DqkVi;KE35i?)R~BO#^UcU`#e>Fx@@7 z4q!q49twQzQ5^^!McuhM1C8YzOez{0lsA3+44ZTn=W&F1uz3FNR|gdxIxoQgmj5VU z;;awsy9bAd<$5s6kZcnj%S=MIT{!A2z6zf;&4kEX(iZtD58?wsO^}_I1%$|9XaDpZ zOzn0p?URs~z1odr8(I+U#ax3QNj8v-gB^~KV@8S1B| zXA7mNNh$@w}AO6 zT<#g-*)*=IR(x*qlIDXVQNVgU4igM^|M6oKE)FhkZqvLAfkuDd(zaCO1q$Ea2z6Dp z*eEO<6d!33Uf!o{82tB}V6fQLrIx~nd>9q|tKdJS3<(RnKTc_F^1E_Xu*|PJjP(=5 zvsARUwz4W{4o0=HH}mH+ZqP9_jFP&iH$F}|`j}xvc8rS1ds+GikgSK&#C5pPF*j^# z<~~ooMj{~j99U|6-)8bQGtfLDaGg>x2(@>KH%5>GO(?Rt8SMlii=Q5eii?v>%hUnC zfGv`!k`PZ?{YmI0);BW~70g%~e2!BkXE-|=&E)suxSY`F$FOZFi{04Bm`_!%moRlq z3r+QhX_SMJk%OEcK&P8Odv_4|>C^N2Y|qW#U&mrtg7)41w&-UPR!da6xLmTNd2b&D z3p21@tt>B(2DRx!z20V%1yWmm^-viLxlz+l1z64j<&ob0L4~1=2?wiEB9@uR=F|rC8}7~eB6ZB4Bo~2 zMGm4eiGY9y5?Y?suFPvCS;8RL$D-m^6WZl;b+(lzmwXRe*k0nWZ&~K z?OsfBgemQV@EP%SYy|Z6>(@No-0D)MR_MJxtCON4B4|LHT{Zjc$rJoMQ7I_`o|`t{ zhSk(mQRyqp4)Cd%X*&*CjU9(ys>L39Pa`D-CdHpQ($J*Qf-^mh;Z-kX46%vQzi<@VdvQy&!$K!dA=Vy zfk&91O{Vc}Ck0&7peLU4nI;HbRxE8Q7DKMjLzd`=-U@{VCkZOawPH`VA~^CZ8E}GN zepJkK>n&!1jWDcz@yoEjoz5TgWyw}wkN9p%U`>=SQAcuiyUWp*9dpmkPJ?Y^S&(jfK9dhs2m!jo^uI_gQzO%mHKu;)H zlE<6Mnh)#Tv;A5RNSnYP9Q>o-hhcS$`2XZ)(EXhBnSuiFMkS>jOsEI zG7~ew={IuG@jEW{^tAf)@yC{9Yh5nrN%jcd z2cu{96S2#J?M5#kci|UQ;!x7`nCNJh!&WMN*aye1eYp&dOqn@<4#`8=<{+a@4ldO} zia0 zWa2!w+ALbU<1rV zjvyRdyq8F=tDbVTY$E~zlX@e964U1<+23pR#-<;kl>m+~CT4A|fIpB{IB?W%M%q`{msBT=#YN z_4U)@!KhD9w>Bcjx1-&|r-FZD0p~EV(koQ5f2)6QW3zd1P^-^VPWH&_hsW#`7JQpT zYVRuBuzzCYI>xa#md00uCRp>${N1ctqsf=tjt`OeM0R82UiM1f68Zk%7XpIm*{T@+ zohgaOJ3EUqBI!E{_<`*tStMr7*Fi9>o>wn}&={3}R+uKCeI0RfsWd(;s!Yo{oGvr{rzgHV5SQ~A-D(n4`$uwvf;Nq z@LLAt%H>e>a_;hSM=li-j=Uh0x?Ef>PK=4EU3PO76ciGwBFS@{x3sfZr0VNcexarY zPIXmPbyXkzmkFs!c*Ds2)zyTAgv+k3%MgWpoZf`cf4F%l-YT}XFI|70T%APN$nVeN zH$Pfm+EV%B&w}(7pJaIBKNG+?Ar-d__a^j)HW|YXb|!3u-^|V$&(D)NGvmX$hrV9o zcj8SvBEN&NiBSB(;ckXsO6Ci6;oUY~-*EP`=bG%QUlA%SP4dW~Vku{ z)t;d*VR9TWk<}%$4lddB(b6(CU98!e#`5bfq8TzbFbL&ea z+Z;hRI{tz=l9Vz>`V#YE{~zwg-aQMZGk8Sw84rg%{4(BVqmdF6y3G-X5x!;@`i7>K ztHDK>NtLlNfOQlZ%D9o=B*2pu^NG0>uNqo|)MhpT_48>krofnhz}VO-6YRw4(+yze ztpE`T)zuFwl|u+u5FvezWFQThc(SRxy89s5Tjw%VsF4j#V0<@Ze98vKEdq@3Gg!;u zo6IfZMJhgGcC>4#YN)6TDKCsI$M&O;AHVUXKlsI74;$XISX(Pl4`8Dt58>7V%`A7g ziva4KfY3-g&TsQB6BmJ+-8{XBKuTp4ufhl!;E&UGA*O+TrAKRnkq~~@uJ}%thWf1T z1H>L#`lA@5i9sErUS3{6eJEM1Gxr}i>XFnA)LLG{!=DLDjtyc9`J9CQNA=yiTAh!r zNoJOU(yrp_EeV9*(2BI(3K+^X3hAh*sOV{!+N8U#yS#n5_V`@4B#3Y3GpPE*?*ZCE zJZ3kYom<_#_V5v)u)ScLNKmc@baeC@F;xIkh3CL^HaAMd-P_8ytl=A{5^M_ zbF+j2^O=8kc3O7tLkT&zS!nCZkto3FJAt9j?#g)^+!Amr?#pu}B z*ytFPx1K*SDu&AV17=1)x%(R%$CZe%h)8X< zFgU@o_JlQqGUNpN#SH25vM@#N3x=WqL=tB?2W9+ih2y>8`~u>WF`W4TJ ziGG}kKIDTmB2=*ICu|87%hmklOLOq%`tU&RB~yrB-0cRB%u&&PPgl2rO5pM`y@FCt z_wV0b-L5n6!Qf3Bs;VMPLVbh>G5~ysuToDqgTMRQ$I_Hq(I8tw&5?)*<Ld_f42X%uNwyyZ8U-AUv;LgrZ^O!Q!oXZ;BKMQz*ffaK6((Hs zR_I1UMSDufKuLN-WJfp@5K@oIOyq-9P9l7S?V;&_fT8^@XyC@+CVDO7u!Vi7>Dpa! zx4IAkg)SSTqCg`S?_jgoxXr$lTmGL5u?bqdg)Y~J(!2KLJAklG=`djV^mz`R5FgVO z<6c{Y?=!#JmzXDk8E~-i+WeWw103M$YGm@x%w0o)EcgiaYUTBo zh*iu3B}I}JC(2g7Hh;h`GM$w_qYJQ#{syq%mRBffEZDwU9$no7i=X)0fb5@L}< zf{`qw{~QquCMd#+e+QIBDiIC@oTjG`adB^>Om1f8;ObDB?cWOM^`@piJ{Z?nalXI_ zvjl}c&Y^(^$VxrAeibEMcgta@1!iFSXusf}898`(ER!RG{I&%-)^!lQ)G+QgF57aAG>dI|Vczd(<4db+%Wn!3ud7^#ggXGz<5gyN$)& zZ#59V8k;E9Snsh~K4Fw$)|H6+mSganhT%{s$G;lsxhV4ii+0Y zF8gqTAQ&GX1CRovs`~P!AWhup7FuIgQU)z0gQmAb*HEvHl2Q+XHO_T8?I_ zSA>ZN>3uXWCnpammMGx0z9(TL z>esRxs?@OK1kt*vlx3hOkbTv$36qcO2hM>^*JQUCV)`7sup7BGjAv$XSj5MRA~5u> zjgeW51HtlT`iQ$okMRC0j-{vjRUn{)^3Wn z`z8Dc@%ye-q^+%OYPV{CdY0^KYF43cnxr@xaqO(v2u*McMZkzWiNC)ONlP;~lbP`y z#Gnkp=%XeT4*Uz|%VfvwT9@y1k+;vb(IL2Ayoq}j*cct;t}!rfmbgNSB3d)LYC4gu z6i7+?T`a>>feFAar~8PqybU=*BQnbQoPw=j$+?nMhv@AJDSd0g6uPU)*9<7@AvPAPOr^TPgd4vWhZtQmv0X{P9tpH{Q|sf%=`wyMK-=n zvg{=$2Aqne_V%f1m!a#QUT2{hU^Iu6S65d9OyQtuh_K_6_$UNg@7&W@sB%cuB}Q${^*3 zrJ$u^i>-ARfd^{Hs(94FKIlI_{kx?-`J3=AKcT9h_<8}B zeBF$M*ux1T_rR*3Avr)#K=5vNZ)E1bY!7#!EPJ}KyX(H60rKi5>fMHUc{lis>YR_3 z3kFj79*oQ51*MLTwT=sHp(cW>7fc<+!Nl~~t|a*{Bk1Bpl#UsOp=%?3A+KK0rJKD6pc50izhTq~0VJmZV??*xrjlw~Z6|qmzZ7`0>yKIrDM1%;w2x!>m3LzF2-lA5R z{dD_HkC*SIT#!>#@72NI77KEey3Wh{?=F5ERlhAAm$-?7iJVc@OTwq;ilYdN&u%Ub zl;&eu#-#VDLk?2epJjZ+0u}1mn1iNb&p^TLvR5fR5BC`6b0sAu$%v1@=ccZ@3JD6) zAB%{Jjfz?rBgGQVPW{Ai4~;+Wcm#VgpRr5(4I?RDr5E{qP`y-2%bLv=CB7WOB<)3F zuBK_@H#3(Dj%D+t6LDughOfZcX6)$5OV;EHRC!fZz5fwSQ0As1P6HJtCEozpO8WWR z;H?wazmR&AO*sTEwAe%k4F-vz@94+DAP>1JFaA*sOkPD!kYv69E^oAjsj>M$)6BbE zRo}Srcf{~_>`OPwLYd0&N;)&im)A)fl%4lrm6oR{)nPfpWQSz<1J$mUuhMtIO3`2& zn3jU2*dX;Cm^*M-aJyU+j zhI}bExA~mhu@(34<|@#R0^YgV`#83iu}t+&fEfUYqs?bk29TrX*1VIU2Z0y;u~ zkg`G-fcObKk$g@aU?~xS>SF1RMntGC*hS9ZGJqc(48o=O#fo+g^s0_qfws-Anx^0- z=Rx8Hs9gIkfpUS4E)$ls3GPo|uCTJ;;e`Q+gvt)U6ZB_tQrXVlKI>T}j+uaD%y!S8 zKf19Dz?G&a8DMm>s`6rp^fSBHF`6Tk_&yI!OG`gOs%PK(m^)`-wN6vE30a>D^jo)d>bd-4XcK_qFzeIG{L2VqW-#jc!J!{2IFQ`nI;6} zv-rjN5tu}tgY)$G_^8JGFTw|vk$pMaHVhxC*LkP*1nsK+IR zs3TfD$6=;~`c^nL(pe1UfNxP9o|`Xeaiykg8gfgEr~DXFw9wg*mrfx6eFbyOIN|P7 zRMZP)Wk?1;n@CtuNWvw|T83$LWc__5H8t#91ym*)Bn3pKh>7v>mkEi~d_)`M8V~D< zs$7+fD$ai~ytLccEPRZTlAkUv-sM>_e4~+w&<_#fY#s^0sQRgYn3b;fQxa@Nl4I$0 zb#!vCdCeE!Q7c{Fc)e?t8WeeQ5yco%amT^Le^ik{D+ZmchnPS+pjH_=h&jy7wTF;R zJ?YQNEWI~*jf7j8OqVOFwq=V~?NSdxp*ZhyoAa9U@-j2R03W3(-;ueFk}fmTnG9r* z)E$nb-{2YB z1^`g-vWGxIbfqF1+iZk~M})9CgkyVEkceYOQolGwe)41RG6GWh!xWsy06u?Rac#LS z+wx36JTMyH>RtebB1@2?*m>yh{OHL;W(7!OtT@Vin>kL;X{S6h{x~_eg9Z3yB*z_# z$y}sa;B#~TfvQHE*v7^oNts2Hg+%#Q5@_y-%D$IcJVDGEoVUTFudlhP=_ws)|rq1x)eb{9Yz`y7zPW~r|$Zmhsh{7;W7-dAmV|Fg8Vls zD}+QQ34588q~cLOGNd9ZBD#)}SmQGW)Dt`_K-AF_L)W;xy}iD_zqf+QjpeK2A~KjN z))Uh$?n0A+$0%g+tWwj%W%ayoSo>2p$#KPXQXu7j+S@t7P>j#>b5crQkID=u^E8`e z6hsv^H24E@&*h^vVuGm1l^roNf2L+_4rzbFqirjOV~_M=?o48Jx7IoyeMfywMoa6r zasEE_!!jBk0YMA7pBf$|?1fY#K3Tggq?b>9gIAryQT|!NdwZML8%}cqaS*DqG8!)SJMEt%nRkyM_>3>{qTarmX^%e(4XLto5wigW~T+iB3V6#DE8KEx=u0XJh=Pl828keu^$Db!gpc zK(Js#8zm=1^ax$MVh9bcvR@8>Y!ww1AjfF7>Rp%Usec6i=m1z}laqD5BqMGr3AZlW ztq-vF3k#22K~mnl4NK@UNVMsNj2{n1t)nMmC*!J{3)5_T0>1LW_%u<2X>n0$QP2ta zO@jF<2ZF+m+65{iLVUu`EdYa|DSMqa2Eoi3v-xA^ku5hi@S6qX-sDVAkIrDRntZdI zo$0MB+1cKHMDyiKdfEa=(;lD0*;KVRGz6!cni}Xk%3ClO{a~%Bi3Of^@#O=MY=>Cs zs2?5}7+K>9NJHSe2`ubpI6NB}?C&3*m0eFnd7L;nsE!r+t?*SAOn{RUU`-hXh*idB z^c6R8Czt~0y^;5z%@nuyj?N$hdSUFP&~Y}{bLm*?2nCgqNsH@R zNqYLupVEDLdT`$}Ha3PW{5D}QG2wG^YFYx95IC2%w)U4?{tV%dNM0&+F`OQscVlje z`>e_sp&HT4!R?Ap&Q3a&oVD%`gNr$KV$jk4 zH8E<)!KfiDXdaKu1(X98PiJj?yuEKOiQ{|51Xl4^ARPIks396YZO}giS(9S|0$tWg=k>itUI96BHsQF~yzW zwB{w!3}kUjK>g;Bp>T*IU>A7aBO8w%xboap^);`Yu?vjee4DL}LYSG~!Sp#gDNj)e zQ-Z8&EQ%&#W7QZU19GegeN4SbZ1~X9iHu2FaDsb-{~P?dh5Ut^ZbedkhmDzCsD0!{XIMr$tx2_FR+r@g(sMCDpUn#4ue z!8BF;1WS7!e75p z<`npUT7XXdH)by-o2W-@EN`EkDCSYIsPwmP4l5Nv!ez`Acy@7%n4XbTzk{(04S%|x zxtw{(Y#SX#1pdCL*l9a%A9z(f^8(y#>bTYa=;dh_Ps(6$h&|O|O9Xs`C+`YnB_KcC z`C9P(ahTHN*m8iT0_2gGH8%s5%9y^0D3Ci$0ZJ#X!P{5zzf72l%Sy}POz}E;)6aZ# zbk8mYHbwp~HW@U@86-=%K>8sZj|f6WL9ymits!g181c;2)$Qc?M0UHU4GQ8l)6&x3 zq-{}Ca~2n?D8FE1XRY`yqwZkD1Az)uR@!JcKBA!b5S!dg9Q^X0meIkPpE>v4_i9<$ zNEMZ`_wRwR4D3Y2Jg%#O%&WQiZcFH$M#eMm<*rBzkXuqyRVDFZcc#)pX|@&Iwxk38 z_xYGvI*AWp`!nZyJXs8qFaHC3o(xHVtb?C}H7zCvFugt8U30_1M(3S)!s7QlW)brN zu-NU<(H?RZx@`=LOJuu!kn-H8^&cJUi`=tzavIUbf^cL=xvigq0zo6Y8->cby?yC4 zO~@-qWjIq}V|O{|$zomV2KbDFNQ=W@q|j6EZL$wy;N5$1@9oJ*Qg&yBr@|j3Jg$ZP zr{?FgzOq(0`1;<_TlK(cqxJsf!e6N&bZ$7_n$v!#{~nad*cOadOwWn{jPq3 zWi$)aMc^=Q@w*Zv@IGS7s&6m`b|*Jaw??0{?LVb@V8r#d+{g@RR1Ywr{UQww$D(DO zL!ag?ah)wX|8;hvY9xczhmRou99nHCwaYd3(?8yJy4{AxDjD|X9ba7Z0GI{}edUiz4M<0y0h-nmnZRrtJn(Hj-KjBx z3COk+5YF{*#70wjiNE_lz_h6;#4!m3CUKAf{7LB_!FZ)ijnY4X*KfcEDVZzuCAJWu zh%NsF^byBO4SZM^+^PTl1NICc$oxAVR1Np{mzTG&@JF>jmrq0Xk*auj;1}J24#WVM zb*T-E9zRT#jf^rhCF@vOK?a*L#;RNK@NE!^-z&5ADrJw1LbR7)gCHab7AkLB=`^l8 za2Xfd+JbTkJOI}yc4TCw%TY9xAET)4GId4_GQ36XEzPxSEQaO0>7?&fCzi-SM|x6J zQUW-yINxyb*Boxmn3yts%}+C^%#@PE+9fz02wC9-|G%>(b9Dq*S7v5X0Vvq6hO7?0 zV2-Mv`@Gu7jummj=y+;kZ2XG{^&Im$kd7M5B`*iPIp~%Li>fb@lBx#fiW-+KkxG~! z*zXxp?3NEMyTqKX8R+?`wC)*SU`>(1QqE4#QNnP3`fO@cd%EjC()jAcot*nBCy(@w z^E+-%RaK(CBSg;@@QWQBzLz`2O-@P0D}xaF(h_<-o!75rSn?jI{JEfsgoQ?jjt-u1 zXL3d~Ev93NrQNSe?1Zh9P0~bk2vBtv+v1U6^|yj)`{gBC^;yqvcF0I^uZElE!p^2mzSBW zQI(0{OJ-kBNLpq9KCGXmXc_f0b{DDupE>g$KbRX2N1+4bg~iHB(OvLuk10PM6&@-E z>PI0woSZS3JYu!gXny*?|J0gDcd_G;@85$>IOO{jNJpET1d423on@rPa}Z+&u+-13 ztZZ#LIfTJn6)+-Vrnl{$fvpPoBYs`KsB3s2{YwP@6&2-+wLQ1A%;``0JdEO>v`*zi z^K=cO!p_FR#`?Rb$3mjcUuz702zv^o=mei$y10Nl26(}_xEvLs=iF+_ayc~wZ{aHu zQGPfuj!^Do6Xw}ZCADpn<&YZC$DmNFQRxYOTBg62@5`3&V_1f;p~xP6 z{@(`=AD6CE)f{0K35oc7qhzYcYD zK^w;YQSg0@XLW5&ZEbZmks2(Iz`>T0tsR8gH8g;oAcipRWTd4CLRcYP#^c3{C+rho zPyIwQ3+CT*>weC`$H~db{Z1|jXgjsAY9br6?E{i!${P2;(UK9&lebPS-5F3a#dK;j*Rg5;JvfP5jU9*FL_fweVuZq_$Rns z?BK#lb^{6%ppX%~PE7#va=gRmR%)<5wHacy{(*ij5UQ;3hIjN&)fbyLw&>axj4MO+ zN5yRYn;TwF%2Eq^w*EbYOX?cFW?Qb{E|^L2-6dB>OoX}aB+!5KR7CD4exY%m9Yywo zNZc^VJmOr@PwH0~Qv}Gyok!cUA=5KxTw~upzdj!r{X4FHv$Bz!86PF2_{_ob)f3^# zX;FkzcVw_tC2`ffhaN8dN>N84zA_b46(Yg#{= z2^ik*iSt~sSh*B4*TCV4EXtPqU}wYb+h8txkWG$P3-0s38Wbgc7muDd(psR*%mqQ8 z4>YG9GK>fi8eosUICOU-ef8eaQ3`%fn_CCAJbw4L^`>B~{p%MnOyc0;L@TZkvWXyC zbtZeRz>8PfzajF>|M)@gb-e`&5~=&k-w({nt-@CSqh(eatT8(1v{^i*T!aC&P!uk^ zTa{bV2>UGSwhc@U!};!*0OfOKT3Tl2moJLEfmHCDGaqm%>2Dx@>~8M|9XUFh@dvo) zfx)or%|8f}ef2My!-vryC|V?$S3oHyy9q^=k&IiiLmA^EuoQ!c1ws*qsH_Aa{LOUn z2}nxpJw* zsZ)dUXa;I(3LDAImO1DyhoqVZrFP>D4YTn4h-$qH(SGJ}r{zTICLQ4qMiF zxO?7m<3N{y^jtUW1VXk*i!5lq%^%R9Fd7W?L_FA`rIvM2rrzha{9ad+IoOz(!kjQV zlUI*!bDrE}lsdp3QZn7s!5z?@HDV>DPEkNn97+;7oQsDBVd+DwKVLW8U+XUQEaaMm z#DS)r(AI;@gsJf=8i9ecr!?33Ny1` zg@sO_TGhIOtqMGuxqiXe;gYZLbpe6zQU)iiIh`mMPJL+f=Q($q66`kBJ=SZSpDbtw zJIIN3_o#0?t2uy8!U4;0sg#VI!tIRdOFOfBIUJqK)E`adEq|(~sq;RqJoc!+K^Xj! zF<`FOV3JZ%;L;XlJxtL|HT_|0ZH=EtLhX?Mdu~4u#D#=_2UJ;Fdf*pu@jqVE$u^7>KLF6H@wGja*;rX<>IEH2uA~xhqpdzL$)WSjS4or|5&tBqG6MEn@cx$aDpn2Nb(lRnSiUaF}r@HCX z*{PlpQg%`wd3d@UH|y|P%~nmzl@vSyTH>Dkx*at=C&gH?at!Tnt33557_J3`O%BlTG)Xpl-&9wv5k(z*#?LRe1!uP4RNktv zjfD34cSim!XwU738X6i_R@W5>$$LbYa|Re$6OHnojmug26zhE9f6FmCd`G43su9I6 z4*yxo<`TaZhD|LN1wnnJP9$VFx`v&Q4E1wXoo}GI#@2bpd68EQCdK z%?TPhIuP+t(1bb|PG%pR{c413TuNycFi`7FK2;+Psv8Xdt9bNBNMbN&okNHzw30}G ziN8>rpBo$QqLOm&z;!cfb(O8Ht&O)x^46_GR}^;4d*9sNJ}*6;7wAq9JA)c%>FIy1 z#E2gaKK<(uaDhbb5b*IfHZ}l$q=nV!d`rG&20srEXS*luPSnitV*G1a^jri3!AELlmb8uKgb)1a;LBSnq1FoLl-!4#^JAf^?%>b#w~fIVDu8 zM?Sx1b@W6F69$+O?T*A7o>~UC%)%9wd&7`Lwy$terGh!aggx%=fq{LgpOm{0Tq=FpuoK{cIg=K2EYQ`SAwu`` zM{>B4>S=}9*{ZysFD1(*$g^;qBA>-|^&kKdZ>kJ@cQn%6-F+CqB#@+cwYM*G+x(uI zI={O-02zq_`jrb)Q(GWt0`>zAS{#CA5JvjN`atzzh=ZyLCwbZo;wG&ZXPu9cTxw66 zs6}1kMPa_>F;}z<+?a~>>X`WOBn~E;un%^wt^;PGkaRcb@pEi|gbj2L_R@Th2ODMQ z+F9`9p%tUsVx9mw2iUCn`SL$zLFN#Ygs)t=f<+Hewi2^Tsu4=HJOP0bOmyR0bzutI z#22ij_G7QkfVN|0S;S{b@o#Z%6V;c za7UhCfj0t6Hn2E=6=V@{uD!XsT~5{2<;y1*Sb7l$6O`iOB`Q$@kjePFaNQRp_>Hl_ zy|Lcn@)vP^1A`J?YKK$FeMLnfAt4X|0hx+E3kXX?mJ+b8+U6RLMk=?`@m!_iB)q(e zF%^R#9-!7204M~E3zEK|!tho1?=XykwVt7sqhD%AjtsvCOt!DF-(tVU$`znD{s={o zGnEs(PpiP}{aHI@U>WdCwAa2KFq|Np0jw8Sw3j#^pCwE&7cVB2lrVnw?j-9Q7*MgW zjP^~qLxHh;Cm$Xgi#wc87^zeS`W%$E>FVk#RB&?InVK^8^u!Td_mm6?$k={jCyuz# z7`8Qv3R_>&5hwQFU9+SMrq63bA1cx1mU)8YCV^H zGfpvV7^y)9SrkVu>gwrrX?=0b$>dHtc$B@^VFk90I)JQ;Oz~TDhqO!6w z#K$j_XMOv|Qu*pGf$`jV0Vc*@WeN9o9`%-77vy@warW#^Ej5SF_x!}fiJ|`Ul9>L} z{W;lL*_oI8Q8~R6l}|TZVV;Ce434$EeE=`vU~`? zOLxMoQ2U*H7Ze|xWsiQ4{fKvVOEyJ)2#q#)`gAu2jhh2l`{Un6h!=sMNNQA=szg{G zk_w?JDZye6wNO*Y$tcWhY)s}5GI+9W=s28Wdmbe#%tA}=%Xufc4}Ys=QjLF%=qScr zHaJkOMi;ROGEkR^Z#-CS71Ipx{u&a{AD1-}t~&3N=B{Y-KxSqhaQO5>=dQ{zpoalc!;##r7l+ zc982A5cowk==u#0y%wecy%uL-ekFKuug$~}?m@G$ZyFPQFB}Wfx4gW(?x*AfR{ft0 zb{-;w4w)TQg-BReDOBTJ{s$qKmlobBr!*3z7gd~^&U}}4NHb{@scl^rA$bhm8 z_bec&n%JDwWd@STNFo-!Ue0HL+#V=ho5%oRFOPa>Y9zkRz-;FTz|->a*&U+Qbw&>dC91kORU>T&aXPH+Q@?3<38 zh!2M0*mZdGomUoXHM-t3tY0|$mWAGR@KRTHvFV^p)b6gV(x~fGu^hj)NQB{B)%f3W zK9`AWb?UIX08n&w`bVvYh#yIVtyf&ctZ;ZI zI9L#ca~RM&>@grrXd8(W85`=6#p_~V$d$>aTTkgl-R1ggt(HmQpbvmU3&q8$~k{#NY2$j~5Yt>)RWgir))Sdx`at@~%nUt3c&Qhuadb49zCY+`n8QoI~oN zU@YFakyl*27KX%4XO}>VVg=C3-n@AObiR;~lQf59AL(-AkLnRsRQS%TxVuAgxs0I>bzY~}GmwabWmLO#xN0#Oh~EdoJNUQvr8RT1Fa!3F?zq4sKT?+#d^ zVr2{Hp0>84-QA*H=SLJT99rQPTsV7nW_5LSW<~@IKahrWpe))73kiAC!Euh92E$B6 z4b=`=RQSj%$_cY)CN77O+Y+|q5u7q4HJK`0_8AqS?Hf`aXp z(b&qI9H8fxr2sX&2oQvnq7=xW_!a}vQz+tF$5jUvfsqGwHVTAUtpo!B>^*!_{+eLV zo@_7jdWG&@SCqlfkrUN2+iJIoD$OZ+MLu3lmJNgoWdFX+=)lZMXtnj#L<5<=wXsZ1 zj5A{~@kE97H*2nY>lbp<{T}2v&ksaD-wD@}AVW`mLT&5V&W2v7oU=70xO#c)Gx>L#f`MBLC*!B!@P)&@K= z*map#F^o%z%S5(TVzn#I`N3{`M&ACN@rj5v!UyNyQRTXt#krr`4!2qa6Qsj`fZkws zc3_FlQRH!2TCq}HL4tY*dk_8iGvwyfT}{m|U{FK+5f*B8SVSGQaks&!2~KuqDMz868a=NF1s9j-b9LZnLQ)0l~q} zu9@~k!O!o0ii4+T?1DeYjDT3gJO4M7{MWBDF`l2BOPnMrKPQBT+wU$t1#UZFI%(JR zu^g;1Ut?n_9vParlhAi>n?Ui?p=u9mMwsSwQD85u-DhqC~7gh?ZBa{(Qume$t=5ONN0^Bf5q11Rl`MiLGNYvsm#bTFiGo%=H|**Bf)_K z+8zu)P#~S1eMP_B5HXnO~M=VbVrn!_?8bQrrpxV#T9IqGrie-9BE8T6-kk5vA}pjH?mJQEQ8KHHq)4KZbMF3%+8Ev_P!j4$>Fs(%5klPDLBC}Pt z5$f(5rqsHmmP*x>C||>>MD*}qkjp83RD8a8`4e(sfv9tpNLR{|bKqOmer0SPTa$$- zAr`MMz>vDY$Vl8BFz7i?f(VU_ZVN<_itYleb3OM|dBfd;=EB^<`fms1q_k9(78RWi z(oKYX+Og5maU>B>O~+ni!jL6sg!jR5?;-}h?Y9!h$sS(ybb$!y>j1Ax;5t=T>!+xz zq^MWvhiOuP4lX-6llN8HRwFO7Bb>v?Wc!P^Zb4arU1(FNob}koW)0w-jSZpFJrFA) zdEF>*NFcg9ahRIaZ}t&KU6i1h@igrT6d_4_vUlw)?Yr+NR6;8XlrHY4EG@3We-&;9 zSAogD4lHF$Tg&wYx!%BSLy^YnC*;o_Kxn%iB<^+(_9#@VcQfx!UI z^~msWy-sE(@&h3tb$zk15e!Wd*<=$um3WyEPmB}LXQ#sugag_5+r@KKTRE+4)?()SV+Q^PYWaW+ltvNWEZYW%Y#TY&Uxvl$N#j@-jDX z&NsdxDBx>2Xr`OkdgSW|6NyLm4i*;fCho=##(>(|wiZ~Kf;VQS= zrAQi(a5^57zuzS$CN?)W=WXTz9+fniRGU;w?suEc0SzKU_!CT25J7$a-d|Bs$sYF#34$AtSOgp7fWiOuS`_K$Fs2@Pd6U^NIClqdI_F)|z$*e!(%BifxysgA-iDk~ zQu7^9uq_+AgUyk3{CntJXQ4i1yfLm+XAz;%o`b47M6+8BJ^qb8?EZL3bp`cFcZ$RF zr^{6jR1f#yI~>xv1--|5vBa97kBF3r+@&AM7ndVIiA_Z0<>5PLS00FQDe#ACS+^R- zH7FK)uM@C27J|NB^T$w2x>igL_?$u6)XMEU<0lSL#J9E%O5^?erX4L+=)SnCPGQbP z=km6!b3$-{V|u4Wevp@a{JTc6cwq06>t$w_nJ(8oa=Xhl?T(T(heUVHo;~JO*F?NP zuPk=t;N)w{8RBvb(QEA>esFY82df{VHC{1*ix_N|TD=x(wayIH4?(p1FK8LS^d^q> zOztwHr>7?gKqPnZL^T751Y9yBY(P+yC7xeEfIqs;Wcv5-%B~NYpz*Ek?OmOkY6?V> zAuB3;X!FZUdK~?dW_g^=9Xt5z=X)1#z^vYxMi(N>LlGPj($?B;A@U9U+am3gKc34)G9uJh((WnMnH<%w+${3Wac#{X=&W`~_xo~j{Na<&sv;|~ zFNo?G0Uehb0(F+f{#2ZG^=`TFx@*Ju!6&+BbzopeZ=78JGjd2+aW|i?u=sT6?(7_U z=lyDIj5VvdttAnkzFu6c!S>8})HYm-(WJn)l26LoBU-AlM2J|dRZV zt~pT(SZbi>A8JGGzkBNuM=1_FV8(6T8ys8!etLarOUe33O815(9Ro1$- zjUMbf&1f;s62CB1hZrqaU|4Rg+TOs{_H7Qt&#kW@dg(m=+OCJLGb6V>QXlceeESA~ z_ctI8Yigc~yG2?F|B0eTeMi0}01Cj$ss_f3Tm)T@YJNWH zwvN@;)Hr`<7$2rE-N|QNsKrHh> zC>z4KAXRq!Y7ePTA4o`2z+4A}6e=oUt@wExCIY`)J>#iG(ft&PDUK69|B7xgpLqXN zw&q(WsK@kKEMg$Y2(}d5{iH{Kuoy5}z$Rgfivy1%B;;YpLsGNBUO_?O`OuKZwP?al z6vE0Yul%4ncyYFOylIMCSjQl9t!JOuI4$*!sIW||3gzg;6}-iaNUiB4Y^>77p!-Tp zPPjx-RM&|hO^Hd~6B-0UD-lf?_rffuf=H*x+0xz8ZQYJ#Z#7IiqM+tCT?%qyRb^WK(hc6tN*M3TMG*dF!lfK-^@Tkh|5Jq_izX> zC_Q(6eb3L&tgJ+#5Rf9Bb4bL*pa_eC$b+qn4kWS9)Puvod-pJbu;uUdj5zM*skBXG z$wk#|&P`K`5`E1o-}zPV9zzpy5B0o`HyXNr-;)YK)e#>dHm)I&b3`Tgb&{()`zlC3 zM^HdOae+>!0vu5@F~3#nU5~GJmni@bcC9@CMhM~3X@k=zdAmF8UOf0qF{Aup|1H0g z9VFG{dIzb1joVym2#sKB0)I^B0rUuIgxR(1unY>QlJ=w*&*>S-+;NkC z#AzlXtgrshDdchn7BLVHQ%_l1*5G-{3rHp()Y{9}kIn=eO@|kF3(rFWAezda*`i%JR^U*I7NkU+S9nuq33_{$Ucm$DX)7CO=cnyokE}dO(Bjn zH$NBBit90uIDK}Ri%938wox@14iVDu@UAmwcd4nVnSUegPG=Id<1}*y7^>+;k}#Ct z4dPU((>WlLrJeyvJnAX(nkcQS%ZrPvi>suho~o)bb#qg#D3DmXpP~nrtGc<8k}2dZ z=y-ToTJqf^u*Q4?hwUSYH3NC)*Lipjps?$Mp|Af7_4!XzGm#t|z!$*5dHwnj(4~@M za9T#+y00%SrN8_0`b*TK-%YvCzFcFq)fTFqVaTIP$1+W6%H3c;J;$M;eTPzz`D@Kf z?X$;DQ%Zj*0jf^xN-KGs9UsPCH98 z?dwm-6IU3et3wDGLbN>*moWg+Zg@P#yGXH)hU^KibhzJ{dsmmg-1n$Cewh^2Cf(JN z65x~mfH{8Ron;8P%*{Oyy8W2t|3#4h%M@rg&I1)N$g=$W z`P`t^<;gaaZC~>@?uVQyb(!nFXI@i0n*#9b?*_wfNZXuC8YL5)7rYpr+iw0|t1JP+ zyP8^>pm|02LHW%C6X7jl<58}FGZ#k_Vq@QOfT|w(0}d8)u=i@WVYG?L2veGng5+t7 z=9iBFdIPLH*dCo_0)7b8OL{=IeQXB<kF2n`NDO>MQSsz+xz*LxX=Sh7+)A^v1;hmfVUB{Zs06hLnU81(G0ojF6=kv?(;p1&q~ySW{Vy1wdz`U*Z874u7&&|#*&N}zCwE?NT1?cCHP7|1s0qHacDr(uK<-kv$?VfFk5-_N? z+q|?o=f0Z`=L?URiVAr6r*g-?koloExj6f7+W6#3=lHlADSM`AfRQeiv40(Y6V76C zF~E6BZl}AY<+zf)qpR!w#>VBi6C516r-whx7pl529z;~$;boPDdsba-MgL%Kd3kzS zCE@}TaKqwpM>hM39pe=;qT#DZ;-O$^EU&G}TIGoT>Xu1*X0M~s>g#lG4KzkC7&V{9 zFTD2nNvJsu1uxIvL#9)=A9G%!4xopO9XX04G?zGEaE3>AadIbLXpNt7-CPuTc#7%D z;EhUyyKR@fo{cH5cxUZCPbm6EmE#arjoa=(EC#Rxq7WDyT*+j=2@$oz4ZtjrghTGC zD`r?kdt7p~fF8#sY1nZ48zi@DAK^{JhoL*-q?B~jZ;`(H;RA`g2GCX60O{p_HaMWu zCOumaKXD&Hz^@|{@Z#9p@XdWRk~lreG5@-#_drET!1O)|zh6VEh3y#HXM^ z1&J%qpAW%Tp_&`Yep=DOVhO_XaQ;w-&V6;b2x7-SbzgP7X_w~b50UZ*E5hnaY-?*K z{F>=#JLy2+e!|w#BKSw}PshIvoX8HrKhQ@(@(+S^*RS(JY981!FiG<9C6ly!VMxFp zM3pKa5rtHte!R*9eHYB_SFdIv4GrvDP*mm_!~in^mIZ*3ph%KfulQ*|w81N-W0(Gw z^@`g#wKePgd(wYF8UQZ<74o+QfqR`M=kCd8t*fL_Dn?@UbZRWeOM8$0+cMSDz{?pP z8HQr&Qo|#V;|!ab%@7H=_^0j+5=1t?hrnuqM3nC1f5jn!4>KPmBo<_V>_LzR=EO-^ zSd-j`4-4TOQ&au-Qs(DQ7glIzt-%=lJ2^8;|2*v8_4BMDjRMYEs;Z}Cn8R@Q&p3Na z;td212)kQ?;6KJD1>gqHBlgY6+#l9@ie(_Z<6{s$d@iF605-sU_-i}<%g3)Lsp@{! z%3b1M9;0M-he(N2uTbmU+R4cEwAgEmPtHs>a&2&*n24YT;eYOUHEEB%;7}VOrX36{ z=Uqt`Wt>kwGE7uTCRt~;wpt`OCx5dm`F&%fKeXTS(!zWyBOPRv1Dde}^Vu>7(EopB zJ(YHoBL{=#{m*|JG(CJ;l=g0j{Nsl9Bf3z%MEJ)*Sa5K+e@aIR2m4$jS3Vc3lH$VA zS$=hluYL}P(?b;@IXdnih%8NUp+w+e6%Bi5efiB<<>j?AD!+z4b4A<{6IdaK|0 zv}?*YFhyQlTW7HojPZE?#5#xC&tO2M#>K5>GyFshKW>nE%KJs8j$7?h7>A>VfS9a1 zc(n(AKt1K<{7a|X^SF9Zn}V{kXxbliHC~9qx9^>Er==65C$o`jNB3ttJ%)|S~l?L)_}TUn5^%76AAl;S;7tT=}V_x z5p+MktOWUY9$kx0VA||_%a)ZC(DPieU#xfAe`TTFxziR=w%L zrH^)jCfOMO0h=|-%$n<~z{y^ymmUV=ORXDiJZO%B>pD)J47*JtfFW%Mq z-k`DQ2_j}WCn|gH(nBLt6OL!Z!1ZgiwgvdEpq-7<6u>W_nxK0XNGV}N(U=%Z5f*a{+n$yP z6ToOc8Qn$C_cpW-J1rf3MO8{gE0D{zZ(@2aSZytpo2ZU{0~J@w6Zfh>T>dKg z4;x<)kzNzsy_)-~eJzwCF`BA0{OBm)UU&bzIvW}m#(E;zy~?{r)LacQ^3oK$o>0E9 zV!Zr($!2f(X9bxIO=3#QI~%zt&tI^msoSgZfa9btt0`-2AZ?@08}sQe2IjT*@0A`t ze3CJv4@2{ZDS?+=`VkL%?$iDyi}cfv+9qoKsP{9{flmUh(j0o0N05{SaT%EWw8yt0 zKtxOH05ln4VOt=``7sKE)`M!L!z{=WO?~DI(8|7ig_|25SFd*Km>XZ>Px?cSzmocc z^zK#tq>{7uK0;Ll?#uPcmm#JIawDV-p5@t}Rs9ee5CD&WkQCIy*M%1F9Uea>*)A8D zl`WHvjlaJz7En^uuXGBapp1eANI+j2Ir%$bzC)vMLjF#DN=wvY80rOv_zb7wRJ3TnjAZ9#0Wfmh5aG6(9@~W7a>ZO3lduG#L0VVm$ zBzeU3lS-6e&u`c%-Bs#29;kmu@$Td12cFWOnYqI+-%Fu?A1-vli+|_RZQc}H(c8SQ zKs}FvzLa>MfniY;-(?#P3#eB~Ay?LS*MT5@$9d z0`d$aK7G0)2>;55Oz!~&_De^40#q1O(+Bd3p_l5RsQVRKQHqkE(qidNUnoDz-!(4 z-42jF_4+VQ>KxP!NV?)sh9g;A(3!y$fG0B!uR*Z83dmDGW`Rfo6|l2@{YrcaPr+(% zD61?i%)FziuK#b-)>iaSehMFf>EU%retvrT5*QP@8Wnq!3KnX7X9Jg(SSAPfvUMc?@GuSJ&KMtfh_3PaAJq zYMIOo4QaQH#Ny)W>i9;2%&)90h#aP;XIGN$f(Oh>uz~${Z*NB{`O#k%LyNzkXMVtF>*fpS1_^sU zt*I3CEsfw7nIDSuEwe3|i_@1QN+pzI@BFr|WBzpZ?iU!_mlzB}gJ-U@a)%3TZqBG+ zxE^-f-)hsyd-CR>BLPVuKp-z3(XlIaN4L`L*3Is$a=BOP2{c3d_tojEt~5BzYQ(*F znIAA*uu8M3^*Swi#4GUEg4yubKD(DQ?>>CkW)z6Ub|BmF^N6!*sYC}sf;5rr6eGmoMomacRBnsi4JuKW1oVPS-NKhH?HFsVc6B}Rn`{RO(5rtNX8+ua!s*r5 z-?@ADE_lj{isgX;mt-8@xGu2;I$sRG_Xh)ZI85#3pAzn2)RdLW+(q9ND%qCRNLb6g zG1P`Z=&-s}^yyPZrB{ynncsH2srx?%gmNNhWS ziohJAXp<>j^uxRpAs!eOv*aI?VHe-^dn%+r~f4>bR25W|jitDoq?{?Vh4P)8#sDk{N1 zZk6Es`?qm)@TkEPU0`LaUynR@_Pn|bryv5sEzZp?E-uCGUyoegTa8&6g_Y7eX8 zzbICxDk93htUc2L#pqn}B7<9^4%MkwuN6HsCzZ*Fy(2%e9rNeq&ey?bH)R4=aqo`z zHsfHd=_c$G_mPASR7X4>ABHHcc2NyHdpVKurBe2IO$wu3Re!8T2Mp3!%UdTMC#i4B z#IZ&()fN?)U-Wt=d3tdVK$^2ci8Rz~amTgounrdkp-RBl(r0TG-;12ira!gYJ+{X8 zz5GlG`TjTT*5b&VYCur(BMuzLd^_I%CO>pJwc5|e8Gm(PhKkx%280p_9 zy(@$i1Tk6JczM=BFDR@6WbK9%5Tg7=^$||*Bj@a;lge5V5qeJpp-`2jRt`v0a9_Ei zr(7w|wyH&+^XQeYlgbo;3)&VIpyMZ6Y^xqA2_p*P3gB2uNyTXI^`^vu8BrIps{K)U z)i(X~D5PAFYyg|wxFKHdJNy9nPBRnqhq0$cq9sm%o zT?=o139drQzrroL+~p6h!m*M|p7bRSJh{sQ;DUg_Pa(etn({3{fq}=5A5Xi_8QEOA zb~t6s%;p<R=jtjq^CTrwAz zl>!u4a4?P#QFH8+VNJeQvpqlEks#x-XJxf=c(5HX-vM^@uY#Bij^GfP8L*@kWNm8lw`0a zg#y5!rm6}S$^6WW91-5Cc`SGNd=Sv~7vH#bBH#_Xv!??ZreH-}-gO7Dbv)^dO61|s zu6qMl3_u`nZ`mC>c<;i|L1iT9Oq|A1!>dr5OL@=0xB?pp6ac9H-?!r9;!>0d|NSW+ zYdlCUqRSkqG5@|qQ4vkjYW_dJ3sgJ^7Ueta{O1T1@VoKR(Ti(q0@ts(eB!K^`7ePR5v~H zaH?C~BNb!xq9>UhU+Sst{-xL=&&3pFN zVFTnL+DsT`I?3xz3gAG@A&sEuh0)B?Tn5Ti=lpDFn~5+>hbP+EBT1D1`tAJt~UHSLg@LyDDw0t;;3*!wm6*~p49o14)f zrwFm^Kl?xk089v4?)E`hB9cswr}h-CXL*<09G@Xt1=k1{OMa`&hv78dL+@4IGtl-x z*~jd93=nsAc1B}`&ei6S!Totl58X)`sx(3&8mEbVBD8Q_<4dOG`~D_MP#BGSfgV&mg!KW`t=54zdWX0J@xQBwY(ziaC*I)D%w z90&RQRpH>UbzvSuBz|Od58AOrfGnutCKw73oe&c~E0+qYmJRad1H% zsHyd=#RquDV^UX1wa8#=YMP&$8>OJXEF{S{-1_q7YZslt)jgP@;Fyv-qoc87HuD_s ze(x$KhtuY1XPxqY;po`&@@7aG&>}~oL{erMP#AA-VT`)P}Q z+eiw@*TZrBUfeQ{~&pSaI`7P}&wJ3AssR!Ih% ztOnzxE4XA}{AdWumb}ee*$v7cvI3`6U#sjkN1)W{{iFl#2Pq)exaDk<_v~Q)fin$3 zE3dk`{Tw)qH)Unp5jq`LY%u82(HRbVxOR7U7Eze7q_NlHOKeQDb>dlELHDTCs$pW8 zZD2`<&2Q|c`OHv({ZRTMT}#KgA$!w~##el$v%;St{c>Z zh~RxS?EVj=;m38WlG!%v)l>e{FjeK|nHw*?8xbqS8NFcGJ=-}Wk2^W)*@3Kp@d$G% zd|dN!RBdFjOI?9|N94uodqgk<;;Se5-VS%-iZnAmtGv_E)!hk>9a(iD9NUO96=A6o zlQCz^j+PP=n|rNUcyc}qLbPg(TXpAb_OARAT}j1BBPz6b9!-{wW+gOeU5CV@t|eOB z2A{sovnxzp{9j~q&1M>ZvmO`RM@wdWGuq8@tFE4gR*(XjV&!F|i;Ebi8ZNn-=pXXX zKqQrYb*gN1$0Ag7NY*n^u`HFqc8a=a-TbrZ<0puTZwH=u$EUO-{&bNVJ$OpL1h&wF zyBOf?Y+s;}vvzlX*yTXbCM?`GxHQSwGC-6y_kk9Pt@E^QyoU7f7+E$g9I}e#@+ul5 z%_LLA>fC472{Skk>rV{0rO=!jKbU-q^F~(TOZp6Kqvckk_a}U>*gO^Q;0&yre5R{= z2gl*V^|u{R3t)S1R1#K(F!MXZdhaT{i9$WMf%LeIL1cVOBCfwF9)B&99+BD@7ozex zLlMPu0C`bDLPB;M{fLHfuh-wMj%irBr>Vr3@0HC}nEJ&85m80j*Rm&yo@w8mbhq)a z$y%+W_GU=PA;Rb=T*~TO5^d{@>{v>uz-o95Gmh?gHo=m$HIk@%>KnKI=k=oETDEtN z5rTNu5Jhd(1bd?Jyk*}u-}fWsFhP2~zg_Mueqfv+c@ibM9tKV7KW0;+odq+2jlGA# z8-}KF_6y>8EG7`volr3weq7m_PT>4<#qy=6j?kay_uKO~-H|#()2G6xnsk-!|M;6c zTVHhG+MmFuekmZmrX#hW@w(k&G+L&^KZE&^lFvXU zh4x)hO82%D3K$w-SSp3Z+^X;>CZKpUH=pCTDG69xWkAU6LMC8hyoDtQYwEn4}9!&R!@6iYL@MY*^Bo0v%m z0Mo+Dm>wG|)Ff*_fjUY|!El4L(lwuT(Kg?a_c{%Tz=29tGO~@|{?VtxDsgeMx6AmD z#Mh`5tJ;bDIu87{ippo#R?m&H719j=BLAQJ_bG19BL+;-<7s;=ly*G`R2aJ8LLjZ( zEu76JsqUVV+7p|{Z~BUyZF&<%_Tr<7xYpW^{4heAWi;9o%{>s@0a{KclU8B&=JwKlQMBam-lU(A=P18YhvN}q zd%LQmjueuXbY30JWK{l2p6{4Tg~*UeF$7l^m(n|Lo72oQoQEduDvPweX2~$!h>$<7 z+v3Wz1>|HAALlO%yHwx{K%gBTnK4?NDu3`f0>u5md97ZO<`X#EI5^*0WznY11Cw?u zD=T&di{Sk=1TZJzip<>J-rO23PP#}I#x7psntpP68fGU3 z4M9)m8y^?PEY68!Bx?6SXSr8PWrN{dyL__4Qx&I#DJh*mq_$hKMrn_qrQ)o-8Vwo1 z9&B!k$t}P5+gH~tM^Sa3rT-(Pmkna#G@a) ze6`lHX?m6{g{E`hcbph;ByrJcRQA)zF>Eq1FYgB7@vrrS42Ho~7fB-4gmHd6)fI^QP^E(Y?_ZI|q+;e36>f zr`phs7DZd)Be04Nza&``s?M)oHFVq}8?C;Sllsu{Vo$W&Nny9u%2fSxIZB(m3M&O~ z#%%U=OUufXi+%}W{`@;hE+iE-HJ$8++OzXfU)W_Hm!+I`(4!*E@4pkn&UC}DmyleM z-CVEeN6eDBdku27aZG$UV)yb5$@!5eeS?&Y%Z8vAwhs|mhMbY(ldBW-d;?h5t=mM@EsqE8;KYla?gK+=;E%w{oOH^s$kNzER zmz%_7STt(QoZ%QGH?Fi$LP%&skD3@zqREH+W#p0iEq;M(=QW%GS#87G57h0O8zdKy z%s&MB7r?IB$yWsueiD3xx#+N+lLna3kyPz?lOV4xlf!!OET*nduZTfItYSfW6L3S1 z-f2uZT+8VuuhIYCfs5CH?}U1$c>?dpdHcVIb{R#wx{6~6w>LOoz0{XI-p^C=?Qsf$I*kp*iR>kXf2q7kr)*HB*O|T+OYPm~Pr0 zRBiA5Y%4CqzoPauz2H{-68pR0NNtA^P6tF82_TPZ^y&Iph$L(4>z~J5CnG~&vDz@m z8GyrM(}*NJq>vD6TaR>i=k$;PpOWV%*mfV*^^GwcKlyyc1^d(a z{`QY>iX(L@G3=1}5@Rxe=wi}yLF=-tnHe*xJ{LbsqAI4%wfIrB#5z)ad4<)xORUnS@qB%uAfN96q~LWueywMfBgHD z_dwtlw7O=-Q#&1M+<$)4kmE49XAu{@r<5}Qazp54uU&k_cQSArZO~trCho3SH@0I! z(@tWu38ZJ-RK+cZ9-10xv7)8wp2|1nIGG)URh~6C@2hQss$68X^>9bH@%~KzuE=m6 z2W8%0A0{Mh6I-;2{~ujn9uL+2|F2S%ij*aak~N{ozD$W^4AJzv}NpmE)zTqK+ zHF;*L93 z?uWp0SV?#WS&c!4NTkbA){ScHpKM@YHf$Y?W0;U8_lv9&4Yo^1vSUO z75o<3oVrC%hS(2|%SdyRLGY7*Yy4~&Sj)SS%P8GxGbEBIvat&an#@YAI&d(ccH;?X z(n07t zfup?s+MSTWHO|i`8Q$~nOaxX-ZC2jXu&|fomAS(<-8%peuG~Kp({8!Hnqb0nzhw*X zt$+@X@^+;4BPcX6K15+Y^uV#5k1ZO3XeC39yF}&Wd=c0$FKZEt!P$53h)GOD{qqoU zUd^P7{^a{;`q9%n?)=1T)14{kk~~~ceEzNL6G`jB622b}f`xdm3!H&y)Xlf7$BxO( zU;1Pr5l25>zLUzA#ZzMMr$>Sq{|f}FT*eFHRPFXk7qaDIxrY)$o1do~Sdd+K z;88wE1~|T(>?1*TkYF!`tgLoZoW2a+{6|8-^D}pVLMTTkr%>?mEJK&d*c8zPdNg9% zNx_dGNzlvd=UnmsvekL(=ORp0FA&=i51kDBXsP2y;N4B8UcH1zmC9Ea_3^fpzFv#k@nZJ$J z6zPK$EsBf*F10n*H$VWrGobyUY1?9PeRUl&`ZtTo$lRQF5>$k?w6$Tp33i|#KQod@B=+qkk&s^H3Dqg=T2mwmd%tf1f*RE3kfAr0M-dS*eXiq zGg!4=vIzLNbwMd;q`*r@HbQ`X3X(Rq*Zbt&83-g_gu}q|0IKOITczqfIb1X04LGFx4k_Oh1C-tqZ2PLPm z4mAjfcK|H-d+nD&QW;!<0T|`Zc{afH4HBY~pkEXGVTS05m;+wqTQ)YAkYBr6TMnZu zJlmHGcjf&0)Ri$a1Y}SFclxJKtfa>j!1<74gn*Ki?9J8W*#Or zLIAGaB4dXO#z>a8bnRKS&XLq6@6yN@jfAE|5sFtG1QDKkkmH(vf=ccCg>dBzHd*M_ zYu8lgro^4w`+K4qtbffINXLwBntc2g5sDCvun>*n#Y}8(57mx(m7f@AkU+tAA_j3J zp^p=SNj@aMHpaDQ4ShV?QKI`JUK(Yb>t8!oE0O2KuVG=Edavu_R}q!lrf(=@+>_IC zyG)nV{Sp1*xB<_mcg?rnl@<*MqzE%}$%ErerK)uC znzLHonMhGxBEM5_WTQ?=*(2^#r})w{Z6+q08j@6qdl?%57aENLVlQIXnS8SYS2+8K zW^(|SW=85oV6%U(kwPFsM+c7R4lVL36JcEi*;3ju=vfqLVd@r4%yqpyaQ17?aBQa4 zYuC)kX1&&_6S1>b~Cb)q)=c~f?#6F&vjJZ?E!DJ$j?_t`KTsPh7#XN+cfyv#c z8aG3rN?&4Offz8Nu97@nWQuy-`GH4oLBn7M(GXt?RI zf%_(G!=cLUY_-dYVtmB=b3b{~X#bsUKd2-I=8ds~PxI03qx$}b%P6IdMgAEo7dQeI z6u+FJgI(&*K`IZTd!j#tt{Dis6J^Ml_C}COX>Xe8XWe1!=Z7}ppE=(*DtFb`Z z0@+5jPHywC66L5RV@$yTBXW?-+fI*`--7UH*??vU%qQttTNkC;tCn>glO3%-tVp$L zHiLc-MW{8Njl`;m`0c0LDjS7f{A8jamJ{##*I%G^?|v+?n8>p8!!tv%r;#OJKbkMT zAhu~Qik?1C7NpHu-{8bno` z20j^KlyOv?Egj$ZG7^t>4G(~H9sPccEbY+e(^V_G4C;`auMJ*(`LZ>ae3M+T3*m~1 zPh0!}D$dvHCvA{K#PHuu2R9Xz$%0JXhzOVKE^zr{s#hxP`#ksvF;k>9Ba3v8qe7%% zp|Ynw56Hz{y=uHX1@DZq zs@1;}o|t%x`}MWJjrmQ8gcs07-Xgez>gu2i!XY8%q&6T@1KAUIAs}a6OjPtO^0pmQ z%Ck^%7!?e7@2P(Et>M5=w{{RxV+T%&vC*UF-V`=(umpRk!RYul%vArTcJ z-Nx#lt{h1kSs{Co+cCnL+#wFPrOmomAC3mU9u1v56q6aua=9i2mX#BUl^5oCWYPEczZ3x4Fc zp6}gqUvd3P=T3YjbF-wYQl(F~{?QXH=E_kFK?+t#B`QdoWq+H564P&glUcM|V*eQS zxVKB`Q)FJ~a8Y4fUVUv{dHoWxw8yx{EWN0Q(O)hrZH(I_u=MA7lcUQ-FLF1s7(f|H z9>6gN!?ekuHa!W4l(!?fOlQQ?m6Or-&Gy2^H!E>1B5t@tuD7|`!*AQ=K4f9?v6@7r z{V%(WXzN9T{E&g4w}Se0T}VExOa{0KOR`ENs`%3-Uwp@R0vd?G+l$c7Clk{lu zLbZUCGkq&4hHY$5dZ(%LOTRg?$p#mBbHfxInPl*q&gX7&6;>^!sbP03haH?YH!2=y zL7W(FV6h!ZEPw&Wlnu>T0OsPi^)#d{9WI?i^{DIlBIR1 za0-MM+NP%DO~If-zrUi`9>-J#5fwD&?zf z5@PFu4lLN%Rj*jQm-eGntJI)U8Is#=i$luF>3;iN@g~ToUoK_(z1i7lGx7c4WJM?| z=bJh`z_o4albp8PwAmhsyuw6_0Vv-+1~yY%WOz+=j}2f_wvK};xY+Z!>`>)^&dW%@w7KfGd*Hqewk1@ zb|fwjQ*149-Z}wpscf%6-*zc!-Bx+Z8}?{f^YQ9ll7zg5wPO2S=F8%H`04 z+Cig?lBLHLb6|OEI<61qWOQngPcV`{*qiolc(OU?uH6yE>9N)Olh(ZeOZDc}E%_9j z{Kif)ZoPD^Rccha;eH-v9X(`vJtIyQPTTpDhdx2!+n3^D+ieKg!$G!WrmyZxJFMtp;40CtjL(?w==rJES>9~EK1PFJ5B5&- zsg8}kT(MJmrQlzfb2Kw0|a< z2@4C7WsJNfHVxJ$9M4sTIlb+RjM2p`(UB8%6BoB3|A9pMEk6=IgDM8$t5NS2k+3uCP&ss7|i*4 zI4if^7)CVD93e>p=PH29dC~}9;(1AsG0dct1GP>tI|$Ehfo17`z(V#}{5k_!y*E|- z0_^;{x|eXTjclh&S6y#e$k<{`(C62PkN#hXLP239TX&eWMq2M*cLMwMZb`=>*Pd?B zQ+)gd>vS#@Ub$8XH$B37y@NGTTBUiN`cs7%)9Yualwe`J6No}lQL_S(vG?#6Hjap~ zIyww~W(#Sks`f{fJ5(6Qs`?>oIacFcTd%;-=D!li7|0;%ZX5~t<}h#A;`-kLXX!z110|;G>mWIUX7Ce2+4f^a3&MRLrv+BgvBrIGOFX+Vo8}eG zNK5L)#;tHntYKDyB?0NVi#naraA6m+IKuevig}PhesYCTYU8(Wv<*hc*~M*Z#s=d| z>CpTgwGI97Oa1HE9twIp@W*Zg0dSwKi6<9aU{0A|ASVpR2fMjVPEGw~XFa=Y34GG{ zZF_`P6L0#oANT~w^6#p1sSH@1QLf)j?HhchOQQ+>$Cmv+Lp${Nk z$}j7cR@L|57SJ45$Pcy+K5hk3gpW@@^OeRw<|?hSZyHQ#EdWv=lTsmIz(uBsbRajH zF)+{X9ri;&CuvYziKU=y=>l~%YpZBy@w=%`{dY{zeIyLDp8GGVte8WHt4{z!G3^Ud zNoKT|TEtv}gSlX|wcVKj`8|P@+Yu(EV1j^Z1KMs97I=iC=L{Es%hbB#n_lYvQ<_os zKTAvn10hp@SP7BS5_S?Foc@Nd~B0rl*592gNun?(m z)p5hW=Gnp3wV#_AG#YlpUsgcwv7n~WHQk6E+qgLy;O5cq-)qatKmjU^CKmBEl<7QM zq|9zqSo*Z1knj8lL62Nk$7iYySM+)5$-SroNB@ETe&VP6WsA6Oj3Q>q(a+Cna$!r( zn6he$%;@l~m?P5xn8nW!3xM~aS_qXhN`s)o&kwg;dBr_kwQAePw3W!7Em$7I5cT}D zp6jU%ic#YVkRRZ|1O4H5U&3NSfhe;3Tn=<03G0jauYz7{!1M1bYv6I}qQ!OWkT2%r zM~i~~o?dKB;d(^!#IW`DnYA4gBj7+1d5sOc`Wn|5*+_UL1pM43E!r^#3pC#`MVJe^ zBZxP6T*9*$zW$e+j|eTju|s`~MP7g!%5TPK_?bG-_7%FC3c3p1ym=Fp=7K00xChn+ zM!*dV@=JS$vKyP5_j7=BwyZi~ugu8ALwE0wB8^R26068sLOLV_5TK)DRm>erF+|Tw zvzC_{p?EP)D_E1oSR)(j6{!)-?(Hz#XfQCw@^mvmWd-G;O3?zUdtiCUeYY#B8A(`+ z!?VzD1;L+p%uG)+kdA;5V(_RW4UUA&nxg*eH+}2Oo_B^28HThqitKa0ZNAP|2&O?)(%|0-Po= zNRR?79dYvC=LjGnqfL<-Q?rBW^ONlD3~&x1-#~u?#VTNw{$vJCn6DnF z{@>4X9zA;77tgo`)e7l=od z!+91^6^QRp(fSANq6$3x+36rKP^F}<1LGc$Cj!%vh0%UE#uCvuI{Xu(eRR?(zug_~ zLxu7ZLoe%v9p4l%FKIzr=g7Lu_r=J}EwU@*GRXmxkT`~bpw4aS-N;@@ApmW-@hWM$ zm$OzW8O<&PM|uTMcPJ&FA6_?-p~A9U%tQc9lw{U5(Bk|NaN?cJ?@5qeYlGfo#Qvzn zrXbJie=G|KVCI(*+s1sr_BACBqu$5_vNs;r1YObnF2G<+C0+BMtvA;R=R?Q^#+&sP>slQlz(&LP8oiK07b z%>u6>ha(~!92_o+FJj-_VohyuEXkXld}@j*o9bRPhVH`{cGKh$4ozu2@p0kBpiDGu z*!{F~X2@2yZEl{g-D%F0vPWp~ao0{1C#U`n$5GB1?2o&?w>z=K$sEWm_6o(`)X=$i zsprWDV08hgdmC7x^GdG*hUMmaLs;y=6modD&O9CV(4NDZI)^M=z22FcKpU#>L>zDe zM$EiYtYgcJYc9aPp0kODktgSlmne(@fqZ*9K-2G4qzLIlZS=}JC-zSFZp)fdl zfH(uxwFw&L(k4T>N1fty)70(j9XL!pXA$+KWJI#kE1zc|HN1H8$$)(?hqfZ}c?Y^n zOdmOn=E3@*jPPxh#LcOG>K$766ajD%#Pp}KnB&nYP#|gs2PhdWG)QxLNZI)+Z8L9z z3Q|!;m8(*{>F*=9r?1wfymQUIEO|)}^(ct!dAcHaCm~S?+du@{)xor9RAFd@zD~CI z-+|`aSZdbb$+&jRu%oVM(16ui6zVeoDh>KG;w#q#YOm;J5tt=-KZt8L>F#4MOz_~* zEbt*<`r?>OWQ&*EizRe?a;}i~k?0Su1$+CPpPWkczgNkrNt}t!-0j!|ghYJ}r_*<~ z=xG(bWL@2FFggd>#br8&I@2=h#VP`f#;9jgKF8izNt!_|S6z)=lc$3?CuI|~7$M8A z4SKoURGc|T5z8!#{YVq_3D>>4WWXpudwwC@XAY&8i+0zi2Dde$|7?+3oye`;!Md!} zm~9hd@M+`zCy(KhXd3m`u3lCb?`f@mVF%tBI>+P@a2sYJ6BeX%q@6TawvqS^k!#mx zCU+9N{V2de>c`pLNlOA*W_nG}H!eKiYo#Ri_nm7I$Fn#z_IXgxTL{3!DECE5kg>p%w z-j?in)@prr4~HG?R=SYK%4UQdK zhz-Cy1_;P*-^Tj4B9)(^IGvaVcbHQN@u4GhA+ml(TRD$1qIhyN?iIL?4L1%ZHL7<% z0DArG9~If=i~t5RMJsq!Who}|>9wNzIdlr5BYld~ZIae|8W~f}`stb3yHd;QH!*08F0Y;8br>nD}#~mb!_-a=0L!5k8$JNTL z4S|8#|B5fRfa}y@PS)@1^gI7jl42X^%_R#pwGa@-?&zx7W4H(h;GIhHv;^q3E!cKf z`K9?y^c%-$tQz1)eDWeKN+OGsMae9yr|l9l`u2}5Rlw)MVJWQ{QS_?J{l@&zCi4;3b1JHpvV;+tO{XvgJhNs z>p#O9A>{SW0fvp-z0Zlre*x0DIqcF>tqjF$Or!h!ucnp;EU4}wq z2N3r0eA{goR4GfOh3VRAqJ3}86_nOASzn;m*LKs7=wUnYSk-O?r%Q< zRAk%ipsIdABzjrkv_=S^wpUn0gg*5)v~d9$jRRd~1n~mE&3tGZmE?8XG0}6FR(ex# z*5`{1zo+UezvNMvl=Q>fwv|Xve&F=@58N?BKpvk6=H zp>aINWLislH9Rql(yDptE^)ffQHL&k93s|h*N2hu;pnkB>tcEOyYuPL79+N}TK2rj z&o`Ff4`Rx$Y+Gh#@1A}fOn-W>eo`wKGEJwTC@Ageq8Fz8a5!MKtJ(m(MNY zJl#JBsDB-F%!qpZnwHF|PcE+jq~}tzQ;B9Zc#Ecn=haUY#YPxj&P<|LcUwQe2B8`SJIc~6?;bJwFlVS(_$3}@ZE{M7#E-NFFPD;1?jLy*^rkoN)2( zzd&Ds1_m|rmu5NfI9D06EgAQ*@@+XWG%IEiJ%=s5S`>F=RCWW``WgJ$@fZH_V0c{n zxn+Hdd9A(>MRkAIFF^~ML;~X&^q2;*XJl#6W}&gH ziNuVy`53&1tu1W?ypw8YAh2^1T>y&TU;sCo+_;6cm4&u6y19$6lCA=%@`*Y0PpF!J zdqG4xMRKnpm3w!m2%r#rrwtyu@JE7j#vqlIfU^noX$$DUs0vuIULLqHG$vxX@$Qzv zX%!)p3VpQ*%-1}&@`()HM=#PE#aB%d2M6fMnAr&I6-}WzZ|7Ttwr87&bj7rK9T|5V zi)s4=`FmbuX-g&SRzh0+>dC8X1GcghztF=A+eX~Ft@q+ECYtp_0CWp_WWknJ>`4bI zWA|n0Q&@p?4W`tNUOvrfRs?0h(M3V~Yr#OGx=VF0llq6Z+98{f)ZpOwSEn)|hO>X> z0sLgGOhx1^RV!S-R*eN46-P())RzC~a9XRW)hVZT(H*L7H)`J#w3!`i~W-BbxEb>XxXr4JRWgKiQ0cE%09g? zNV>`j!-3M9@p^dWYk>r!dgzoo<~XEUxZERch+9vW_+U5K}~*8tMW%=*<>v z5AMQ%&ziGi%AWTV5Cx*2*h!8 zI#x2L^cMtrlN)ZaUe1j{7lGVJeIU3MW;1k*JSwJV3 z%ufAQ<{OKfd6`-o`@0HJpLg?;4;+1}EjYUegr7sne=W(uEkOaHXCYz0$ACC}@kuGm z)ddtEx2*JAA~yNSaU_Qa_^wm!LJZ#(v3CIp`iKC0sLFTwiq3b@bo?ATOx@krxEQa9(EF%SdXV zwy}Y?Q?qp|zf=x$IC1l63tBzP(avh5aEr`lET^~_>G7O0oilQS;(N~<=+2L3ELQ?e zC~Tj3I#yge%akrdX4(3_M1;P?!3yW&?3CEHJp~%ZuVTu|WCWy-0$GaL!EwO57raKp zAl2xjYyyriis@tB@y|#*UJG~(Pf&D_nz}mRWCDZ^fa674R@Po34pfJ402#9q67ujNaB7o~Jpe3d zwW!bTGb(5=hu8Uibp^6!(SrpE(Fb=q9s+=nfr_GwO%R8L1*=Ibz)%PrX~=;kb);1_ zRYQ6~AMsg7>lzwlMgz}t%cqxRj9@|bgW1tO+IENa&bk7|x3?S{>~}687wZ9%kB`OL zp7xsoy$O-c9YNV@m4z)P)RZN^xGkbVus=lB17zxxA;BRr#Om2NX99;G;`V603qsqx zRZ>9vA+c8Jl4O_i0jze3K{7~IV>tK=FZB0=^fwMpPI}62(dLcA=}{j;E~h4Ifb()X zYP=9X?&=bxVUXdq+TRR?k0L6kBpGjv-j33co}mXB`LHW$gYhkk%j# z_V5Y{02VC(LIK48^CIkLs5Jf}fC~TPG93@V?Aisk<>uWzh#}$LBZ~=Acb?D<5Q2Si zCDs}|Vfv4l1Mq&NUB}q?^xo=16fX~jZpz`TqZezu1#9X|Q)ppKAyA$6ANxTJK{X;S zE-s6hd^5WZfZw=8qyK%9LOmI~xA_!wos1(OAB&$8y~#cTNK?qgB~D*4Fyzk^(ojEf zw*@kSU-W_l%@@zzD=4+}t=~B4W99c|+3G3m-RGuGU+m-2TgaS2b3us7=t@*!NOEAZ zqNRA@mHQVMHFmZiIS|Ykd4vBtf1@fRgY<626MM6(NoLvH+!jI;45orV$ZSWCT3hEv z9UY6$BgCZdrb7LwaK15!FDYK?MPJHa(9VlJ5e#S!1FyP!l2LZw*545;9B|^16QQ($ z-=w1Q&L5N}H#^?Z`x#2hH6HnHWS(0U(+Z(Ecm9xMJv)%}D!%aqtzx9!JKRTMX2oR5 z_p73`Zh6oY)Yd{_nTew}IW2r#Ta!m=EY`+yn^<22E;I4u6&INKZljuBN)Ebkz1Hgh z4b{@&R(6^mb{GbXclBY%>7KznYE~o3h{^#XwN7#`nlJ->d#=s!a5_^rOQ@~wMDYG6 z4YPAn*EM)Q609Mztcp+hEEw{MklAu?l?3jY`K#6pN|()_7-tDLo}}ec8Jshn4`i5} zh$=CPo<%KuX=Ac;Q{{N!J@Kt!wzXmTN6*W{k6%XUwV8`3yyTu(W7%nOU`qTLRkgO1 z)3Mitymnh*bSYzqJaU`NUcMg;sl%F2_1?GVDh!QbVPDNR&E4w8rtc45n>bX-N5UoN z9erw8dD6eS3v<5}A|?3N@~gh&zTIPU82`8E9 zPb;`ifB#XV;>WD6+92)-on+{_YhJs9YU|-@U4RkdT?v;dy?pbA<22bsr4r^8oj7!~ zQFto3nP z0lDG#=K)5Pq}$fEHq_Rz$H^{?%KRIm&ki+n$62f(UVEKs;z-7U$~8j`vPmgi|JWo% z_aF~n=!$=B7wUtsRNfCND*qa<@#;(joolA9DbJ=xj#w*ZOe8;utZ1&mOcJzZdptZu!iCrXMBZ_ILX8ci0Xi2qrYrsIfz zFZtd=bXUq;I86MehDUlAOxq`G#>TKO-J!+>5$YPYR(mO``-yi)q|VNvedY`1tNdjmMU~u0_<>G6NZ*}B!%&?5gF80cWA@_&Zv)`{}71Fsh^E&87kIlCK=kF_j*kNE*CdSGBxXu3>eBIR*p_30wQ9?B*@;- zPZiGCA9#XRN9#pZ6Wh)Qd0BF8i>i6?a-gBP`}{=ISXsIyxFKh~KP_T+{Sso#&UMtv=si&HQS>-P_;Kwyv5m*3@iDP4SB^VpUg+LxPJ#B8U>Ioa|LpL` zh49NR=WsXg(Wb$KOfjG<48>fDKo0EE(!W5v~k)sg{1! z@SO#MD_Pev@hg`!LFhjN$wt680E;E(zOTioR&xjQNA-isc9wbC^_8>x1HWB$msaf< z6nvf|iBUr)k86=SPKKWd=88z*R_(Am-O$vOh2vRp;Gw=a!LJFtY-YS}92Xe^b{$YB z;NnPHSe&7g5d7T&olmbyBWFzsGYTl6&|@316ZO@BcU9doQ`K?BICCn zR1n)=9RH4_S>LWl_qBeNk2E$8(XlAD6LE#0wh8x1?zV%T zwNHG-Ns{4V^_?))tC_9G@I?l| zK*nf9cFH1g(1s))qw2M_ac;o}s2p|Bus1c2%D|KHk92uTHZ+`KIpPaVd@BQP@)U3_ zL|4?hK+$$q#c8f>16xu zJem5EC2gKc&*qY)aI0Bt5vPaCCfyD#RzMyRwCS3SL&5>iFH_jg-;%+i)C^G+C5YS@ zOnjMCTD8xW} z^X6f0UiMCQ=b05pvYYixw}_~p zTtKO=u^^y_NrILK+R!@~IfJCaxS9P6yYt%>ZVwWTLYg0cd{w>anA~d3_?1M2l!T82pD&9%vL`>TRSAPD7kM*V1x%Ip4ZhlX`e%I_TY*z<}3iYd7c zzYf^YCa1_EfE$*qrzkG+ytebu z?OeB~KH7H|H7ebeH zfSeK4wN#h={(bi4pN8NNT}mBHDULpPuFAM!v&ni|(6@ebOrNrqN8Ze$Qw<(F{zTzu zfm`~n5nJ0A<$6`2BTUKD(A=KONM7^t>pMg2MEYhE@KdOd`JN9oqhBT$2puNpWn13o z9Stvi7!knNVhLeWY9;=f}ChF!hCBY{5u*UH;RODSt#sD7xQf;%P)wai&&4Z+=AQ-F=D{g4*{xW_v25EP$ zQoSo-cGJ}~;I?z1&-~46EnL{|ZCjAlV)ZGFVFn}G&mgJ1ld9vN1)y|frJ2()QLR=|$kp_S`~ zf=u54L-#fC39Tw4C*+J)%OGULqt0s6|EM`ZLGRI%Mzg`d-QjKsRfU3QCKROmA|md= zHU7GgyxXJk&qi%?%e9E!8>p88Q zvZid%@*bOpv1waG-KC;<6uf^+?@MF-jKEkSK43v*sL;q;BBDk;Go70srYJ9^`TbI0 z2I^*sm{z=g7zF*#Yi{&l6f@*}7KZ}9DE(*n%_LA%P29a(;L{}ie;$T5HT7NT%3x^O z@;DgNr7R8177EN9I6tvJpvF+u$&W(N>?fl7@;l1&+3S{{fX(E3VE>(;#cnsv*F_>4 zT5s7vn+qj#y~y()c6%<~nuvN|WbCGDzULx?wD&40s7l7WoBASy9VMTR>g}suh}Ie& z98?2P&A`Kp%ikti6sTZ1j^cGTwH0ORi?`w%4cX)?cUIa3p5XRQF~4{E!3iZ|W3i5X zG@KcHLR3q2wnghDw`!J-BEwNNKeGOz9}VN`yxf=V@2|{Uqi0NsiU&>*_S=SLD6cb= zX5^*lxiY@(dc(NAuQVa6kDVutHdq+ke4c_3>48f3m&ZgW4pW~j%j~jiS{0+^x^(j+ z>0rY*K^^-jL&|t=1v8^3y6A~=IFBaWTK*Vs7Wn~-p27dd#_OkV0>iU_c znZ(5{?T8}Tx6p^rf1Y53FfpwZOtl8x>cr}#d?{IKll2QxfGj8$k2l zzKN{!`e-Y82ydI_?zk>Yp{3!iZa`GKVn20SHl4EGe?RnI5ZC4%erCAckm~q|10aQ8}mp8zF^XR6+( z{l|bVc0*FrKYpwlE@uG}Ly`~hl~&zN3?q;c9W%4a8Xc-X_*RpgCFQYb`o0b(?g7U6 zUds|tGG^GfqM$fL`W(dF?R}Nny|45MHUb8hukxo?Xa{|4p2BCwZcS0U6HDeWi$C+ zwRQ3-sbq%H^5Kdj3;1G=&XIEc!Y@mw*pg0M6jOwv#MJXR`9qA8A>?>BO)axr*8>6B!*r`oGpNWDuI<aE|Ip;{k?xo5hoc6j$*=3W-oGAVF75&vJk;<61-5cD0xtzSCdf^FPVI4-P(&`7w5i9<#_*eMLB*xEHmQ~l5 zgtk%$fGF?7!1qLBKRoBLhV&j8fy{G)r)P7rjh&Jur98AWG^7OV-rq}4gezHgUsLai zgOl4rLK^LEsi!_V?NOzwf?PxYaCZV3*EhyTGvHJn9;0+^%b&>4PE>@}gE~01#+E|` z@$Va$6v%HkZ}4#zPg9{wWsX)BetM`;C^AbpVe$-sr&VM)u#9}Ed?7p$z8&R5vy+#N zj*gvg>x`z3>$|Fg4k-J^d$5XAFF*GimaS@%_P5q$vpPv4N)A0Hc!Ae?0wJ;QHHQIl z5ECeU_Sk0}e}wAB1Aqh7)zk{K5`W@yp}Aco8*3}l)OsJLg*&QC$ffcJO1j#L@oQaL zD5GUy9>dNhEWRWokUNd-@9h(v*xvyVWn!PWcy=l{DR2?n5alT6A`y%=EC#5HsV_8o{7PL~CN(y#CPbK5m zIX9848|6qJM&#UVP9TQAQ4$1rL*e@GITNfTR&7zgj{E&v`!oE4a7siKWX|LXfI2(m zx-r=mTlg5Uz0+IRYZP^>X7>r4^i8b!xmDEZlO;E%u-IF%L~WfebkHCITLT+14#LJ1 zeX^xkXgyd^(47pgX(?JXefgMi_*rSWT!Yg>qZwA1n(97NwH^SC?@%zlbJ3}a60iEe zp{QHCs~*I~UQlTh0KWRDl?b~10@vWqe_VrtP9t&{R&T+5X1&G+S-DV{VWmBfVv->LK>^UdlXzWCn9@FlHSSyQfxvBY0B73Ora09rVf9*7BS!Gdzd|Z5jB`SJ~zdU|UUN0~9IcrY9?ddI= zyaT_L&e=KSva@sA6}+aeoFqwdu0VZY9$z4tW2kur`2HwJNvG>W3?soY_!yIBResSV`Ifx7mly^xhRar!echeh?Im<`Bqd+4N`^`)K z^Vh8?(T=AmVKv!PPJR^1;6Bq8Jy8IO|0m3H^>o`C*%BudOF7&RplUe5r~9Zv;(F#@ z#o@UY%hO!kyn-f~oS!Wb05^LOLYi~gDm33-M8SQ>56+DzcL~y;dDDu#Vg$Zk&cD_P z$fBjkrhjTND(zIT>19^WE zWt&!2kL*bHaOZ!LRi{&Y4P}sms2J*feJ1i265iYA*$1`x8=85$-jVT>2|MWf)uA)YC{9P&^e#_wG z>enS-4}y=2954*OF`sA>Xu9Ad&Be0k`Y_kC(6tFVnOhAGdjbh43XTZ%nZE97Gr``s zS%B#a@t|>h#lR|3G>x^MDR`=>6-K|Vz%S$cX-S6@kHlJ(&n%rzMso5( zU>0YJ$ALIp$x1~faiB?)yg8edClLeFaGJ_OT;h}ExL_AUr;^wCb6ESwE!`)NZI(>8 z+76h_6M+&E0|1}W&W}T$gcGS64Tmq$N{}Z-vXi`*nvrR#+mhoa^SUsa(LnSScb-D_ z3+|@c{t~p@odtS6kjGF65Hm<`qo_QgDQ9zC_6yfDI=pnPfS*ms2;*IFH~Rp=6@uBi zccdNIo#3h}T9~avTVM8?w`|e>;ylo7y*CKc63-Jv9&YorNJux6qK`1A9r_ZD_NNre z085XH-kOZY2&luq zD^%X@?Xo}K$5!)b-Pg;t^c4etzX%je%K48`%lzVyIizZ2bW{ii4ZZ>MaQQF?VX+(s zI8SXtl(1jwDN!Fl?(pq_5h5CMW)45|EBk`NKjd|G4p<||>|EYIY%&v{vk?&PIz;`s zT?m@5eDu;6gf|`n7cAE50Fmay9JG};5J#W*Yi`GSrjhHH_wdERu>|R(j1T&OMS5G3 zo%^WtfVqt=D$ooO;S3IsMD_xaO%;6iH31oPQU>i8$}wIGb1-o9E4{I>J@+}gtrZwY z+^R2uzjxvqu$_VJTl|dji%L5#ZTswKN>p2?s;o(7+eQ5FTXd~mEpUEUG%s%EIp*F>y%0u=o8^<*drq>^>CqdNfE$C`D2P(}sXhBT30EU=D z!@+sY1AKjMa~MNSa9<4LUgU2)D$%Y+m3Q->*p`mT%0-QecsFcTJHM}7l65-sw(I9at@Ov^w;)_^P!AC?a<@6$}9+G zCHU?qMwaLiqh0Q6<+pd2On_&&)kwf1{?g#TX@~4{HqEN2_@mQNl zw39{^wg#=k`gzG0D8NJ`#mS^xiv{;Zkvwu^Dw4oNZ1P$x(L-vj@qMP>F2gDC=Qi^C zex~Nl!0b^eS(-Ja;&GQpi@e*TteLr|P{U3ytIChi5WDY1B04|`t!-=5ycmwoj;0u* zlH!kRtaWvRJ+3mbk{>1JkOD~=8JlDL&?66ipn%Tz@82^xM*h<6Y7BFQHPq?f)HRD4 z>-OJ{MUQ=^aQkM~L5aa|^FrX*8(%M|aCY>kV^2 zL|Mze0afIH}1;UveHrB*%z4B3m3D@GfJ`EuleUL9bmrOG-q1Se*I!N2N>VEZ!XhK=heg1o6wU zv74p|tM7~+^0StFuIbrfE#iv`tAlnoIIi+(RfbICDRCxP z=w)N4mr{+Z@XzV|!ci_AAFoIWU5r?H3)nywg)*_FjfzY!i0jxg=q`6ZgT|juol=5Wq!ix#iB9Rsqa2hRkbRJQIPIEHkFacgYUibR(z4kXl{hCKtgnzfsb!tD|Ae^hOQqZoX`88fQ0?8@4~2C?VU zk#UAknVyi;QR90}$1W9VHoxM1i>2X@+gS0G8IX*hQ`tp7*>-pkqKb-Ap!LcihZY6d zi^cBU`vZs4V4|m!NmW)hC%0~!(zZDVp^5-6qu1p@r7E&dyRaFp+P=LRGou^kLp692{yClu9x3yi*NFhA79Ju|!yz_7_ zQk_X5p4RSg$*pRqOcTFU-uKQLiZg&otS?$b9c@B``Hgo}(Qt2=H$q%TrFd4z_**sV zun}@6rFp;B50!@-{rs_Ft(Hdy3Pu1C%*lniRJR<_F|BZKDcbDd(wB`b~JI|t) z71@lgB4zYIJ5o}qm{LY0-SFPxPbRs2&T20JxvQXS+FQ)HredI`nJRJt(qzMqngzy8 z$9ET*SXUT_kK?o1PPbzFqXwy-A$orQ@W7f!;8vIJ+#!VsE>0FS!|t4jd~%fWdHc}k zp?yDASv$wU)9d2nOLc9u9Sl38CwEq~jR?AJ~=CR`ge z?*EMDX`9YI%ttSb;_M1b(~a#)PU~*YLR2!FCP-ffRrtMT26UtS0%H=cX7{HATcDn6 zUGl9=LOa{-kzDt1Cpq)NF43q2q<{nP_N zS&E4yKIe9k?lISa(gi41AMEGvC~<4H@}Y1J=i^S}^MS-ah3y^hs%14$cyfopj~-$7`8ll&DF%HBgXGV+w z_ro;>$is?>9L3(FXPHYCLf@_O z4(tV0`}A+2Q)ccwWgG`oC!n|7;TF{LkSm2!wwIUBBl`GS8&*O&?!{;i2B=-iKp!Iwx| zr(v!wzYDn|GdjTMfdV7cUqkAU<$P1QOD!9@SvRH3_6L~0jSg1s%XYw>9n+01>OABh zhs5-jo98$;4`@PP4s2PabKW1*sR1L^2le%EVZtp&3+QdyMCl37uSrs7gr3ppiRRB9 zsTrr>8m7bxW%#lVZBe(QrvN2haXtOuK|S}TyemT45f3|Kpn9NG_fXCo zUllKngS?ND-OH|~wR)vTexO&X^h*xYKJ6+aebwxSt)}BzMPXhUGq@w>98U9%&pGR@ z+w>KD#u$^EIsk9YY`w}DF@OCe@;P~|DRwXc)xWKQEPi-)Yapg29j;69-=nNil)0fM z%s%IBgSl(DY-jiS&M94PWsWZOnzjIKSxoF%jiQfZK3g~G+_b9aR(|SVq?2E0apw-O zVli96A5(kuXywMhdx#wyZA74cjF>uH)uVXRbz1@RAMSx1UUh+eD}PFC3Ai);UzK+$3xIN(b_F*1rgH)%_c2$d{Cqz%5Wd8dfLUb@NCH zGWPRFHPM&)WduN93p#NrDj&f|sE7UxlSCt2n9VMW1}jg#+0B#L zyC1M}p3Ht|q*sH5oLE2vCB2{=6EQMBfgWHbjvl@p4YYwJAaB3mHgOXXg}p_wH96AEkHv3?$;&9hRXJ z(DnVhduh$}>S~(I@9kQDz%lgu3*FB&DULW>kg>w0}_q_l!~N=GYCP zqli5X<48D!v@BO6`kZuR0xKc{g1s-)Am$Wd?Wp>krZKKz>+dOQdzQGqR|b$RyQV@_ zpCYuI*rp{r>&y5QX)Uj01WU`QzJE%L1Z{TwZ17n#bFY7KA3%cX=AZzHPFF5pps| z46X2mE>GK$j9k%dTVK@)oKBM8!lb9Fszmp%0;`u{&VgH%q@$z2=^urjra2z*ZmST# z=u26z?;Ftxw>;+bSt!R!4#khD>o@X-Ff&bB7CE~f{9f)?9ZW@F?F|nH)a4+#ZCF*= zv$6uVdSMUM^fk_m-1onjfR05VUAEsL(uSWxZa9dN{wGwQJB(9PQ==_}YP~Z@<8I;i zs74fJnkT1CykZ|iW-P4kbsf8|LgqaFM{-ch~l1qKvO5&5qJDeaN zCj2jWT!#sp`)_#MvxxVZX-SQwl>lO}QaZe`rIH;YY_mNN=cvdjP18itmopEx&HdyMmL-o_%fT*#NJwb|-oMOP!aR3wZDgsv6%P3a-X_TH zrYL<8c!`9H7QggWI!vZwz1N-nu6R4G72PCrhMnzSTiqw<(#J1+9HLCUfWa{902ZJC z*j%Yf&X=dW zXs2seuL(QvjaKg4@ialywu~+uE2=xbv7$XrQGay0U$^cGe_~iV8~jm&|Hzh0ZeO3K za#!9duRSlN)|@(41mV9sg#WN?gipGx-GIJC#CdJ{=xq)Ur**0GGwhsowW?+EUl_C% zZ;am@Df)wZuaXa||56vNWuYSO7<4^VzX|UH`=~1aJs1)Q@4y@sk5;`s& zD;Ang*0OhyE&p&0a3TvbdLEmUXODgT@vU$2c~|68xKnT3c$hGbD!$LHL+_(YHix4z zO=v)?twVeFbBCqCzH`r6!28kc& zjak`JEF=U&0eK>rUdTchW}z?nkzo`Ym?av%rd$P3G^c`EkCb6=E zlV%jZ)id-|z`y%R)OKURec3=T6!S|R4I8Zac9TWlz`MKyUCtrQEojEFN=`hCGs4_b zd0ijONL>he;~2@bkG}i%*$cU$Aeu~0}`V*l8XmI6y}c$*fT5T@Sz%CZyt;}^gcn)n59uis)a z%6Ey1y^Oe1wCZ1^?9R99pyB7W1_F!_O7XmDH!maiJPOo8)VLp^dP)_9<6esb&1M}K zHe+p_t8GdpRawrQl@3xJlTs6)BB)Li%b+Ha0j>zjQZqhr2?pOjq z*F5rlvX_8?fD3sje326lZ3%hV)0U>s+II!b>84xznC;+t|9-EUZ|Fo+7lYoEr^W)n zt+yk#z#{q9wd&yB8B;2zV(($ICfNnddIRZs?DYJ_`5#%=-h5I!{K(K6`8{!h?rLnnM#xt0NIQu7zTi6xk=s@cUIP!yEpM!v{_N8=sMy2KZ)6a(4oy0k5&;r@ox>nW> z_%v3__+h`1M|E|e1QnYSR2@494u=UU_(b;N*4~x+1LGbs#ld*_HSq`^owI|Ws86(X zewt*LW~?q!P8;XR2vCqyExR7;&XbW6zqb|rIF#{|*jqap95tItd)SuI6;a6Ct2`dq z92fDNds#LQ_zirsVo}w)I*a4;jt&<@`5WR~NE-}s74QvKu21)>Ctj9FA z*-z$F{40f`gdy`Jk}n_@RZm1@zlx}~Sxsda?jJ9kJWa#+7#YV)ZR*_&Nlkyiu$7fn z;X-X~Ep!H-a`WyeCslgW8lvRatwR@Ps`I*0qelkuc0$fiLD9ly>GV;?Q0lyDsv+tw zCnCf__dw>d*NWCu_=qW~*yf^bx+)DQQGtii#k~oah^=MGt^drOyrLSCeS&ZwUHNM=H*!tvWTTP9y*1-Nz^@b zI75hB;ZmgErZ%nj=zt^fsrlANHJ8U?bGF+Snv12(R+6--k4r0G-Jh0w-7cjJoVJPh zp{yLjZpyy4h^^w7wlD9M!#HtXSnq#WjXjp`7R{kC3k#eI3oP3&?jQA)UXF_JAmLq{ z&$y*Ja}IMG7MHG+^zNJ**TWsqE78*CmN63NQ5QHn=HtRI)wwHmPUG0Qny^F5Kv@ld z@;X?hY}YjbHh}REdJNxyhzv;k%KR5hUt8gZoA5` z#>>;E1}X;gMo^PmOwZb*mZ*XhAxr#nJF1q5rUzPGwwZca|LaTOzVQC(NDFGW_x~RX zcR|_bc@k%!8QQgy+^oXa&2Z1CxfeG6$_~hsi%AZNy4JG59DGZlMdZF%BqYrtDfJ~= zz77hd8}(~K5#n2o>*i7j;Lswep8Fh}k3f;Smsdg3=7T|kOd~MfhF}+0!4sDPLBw_| z2ohQib1teXcpMS@74_w@s+cZ+PAiRzi9bqDtYYV(TnC@7UxRVMC;=_7 zrtaBux4MIHi4i)y=3YY8WG76##sNW z^}ZwZb#ZS{O3J zPM#FpT&k6ilaG(jwU$1|<@Sq8`|6vQLhg^^_t^V3x)+75S5BUHD8P*nE*_3B6y3;6 z{EU7-m04*5g=E49-cZ)YqAW`SZ#`8;(N^?_fSDb0jLz{iyeH@F+H5=HU}mSmV;rJh1mj?wFHjOV~)%bQxVo4-`Qg6 z{uJQ;O@RAegV=d&zrLZiG11=rL_oBjRdmX2Fel7b%@gbWbeH`SySQN~#VS;Y?A&>g zmno4$r%o)nKX!qg(QZQStl;SlXbPTa7*2E$a%*_3P(g}X#AUphQ{j|l)qPbGt?!>D zC59+`t6+AoJk9`r2kZT!S|VD072Z^KjL*Lvcy@7yk78MqZCApt8L0#_k_D6hqfGoZ zM~pZ-Z3%rx4@T<2=nSjndtNXDp-{R>_p0JhyXCsR%U_XP9})U26wCa+^*2Rxc!|aQ zOwKj`W`2sNnxk^vNoQGE3(Eu6h1Thc3=IvHpmX@&K9F`aJ-3ocGh<~+0Eu+&t})6o z!GRWs@U8J?{`P6pK zwe;sLa-o-j$(+$&wIm*m${$~;PK&#Jz3bZfZpG0$a z>D*tKZ9ufM8|lv&KK-oFZIs)!QHY`s>{G(4jQHkh`Ad1F!znt4BMwMWdjhS8ufomX ziQ?VIkN1}wn;RI2>5iKmDrC}oZPoU6b6^x2z_JtD{$a$jsz5kE;2HyrT3 z8XFz8eX65A=@Z>|X!jW;4WgQq>1WR_=ODY`n|8d14ai5csggmj-a!*lWgX=0{%BYa zEqJGFLkT=k8HZNxska;2dKa++ex}Tq#7vYXtTlqBsUE8MawVREvsuXGgOq!rj-qBJ zb(_EAq~QI?S_F|W>Q*T4A^fBI3@H=w(Zj*S@`8NC!0dlm;R?ZI-ml5OB^4hnnn*8oCJRiIdBzd>^YLcgFM1cJ^7 zkX7NmWkf>*Lg%L00rpO2E8*vWdN@+)E)N8*_q-V*`4wO`$_uE0=DlXG=QXz5z9sMa_0+yC2|jD!u60 zyU;)`e?tqA2w{@=g$>xD$DK`=#W2=L1m4I3=IIpT_u^rawGw)td3x1buWF*1;-EK) zfLTMs$yZG*n6XxyVxvmCM=$USkw}=141u1NZO5KDdgHu zM5*|km$@?;V*xk|?h-3Mk5*xsD}eO6B<~@MdVZbcQaFPi1HfwcD+YlDPuOO#0X5)& zoh)~gBN?L|QZ$D6`T1E`?0gmnEMHgNSroinj^V65M_^4hY9F=rDf!LtD4w&4V^EUJ z`Hp{iHN!#csB%$oIi-vBlHkua&z~ZPjAE47AU2=%o@Wu?Mj|cPO;9Cy@S)iqGoI@E z%&L8NT!)c&B}5hpzC0?Q!)^M*W%A2LHnfY?zp=LTb&5yp$W5JbJHL9;!fSldCa)&0 z8Mpx??EJ~YnthvhbFr){-aj{q8zKQbbFX|NE*UP0VPW62*aRWLixabWaYEn03D~Cc zEIcTLjM^(TEh~TQ(U-RGJt`t8pek3?LOfbTn_Cc;JfxXHuxbos$C`&Q@!xCDlEnOu zN?_qT{-fKyweGUlJ*&`NjXXTZk1EUEbTl@IGV`d&e)5NYf6tMc#qeU!(NJe=$U;T?1NZrUsqV-tC!ynkyz!{=-7 zzDBw?k3Ja9j@r}(9V176lqa}zW8KSrThWXh?Uyi@;Q*6n9m$vbR4_mrLm>-ThM(Oo zU+;Y|tADcO_5M%3j|ThURauc@Qgn}&5AS!0AF1Nq3CnceR(3C{9DS zTJ~z5C>OOI6&HwAu{E9gel%*2{H?`sQ%gCi9KVrwmwVFM-TPaJiulGa7>`KlW-+yo8f2_x z?2QH}lV4llR7S#IHC%imLTmP!to!m}Ki8Gx<^AII?uX2`df)>;xvyLKSzqBU2-$#1 zSc)6e>Od!mD66n}p75Ji)K)kyAFX-*aHmNdlZL6Ej81*wJn~O)tk;#)v~`9CRiBHu zT1L=@)3sA^WUY~#F4~ny?rc7+O&%jhA+9&>Lt~fUg&r_X)RHdm*UGuhGg&t4@`2TO zp)tzEgCtogt^ec;^x{o`Nj>G(eA3auPhqHdKz7IHWPT_1Z$V5$ATd}Q44k`8RB7U+ zZqUH8wwj=h7>rqCL6Oh3+fGgWS%nUA7V@nptW$a8q=FsgB7O1?^L(h3QwjI|?3j&DD4L#9qHdbX1n7#|=drrdIeJ>M*TSkYXD z3b8vJGaVz(vWxh%&s&Kktd82WNA)}J34+3g{QMFS& zzQCH$UoI`k3ZG~AbD6uxZ;xVHvG>`Gs7vTR1ttdL$EQh?mKMzrEYtq-&2gY z=PwLRs>nT^{JPf)DnWrk!l$McBmjQR{KiO#Z_A^h(k4N!JupeN+{@q*Qz1wDzP<8$p41auvPW%%H z=HrU6PwFw`rpBhGCQZV1rrF`px=&(u(-V$r@{%aKoN$XI3e}Oinh;-!1*8(w7cIdp z5rWP!zQPBudkAN6I3k%$zPL(4UcSG(yJQ7w#pa=rH!w>A=IDkImB70+aigUSaU8*R zG5lLDbSuztxQ&(h&O0s(qm-IUX7ov0lw*(TJd}}E;(`VwdsH3XqubfcIS=+SuUG>q zvNkXsv9oxwQ%`4R{nEl| z92?wN(2#2kfC1z}5%1Q%BNXMGAHn_tpxeP-u=wHyn9jAZ3~u08SU^>{zHtd1O$_jbS1BkcI7uRrrlu-x4z2l1RiE#ZA(kOIRzhCCp&iR`h2H*$ z9EV=4mnAQw^5ZfGW6{#mf1#xg#Mon!*59a;i?x9mR&e-lCIK^s>2zqKWx4C^=kG8a z?`j?aw~dFIq<>{TR66}=8LY;m6S&`yw+os7QUL50Iyw~c8!cy{kUY&`=Xmc+-i%Tl z-weI!_!>D=A?((d=EN8Hg_Fq}<&<6w3DCTIO7fwQ%Dt7l);+|2pp zX)tsmGs?zG2D*gQI_?W$PGqEr#d(m$#gTM-42-2?&X~rzL&Lv1N-1uHXx7TgcPj?N z8^h!$op7X_IEz2wPN?Yp??8g{g-8*khe<1N&>Oak~r?JQq9HL(6Onk^hyJ$R%`)T>A>fyhVD*Lk|r?nCimIL%oZO6&ASKLRfIOz5^S= z6bc?t2{uKbLuv^vT#ySA>;r>8%}0$BQ`u8eK7jZrtkG90({=3DGC#NnKwIEPG0_qIBvzx?UB|q5R?Kq62 zR!Pob30$;6sCCB&WZvO!F*`bxPMZS?Rajbf| zt?X0w0wxrm@Yo@76VB14Zd7(Qn~9>mr4fCFx7i*KmBw|4U7NR0ezi08+VP}8uKtA+ z_A3OIqGua!1z}!U)mT~{H$MMzjmqVx3CD-&LRIEVBiT7jYvb@es@%$A=-o@R7&bPq zv{*Dj)chyb&tr5pkzn7ylxoATtj)FRg|piJpYTQ*i4$IY0XGG$jSaY!jl-p*Q=s2{ zYkz-jZ?9bwfotU=5V+<#i(1c;LebW1NBGDJsLm%h=G;lG_Cm(A_{;= zyjWlUdV%T|xU^0A=Fds2@KqRF1;9?B7<+XF8BgP!z-v0 zpS51IZDnI?GM+pH6%k(9_udq2o)b-eoZo4iEWD)JPR^Uf>32#E6TE5S!InjAJMPZ- z;)2)&Yx57<UKc=Z5wxTUB=U3k4df_a^ zapBDE!z;JFg$@RSe&^T5XAsPD{dcUbKJsvv6bW01(_lbgEm@1%p}6sL&kLZI*i@A@ Km2wm>-T8lGZ)9Kq literal 0 HcmV?d00001 diff --git a/docs/getting-started-guides/mesos/k8s-guestbook.png b/docs/getting-started-guides/mesos/k8s-guestbook.png new file mode 100755 index 0000000000000000000000000000000000000000..07d2458b3b54fd0975f35f0a6f78fb0e48e287ef GIT binary patch literal 44000 zcmagFXEa>j8}~gk2Ez^yQ}J}0sxKa&*B zCj31D=T3e&Jk7RO!xzsjF~-ukjB zz^|f>6#w?X;dASv-K?E^N#N3I6{=d{eq=g$mpD=VoCShM z2t`3_YwJVCcb@EYtUx|=s{$TH{r7KMy0Wyhh9@h*+umNH?a!ky=vZk3#SO8owCX!Z zjFu#{fv2Bh-pnd;9IUOT$W@c7GNi;f@9xYg(@b--BZhimk&azd!--+ zG!A^;_J{p?FR0Z%Ry$&pnlg&TxuO!l*qqokD{HhjevES)RO)rl%Qz?mUub5gkSv#5 zUHym=65653-{lb$LQ~mqey^0=bG>m}yb8wU$G7IoVxy73LvVqK7!!d*vV$~Uh*HgX z^LMurOWDcZ@7`O~I%2e-xRwg!c_W?$pw|I6kFp!no^IvGNOL-1xN7g31=6|SoXttv z_|+GSvZinwsg`9hmT6Va*9=L z8QqF`nv0tSEZc%n`_r5;1e$B~pon7gM|5u+B7cc>2Yu&86`;N$!*16fJ(I6D zz1gFMjE_#X?7Lx<^A zxMjkAHTowZnFQ6OsH6SdpCG0vw<+eE2+o8C#5>Fez+pK=k|-?59w}L&Ye59)B&Tc* zIvxEuy(2KA=~cxILLPx{Z5l&WoCm(Jfo*jkDda`P=`3ozxPz<;Kod5EZHU(K)!ynK z-AMG~(t00d99ZO|H{?E~f&8_>gUB3m_N|W^x>7)^SJl-uHKHC`&>|0YYgM%y0~$*9 z2nr~TGI>*jG;qfm0)(8|X%p(5)tNZdRxEWj63d9Z@hvZ6jNa381~>w9^5yWR$XrHZ zpw=I;WDx72$3klJ^uiq9Ki{~n?y2V#bVF8y2&X)n4@f55?A#kJ*Qs~H)DUEzg#((9 zln-q1(mKf3(O8WzEBQU{K6(4{qQ1WVTm5}{?*9^=Qly!#^@u@I?n%iK1*2(n((aqr z9ai)+nZZ2yCn{Q1Y0h%W7k>QAwuz(diQk_VLB;^i60eJTof?R1-yy~!b!VUJH^=r$ z`AnH*Cd|joN zrq*zj-1;eKPN^gem5wlT?P<7ruag4nbKOV%pu!Z+dr6aqjyAF*Z$+XyU_cr)Photz zzLeHAtpI6H3v=%p#Przk z3UB;^U2mFt_hES?y+RW-6NKejuzrpb_c7lQX|s1rr^r&}ny?luJDIW};#!LdKVzxx zvCnK>tJ0n11{yj?LD@47IySg6xD7@bpFMF31S7;e6@%_1+XrVhD%~BpZ_=QTvv68E z74uDicIPrm3(wVgu?_c1GtLV=sU-yrR7mUdm$$@GQWoNP&HnP7lgI)|Ekb4YSs`-V zny~g{Dhm#m!}`A_JyNPZT=F&tx>A`z&=AQbV=#)nLqN?u^7^|0bpUF0Q{@Jc-&U+Q z-C?d3iCGZ|+F`Q|n65UfDfKaMBTHk7WUCUIWNR`)QW3V0#%ys0;qR28pFV`o-WxB*a%u-z=k5|6n^ZjG{N zJzALmDic4wv-9GOZgCrIRacarlUrRSW*EDRP8X?^VhK;yyyr@*`4O0>N-!pvuVq%S zmasM&dHM{2VzIvb0#l$^O5NpH>oZ?uG?unfk7&uL#^L+SA+N&U4^hz5Qd%bTT#qiVU4iFDp2pQ9B8 zdn+L+gNfo^mNhvsPrjrYyTkYfZwKpgpIyPr=5H1GRy@;=Te!uQrd3}B|Y_kIef z_j2hengtGj*K;UDG|DxyFgR(BWfUy8Qc?eGGBL9|4$dB*Bs<-mmFPKhx~Cujch%#4 zlcDa&E0-T|A>rN?l3puH8t?!nA){2dK7Sh}Jkm@t|td7Q)IuQ zabUtUFNG>JiCy&xf!z4}nS-Oohj)+#wYY^Ft|w{XvNCoc-zDXbB9e~r#@%3V>{+eF zyz-{J?2^V}WNta6f59@d$!Jio`6E|=v&S^vI)ygqS?rSx^Vl ztPY>D$~cu@l~6y*_EqYk06iwgyh?_w(#8)!>@XoQ$G)6tz@d&Q*`1$&r(BhUr=vE6 zto`OYMujE3quxG%>+SUe4qwZcY(e+Z%yl1h(Ij8=zV2expGB0n$dRK3*Aand z0FWj##>4n6|I@@j5}!J_o3(Bhj2K5h5tP==N>rt-^*eQBU!mdS zCe-vH=ytfHYD{r`HTk!OM*+}r)-?l;-7^+Us!uJ|Wm6G$03Q>!^1VII(=1l7hm46x z%ElwgYz{U3U)^krP{HU0RhAizF};Ia@ovU2 zW$RC6W#%s-*Z)BsJ!Xc?hHkVfk$Y8pGhydk&-Is?CPbT_E~8`#JLo3JF6UWz~bniF=Sj!)6}1V0oxO|plJ;C@x1(=eSIuhz2&e_ zL_#h@=-fhScL-9gsR4E%j(Nu}!%PMawcC7bXC)CAU~?AtcoZ1a&Yde44b)(6StXL zKiY9xO5Rh=hH}rFn^SuPJW_`WIxO?pjuH`A%<(1@HlQZ=YQA*07ElQmTg z-TGf$KgdBi069Lg*jn@15NX)b`Vsblc5O;q|NX}u`$dzJIFXoeHx`R=IDNq^Tb@=J z`fgAil0&1Q0W+^3HhQl?+RUNh zyOtR+N8Wxn`RM6fK;_AXGuWyAYU&yuR`o59%HviG#d9ar>>f^D`|ml^pTdIs(Tj5n zUr($F&;o#F+0T2gy-v`pd*c(gCFupT9OpP^TT<3^=+&t(g9Z5-4r{QvAEK|?o0@B8 zZS~-SO!xJB1)Zw++|ZUwEi{}t(46NIY4)6gy&0~w_IF73$GNI^YbpB@uHP~wI=Q;Y zvV$De$U&o$L|)0I++?gMv2$2IPVd$@Z%P6^=y(Xub2u)r26U}s=~u3`a=VJsNd}40u43D|z_k0~qR6qII&Ym?aET!^X+na=L?PP8iW9CaoBm5`8(j zt<+$SW4*9TjzE9_+ySq_k)^DX_1+{+CC37jJD>j;zt46w9Qnz?D94f6VN`v6@HRC; zTp~dp%iD-t@s`em?=q=~ZGS`av&RxBqF|0JX&HkUX;}>H#+iZ%;STPlh;n64Ju5q9 zU|`@Br-Nmz>Kk!{k5JYO+yZYpE7AR9#;nGC&<@7>;W%8cRQR=|j`Y#)u!^2+Sn?e< zA9tOw(+hEeTdZR0t8my)sEK4Ucv?69_8R-=>CqfLv<3}yVscm}viu5rOI zn`nP14z<~;{36v-i`#WQE4*t@11bqa^nJ`sWjp3>vL9rA!l-lF&^(ptFpejv!QzGG z&3qduo;ODA1MTJY+^1Sydo+6Wjrk*hEpMoZCs>y`&&%o)(- z6E6ixn;)qxx!$qOKhR2C>84(@77ZVj*_F%9RI(88An_@JY9L0PB?;&`eGz$El0X*+DI2cm_ zFt@!UvfpiK%=1|5HCM6}NNG2WzKY+hbga9-of?8)`@S&20&TaQ(p$=$_h?g9!K)1e zFD%W)_>YoR&@|b671^Tpf-S060s#5siM$+GyosUlC$%I>n@G*&Eu+i-bNF8%H1NdxYV;UPs6C5@<6 zZ?l?g1Un~sARs|?cH~v5foIBwZ5wND$079vN@=PqdZG5@(ind4oeTX9A7?9 z%bcAt>0Q@w!g)n>-E!wgH~(ngx02c%pvO#UQygsjq!r1D zcL;{ZQpz1>hU5Nfz30iw*F*U@^$lV3@}uMr0UD<&l7 zQb^*@zTrVYvDs03@GE)kTYJ@Pnp3-X$KB~tdBf%4h&URcH$t)|Y>watA;LBk2HN)D z+}NCls9xiqlal%RrI$ZS1&dHN4p7dy4oe!%dwrwqO@H5u6VTHqPFg8Rb-i*{WtP7q z|7fnPtZ2TZ8ywzHd0R3VC4AGbl{EH(K6-DtXm7rJO)nHU$rmo*yenZsE?D$47zzg?tS7w6&t*cSM-%cy;tfsa4-FK+{ad z<#qvClIFrhTxLaU9wnCAv8pAgl=}~FJ+i%Bd>zJ^X)`cL z;SSl&i}-Zvx810f#L90VuY%r30>J7Cj&y$biKe#_qxE^ol(lMODISBk*Oq$<>-g{# za$mUsYfI`db7E>R|3!9u?I)<7y>N;;#j38wNSbDDckGlklYjin13X6CM6=H+7{J$j&->kU(qMAEaP?EtY5)wF82lj3hF60|l@^W->RhBep`m z665`Ls^yTq67D^g$@H4fHeR)*>8xn3o;$+0)jI&j&X!qKN;cx zQPv>QmtJRg$l;@jZzWvEQ#s4L?6A6?R0LzfP!E0S2{d-*6@bT$%&gC>|C&u;#OOhu zs8M?oNl~Au^pm-slzCx1uFV~11Lvh(G$_d;I+Ixdlo61aB<0rVv?0Zc1G8T+)`SlM ze|o^sZ{FE@;a^ctvKw3lU~(-0gD3|PYTCi?F90%s$s^f{=5=Z` zJvo-4n+lv`0eYq4Ne?Qsb){UEZ#d4)_GU_Oq%E0%9G@hkzjTmsl9lE2CQFXwDRa}C zvde#TN*o?G)aaGocjo@@GOya=XP{r9O7` zf2(s-iV~HH0IdWff}P4V`Q*brp|@>(e2*J>pYkx>e|wRR&2-d1+7uKa%C2h1^S@$R zP_y7z)6iY|Pyu)7Pf3Y!jBF}ASU20Aii6nYIZ71!UW*K^Q4L2)k<~CcnLqqS2E}Sb7F%*c+E8`CmIyAj^oTl+M8|vAX>j2}MLi~UF%e|vv z87+_`lk_((x;c?r1MYFXb~{e>$iT9J#R_Rr26U|o=VJi5g{C)Pqn;o~-}^<-Z`59; zJn7b0ik(v>jI!{=xevge8mLX@NBKI1hR36+^y74s`ZfG-sRm)|pvB;add__J{=~8c z1q3WL9=P48q7W?*NM6_unRa|{Xe$@6h54lJ=MYf5u6j#_DN%-{dDVM#*Ta#P9a3QT zhdZlPZqG$Q3yQsCy#(AwnG4}H|KrOI;Mc##Ty{qdcRhvr#>4Zwi+n9fiRd!Y1nb6& zSxqW?jT%8k(`^R@4jLSo{u&04G~Tj$t?;{54z*=v0VQ(rmoN7w+|{P#-{!4r`dy|Y z&>t-;1~5rNO9=-fVIW7bSY;A7ndI@7oMeD<7a)aOV z=K*3KA_(km<|Vc*!|EvE?@`e-HqmQXk@DYU0)gjF83rKO$d)-QW;%I`6H4sWT(%XD1=BJ7Cmpc|>G=Vh4iLW)#M zLPczUK~O$y%pA4Jo1a7jR~TYlI{g`c0N5d3N;tW-$l*( z+JT*IKD61(O*v_XupfA+6U_SNIAv@ha^O;-W(Uvam;|U628LE&gVI0zCh`EC5)NxF{~kBo}lHWdhj%tqv9QgtjDnpiBzz8TD_Z9N@He zBxR6V5eNrfLpGCPtV?ga=l@{Q&9Mw}P>kKga^BXX?>1HE!aU_o3H?j7NAdh_tvFoI zGoGbXO~%@F(X{gV_}H3mhAhC-%8xx5`OE*XWkMTIKQUGRhI~{D)sFQNV}zPjwY5z; zAvN9}Zsjbq)Ulj=j7#u*HRR>_la5Ak^fZ&?6)3uEza$3s2&{0vb+*u0z^??W=7^)* z77gvjWx}La!2!2MM9ePcYtlQn91&?Qm{-S^y@HLpsfRYRot>RuQI7J`>Zu#hClp;; zn%Vfs*!YD`sqZ~}l-x3;%Ab^!Q##~lXr>wO)R#qitfVT^Jyr!HLv3!UfN9Yuz; z%2wIuhpKF9dn{w2EvGE9()!HUM&>(>=Rr3=dD5a$KYi0vy36ldwCU}s!|!Iz_syX5 z701Aqy`!zglsh||!@AeNm;N?1(qk%C#u+x+MGiv8sFF;f!sm3#g(C~%9qXI%41p!t z0l*b9N@}~@$qU}%JDh@>@Johuw82WbT;4oF7*% zlybH*gOr>H97>ux{Hlh6Q7?q;Nk(rntw z4-3?chxmIc<9i*6@s?q~(G-*E^2pgVx!W$h$M0FVm@q1qA6~7oWVagq`ew~-3*-c- z)5A&)*m!we7SJ!-7E&X3J zLy`@-ypXu0!l`=8#`Udy`}+?b#JjQV^){+4fw0V_x;K^FlAj@#+T{h{F+$^U?35;x zuelu9y?a%tI2w_e+OkO>rZD@5Wj$6JxzvgDheB!Ov{IHhb^3x6`lzzj9ktm|-!vX)p`%f}#%ukn!=J7%F$+>PiALD5bq&Obksf~xn5h&;!F~Ksg;1pBaCLEX@59nxiN*V#E1H2G1?}fnS z-w+t7El%nf1d(o;tqw?S+&ZlZ(c!V~h-Cz+>46-nq3!f^BL#;UEfi9T^=w%f4~6mA z5V)4ah1;{j%+`U*?*;5Xb0pVWdP$20chI)ECAs zcknDH#2>3Wwy9uKAMlC#BXPtj++PO>}V(lwus^MqUWGtuEqXZ_Xtd<`^v z0($2ncrCgix4_rFdil&Pa;>~nq(*gUi8u+5GytAIZ{^!0G2R%e>b?cKvg1CZ0)^}V z$eCIY-rM-*0T0Y9VjS;MZfFXuTQDWaIwOUR0r*Xf-XQYt4XFIcZHj<5Cz_V_~O|)P1yEYYEey7c^i|cZc=-Rhx1h z)(h4r8iJcl#0EBb{n=u$w`eomA!#NW*Yx&>QPR!j77!Ez5LFA+FrIyTzViKUtrb?N zKa(Ec`r#gZFykY$pJYvF;e5~ew~zlV{5QWsiMvm3@!rC*MC+ly)>AaeERXxiN#W>^ zt=+br8;qnKoS4Z$SB!%4#H;=GB*4I!hP*<7QeSW7-9soJryWEKqh7P8`iK4%vKh&l zzS~WDraNO?ne3#l-akcB|NMGP{Oy}x|5?~s$$Bp#lxgo9=l^Zsi~Ypqlb0$sT$>Tf zOQ;Wwk_V(wIw**Ek>OJPdK zVA@AAqolU{9Mrb6+#1eJX2bd@C*{RIsPa|1CqZS4kMpf6+g685P2Ej5RRO$1I{xJK zN~?>)ipTw9s5S^5A3@(5*%-X9bE90&5pOA5z$f*lH6t}W8IY&0D~XD!sG>?*g>t#i zhIizt0`lIA)lgUyzMD`<0;+~(_3jEjdDAtY2vms&Vgx%Xp}@J3tY^0ye?bzBjizGR zSkM*Eb;YP{EqKos21`>rp`X9)JW9)x_IVQI=vZ15c{Ir8?^fpKx~XTP`y>74zTex{ z&Z;ldIG(>tL(1Eo;#X&dstN}^%EkKzv4a^Lb>qC*uM<}wE(KX);S?Qp-p{o}U&Q?X z)&?incdZX~I}KKh(0VCk#h~(Wo1T~Cxh5dZ;m;is2lZ77uu!h zI!2*MUsAl~FjtgDhlLKkhu6t-tE~uWMRoR`M#bw{8;i?5bIH$K5=Rqo`1CudXzG;c z=&!x2&J$2pG6IDRTvqjPpM4$sQ1kIIu&d$!*p))ZfGC8JnB|Q4z~=P@LG`mX8G2o} zeCHfPIE`GBUd9`*I3qH@1fxWq4fc+%Qmj{Slyl4mmXP8f9?>(P8GT>FF;EBpAA(RG zvq2h%%nrY9@#U3UVcO&gFW6=#xN0L93)mxF3+t1a^ipOgx(4b_GsYh}1Nr^wJI?qG z{FFpAN2Kc6h(~E`M^@g9U0qbTKVoxBAFYtLqNAr?Bm*S;v`~cWys?KumHX1)x-WKc zwks4}bz5uDtB6@P0GxiU1l#w7%M(pvX!LbIK9Y}nP|4+BvH&{WMF&{C z$AD=yA+^&*hs%M;03T`r3_^^Z8gyk=)QVHMz=Z^9t-||Y!jD%AZ#~kE(YBnFLN&@g zVLbVtSy4~6+@2%S(A|Dm7-}Nrq-lh-Rly+C71T4b_b#Hv#w@c79!Xt`#CTS<0H9M# zl{?EEYrJ~x%Q6~E>t3xRc~uwYkK-3#+`>&1{XB!7*drCZ3&8O7I{8XYEqVRw6RQ)E zv??TLUwLW!BDE2<9`$#)FE8ix3zgnXuYA(ZBl$v_9ngoQmV)&9I@KvMn&#h9-eXOU zOx49aU`>gm7waw@hIYPU9vUEpFLq1_tVhIrFH}=slCxWV;@nHRGM9MRJ3QEZYaw(c z;$X6G=>Z7jf>)3Vt=`C1ki&;;gT)dpNh2=~8el@QjlR!WBLi}2 zA$-I5iVqn5b~^ARk4kv3=hUzG6L`B88;Gb8r1w%2R-VB2lKL|!Pt4Q>;dQsD7lxBm+ZYg#8_bFd zbG`-eM5Qr@{N zT12iy>!I)3s80{Y#1P9LKE&#wwH6K5rpP}sa6m99Ar^RjB8waJZKgBF2{JyxiO z&>sXiIqqGvTN_pbaNJ_pn7a}R=|>C>2Fu$#3wdQbwow4z5l)SP`e;TEux}bLwzp}J zM@sUl3g*jE!(8itbMW)KS_1v?mU>BN`p|9R>WqeG9RN_Ep{gzNC_dgspvG zXha2@UfbM!^7~5(&&~Uv)(4ZJpDn5Fq)o#^Et#pgLr^3XLn?ux`U!;IPI~3K zzBzClj{+IJ+mgE7=@iGWof*RZK{=cNs1{+*WSe0Cz8W?d-Vl$7x5&E*(d6wrOw1g< zVO1LM87I4=*!3=rD{Qas_1&N-`C^wkpSqG~SjV%&HKF)eIrRl_T0>T05)JR2nH>>7 zKL+2mXv)VShpPWc+;98fCU)Gh+@Y3b>aZ^bbZkg z*Q(n=cijKV1PIq6Vfr%);eg7{f+kftH?_{e9AgiwC5^mqHveV8Si#{em+k}>$6_ckB+9Gbpf|=*2nRr4vvz{q!HjFv;ck$X}YX5LSW>6S$l>nC_Hu2k{_=NtH zsnxt#uTf}S-#DLXj_3&9vixN>NUfDg`RGyDwz6Vy)zhHmpeH`&f%QpKTSy>3U-b;n zzr%Rv^jqepX*Km7)h}OuEsdc+&f(R-*t2;-N|75b1d7t9lpYXh>GSrD9PDj#6#>`~ z^m;hxIUzUQQK9e4+uL@h|K8JolKwJw?!s`RPeOd7y@7AKIs)%AUR1A3aqJ?xyIgyD z8FgvY@$g+@n6NfW#kB_zR6t)qpvtWzF7S=`3fE_Mip&e#h&Nr$Ll=Us0Vf6D{U{DY zMqL>QYlT)m zM-%UY(yEv8(saKy#!nN=q>H%RKyJx)7S`t_mp|?Iv2ieVHKG(UwEX8M<0K_$dtT-$ zKns(|s|P87E1)D@La#~sR1WUb^BQ4CX>RB(s0mO_&7wf>vu0?Ag zXBpUH^Zu&MlHKPVdxi00XDMy8P7e{^kkKfl6{&v!j-<3{p!VL^1*+#-$%tF^Kw*rznn= zjF?6(FfEXC8N^Kk*0q6JUpo{&NI2b(Dwm-%b^k!rR z4}iOVM+H26yS$7~L6kxdjF|xf>PwAYaxJ}3zVnoT`D(L4tj%l-l}xnH+1b&?2X-_; zzKTF1f1+TxSUV}XUEkb1CX$*;H}_*B{gNyZd6fM2_mgzqRc5Lcva+=5kGJ7No; zEX*n5?WO;&^J#hS2Zvz(Y%QL4o=?S#|JgL^prA;%kv(4e_xSO}T- zu(L9T+yVp!kjzrw#Wk^vEzw99v(Gj5XBc{V#Ch{LUWpO1BZB4dfQi?X%e*BB0AwkR zCcQ(;lFM@3_>C7%ZPY4O{v&AoTsKzB_LF2zhlbJ4ULw)#K_|uG^6e z)a;swNG>MHHL6fKbFuI~9HnxvWHo$yHC8UBJi!G`3(^zgWp?6d!cwg{a26gGUPl!0 z@k#z354)=J)m1CkUaAD<`Fvh6n&W&EH>#Xu%#0-#>k&-`EHhkE77G$UH;got_8|yJ zWLpYGj}40-pvVSPFefQ0`yz(-51_O*adns_`IWuivy}$QuZL(j9fXd99 zbgKF#az`hx>c3819`RfrO{a4cT{fc$1#MX)Q)90$HdRMYlaqHJC>gt$R^e({16aR!UBzeJ;Nm~imwyHxNStlvxx6<0xn!s;-MIU%s>Lw9 zAOcDjPufqQU~Gv&^JOKD_i`dbE{gtMTKtrGtmJ685|Q@Y3)TE5^cpWoGZr|%9^S^0 zbdD`h)B2~Ka9Mw~pOsDuPSz8K#b7j1GGdll7BoZu=F6vvn-pxh2;(<5*UwSV8es}-7dE_zw$8x`Mh@0W| z#bGohv~9^L`Olw60UIHwT!C|$eh=<^^I-KUE;+8z8wo?G!_V4W72q1Yc&}Q&cPZ== zU1#g1R~1}l_&l%(A!IegrXg{NQnB~h;(Y$Y`dyKJRg8L zlq>U0EWci!V0=z1`ERVO38{zk;%jN&;=B$6MUnjN>sboEbECYW1%vhKQp~@Fp6Lnt zyEG3M5fZsP{Q^fyd~o>-INuSo;Lrju8F+IAuTN( zxz5a!Uc6h&$9&whd76EC%QO8WTX9oK<-tz(j;_S#pqa?vi10s}%x!5YN{@bv;~1_& zly2A!*~4L0o{NiE$J2(9x1AwmS$MjwwTIf1A0;(?1W=Bm7LqJ4_#4>_+_i!Vn{pYw+gJ z8gr0^XUN6pci}Gswo;f+tGO*7zJJU4PvSCg#*q80B&`&8gvStAz#jDa&cy zd;le4IT?Zxn$uqr*!86?BxlliKWf=QfLcmd!xHl9aa@Goc~K2eaDfz9YGDBW3QRFe zc=^&#=$J&#jAFdX@G4tn28@D{&$x&iFY9WY0rc^x+0;N4d8ZmVL$`lB5koouoc~)? zw)r`n9{f*lUgB9v%gyP@;y?dGtK^YypN8>=^z zLss@LZ{1s+uF?AY_lj|DUj7-u6nP`kL4T(Ze4~i zz1@Gbl_C+Y{eYt&XZE@L=9emR+Q#$=_my7d*dNy@SrU2qch0LL**@|&fr7r&SHi9d zYss%q4Le72w9E8mSc4NjON#`MRzO6k7eEJp z)-s&)I$^CS(OP+tm-|gKQiwrHY3bE~H#q(7*aV2j2NLH|2XPG4E6nf!guZ9=J$0gF zO=j%SPflwY6W~^lXf@})hPO2FgSa)C?t@r8An~+79-tseCwSrb{JZOF!45Mn!_o}q9(z5VO%b@DqFjMFAj-RvZa zA$*m&eiXt}cA)L|wbFxIw@RPlFsxeCy)UX*_=U_iJb5pZ)X8HIq)dX;`&= zQRq?X{7^|`Z^G?=L+*+K2EUKvi?$--m;XQwL;XHozWDcE)Jv-z!|Mq9AUwMgz-Rf*e_W`$mzgfJ`wHlYsJlki#7alJ}27SM6SZMn9`B={n z(!1f+(qG>{O8u&NyZ>*z=kc41iXsWazq@un84dfz!w#gs&nbqbDQ!mhr2qTTHdDnA zzTbY?ys7v);xAw7v)Z@y+AN}$G>Oc7Y0SE>dV12@wLv(x?;J^#eFMWJHX%hFp)9NP zM{}T#Ve+e2vn2;Y)2&V8@N-w)h51EYi&sBAtk~Mr$hMYM$$Zi3St$9<2G+RV>QMtz z-cu4V=K*sEA6xn!fQLu?5QD+!PfWbO*pje)`t(YvSZfdu+hPUvGQN2A<=(UDz}D6n zD!S!|W0;b#Z*S1*aY8p_WKvU8p$M_FDef>KU1iU;CY4s+=J(tK?+x)3Ksf`p<}o)S zO0>Q0bdb%im(gUQ_Gxz)H{%<(zO1`fbxy(^Z%cG_-Q5MpxXZe`SBUPTF9NrOUYW?! z7a(kJ>*y>FtoFsw5agxqEGDqH;E2Yv^zUypLC--!xV_L$FIGdFPfba+8=0Sv2<(Ds zz2U&-lK>|gXu=1G$g>;_&S@S?=wJgT@w=G03s#nT#$c3tS=|aUZ}jeeqUToPU+&W@ zu`rKTN#fv8@-j?P>Fn%Ch$>feXk$D&GAn^vyf`)mGB1KazO|JGm?2Ud@XBy@|fyBIcik6 z-6}{G+||$^um{n6vA*ZKGaLvz=jZzIW}*lUbuOGX&glsHSY+k2HRAQ>$oSsO&tEz|&33&<$FCgyr1$#Oir{77e#f@UDO*>hGbLZ!;Ki&T|^NOwK$FXj3?y2{O#OrcG z#dE1kq3JU#x_KCUNbuK~tKIGCv}R$(@cESD`6a%c5E%D#?B5zsj zp=OFcb?Bm1Szvjqvdgf8Y}UlYgnB&vNAKnd+0Wk1YyI&LIwxeUAGFS2-I@mx*51ki zLLUTviv02$9^pI66qVBIDmJY1@i{oCc-zglYk(UP`2#QZia3h30ga@0E3sz9FvXXc zZSIdVV1&JVbbcj|wqC{}AGBaYHipM&f|w{r9CB0to0 z`LjAbCBl*|+2csQfW{r}!o5zFnyPkVv{k9ro1? z4C3efL#YNLK&tccVGQJkiNK3-oYk)^s*3!JS$cN85iU@PDaxd0Szh|;j~o1W1`WR? zgfv&z<3W@8@M-G6`%kwd1RxWS0OOEeQlKnqm|ulz>iUBh&hZG=E#AqXS+BzA243I$ zXa|Mg^zW)FoZS!u-#w)i-!m%rHUGEssA#IlvpX-|y-J3eja%1gn4Pai*nvVXtcN_4$YWW1WUO?p7JhFDiD0Z}b_@W^^Z*jJ1{=QANi$V8c`RDD=IF zsaQ9l`q|#MwSZ~XVaZ+}4T%zth0MMVUa0S=Z&m{>BjQnxN)`}p{b5-ZDPsX%yDzv>@2bSYs;FwB{J6J;qu*jaY@<*9%Rt-XE*!t$z^BwBer>`8}yUgPiho~RM&^+ zZk{=Ka5?9=)Nk_6c4Q^Cw6$iv&%eTk=lOQ{24CPoS*(qvon6~LYc;or?VBcBR-5Uo z<(PHDB7a6GNI6(u!^lASdQ08Y$39Kz<~YTn^wC0B@4<)iIWjcEAzq?=MkFVwXQA7LLp))ye;Ji%nwzxdHWXr{<)P+jyg+j}hGl zdE*bMKffezinE;S>Z>_ZPuR48H&#T83nHnuzp*Ckl)R7yF`An}w4tG;g8~k7?wRUp zVtMzx`6-UKela!L&w51$l_m&F1gb0!4!>h}YT}L+;vpx*@?|rLz6NA$UVPDR{LhrF zY*gsG-$N3pfx-x__Vkibp8YU4x1xc+1!Zk*q0+cJoGvlwcZDRiN}7`!(Q7u2bme*o zHX>0mO@NW9=op^(P~fHbPEZp=LhI-nKh?to#&MCTKa|(~N_Ct#rT=oVH~Xsb=Fg7? znM%)p24s5N?O9{*1o=^oIome@5p>UWZn4Nd7g->2G_$e;HGP2EH z?Aof8gMHsX_G}W)$jiiZ+2c7RXDN-osgBwIjY-c82vvXm7AL*OfJbAx{*&jaHP6y| zBG!s7{6~FN*LJcSE!jjSv#Rc)H(mSI*6uOts*U0A1qON#TpiKBTHD$@1&(YOk0Wc; z@c>&J)m_&+D){HE;eq6Rk|BjVdQgqc=wO|{QP%5VxVtdtr9LsAHDVe zF&-Rn%lqwGe3L(!ddYCLV#DQS9`!|2?%Saedv6C>jM5742`SD_n$_PC14|83J~1_s ze!ZMj^k&|?W5>I7%U5>41MA(mm1nc{lgHxRQGJ5|OZ&Q)p;hCZQ6U$Wdy^i_x$bg* zbCg;q*-lmBtPDANMNX;))^=Mswr$(CZFFoq>DacNj&0kv)3H0YZTLpUTe{7dZID%b{vVs zR#3U^oHVBYE1LCro(j868IQeA00*TIHk z$vvJVKqMN}(YTUeqN%olzG`Bd<={Bc6#EhQUba}S(tEZc9Wxx82@1NuWSPQQphwtL z%oL+oS)?tw`j-IrH8Be$i9>3GO3PE(Y_H4UOZ>)}r2mqcwq(%HK@vaHXMmzY`va%H z?!+SwPHy}A+612Ov1;3o@D7#y_ot$HqCJA?)_qsq=W^1ByYBatIFB~j?`voGREgg$ z=oVW-XMazQX>M}RFt^-37oB12m{GV7vRn_Pz*L`SBiY{&=24OI{19SyBDLf#8G}*rj#P@jm@y!21&T z`)Wxqju}L09wuUNQn;}4bvzLS1Lp*t*b2G1MfrG5(8hjydwzN1*QvQIZdC|r!B6wg zX(?UhX#%tMFUc^j7A@Xp7VM``Q>PfM`|SK_RY{(|v@MP+z$+5jRO_L!_;Ogq*(1m? zGR30k;K;#v#(>4XG8cHa(uZ2x@E`W&-`nwt2dEf#|f z9DWwKHLX*|(y`)gvPgeSvCcZc&;+I9hmV%wxe{3`hm8kr%O>L8WTgP#q=-hR++73H zLT#pKfF!8*+iVKKsTG!pQyiG{szy2zG`G5Xd0J`|C{@q1}W%w#6cRv-5R57t|!%? z0K+EF>e?dkG`CnCc}{S)jbpJcpE!~(VMLZn>k>8Q*v)UqlBR8MEumlWWTTu;DE@I$ZI4>u^ZF-S{Q)?I0rWeZUtJuN9q{{_V-i-}}98gy(9s48g;)?I4%$ z^OpCfn>FO`>)T@T^Re1P4!7$|Bqn5=D$i5r`cHlQv1H)uf#S>i$EVh4-Phv)szhYE z>y_lU`!YN0&q=t{=+w*S1G2@>%UG%WM;~!f+umU~OLQlRGM%F_ zkYE)3Zz63Vl2n=2uCfv#M!ilrjn?iX44v+7Q|K}FznnSeK;qs_)3sgKoi1m;zU0q1 zHp^XUQ#68Qi2BGlb(@h+CYUnB{NiHz28=*TBzI7K zRQ%WGg9;jCx?-P!>@3r=4G=C!!68;gXz+_kkzSB<_ojP5r4aCtenxY^b0sZ>(Fi^g zdz)xf!^L>3R@$b!SmleZHrN3qrPT=chjqvhJ|HvE0YW@Tx9c{r3wIsZsLGY18yLn+ zaqXDu0l$d@m}r!c#Zw)itpcGvA~gcZjI~>S2@(>++QM65FLZr>>hwk?D9=#!jR>dt zOOcFb1h}U|0bTOKS-v7QgH6&x#ugWm!z!!s(_}F-$9S=t>Ult+axWHCErp6Cf@u%$ zB|@t=vY=SNurS$y0rI60WhjAk4!g#o-Z|yb!k77AX;TEWG;&7z;Y2|xRRP=zECG)( z0pxeMq<4B|llkmz5!Lk~Rc4&|7RTJ1zk7}Zd*7Gq zydIa^FB_)>-X`ueN1iz$au0 zC|eL=#Oa7ey&^kIJ=;2mKiy(V2n74!B^^gGWgV55i4of^2a?Vrz;-cj92=HPxVfS- zV405bnHFyYRj{-XVYcI3(t|-00v*bklV%hYq_wh3l!MT#@j`{4oMDmkxssL^qx2_X zh@Rm=w5M~lgf{)kjp<9VXT8xy!Lau({2%aJ^}3>g!0>=(o}ek%#PFhMo^8AV{smz6 z?gT{r8o5mt1A(&YkT>JF4o$-Ak(gph9Fa0aBE5hm<>>m6-PyB!EI^E=be*qSHh*{3 zm;epOV?u^~6$Sw*24dgE1(H@ld1UM@AJFGA(-?V@R> zuN18`c9UD#ErQ`S+pcq*W#)s2GS$n2wARny;4qvSkJ2Qnh?%r_pdi3;aXcKTMcJ$u zjP!!m7u{nekA#)#Vvw<#6MCg;31CNK751I;0TW>Neb)Ci1f zf?{KaXvEN)0(FTNh0tSy z#W73>out6mLWsE}se#qcK};Tm&p(<}U?x#`i!HQAzgc8F!iTZPKsw?ho9ZUiY_>!W z{RYULnkexE-eLFCcq9eIB(`VIJj=I#c}*0%{qX!=%WNt4RiyxgS&+2NNAn0DU?042 z5rZ?OnZh)V@e;^Ka$dE`#1KeAiDsNy?tm{0c|#eT9JR&Ev{jZN0ue92Fu7fi%>+!$b5J39oRN;saqR z@N%qx+Vs>ZAYqvLjz`2s_y#cY_I+L1V>0UhTUt77+q9$yB($Q9hMw0g0JcyJ603QR z89P`$z4TOqRgaL3CbZ-9e}=+xj{EPvAq1v7SO*xwn~;!nSJ*+8QeYMYf*NCqZexNC zJPFu05eNilF--A@xdkMj0C!kWcV-b-ctX!t2U~1CM6IwHQ6A1kBSZK#E_w^1 zo9l$b>Om|yk`P2loDg@PNc+4{%;6p{qe+cynenEVdAR#=jj0n!><<_IsC3J1_~BXu zCt|H;BGb8LnO44#Sgt%NbGJ!8mf|Ls?Y|^MHfzyl(M&A2iO|d$p+Z&ZyhVRmWTs`# zjR-XX&bf_BF(zyVzV&;&-y`WwO*)e+9`y5Zlf}&U?I|KB8=H_fE5c4Ga~#x{0Q6QH`P)kqANHiuEn9`5tUP1zTpqpO}R0fj#lU7ok8(92fuY$x$rPI zsnO&7AZj2rLEawEZKvSSo)DHml6F3`U9AM2HrMcAd2n$spPm9zH?agWg}w{3lvGR- za42Dw?ujOz{ud}f z%woq)(D~h?)@E?8Q{G(7;J+VFcD%bQ;o=gRp)o^>W}5ILvU=Y98aza9G)3QL8*>vN zMnG@DxIeitbYQ7gN0zRl`xzcn*uVTrK61E#qPxCP+5HY9Ogw-rDB*5PjhWwi8-ivjLGv{y_ z(VQTczVCpvuTW%GEfvlGUXV$i!Nih5HDJCFg=!9rw`3LSk!4wGG*xv4HW=LL(NxJT zR$?ca&&S%l@@F9MyHlG#a|GIq985&CAEYKNq-DJOYOzH=(rmkldoX;Cou=EvzdGpP zqd`tR(AvaAXp!(00z@09ToDtayv(yhpknedw zvC1KB)RG&@G|$G;f#0^gz=^G{-r6o5J7$hxyvQ|}nlvlp*3kD0W1(9z@Ns}I#)bTA zjDvrRwUz>Mgz|+Y&#jK)6?v6tE(<5AvUNx~9bBrue=3fqvmlCkQ!q#pH-n20$@d(2 zCt^8bDy0f`93usBN>-zS0xHAXpUh-ojMgS|yoz}$iyUqPm2h#2PLHJu*j;?0l0$Vs zik7ydxuqu!!X824JIxQS?LDs~zH4xPUClVaE(n3-nzNP1A;oXVdQ`%riqWu zZTkT;JeX^S>wu+`1_V&*M728w_LHV&Ic+zJbWmFjfeG<#=0n|Jkr=82_Z{X)RJKXN zE(%RCwR7u6{)M_F>=8fM2(S^HVVDOZf&nseYT@3UrzD_|%C^!%`;PU^yQFVYV8a?M z3oe6pekPb9UKFBCcw_%amG`!i9Jr{cMCxy^VhO-H#e~UD*I7`Ql;acB6p=;O7MmGS zNvKapR)|wULqw)mb6ad_R2v_)e^F*~kic-oI4o%*X7iHC{Oo$e6jTOFVGh>JSPyD% zA^)!iz>+aBvGHXL{$X~b4_84>sLUGv*>YDI+ zgg{BR8Khg-_<(wcd?FG=oRME{;s7{@dVb3po$)4a~i2gnYiR8#fJW~YH51N>dA1rAMbT!kZ zrS!3dh{t4MNigfHgDE8fGs_A)sw<^>?GWc*zRDC;jhoJkT3C$k*N>eqzgKmg0i%j` z7*SFQ*lGnE@iqrKm3|IK9BC!8P$s!BD2No?;%J~Gs5x@gO&1&b_yrQ%AYw~(?+%8L zz@x#SS$$SJLW&Tb7$z_y1UE}~R%}Q0S6a5r7f)dXuh%d`awB!;W(ru&Vd1-aCl^JOlJ6d|3uN~(2Rcw&Z|Yv>$o-& zkysRZH$8N4gv9nkCAcs9Iq(6#WXj=1u2{Kz=Xln1NpO&^Ubon{)qCD<&HWE0Lzzw3t(bUK{K;I ztJu0OGLzP5gVEjmo&!tCkrF%JrL!Uu)j+sMkCeB(tE!?7#Rzx~Y<6&n5awAsl!`{B z&3(ME^n4n=0-qS82|@iZ;1{p5m6gCyY6h_Xs&dH!^9P*Aipl&57M|zkk6>D4f^Q@k zkfB_BFkJD3w56mWT&_cLL1RK_u=Zi>=PZ?4)~x94s%gy36QIhWq*^PL6A0YjcQIyq zK>I@Z3P?~HqI@{9QT<5F2M~ljv25}=h%q$d*`tk$pqEAD{@j6#sqiQOOO0^beJ9Jl z!)#)IL>>@W9D9#b3q;l?CocqZ*jIyYE8=rH8pE@66dB0?17u^cp%%j$_e8OoMNOKJ zXs1D(xuE2xAdIZ^z~t2;2cTG>swWg;Xt3C|X!|^omHqA)AwApR*6U)ks7^^lOLm|RbYvBt|4~&5)z?4>O( zEe|BZu#WyJ03RTsfJAaLIsn2ELc=pmz1aZvS_`OKWS2o@OO;BuK{MaOR0f9P;-NUL zqO(L3=pcu+V}S=&R`#`OX&4&dI~yBZEn3fSv#VnOCVi-AhI#+g&3WLSP2-a{F0=2dC1uVMAomF*bgka{M>B< zyn=G8fP`fjV;s*^LssyU@LFdq^7Mcn&S{L2{Rlj2LvbRaNbsp4F}+b|0k9R*KFPjA zX!w-m?{C1Yx#s(_Wx@abnwRU7)$_-c4vObG)Zg%Qk*`^<#}bgV#aJT6hm#NL4n+Nj6443%1x;q+7a z44Zc?p_7zOiL*d{R*}i6%FTx$3@I2f7;d^|Dcf|L&B$(eR#4R1prFj{JSYZw)4L;g z!%$)zk`%eD=uMj>1cu^mlt*s1NjfW$c27Hfm zqTc(6%-zKa4@SHAA@Ex8L4(#^9FQ=;f9)h1fb3jfV=@nZJAMyILUd7?e z`Uk^R*DxpG?(~z2w3EGf9oPjO!g*Q|>;{l^QT~dLHPMRNR#(3qg*k&ELx7d^!4o=Rt=4RUtz{I1FWsWVPVT_IvE=0(%N4}6!5iqTa z)GmZ26rS3zB-}4Kf1esMi#^MS9RzJV3@k*%nBgD~o4`Cn)uEQ;49v9=a=g=}cWVV3 z2O8iI%>)+p{JS*tYW!2AcQ;=X?7%kvs5}9Tz^pu7{4GJ!4qhb$egG*c^hgL#)j)SB zliqOLvA=;m9fslgk0vU*P*^HKtzj77n5wh{2q{wkdAmhT7{LgzR$yF+FNE5e@GDaU z2dsWLO0Ns)E0hd?)Kx}C48|Ic%ivNi_&ZEac#?P=0(3&hVkE*I#VWIkO_&y;bq~yi z5&@Uyxk@7>V43r$5xaS6~Ce*|wx2^3W71=dLxzK`{@6 zu&;x33PzIIh2*%bWRxfa71O>GdKmCRwd!H^)k4}?q0mHgEY%T01q#}`n8x6X z61#x%KT92^>LU!`FeD-%E~Hktgko7h!4Cux5<@eRLo|CjRT7bUCcO}?7ti<@?Hn+? zEB31KgLfkx<%ad}?@=U_Oza0`UGxwR&ysPWl=dQf25`uS&K0`OcgWh+!`L;XDE>SuxBQ1tfzWsxDVjV~H0nTpAs9cR%8QJhw}s-2 z^zRSJ{sOoCpzACJh-4R3Bu_?QI%1+JNZJgkB3$X2d@Pw(emv;(gK%0v2xp*3;wf;E zrAJ(bs~VxVhqaeV5tZwojfeZST7FJR(e;$UE>?l~dubR4x77L+G3jQlbUMGoe}j4U(PS(Zg434%xq4v@-l z=;%(-Xb_3b-MV)VI-klS;Lk_>kbIz%Zi|hRPs0_n3scR1A@GuY|nO(2^pI z#)_I902rpwg&|N;+>WS2b!a!b5eZP<*|>0fz1A!&krGo4fENrL9_ICRGAWg`g9P4$ zByf@{wE`9`l^-`RAEE{n5%hy`@N2uk0&6SX@{7r8oIB<KcqC=C@GhRefRu;CE}QS1GzF;7<)w-lEJvZ3`X1-A zim`-(y@0kjj3bmlCTo1&e1TK0BMU~nEQv6Tes=}({FIvQGbR}}AXgy4ymRZ1)M9e6 z_`z@oRVKS(8o;2B96&#(JynxbuA(RzK_hAHWnyuz3@Ha@Is==S-caI&R_|+&ARQqB zv$}o-A_DEo1w*7Aq`EDlcdtz1Dh{m>FL1Z?Ko3V{Q3C8A7)&o#o(U42+mJ9n0xC|< zpqL1EgweQ2N}&(M1*Bl89bRUJ+)p`$3EYy-!b#A(1RpgX=!9Ys5vYt}%$TXS7PG9l zyn-r_jm~s88!{nnuFQN6j?gJ^%_14qC|WcHedkdE{5JWPDUwYPvmeBvV-$i6Ng_rD zH%mN`^F@Nbja;vC0W(c)$7P^Z>o0M=zC3@SUsuYQW?qvhi+WHwKRt8K9$2_#nkCK@k zU)d*PvW7^NgOg(LJ>_KVI*KOK&w-B%!^372zV7vl0DRj@OI*(kw}K9IQUOYo6-Qxs zjwUhy=u;#e7UX-dX0j+97z-+cAt0L%yq*-FP-Z4Av{wb#S9}o?; zn9XFrP7v)p!1Y8j9*V;&02%`;t{8Wvot!rmnGQfKYeba};ZSyZ&Q7wK0HhTZ5*?3Mp@gpyJ{eRL zT?0L?V!av>0-rai&VD~qkSHnC2fSYrC9RnSHC|tuNWy5({dRZ2-|xIa92PUlbA~uC zVzrwjKr(MNA)ZxnjNR5cmge;~=}%IEc!b7qK_pJuILS-@t--pSw6|QLRSKZAd|@TCJ%d;((vEJQPCqgyI>e zF%wFhtv%ad9F8HNM<^CzLKaMcqLQpYcR-gsV5B?{l3G`|A;E;i^#`1cl*Sp!*@-Nb zu-zD{pUsFOVMYo>ZbJk}2s4h(#3;-(BVj&jMf7$k$|JRbhu*~zr!!P5|8#>3&Rhah zud_LHvHWUX_mCaa0qa*o1($SAgOl{8BP<_{h;o(9gh+n@4io=5gS2ux_pUb%K`fRF z79UKwdfKx?#SlhTt8oN|LD(a{&NS>emy&$n<^9Qt;7=JqcS!>2p*M=V55f zAVQQ&!8J+|9q1^W815vAOb1^}!_Q_xidqX9tvb9Px9T8Eu#O?7gk_e0g#PRFms%Su zkopCgh7FI=Xol<=q#Gmz7YFxbCa24EA_)-5X<+CzBLTdEz{msuc8{QI0Dxe_m1?ae zm&xq*w;d=YuRa&wx|XjZETQ*AUbKB zmE-g#=_{i0G_0dLTy5tfxXnY1ydoInc3+97>2y_6^!z&@&T~ckeefP1pYNXo^P?$u zwm{^B7fmPwNYI>9GCX?qU))k4|IvW13A5^={$Y?!il5;v-L>z1JNExPn2yZLz{CH*D&9sMvzPHj6)_Q@te5kZbbFkf>5N+(@o+Cls=b-sf|?KSNBtY9A@bRDI28 zW@U{a%7B%sWvzaG52;_6G$>4b``kc?06zIfI%U*7ZqspPezcq~|H>YzoFn2#il6V3 zj4s$y=%<_*CZIi%V|pkO4{Er>aBtA}Wj*{6#Z*K@WKcrn|C~E}dwYXMQ)ss3aBb1e zBEj|vznUUeKP-){0RP_XuS4$ft*uIN7Wc7yy_c420#cjYk>}F z7)I$xQAB=c5=Sh4nj!u9)YBn{4D()S2rcOJkQ2*LHX$SA;R#rcjQL7cPpFwt0QEv- z>feNr&u7Js#}FOEC$D`-0pW<1{mZNxH`x|-`a3pigrk7(yiupai`|-e@cxVb>Fq4#M zuBk?wt}H9iM)@62%fHuW(>r8DL|;*euq$;xu?Wo9&5MF2EY&`IjG!dh;XWX<54jj{ z2jVvw;n>-h#_B!+RXHf=_?Oc;yf0~epGP%q+g>L*?h1JVUjR}h-T(ntqC@-(a7es` z3-aMGd!O{R!%cwSjs!GD%Uga45ETqlCGItJbYKVbUUfSc`K3GFx&s)UaErfq?zaQF zFicWdhf*DJ8K>Hk30|~|t8;4+`+gr+W5q~PfX|v#xtmI0OT*Q*KDJ>0cO-x7M z*nDK&xobL=6V*s7&#k;{X_31c1Gv6`kXcU#tA%m)xm*aM!0WDi??X<73L|A$*6+?U zjlOp^jWoYo&RQWa#{OJg$5H!PP!MtjZM)3_6P;R#FVHXB30z3{_ZfRX=Yd05zpp9- zM_=QgZ8lRWXBE^^7f}iz_)i2;&aFQ1`S7|6$s&f?mNC|`d-wDiwapB@(>1u zh(3Ll=MRE9DNox@bnojYt4=T0e{F>KVC+rCq3c3dH3jV-7ADh1UkJyspzYx*&+dkr zu*?5$XdLJ9|7s71!v>N6T_Y4(oUQCw-Vx7tKOTWhdfoTqcg45vJ&N!4#bNF1tD>@a zoMqp;Zk`2?%V7&3tIbs|^HxSSc7b}4R$AMX`!MXpsGkY(o~AR|ZQ8cnydUQ!{ofb! zNMgw7F(44<QQw8uu?=hLqH63&MczbLvWD2|4$Al#gf@pMI^46u80e9Y+hf)QZ0n> z&$Io{^XATXs;$p$FrjJOf5_N30LA%={qyu>RO9TFoL%^EyrsEOX0};P*_|o+$Kb_t zqCcH=HmhLi^YY;c$KTZ>YwBo}6y`dsy47CkrQhDlXqxM0n%SO=bwlP*bLx2%v1dJ_ zmsLr=tli^g8bY1zW4o2ZzI>gRQKz-~&)>;EOzZM0q2-eQy*(jWQJ`Z{fmgW&Km|I; z@Xg8N@RL$?E%^$bpJWRurR%oq4O>+dq?k+0K&mQ6ioXe*mlu8dD#r6aW7$I$hMmVR z63vNVVRx$>AqTq^jDd{clj<6tU!2!=p3@YTq&Vxn4Py2Rf&Z+u^@@9c-i;v=3iyQM zyX_7(9mMfEO%&)lkz>t)-hf&*Wgaq93?Uf?<>b6SPrkbPe=Qm4*=QOWZh~+ zfhuBTN9ijcF<8ZtwNDZm!D_C-8m>kPR^f*wZ+{RLT^d!vDZ2XFvb^4-<1Ffa zneQb2Mhb~vxudMauZj6z%;?2&O1SH3J~pXPZhLIU{M!%Q$#m2|8;xAGtmx?|&>R`t zaxf++tfehsEmND^n6c(r&e#{Gr~T~)YRPfD(dq7@WWkNJ)bTJYm^V3ykjCorcseiX z_r5r;gj7-C@EFbx3b2R2mo|NLg5%(Of9JHdoC|zR1%kt3ocaps=CKiG5d-K3`DN0dS%{>j>#dZ8!hgtH+~tk}od&^&QRkHqnsC zUjZyOtHgfrO2NkbSTBJWz&HME^Z(#1B)3IlKB}arCCV*0#F6OC_JnbhNC|WoUwW)g zHp#MU4a{@ZhkS_QjC)%NKIb`}MyYZWt5@W)ie+d~Kf>Hww&#wPk2ZYXuva|c?^KsI z2qsO7->!Es+=Cw*fydA%%+WA^SayIztoB>2!dgL9CQ!+zt8b4nrBb0n)nEMUb+{VL zNnq5WcZ-9(U~r)cb&;9VgE?kdGt_{wg2I5vfII1B;~wg&PyuICl_LJ(m+aHSS;vd}u8V3`e5=eBb~XnjSv3YG!p%U-_cReTvv>r!+-En0 zr>C0Y#K%da;6=RKp{KcW7L-kfFx{NPgHnQ+GL~KI2kuNFSRef zOtgJ{&aSlEeOEToC=vK9b+}3nTk#tkPwdt8p1^SSi86i*tb1xP*z#(fX~uur9KcI( zF>u1F_jwzHkF_Ag#*K7Z05CI&Hv;FcwR+G@6Va9xDaO0rzkj^#x?8*Kbe`9D-!~SV z{=K_w&ncz6EWhXww43k8nr%M2^Qtbc$F{b#GzMQ^+!Q_5XI(~x|7z~LJ~c$0QRuFb z_kIRkFWZl+wlWayjQth>U<+Vr2tSpu-FAFmQd3iRzMtDJJ$oi%jjvCLDvn!vM+m&t z^B11yh7kJz1iGYqd_d%u+h#0}LZle1XC+N709236p^9M9G%%=>m3M8qzCUe}=XWU| zH-=(1$T)vMR0U4{y0j4dNY48_`~c9weV;o&A6NGNfUb+Ty#F4YGW{*fgdOT?1>
NdlsC{ZEfuA7bbgvz`f(d;eP*7 zVil}bL{zQ3m;86j`_b+eH2W8TQEWk@XG<&RfBClvQU_lt>egxF*n7P-zAVIC*SYEU zz4Rbsp`X2qa?Y$=tt?g;khXg1>Y#sM>a+V{@|W*AEZX2{{WNo6aYo$53G;(0&wO3! zpL1;6jEz18!e=Ud4e_)p!{3or(jOiV6J!Dp!;G3`?|#lhNB$<*6#Syqv3JB%8zaz` z6*)#{uY)^q{jr?mJ`9_g<@N@2AN*aGsV%fK`ZEN43xci0^|}ZtE7G zW!Qb0crEYO*j_xLU@mp%Xvdne^!f(r{tB4*Rr^64ZEH`c&uHYAYxlX49bJ=nnOl+N z?h@FjqlT(l1Zg~9lGA#1yhN6Av6yj6K;dNC`+n6jFecn2@%?^2b^zSXQQJQ!*cw6nkr420gzn=)e^Bx*cVcc@7r~y-Y*U@Fl(N{( zh%!Bw__E8hOsZ0m;p`q(>iSd%3IcSuock<<+=~1x+Ca{IN8hGRsqVkPdWiZO7iwo) zRMkT{G_NBiDiiyiJ508`*HrY)%fX%F>0ZvFb0ZeTG*@1fcVXAiJ-i9NTov$5y%b#E zrh>gyAt(@ee7~e!A41QyG(G@i>dzWrA64xzcJAEq`_MybKP`;5q*Tn6`vCk?b`1a( zErNHEitT8X>Jl?i#+!=Lu7BX-&)sUZ+4mwy|MK4OHLvk5<`M=3OvEjJ7|5KYGn>S5 z?L)bsH~iLB8uqUE(^)gWwKYG#UD5aOXy9FW#1G~}M(D8%OvBFBdFkz{92M@cwyL5Y z&IVsFd~STODIXiFtLidD{##<+LRE!``>vl?MZ3|1d0+_4cH2|q?(<| z_$@7N4INdrN(kZ+RLpV4yPq+;^TQ3rsE_1ef8q!4OKDY9A29RWWf;N25vF&#Dm)MPE?V3`k^wOmAnvxBO69Ss4PJ9Ds#v z4={sCf{z{p3KOTgz+N3mspe|?OhnS`T5=s9-FhVyZ%)4@T%Uj;38cu$12*mZe9s!q zhLtKSjosdZzyJOPrKyoS+Z7alp`6L*3BRzz@2XPeolbRCxx`vvNh)VFii?C zAOM!WI^QcBi(F3%`bSE^H|gV`f&Z4jE1Ct&t3Mw&i^H+c|2o`%}C%eouKucl4Q>YspK41E>a!mVP> z#hl*@HGTqS{Nu?uTn$I_jfzejKBsC~0qw4>d-jgo#O0+bo)<|sLQwIY6cxE{~BYtih%~l%vp5$4-m#CXk_?H{0g3xqPH&RD^5udXT z5#l@Hk##(vNQ0pL6xC_#I?pR134M?-(H0Q>Ux)I(AIt8ffNXuJgILfxPhAPiB>Jc; z6+xKRHf70{)SFweTERDcLXZ#W0fXC1SKue^O99|(geWM3voqem0l@0P{Scn-$GE?n zSet@BLlrLh#?Zq#BSA!LFDS&?ODtNg#x;okY2)(AISX0tawH_LE3B;f!MA{sCf3^0 z#r`;HyjWuQmd8QtEZ>9NsIN^vFZG4L?ANHQSD?n0h!V3$vSLI{uXm$>>Brt$OdWl9 zZ5O!Ny*=v6&E1p5i}~#aq>HbIZ;d?8Rn^v+q21aa%Np;1BaZ{oIspdzYo}KPp_a%f z4wq9UYix8@j<`zRU>r!h*7f7g4*a%OUs$Y+<5Kh1fu9TiuR1`nOZY-7fQ|a-;K%f} zACj-)8hs#3A)ZXtNfXqTEj+boh7NXzt~|O!Ccl&<^hG6Pe4hNg#{awm1cHYc*1zpnt()oh z01mNdRo@#QaL{x@7j->6weR|ZVzMxpiV|7jvFfqmCmd>oj0)2C7@bta)%n=Tha!28q}K+ef&Xh z`0M)9P`z57;^CElfXRV0bNMNrYOWK2SToL1XbT z=`anT+!mo&Ljne#^~Di}75eC}4*?vh5yk;vl-dD2+FDI7|6rz-jG!R^&^(xS#h$`5l^8Y-?AuG zi>ieT2d~TXvI3Nx&mQDXH|VM5D(VXteK#sE9*7DSaJ<_XYua#3I@!T&CL2g;`Q=eZ>^o4Pj})+Udgnt*RfgtpHF}UFZ@&tqhh|M`pc6_RBDc;#&`$1 z#%oyC^=Y@xW)4H-ItzXNu236wB6g3lqs zZzKvgrjMsO|D5LW-C3-WAaWt5| z>}7Cp`iu8XuGy$gTA9ki7P8_-pa`eD;j4&^B&Bz?LDuhxez%}eTL2;Ftz*{r?@N0H zud3VDP=ubTvZvl941y~Rvo@wm7~eU~M1oE=@oo}4B+j5VOOOnoJ!l#XeBNNV@2FTf zsW!(>h@pyy(YQoCqTvQk0AoY@vcjskL7Nv2E`1<U+e4wlZ)1ZVOAoOV@XO$u7FR)6WdkL5|jUZI%HE&7eu=o1}+)m%>R83X{qqWsq z4iI>!J$3_=d*9UqAiUqw~t3heXrma!FKNJhZ} zFkorQ@BSR2{mzdEgG4MH1)i+>*N!0xUW`BGcfGaM7ol) zp~1-Bdf^4wv~Bl580R!d6q9tVO>3}7zb>W@VMQ-7H}{BEoldL!_2{*AJ8gdtBP?Pk z4V5C_22sVZuNuhu-1=he@1{UgJ**yIO7kztu}W@5!LO6DTrcZ5plT;T-9v zHAKit^@Y&7!7`kh&P!kJq*BWo@GA@=+ulG&ZD-Y>9`M$>HFp&rersu^i1|*~ns_15 z+ja&^O#DwMF2sC)p|%Ox&LNX5CUVsLXsnQ{4~}RvmyC(wHT4&-v{=dFj$0xzyah3N z#p(l0eyb!rZ1}~w2uqB;6q__F+yOEeIKWZ$chO?{bwiqx`K~@G0pJ`c&rx)?a(u90 z2#G%cu{}kq6JCt?qMXM~=3Of&L04$6FVa!0jgss#uo;HSPGRW8ryLr@R-H&>K7B&f za6ACJ4FhF3RC-@{6g;(A3csj;>GAZST6Aj=hWx_#0;UN5EStpgm&Z{|atHi3{JWD+ zuP&B3y+_1?vUX5TUh+oSC>)~ppj!~q9h1|eahZ^siMUagb==S0CZEA|0)O$>%50bhWU z@AW$bMzSGtaOh`AX5Lny=vFJ%j+t*tNy4VlSpR@WyrGCiJBdO2R*!ja4GK?9!#DVk z+YT`;AoJ>~t8n1vbFG^F4skAd}3S@5# z!vG5Q{U7^&L$}*e!^cV$8o$|LkbKJi@So8prRF`^B8C-Vm3`&;Ws)o?Vl_a2mU=Ws zPqXRWK0pcG!+#0^d}0A1D**CQ^xvopP-7PVt4Or{RaY*&gdc^59*t7+Raa7USfP*b zsBwX+%1>PM+eh6B8w$O#U2&fniY^^{L}@iV6KCCY9K$CZ5(IdXggySAGAm^_mtYtO zLsv9$Pn?h=OnmF$_N23=Mb%|Eze9H%UL%o-s%hCvAK3pVGX-eC9>uAlG8%N$EQ`MN468Vvgore)0Dy+Sj&xJ?F&0&s|;mhOb z|Ht1!L=MBGaZlC!Q47C8n#8Yo;9>CxPW4Ef+ENHEQ{NRn%p=jyw4WI$vQcdrl|(E1 zSd^-I98?c#Kb%8Yhiu>kL9hL9IvKVQk7A2X5cC@zWuhnhr#*HUA51KRACl?2W|2-U zED_-KeI18MZ-9OO>9x9OU(uUxU|l3SP8>a4-xZR3ftGZS*ZVq@S=*8~3^AOcX<-LJixI*>>o`dr%#w4-~^`B_+`lQH-tn;haGr_X^g4Q2MxRq z?{r!_1CEAghdcV=kpxs3jI>BN1rz~<0S#+jdxk&LL?lxnPIyRcDpi;wfzrUP8sjy$ zH`b6~fQcJmdFI|{{Mfa zTT)3vsE*N6anFImb%n$ZeQ&OU@bQ7-KVj zulGKmK7D`hzgo7}`}KOhp0CIAd_A8N=f0jl!$&Hz-hhJ<9il!Izpzoa**$8XX%xY7SSPWlJot&D_ag0R~0 zEsu9^PCDr0Ua~>3?yi8J@`(=r@1P5exZu-QpSP<_O-GdO-m6&aGW3RAHcAb%);s02 zviPZPUyA*4QqZF>DJlm)XQ;&Y-iX~JzoT|@UaDV%p;lpg*JgVqom=wR)n~149M)2m zu|IL}PSCS!SEp6I9{>4tv(S$%WnpM%y4ArOgRgB5mLN0_P1HOoJ`=v7#%MrXYJ1$L zO)-U!1hCJyjNGnyTzlqeqLoA(LbgD4`?8~U*=7D)Z}Mf^V(6O)`hCS)+(VCm2I?9k z*ET%faOd`)g_a#UDHG=oqj9H==CsmY$US;y^i)32Am{}5^tnI7PIIq3g{SMKHtf%iNv-*Zmj5vVz}hFP+^}Euy%0`G&!Uy&8X{4nI*G)mjcus~;(D8=lE` zkbf3j<8o2)QtXv0j}HI+;fC&}2m6OQYKK0=sAO)~-r49?f8xB+_T>xdk9|AE!+JFb zTXFC>`OnE$tcv_ozC_HTRB?wxCk zxT0!cg?e9b!TWJUo2umotBsfc&dc{EJ%4lY%Zh(T^!&b{Lq{Cg3XcK~*?;@f=F-j` zk!M5xVC{UX9d?qSp`oF=>Bg;-QLP+5vZhG}y5-S8PcD_DRIn00VY)@p}rhCpB`=9&);*57pFT2@D z{H)gOeiSTWbbFiFm1IH5VA1^jCBua$g^Egk9I>(v3c0oYvby#c1taN(JqM<~o!A$- zA+6+=pU}P+(KUtn-zNI94D-^qNEUC%6XqBGF}%r4?A8Ik4ama2hT|D`k8M@hlrp#d z@WJA(F^$<{Y0=pJ_F{`N zX`qUc3WwQ?cfw5BOVOvMns$vEQZ-Yi`IHE4!5f~2>(10%YdZJw$Jw^aaO!9dM)KP; zYtE^siTvVq`$)kdIwGQ9?qm#sevm=x_6D~%T-oKKq<8DyJ-DIF&cEjmI&QrwDr&0U zlG5Yv7Qe*>*ApJUHCeyNGw!f=hOnCV=6G2h@8Q2wK3pyndvPRk%geeL-1|rN=fCb3 zmO0_~#`SC3u}!C=(awjTD^aewRF{aJ?7-@5;2hu=+(bx992nK{?KCRS{8}pJ`|NtR z*Wn*&*X5@hyWX6An38AS)Ut7ZO}+f)jZ&L5pUTGBw;elb7QL!&~{ts5_u zAHB0D=3@0D!=ORSoo6h0v!|VI%ICI5NDwlQ+{)+@{=h(vAF+INq5CZcx5Y2-gm-11 zaP~q-;*C$aM&X}I9ZC=0U!C0N8Yo=E-+APR^7KT_#U%X~mOAoB9=Lx~e(vm@m=bch zIeggrnQ2ha!P0NnzG~XQpT=AQK0Ke017}}{q&=8eic~yn_SxjY}ouzwA3Lz`sg>RPYCSsHHV|b*Pk2q;|~j`hE|Tgel@F| zu(y+$Ws3U<$AFvkP(>*&YyH_GNDJGy@6_&Wnifc75IJ*Wb_7_HoGJWxGA8f>q_)2_ zacRr1mRb8M75A3+G1bJF8nDoukg%Be!x2)3kD8q@BjpU^!wAT=oQ=EzFYE+)AJdOd ztEs9s89)g&nI@STu!&eS6K(zLHy^RK78X*1TE}@q$eWS4=nreBA3;j$)9=9`Ro8wP zlocd-$?X8o+udEv>}7tl^#}w=pwiYJ)mBP$-OaQ)1O0Kw+g)AW+*~UsC!_^it%El} zh^xQH>L*A4nPVVRFSR2TJA*)7X|3YDOVy0D=uov04=sJ)sr}isplyPZ=kDt3=mc5v z#uMJz#Vjc)0iNViWKqI@%O-~fPMjP>U4ho)epd;7>sE?PV2d(%!I9JluLj|uJYF*v zi~Y6YTna7$TDyMc&CCoW=!@*_?fuXv{=ea*_B$K3o`aTerfdfFQR~g-`#mRsmRv5^ z+}!*Zy#S4Ka=5~ijvjZsWql`StdA!v(U$FNr(qGqOmn;*?EP(V~)kQD$yK z5^2OzblB!G(#Z@g3ZY^~qJ9jujcKYJi=GP&VDnQo86hag7QcL1``Esu1TSjdIGV$s ztU0pm<%!8w8m^{)_C6k=Hu6Hn5(ZW{WD}a3frmSldUj`Aq&Ja{LTrH0<^|T`!$CCrH?Hh9YWs+^x?<&s3U2(F} zLR0(;L~QsSDGz37z%X~4D;$nW_U?}Ecgh>;nyK~;LcszQj*^E1#*}dVN-pU+cV4(@?{TJ#yZ>~Q8h2)ks2tdWs-WBqa(DN24=h#_F(w-*f45ag za8ULcS@+9?oE>Jth6N{bqRDyd1!4KP{N(d9Z|qiE1X#j?y+s>)cT#{&0dsScGm+_}hB_59l%!+w4fU=zL zJRi$!uM77JQnL3oY@9WoE#7Q{KQ6T%%uimpWx#qTUajepH|~~17>hfq*BVEZmV~5JsT9h58x1qNCbdV7@LO*=SH=%+&-cMVm zmd)CP_EG32#)zHnSZcntvQx(MSF#91m+xeVm*=d_YDT+^T}_J?>g!lOy+yM|uty#F z<$VLmLZsRfdGp8$J58<}HECHFE+uhLZ7ABSfVkGAaZH{QBf7w}67wOPvC&XaO@D3? zIur_*8Vw9l^QnLt_}z|4bGaGJkF1QBaB z7khI&7VTU)B<5!SxyUPEqu~|}u%7~5%%!Cz&>6kZ8s;|KHeEGRA2a5QPH0n|?lVlA zh)`y~PQWE8m-MW_c2PZQRLInTrGsx5XRHxrGQ=x|cPW)V41dG)82kL^SU;|UtmwGF z?eXWP1)5GZp7G-$)4CE`D@%qyzdY%vIqi9JrzgFLju2>F3=J$$?+h7|wTHj+UpS^0 zWm3TDrsGCg4X~T`#zigf0=|nPUncR$xnN>kkd>8{lar?8_9mrV^F?pYOH5_??p)47 z8|-PmC5f@)=Z`mV(Td1%_mYHr^Ra2O&1FuGV|RVSkc`37q`7bSi~&t<))qHz`})>9 ziOJhxQ5tj1o&_XTL4&hE%C}Tx@c;aQZ#}d=X%ZTxF**_yL||0MpKTYWl`U~q^0tQQ*Mx3* z8V-8RlrO2-t=HfLRh*}mg-$XQVG|c$9~#!pTM!zW)`cI;B7+)u0SXN|5*nVJ1RW1a zjkA)BGtJm>&Wd<`p%|%EWi71NCShzk1R;6}-~#GkHjR&u&%FcIj1QOAB~(U_IVeqe zT%@Zro;U@Jzv<0bH?Nnve3Nj#phvdOsGMCi^{}YD%KO47y2DAZ7cuaB+_OidJ>1aL@iv>|EBYuSPcn34o1|XAApzSo-zrZ2b8To?ypHYpl(;tm>s&I7bk(9ZRw?;^Ty3> zg}l0vUmR&2XL_cv?F0K*cMQA^bD@A zbKAEuvrs5f%WYnkG{d%acQ@pBYVk`$zI1G{92a)~Jmqqq5A${6fSsXk_`QdYR}+e7 za{i>}x&$67axkP8_uh->a&AaEA)&d}q-bgm*Iz#U$kpEtk&Gb56&Lo*JR`xM6^wQ0 zkn92yweC+@$uA`NtiIwti)o)DM5-zC&y){!ho1qWY=)dHmdeamd_EBs;k2c~Dz+)Xj}wi*dI;HulNtO`r@DH@He_tdai@=a#|W*L zal}V99)|w@Jek|lWP$jZP$$8V`CJ}u6zAQ{^fP?mWRu75IQXDNgbPz!3{CRQgyjpQ z)k+@OqVe6gMHYX3o6`#ZY{cMV(y(sTPf@?KZy@*-#OR&}`xr@5E~!YU8eR#)&& ztx1YbEg-==Il}!YVQlb0_w(^@R0>fOtDCI8f|7~B1lg0DyaeM%h((tCi}uw>$Hr1( zTHTG~wl+)TcWlo`tL0TbX@O0>FnGL)y81arjF1*8sX&IV%WaPxKy~-5#xGc!)&|b? z{gINQE8{D^FcJb zQ|O~q^NG3^A)KvzU?PUnD?iE1iZ!|!`l8)?HlA$vLD#`UW$Xz#T8lH*wM8PFS*xkj z-z4hQUDiOlH_i(Aryk~;{gEbQ)Zj51U{-TBX-g`&dixwty;1wfd6$L8y zOMSN798)#qPZHE*H)WL$qPR3V+q4IkT6lHp!rUQD-`qgDlg*oKe*m-s1lEI<(O4o# zGQvAHvm5Q!J~qtoRQIEm)AdWVkbkxG7gK@$cAs*27Q;?pF~H$)B$9P}U95ojU!|hi zw5c{2_a6&l2LfcdiHnJC6_ixJTfgVGqMf33OO>a>Yo}sut*xzv>{2|h^NfsOZJ&Vz zW_1x94k_{XGBY?sOk*aO1}U~X7zfQr{e69X{rweqyH)5L`k7|H$T@%h1Dt9`eEteD zmS(SBRI2hPaqF)^1J+XlXuw69|Oza=$tWq`PwulPHyiT>~-3m)JFMWy*z8 z=_hVVIynKNf+L_MCk$9?%pw%@e${p|x%9#4cZbFp^n{L7--v^a68d}eNq&*D?=({~%4R60+7Pd?v=MF;$Pk ze~6S#yDx{fM`S>^gN{}X16?IAFE6UV@%-t3cJ$RWY1q25gN1{>V0OWbq^72Z+VtC^ z3Kf*9?z(R60WM29RxSGe{B9prHUB(!VXH|!BJ&PJGCL98Hmj@qWeM1J`2 zVFz|`>DG!4IdLIf?htB9z|9*AxXyLn0#iA7e-UYSB~m6Q_65tS$qMd_7D~X>E&Yy zB3d`hI7&wvk_A=f^z?1HRfX{NAL168uQZ3CVNHw~;Gxq|3F~9dQ{$1YqBZm-j?_TM z9J4a^$oT3cdo)?CVXC8hTOu=^i(4O0qo0}7Z|Pq*l$lc8PYC3EfwOONF>60l7o*pq zT_H+>MKYP*4F9j$+WNz_H4- z$Db2J@WWtPTPd=;dR;l6M_dq_0ePjohv)Rrek(o+~qCg(26L#|JUU&b2IvH&(l(XR6H11$Hrs z{4u^F@$RA~X@LCIh?QvqxJoip#iA55m>TaOq9L{XQ79<+tUXQ7iEF%hnq-k@ zm9A$;P;LIt3u0km9>nhRW6T#RiU@fIipoASfX){5*7mG#&wfv#>y7 zFvQ<}iDnqJvb>z;v9|jklrsjNk?!tpU`DjGFb50)&uTm9C{^X>M;fzh*K56fd=~px zk4-_#5Z;-6Oi7XMu3CS`!h(E!B{6MmYz$z5Ln1W!r;GOBaCbK4;%O;ZP3N{0&_XJL zierF)K4_)Y@%QDh4o~P{-$tisL;0VeerKwqW3KTwH0%=;NnBYN*IMeaht7L#0Bpl8 zd{h>g7jwTcbc|H5+1NDiPJGhhsA}(_nwepzx%dE0ZOrX$1`NX z7=SRp&gjQ+TVH+?yX0yH&yfQNM(OF%H-=-pAbc>P0c#K81_084WeefMDT!>Fc~_S? zJ1Y(OSx$oIo^sy|(omF?l%jopgZ@+-X?O&p31cHq1@0!}TJ?FD0{AxPx=#tH=3QL; z+gAf@p!^rqqe}ncN~$Vl5C~&~f895#m2h&@tBmBM&{53v&djg?^aW3MH}*eo4}&({ zj-gR_8x$`_1M;E!>#5S{xwqybT>q9PzsCLw_POinxAk>E zFW}VmaZ~A1sS79UAtIS=|A^JZtnq_=6j~T-?vKM>y|`ZEAh`6_)zyuDij}aH&EcR> zC|5)D`dU}i`;Xu%poyi)XkJc%n{#IWGF7^2mh9!f&2^G4eu|xA|5NURnEzJSxU01g z<7e*4Twd~K0Wk3a4zR_HxjMf_uxB5y1_wQtPt1i5xyca#-U9BcGViBt(85(Za@&Nm z2BJ2DEnRcn%V8!ZQ79CqsUEatZzD5lX3Xkp-ksKec6`+$y+Ys@%b3-HvBf6EFRYcv z@)#R=z!Dg-dB)Tq&~BK}>Ss#LEX=zys4Be40l^L--IgYQIs2Tv$R25Fh!H3#9;O!L z=LcG2qP+2WYb%(Ug+|4tbp#j2jIAK%xj>?F{f58WR9%#& zCv{!ZuQUyBo8>~+s4QMzjD4t<^-X<*mFPd>Nlp~AUMrYc5=1f77e`XygY-)9Peek$ zk)|IGEoP`TUA`_C)qi}McOaB9GX!U;pFZ`M!5KHYE&CqE4iZ;}#&XN@&y;i&1k09W z`cEv?X$DSLZZkBl3veAt6|HzkmM~;d=6mW8wgl&TN1=7k8~s6et-x=JtP)tjUFKDk z)43n%Bsuh2drQXxUP^VoHujt?1PmyCNNFuzTwE+JM)V_$h~8_An6ud~V(W#1wwT%p zyFj1TlX^4vCa(APxcyksYF(Kz9Zdfg)|x~ixYxg&^ZRUj9x($SL&lD2aC$oaa-qFg zkh*DZRX+7S*d{#$%X0byS`|hZ?50&$*Is$L$0x%m4bC@1DfWEx0s=$uxP}-E)s+94 zT>W7jhVMT~*I|5&tx%kQo>cPJ^a1IQwn#*a+}l1?)yf^1EEu}$BBT{a+)1`Akb@(f zoxy-#6c-NItTy6~+pN^r71%IiJ5qFNXI=_GXZdL5Ngf>3MRAze#hBqYs5O=-DiFVV zG8R z7#eW9)RgtCRUhW2$rP%PDso_?;m4h%k|TYoWz7Z*DvGRAa<+h$Zlkpvs8L(pRq{r# zCl`<x>-Y`d{Qf(PD`?U(>@=y9|2bo!(iVA4rye)0*cl@1!LrGkiVJAz?-rUSkv z5?4R3FMP&^t}V3U)?XFIwj@{I=^e9PYm8+Pe6VPD!>u!eQOtx|eauckeZ7^NxHI8f}LA8Jh`lN|5>bRH025GP5ga9{x1 zEOGN7y^{R{sTt$LL-UP&IH0ck0^rjL*`@Fce6bV~iPYQMJ4mws7aae{p$!fWR`G;A z&zrt&lexK@mO&gi?+{KI%LzoEnE3b4!2M4bXX?F2472_n4GjKb~Q(`CK1Lw z6(#c$*f(;gnT3VmF88SdBfcjCNCR%ryu88XD1A7mv%c>y1dt4u8QnXob8_${ops*lcnb*rlPVzZH$MV` zE^9#N2L*fEAra~eA@>?U?(za}Z;MHCESQ61=+%qVz@C(F9+UylgsdcpnwEG+j{swy z9pbw!6(t7*p3nb;IM8VHA0+A_9@c|s1IphY=(GME3pD6-2m>9Wwzd}ZirOWq-t(V1 zqIxT5w-HaZh9W%u{r!tv+Aj$PRE+BC>I3h`{}H=~vT3I#ii^8w|3L1}P**~=N+Ra> zhFIj@D1304o^77_JY(BCQ#&{BZbqt0#S zSyAcW)upASL?UJwGY)A@M}`uET~!w=@i}4ZzXmVMV40`Oi1H-5)IVnN;v%q&|C13U z5b{w6N)Bj)`CBUh^?+|y-?@xJRai2%#|#9155x^rAEj4asjLf!*w-D9q2i*21>3e3 zI-Lf34;c=#FOMYrWaw9n4F=9U9|o-}h=s#&b=Hz!Z_xSA5?TkAu&W)fYL=lZVd}>%*x0PS%LJwbQ+(M;8LQS=&^RVxzf)1!}07$ObQ z_ew7{owzMAPd~rb4HC*o(6RGPNcoFG?kD&5mY)oa@~)VBgB!&3ci|#V8&}MRB}-yj zin{;Ga_f$^uZ0utI9fDL?7-bTw-T)oKZU))^5 zN2Q;laW0&4qjA~V^=nT(jR#?g;PQ#7iu3Fb(1I9GIZtp*Qd^2^P^?<8k$0=D)ToOT zqqGf}Xt{O2yH(Oc7<<>?Wnfs*KM{_-$BwbM>ECE7W_gcuclq&RLjehld-shI1SUh% zPhI_;EG+a;C?_YJB|=Of4Nhb)r<)%CY8q%)>BNlYkL-=SgPiY@kyA%hO0KGnhK!ML z?5>9KUxekg8ki%kX323CexC-7-YaVMO{XAAZDyH*~frx@PD}(`5D*B2H~w#!ch>2BRl-j`Ui|ba5_r4WiX9ml1cK`C1HvZ68ZVr&2OAHZ89Ta| zfCSv*x@_}sh=;+UXFm#jUX)G9sQnK^{9~(~OLJt#v0gwC@abq|ErFHOp4xy-e{%w0$UO8>rTTh|;&B<5Rt&m4SY`G(bxn9t2MJJ10Ku4s;r&>S;1!hAogi{!(7e zrj;?=Pva-aO--;t_a+%?Ad5Czm!zm+vE#knG8BO9*#ik~a|?5Gpq#`nI<{AWJ6SBF zq@{&0zEqfoJS|(!2_G(SQ660il?q&@WQDer!t7LXI(S&wOnEOvg8ahed?{1hgO`3a?C_tvjOH*T_#AVjRh`XZ$ zhZF%b-R4WhN)q*~_?r9|oO>7AIyCSnC%qg86g&sMUXxWEJCpLhwG7QHdzIcD>~$bg zve@fdv9)gG+2WD|tXVpqyon~F2|?!t5F1Bxxj=R#{49?h)Qw7jc^v5S9V=^>E)`eG z_o;o??T2TGYIwFrtogMt`I8}GQos-R0*meI!%`xulFx=8^7p3LM4Gqhct$od3e%v~TgNti8OrBh+C;a;5mlUeB21sB|0;Q)w zfsxiQ5xJ{GJuHtm+cc|Sfl=qVE&s`)jEoE{h6YpMNiZM>6B84;Xwbv*Ay4=Kd-O|) zfH){-13Y8D?6j{{u|UmXhk#2B5XGS6hPS7$=iVs)yVxV5;1VzVov94i&)nSNqWNO-drh!7a= zU$GBXhps~Nak&X2$Vi9)`zRMPGw7x1rW(8n1_09v%$S8Yg;R?YAOiM6KDr$H1WoxA zE);I(rE*jOEvcd+p^ETZ=3^%(vnMAX|7+s`vZe!==F!>}q z>ouz=zd_6ng8|Ov^*{YJArA=TwvnW=I;iA+x8F1dYbY)*R^--tL#qnrX(@v58M8ba z6ZiyR7cT5_adkB!DkMUL7/etc/rkt/net.d/k8s_cluster.conf +{ + "name": "rkt.kubernetes.io", + "type": "flannel" +} +EOF +``` + +While `k8s_cluster.conf` is a rather arbitrary name for the config file itself, +and can be adjusted to suit local conventions, the keys and values should be exactly +as shown above. `name` must be `rkt.kubernetes.io` and `type` should be `flannel`. +More details about the flannel CNI plugin can be found +[in the CNI documentation](https://github.com/appc/cni/blob/master/Documentation/flannel.md). + +#### On GCE + +Each VM on GCE has an additional 256 IP addresses routed to it, so +it is possible to forego flannel in smaller clusters. This makes the +necessary CNI config file a bit more verbose: + +```shell +$ cat </etc/rkt/net.d/k8s_cluster.conf +{ + "name": "rkt.kubernetes.io", + "type": "bridge", + "bridge": "cbr0", + "isGateway": true, + "ipam": { + "type": "host-local", + "subnet": "10.255.228.1/24", + "gateway": "10.255.228.1" + }, + "routes": [ + { "dst": "0.0.0.0/0" } + ] +} +EOF +``` + +This example creates a `bridge` plugin configuration for the CNI network, specifying +the bridge name `cbr0`. It also specifies the CIDR, in the `ipam` field. + +Creating these files for any moderately-sized cluster is at best inconvenient. +Work is in progress to +[enable Kubernetes to use the CNI by default] +(https://github.com/kubernetes/kubernetes/pull/18795/files). +As that work matures, such manual CNI config munging will become unnecessary +for primary use cases. For early adopters, an initial example shows one way to +[automatically generate these CNI configurations] +(https://gist.github.com/yifan-gu/fbb911db83d785915543) +for rkt. + ### Local cluster -To use rkt as the container runtime, we need to supply `--container-runtime=rkt` and `--rkt-path=$PATH_TO_RKT_BINARY` to kubelet. Additionally we can provide `--rkt-stage1-image` flag -as well to select which [stage1 image](https://github.com/coreos/rkt/blob/master/Documentation/running-lkvm-stage1.md) we want to use. +To use rkt as the container runtime, we need to supply the following flags to kubelet: -If you are using the [hack/local-up-cluster.sh](https://releases.k8s.io/{{page.githubbranch}}/hack/local-up-cluster.sh) script to launch the local cluster, then you can edit the environment variable `CONTAINER_RUNTIME`, `RKT_PATH` and `RKT_STAGE1_IMAGE` to +- `--container-runtime=rkt` chooses the container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'. +- `--rkt-path=$PATH_TO_RKT_BINARY` sets the path of rkt binary. Leave empty to use the first rkt in $PATH. +- `--rkt-stage1-image` sets the path of the stage1 image. Local paths and http/https URLs are supported. Leave empty to use the 'stage1.aci' that locates in the same directory as the rkt binary. + +If you are using the [hack/local-up-cluster.sh](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/hack/local-up-cluster.sh) script to launch the local cluster, then you can edit the environment variable `CONTAINER_RUNTIME`, `RKT_PATH` and `RKT_STAGE1_IMAGE` to set these flags: ```shell @@ -40,21 +118,21 @@ To use rkt as the container runtime for your CoreOS cluster on GCE, you need to ```shell $ export KUBE_OS_DISTRIBUTION=coreos -$ export KUBE_GCE_MINION_IMAGE= -$ export KUBE_GCE_MINION_PROJECT=coreos-cloud +$ export KUBE_GCE_NODE_IMAGE= +$ export KUBE_GCE_NODE_PROJECT=coreos-cloud $ export KUBE_CONTAINER_RUNTIME=rkt ``` You can optionally choose the version of rkt used by setting `KUBE_RKT_VERSION`: ```shell -$ export KUBE_RKT_VERSION=0.8.0 +$ export KUBE_RKT_VERSION=0.15.0 ``` Then you can launch the cluster by: ```shell -$ kube-up.sh +$ cluster/kube-up.sh ``` Note that we are still working on making all containerized the master components run smoothly in rkt. Before that we are not able to run the master node with rkt yet. @@ -96,16 +174,19 @@ See [a simple nginx example](/docs/user-guide/simple-nginx) to try out your new For more complete applications, please look in the [examples directory](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/). +### Different UX with rkt container runtime + +rkt and Docker have very different designs, as well as ACI and Docker image format. Users might experience some different experience when switching from one to the other. More information can be found [here](/docs/getting-started-guides/rkt/notes/). ### Debugging -Here are severals tips for you when you run into any issues. +Here are several tips for you when you run into any issues. ##### Check logs By default, the log verbose level is 2. In order to see more logs related to rkt, we can set the verbose level to 4. For local cluster, we can set the environment variable: `LOG_LEVEL=4`. -If the cluster is using salt, we can edit the [logging.sls](https://releases.k8s.io/{{page.githubbranch}}/cluster/saltbase/pillar/logging.sls) in the saltbase. +If the cluster is using salt, we can edit the [logging.sls](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/cluster/saltbase/pillar/logging.sls) in the saltbase. ##### Check rkt pod status diff --git a/docs/getting-started-guides/rkt/notes.md b/docs/getting-started-guides/rkt/notes.md new file mode 100644 index 0000000000..44eb57f90e --- /dev/null +++ b/docs/getting-started-guides/rkt/notes.md @@ -0,0 +1,99 @@ +--- +--- + +# Notes on Different UX with rkt container runtime + +### Doesn't support ENTRYPOINT + CMD feature + +To run a Docker image, rkt will convert it into [App Container Image (ACI) format](https://github.com/appc/spec/blob/master/SPEC.md) first. +However, during the conversion, the `ENTRYPOINT` and `CMD` are concatentated to construct ACI's `Exec` field. +This means after the conversion, we are not able to replace only `ENTRYPOINT` or `CMD` without touching the other part. +So for now, users are recommended to specify the **executable path** in `Command` and **arguments** in `Args`. +(This has the same effect if users specify the **executable path + arguments** in `Command` or `Args` alone). + +For example: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx + labels: + name: nginx +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 +``` + +The above pod yaml file is valid as it's not specifying `Command` or `Args`, so the default `ENTRYPOINT` and `CMD` of the image will be used. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: busybox + labels: + name: busybox +spec: + containers: + - name: busybox + image: busybox + command: + - /bin/sleep + - 1000 +``` + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: busybox + labels: + name: busybox +spec: + containers: + - name: busybox + image: busybox + command: + - /bin/sleep + args: + - 1000 +``` + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: busybox + labels: + name: busybox +spec: + containers: + - name: busybox + image: busybox + args: + - /bin/sleep + - 1000 +``` + +All the three examples above are valid as they contain both the executable path and the arguments. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: busybox + labels: + name: busybox +spec: + containers: + - name: busybox + image: busybox + args: + - 1000 +``` + +The last example is invalid, as we cannot override just the `CMD` of the image alone. diff --git a/docs/getting-started-guides/scratch.md b/docs/getting-started-guides/scratch.md index ed45e30ac5..5e82a7108c 100644 --- a/docs/getting-started-guides/scratch.md +++ b/docs/getting-started-guides/scratch.md @@ -3,7 +3,7 @@ This guide is for people who want to craft a custom Kubernetes cluster. If you can find an existing Getting Started Guide that meets your needs on [this -list](/docs/getting-started-guides/README/), then we recommend using it, as you will be able to benefit +list](/docs/getting-started-guides/), then we recommend using it, as you will be able to benefit from the experience of others. However, if you have specific IaaS, networking, configuration management, or operating system requirements not met by any of those guides, then this guide will provide an outline of the steps you need to @@ -170,9 +170,9 @@ You have several choices for Kubernetes images: command like `docker images` For etcd, you can: - -- Use images hosted on Google Container Registry (GCR), such as `gcr.io/google_containers/etcd:2.0.12` -- Use images hosted on [Docker Hub](https://hub.docker.com/search/?q=etcd) or [Quay.io](https://quay.io/repository/coreos/etcd), such as `quay.io/coreos/etcd:v2.2.0` + +- Use images hosted on Google Container Registry (GCR), such as `gcr.io/google_containers/etcd:2.2.1` +- Use images hosted on [Docker Hub](https://hub.docker.com/search/?q=etcd) or [Quay.io](https://quay.io/repository/coreos/etcd), such as `quay.io/coreos/etcd:v2.2.1` - Use etcd binary included in your OS distro. - Build your own image - You can do: `cd kubernetes/cluster/images/etcd; make` @@ -208,14 +208,14 @@ You need to prepare several certs: - The kubelets optionally need certs to identify themselves as clients of the master, and when serving its own API over HTTPS. -Unless you plan to have a real CA generate your certs, you will need to generate a root cert and use that to sign the master, kubelet, and kubectl certs: - -- See function `create-certs` in `cluster/gce/util.sh` -- See also `cluster/saltbase/salt/generate-cert/make-ca-cert.sh` and +Unless you plan to have a real CA generate your certs, you will need to generate a root cert and use that to sign the master, kubelet, and kubectl certs. + +- see function `create-certs` in `cluster/gce/util.sh` +- see also `cluster/saltbase/salt/generate-cert/make-ca-cert.sh` and `cluster/saltbase/salt/generate-cert/make-cert.sh` -You will end up with the following files (we will use these variables later on): - +You will end up with the following files (we will use these variables later on) + - `CA_CERT` - put in on node where apiserver runs, in e.g. `/srv/kubernetes/ca.crt`. - `MASTER_CERT` @@ -356,7 +356,7 @@ The minimum version required is [v0.5.6](https://github.com/coreos/rkt/releases/ [systemd](http://www.freedesktop.org/wiki/Software/systemd/) is required on your node to run rkt. The minimum version required to match rkt v0.5.6 is -[systemd 215](http://lists.freedesktop.org/archives/systemd-devel/2014-July/020903). +[systemd 215](http://lists.freedesktop.org/archives/systemd-devel/2014-July/020903.html). [rkt metadata service](https://github.com/coreos/rkt/blob/master/Documentation/networking.md) is also required for rkt networking support. You can start rkt metadata service by using command like @@ -378,7 +378,7 @@ Arguments to consider: - Otherwise, if taking the firewall-based security approach - `--api-servers=http://$MASTER_IP` - `--config=/etc/kubernetes/manifests` - - `--cluster-dns=` to the address of the DNS server you will setup (see [Starting Addons](#starting-addons).) + - `--cluster-dns=` to the address of the DNS server you will setup (see [Starting Cluster Services](#starting-cluster-services).) - `--cluster-domain=` to the dns domain prefix to use for cluster DNS addresses. - `--docker-root=` - `--root-dir=` @@ -411,35 +411,34 @@ this `NODE_X_BRIDGE_ADDR`. For example, if `NODE_X_POD_CIDR` is `10.0.0.0/16`, then `NODE_X_BRIDGE_ADDR` is `10.0.0.1/16`. NOTE: this retains the `/16` suffix because of how this is used later. -Recommended, automatic approach: - +- Recommended, automatic approach: + 1. Set `--configure-cbr0=true` option in kubelet init script and restart kubelet service. Kubelet will configure cbr0 automatically. It will wait to do this until the node controller has set Node.Spec.PodCIDR. Since you have not setup apiserver and node controller yet, the bridge will not be setup immediately. +- Alternate, manual approach: -Alternate, manual approach: - 1. Set `--configure-cbr0=false` on kubelet and restart. 1. Create a bridge - - e.g. `brctl addbr cbr0`. - 1. Set appropriate MTU - - `ip link set dev cbr0 mtu 1460` (NOTE: the actual value of MTU will depend on your network environment) - 1. Add the clusters network to the bridge (docker will go on other side of bridge). - - e.g. `ip addr add $NODE_X_BRIDGE_ADDR dev eth0` + - `brctl addbr cbr0`. + 1. Set appropriate MTU. NOTE: the actual value of MTU will depend on your network environment + - `ip link set dev cbr0 mtu 1460` + 1. Add the node's network to the bridge (docker will go on other side of bridge). + - `ip addr add $NODE_X_BRIDGE_ADDR dev cbr0` 1. Turn it on - - e.g. `ip link set dev cbr0 up` + - `ip link set dev cbr0 up` If you have turned off Docker's IP masquerading to allow pods to talk to each other, then you may need to do masquerading just for destination IPs outside the cluster network. For example: ```shell -iptables -w -t nat -A POSTROUTING -o eth0 -j MASQUERADE \! -d ${CLUSTER_SUBNET} +iptables -t nat -A POSTROUTING ! -d ${CLUSTER_SUBNET} -m addrtype ! --dst-type LOCAL -j MASQUERADE ``` This will rewrite the source address from the PodIP to the Node IP for traffic bound outside the cluster, and kernel -[connection tracking](http://www.iptables.info/en/connection-state) +[connection tracking](http://www.iptables.info/en/connection-state.html) will ensure that responses destined to the node still reach the pod. @@ -786,17 +785,29 @@ If you have selected the `--register-node=true` option for kubelets, they will n You should soon be able to see all your nodes by running the `kubectl get nodes` command. Otherwise, you will need to manually create node objects. -### Logging +### Starting Cluster Services -**TODO** talk about starting Logging. +You will want to complete your Kubernetes clusters by adding cluster-wide +services. These are sometimes called *addons*, and [an overview +of their purpose is in the admin guide](/docs/admin/cluster-components/#addons). -### Monitoring +Notes for setting up each cluster service are given below: -**TODO** talk about starting Monitoring. - -### DNS - -**TODO** talk about starting DNS. +* Cluster DNS: + * required for many kubernetes examples + * [Setup instructions](http://releases.k8s.io/release-1.2/cluster/addons/dns/) + * [Admin Guide](../admin/dns.md) +* Cluster-level Logging + * Multiple implementations with different storage backends and UIs. + * [Elasticsearch Backend Setup Instructions](http://releases.k8s.io/release-1.2/cluster/addons/fluentd-elasticsearch/) + * [Google Cloud Logging Backend Setup Instructions](http://releases.k8s.io/release-1.2/cluster/addons/fluentd-gcp/). + * Both require running fluentd on each node. + * [User Guide](../user-guide/logging.md) +* Container Resource Monitoring + * [Setup instructions](http://releases.k8s.io/release-1.2/cluster/addons/cluster-monitoring/) +* GUI + * [Setup instructions](http://releases.k8s.io/release-1.2/cluster/addons/kube-ui/) + cluster. ## Troubleshooting diff --git a/docs/getting-started-guides/ubuntu-calico.md b/docs/getting-started-guides/ubuntu-calico.md index ef4b96ba73..eaf8eb7c18 100644 --- a/docs/getting-started-guides/ubuntu-calico.md +++ b/docs/getting-started-guides/ubuntu-calico.md @@ -1,273 +1,465 @@ --- --- -This document describes how to deploy Kubernetes on Ubuntu bare metal nodes with Calico Networking plugin. See [projectcalico.org](http://projectcalico.org) for more information on what Calico is, and [the calicoctl github](https://github.com/projectcalico/calico-docker) for more information on the command-line tool, `calicoctl`. +This document describes how to deploy Kubernetes with Calico networking from scratch on _bare metal_ Ubuntu. For more information on Project Calico, visit [projectcalico.org](http://projectcalico.org) and the [calico-containers repository](https://github.com/projectcalico/calico-containers). -This guide will set up a simple Kubernetes cluster with a master and two nodes. We will start the following processes with systemd: +To install Calico on an existing Kubernetes cluster, or for more information on deploying Calico with Kubernetes in a number of other environments take a look at our supported [deployment guides](https://github.com/projectcalico/calico-containers/tree/master/docs/cni/kubernetes). -On the Master: +This guide will set up a simple Kubernetes cluster with a single Kubernetes master and two Kubernetes nodes. We'll run Calico's etcd cluster on the master and install the Calico daemon on the master and nodes. -- `etcd` -- `kube-apiserver` -- `kube-controller-manager` -- `kube-scheduler` -- `calico-node` +## Prerequisites and Assumptions -On each Node: +- This guide uses `systemd` for process management. Ubuntu 15.04 supports systemd natively as do a number of other Linux distributions. +- All machines should have Docker >= 1.7.0 installed. + - To install Docker on Ubuntu, follow [these instructions](https://docs.docker.com/installation/ubuntulinux/) +- All machines should have connectivity to each other and the internet. +- This guide assumes a DHCP server on your network to assign server IPs. +- This guide uses `192.168.0.0/16` as the subnet from which pod IP addresses are assigned. If this overlaps with your host subnet, you will need to configure Calico to use a different [IP pool](https://github.com/projectcalico/calico-containers/blob/master/docs/calicoctl/pool.md#calicoctl-pool-commands). +- This guide assumes that none of the hosts have been configured with any Kubernetes or Calico software. +- This guide will set up a secure, TLS-authenticated API server. -- `kube-proxy` -- `kube-kubelet` -- `calico-node` +## Set up the master -## Prerequisites +### Configure TLS -1. This guide uses `systemd` and thus uses Ubuntu 15.04 which supports systemd natively. -2. All machines should have the latest docker stable version installed. At the time of writing, that is Docker 1.7.0. - - To install docker, follow [these instructions](https://docs.docker.com/installation/ubuntulinux/) -3. All hosts should be able to communicate with each other, as well as the internet, to download the necessary files. -4. This demo assumes that none of the hosts have been configured with any Kubernetes or Calico software yet. +The master requires the root CA public key, `ca.pem`; the apiserver certificate, `apiserver.pem` and its private key, `apiserver-key.pem`. -## Setup Master +1. Create the file `openssl.cnf` with the following contents. -First, get the sample configurations for this tutorial + ```conf + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + subjectAltName = @alt_names + [alt_names] + DNS.1 = kubernetes + DNS.2 = kubernetes.default + IP.1 = 10.100.0.1 + IP.2 = ${MASTER_IPV4} + ``` -```shell -wget https://github.com/Metaswitch/calico-kubernetes-ubuntu-demo/archive/master.tar.gz -tar -xvf master.tar.gz -``` +> Replace ${MASTER_IPV4} with the Master's IP address on which the Kubernetes API will be accessible. -### Setup environment variables for systemd services on Master +2. Generate the necessary TLS assets. -Many of the sample systemd services provided rely on environment variables on a per-node basis. Here we'll edit those environment variables and move them into place. + ```shell + # Generate the root CA. + openssl genrsa -out ca-key.pem 2048 + openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" -1.) Copy the network-environment-template from the `master` directory for editing. + # Generate the API server keypair. + openssl genrsa -out apiserver-key.pem 2048 + openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf + openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf + ``` -```shell -cp calico-kubernetes-ubuntu-demo-master/master/network-environment-template network-environment -``` +3. You should now have the following three files: `ca.pem`, `apiserver.pem`, and `apiserver-key.pem`. Send the three files to your master host (using `scp` for example). +4. Move them to the `/etc/kubernetes/ssl` folder and ensure that only the root user can read the key: -2.) Edit `network-environment` to represent your current host's settings. + ```shell + # Move keys + sudo mkdir -p /etc/kubernetes/ssl/ + sudo mv -t /etc/kubernetes/ssl/ ca.pem apiserver.pem apiserver-key.pem + + # Set permissions + sudo chmod 600 /etc/kubernetes/ssl/apiserver-key.pem + sudo chown root:root /etc/kubernetes/ssl/apiserver-key.pem + ``` -3.) Move the `network-environment` into `/etc` +### Install Kubernetes on the Master -```shell -sudo mv -f network-environment /etc -``` +We'll use the `kubelet` to bootstrap the Kubernetes master. -### Install Kubernetes on Master +1. Download and install the `kubelet` and `kubectl` binaries: -1.) Build & Install Kubernetes binaries + ```shell + sudo wget -N -P /usr/bin http://storage.googleapis.com/kubernetes-release/release/v1.1.4/bin/linux/amd64/kubectl + sudo wget -N -P /usr/bin http://storage.googleapis.com/kubernetes-release/release/v1.1.4/bin/linux/amd64/kubelet + sudo chmod +x /usr/bin/kubelet /usr/bin/kubectl + ``` -```shell -# Get the Kubernetes Source -wget https://github.com/kubernetes/kubernetes/releases/download/v1.0.3/kubernetes.tar.gz +2. Install the `kubelet` systemd unit file and start the `kubelet`: -# Untar it -tar -xf kubernetes.tar.gz -tar -xf kubernetes/server/kubernetes-server-linux-amd64.tar.gz -kubernetes/cluster/ubuntu/build.sh + ```shell + # Install the unit file + sudo wget -N -P /etc/systemd https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/master/kubelet.service -# Add binaries to /usr/bin -sudo cp -f binaries/master/* /usr/bin -sudo cp -f binaries/kubectl /usr/bin -``` + # Enable the unit file so that it runs on boot + sudo systemctl enable /etc/systemd/kubelet.service -2.) Install the sample systemd processes settings for launching kubernetes services + # Start the kubelet service + sudo systemctl start kubelet.service + ``` -```shell -sudo cp -f calico-kubernetes-ubuntu-demo-master/master/*.service /etc/systemd -sudo systemctl enable /etc/systemd/etcd.service -sudo systemctl enable /etc/systemd/kube-apiserver.service -sudo systemctl enable /etc/systemd/kube-controller-manager.service -sudo systemctl enable /etc/systemd/kube-scheduler.service -``` +3. Download and install the master manifest file, which will start the Kubernetes master services automatically: -3.) Launch the processes. + ```shell + sudo mkdir -p /etc/kubernetes/manifests + sudo wget -N -P /etc/kubernetes/manifests https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/master/kubernetes-master.manifest + ``` -```shell -sudo systemctl start etcd.service -sudo systemctl start kube-apiserver.service -sudo systemctl start kube-controller-manager.service -sudo systemctl start kube-scheduler.service -``` +4. Check the progress by running `docker ps`. After a while, you should see the `etcd`, `apiserver`, `controller-manager`, `scheduler`, and `kube-proxy` containers running. -### Install Calico on Master + > Note: it may take some time for all the containers to start. Don't worry if `docker ps` doesn't show any containers for a while or if some containers start before others. -In order to allow the master to route to pods on our nodes, we will launch the calico-node daemon on our master. This will allow it to learn routes over BGP from the other calico-node daemons in the cluster. The docker daemon should already be running before calico is started. +### Install Calico's etcd on the master -```shell -# Install the calicoctl binary, which will be used to launch calico -wget https://github.com/projectcalico/calico-docker/releases/download/v0.5.5/calicoctl -chmod +x calicoctl -sudo cp -f calicoctl /usr/bin +Calico needs its own etcd cluster to store its state. In this guide we install a single-node cluster on the master server. -# Install and start the calico service -sudo cp -f calico-kubernetes-ubuntu-demo-master/master/calico-node.service /etc/systemd -sudo systemctl enable /etc/systemd/calico-node.service -sudo systemctl start calico-node.service -``` +> Note: In a production deployment we recommend running a distributed etcd cluster for redundancy. In this guide, we use a single etcd for simplicitly. -> Note: calico-node may take a few minutes on first boot while it downloads the calico-node docker image. +1. Download the template manifest file: -## Setup Nodes + ``` + wget https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/master/calico-etcd.manifest + ``` -Perform these steps **once on each node**, ensuring you appropriately set the environment variables on each node +2. Replace all instances of `` in the `calico-etcd.manifest` file with your master's IP address. -### Setup environment variables for systemd services on the Node +3. Then, move the file to the `/etc/kubernetes/manifests` directory: -1.) Get the sample configurations for this tutorial + ```shell + sudo mv -f calico-etcd.manifest /etc/kubernetes/manifests + ``` -```shell -wget https://github.com/Metaswitch/calico-kubernetes-ubuntu-demo/archive/master.tar.gz -tar -xvf master.tar.gz -``` +### Install Calico on the master -2.) Copy the network-environment-template from the `node` directory +We need to install Calico on the master. This allows the master to route packets to the pods on other nodes. -```shell -cp calico-kubernetes-ubuntu-demo-master/node/network-environment-template network-environment -``` +1. Install the `calicoctl` tool: -3.) Edit `network-environment` to represent your current host's settings. + ```shell + wget https://github.com/projectcalico/calico-containers/releases/download/v0.15.0/calicoctl + chmod +x calicoctl + sudo mv calicoctl /usr/bin + ``` -4.) Move `network-environment` into `/etc` +2. Prefetch the calico/node container (this ensures that the Calico service starts immediately when we enable it): -```shell -sudo mv -f network-environment /etc -``` + ``` + sudo docker pull calico/node:v0.15.0 + ``` -### Configure Docker on the Node +3. Download the `network-environment` template from the `calico-kubernetes` repository: -#### Create the veth + ``` + wget -O network-environment https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/master/network-environment-template + ``` -Instead of using docker's default interface (docker0), we will configure a new one to use desired IP ranges +4. Edit `network-environment` to represent this node's settings: -```shell -sudo apt-get install -y bridge-utils -sudo brctl addbr cbr0 -sudo ifconfig cbr0 up -sudo ifconfig cbr0 /24 -``` + - Replace `` with the IP address of the master. This should be the source IP address used to reach the Kubernetes worker nodes. -> Replace \ with the subnet for this host's containers. Example topology: +5. Move `network-environment` into `/etc`: - Node | cbr0 IP -------- | ------------- -node-1 | 192.168.1.1/24 -node-2 | 192.168.2.1/24 -node-X | 192.168.X.1/24 + ```shell + sudo mv -f network-environment /etc + ``` -#### Start docker on cbr0 +6. Install, enable, and start the `calico-node` service: -The Docker daemon must be started and told to use the already configured cbr0 instead of using the usual docker0, as well as disabling ip-masquerading and modification of the ip-tables. + ```shell + sudo wget -N -P /etc/systemd https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/common/calico-node.service + sudo systemctl enable /etc/systemd/calico-node.service + sudo systemctl start calico-node.service + ``` -1.) Edit the ubuntu-15.04 docker.service for systemd at: `/lib/systemd/system/docker.service` +## Set up the nodes -2.) Find the line that reads `ExecStart=/usr/bin/docker -d -H fd://` and append the following flags: `--bridge=cbr0 --iptables=false --ip-masq=false` +The following steps should be run on each Kubernetes node. -3.) Reload systemctl and restart docker. +### Configure TLS -```shell -sudo systemctl daemon-reload -sudo systemctl restart docker -``` +Worker nodes require three keys: `ca.pem`, `worker.pem`, and `worker-key.pem`. We've already generated +`ca.pem` for use on the Master. The worker public/private keypair should be generated for each Kubernetes node. -### Install Calico on the Node +1. Create the file `worker-openssl.cnf` with the following contents. -1.) Install Calico + ```conf + [req] + req_extensions = v3_req + distinguished_name = req_distinguished_name + [req_distinguished_name] + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + subjectAltName = @alt_names + [alt_names] + IP.1 = $ENV::WORKER_IP + ``` -```shell -# Get the calicoctl binary -wget https://github.com/projectcalico/calico-docker/releases/download/v0.5.5/calicoctl -chmod +x calicoctl -sudo cp -f calicoctl /usr/bin +2. Generate the necessary TLS assets for this worker. This relies on the worker's IP address, and the `ca.pem` file generated earlier in the guide. -# Start calico on this node -sudo cp calico-kubernetes-ubuntu-demo-master/node/calico-node.service /etc/systemd -sudo systemctl enable /etc/systemd/calico-node.service -sudo systemctl start calico-node.service -``` + ```shell + # Export this worker's IP address. + export WORKER_IP= + ``` ->The calico-node service will automatically get the kubernetes-calico plugin binary and install it on the host system. + ```shell + # Generate keys. + openssl genrsa -out worker-key.pem 2048 + openssl req -new -key worker-key.pem -out worker.csr -subj "/CN=worker-key" -config worker-openssl.cnf + openssl x509 -req -in worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out worker.pem -days 365 -extensions v3_req -extfile worker-openssl.cnf + ``` -2.) Use calicoctl to add an IP pool. We must specify the IP and port that the master's etcd is listening on. -**NOTE: This step only needs to be performed once per Kubernetes deployment, as it covers all the node's IP ranges.** +3. Send the three files (`ca.pem`, `worker.pem`, and `worker-key.pem`) to the host (using scp, for example). -```shell -ETCD_AUTHORITY=:4001 calicoctl pool add 192.168.0.0/16 -``` +4. Move the files to the `/etc/kubernetes/ssl` folder with the appropriate permissions: + + ``` + # Move keys + sudo mkdir -p /etc/kubernetes/ssl/ + sudo mv -t /etc/kubernetes/ssl/ ca.pem worker.pem worker-key.pem + + # Set permissions + sudo chmod 600 /etc/kubernetes/ssl/worker-key.pem + sudo chown root:root /etc/kubernetes/ssl/worker-key.pem + ``` + +### Configure the kubelet worker + +1. With your certs in place, create a kubeconfig for worker authentication in `/etc/kubernetes/worker-kubeconfig.yaml`; replace `` with the IP address of the master: + + ``` + apiVersion: v1 + kind: Config + clusters: + - name: local + cluster: + server: https://:443 + certificate-authority: /etc/kubernetes/ssl/ca.pem + users: + - name: kubelet + user: + client-certificate: /etc/kubernetes/ssl/worker.pem + client-key: /etc/kubernetes/ssl/worker-key.pem + contexts: + - context: + cluster: local + user: kubelet + name: kubelet-context + current-context: kubelet-context + ``` + +### Install Calico on the node + +On your compute nodes, it is important that you install Calico before Kubernetes. We'll install Calico using the provided `calico-node.service` systemd unit file: + +1. Install the `calicoctl` binary: + + ```shell + wget https://github.com/projectcalico/calico-containers/releases/download/v0.15.0/calicoctl + chmod +x calicoctl + sudo mv calicoctl /usr/bin + ``` + +2. Fetch the calico/node container: + + ```shell + sudo docker pull calico/node:v0.15.0 + ``` + +3. Download the `network-environment` template from the `calico-cni` repository: + + ```shell + wget -O network-environment https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/node/network-environment-template + ``` + +4. Edit `network-environment` to represent this node's settings: + + - Replace `` with the IP address of the node. + - Replace `` with the IP or hostname of the master. + +5. Move `network-environment` into `/etc`: + + ```shell + sudo mv -f network-environment /etc + ``` + +6. Install the `calico-node` service: + + ```shell + sudo wget -N -P /etc/systemd https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/common/calico-node.service + sudo systemctl enable /etc/systemd/calico-node.service + sudo systemctl start calico-node.service + ``` + +7. Install the Calico CNI plugins: + + ```shell + sudo mkdir -p /opt/cni/bin/ + sudo wget -N -P /opt/cni/bin/ https://github.com/projectcalico/calico-cni/releases/download/v1.0.0/calico + sudo wget -N -P /opt/cni/bin/ https://github.com/projectcalico/calico-cni/releases/download/v1.0.0/calico-ipam + sudo chmod +x /opt/cni/bin/calico /opt/cni/bin/calico-ipam + ``` + +8. Create a CNI network configuration file, which tells Kubernetes to create a network named `calico-k8s-network` and to use the calico plugins for that network. Create file `/etc/cni/net.d/10-calico.conf` with the following contents, replacing `` with the IP of the master (this file should be the same on each node): + + ```shell + # Make the directory structure. + mkdir -p /etc/cni/net.d + + # Make the network configuration file + cat >/etc/rkt/net.d/10-calico.conf <:6666", + "log_level": "info", + "ipam": { + "type": "calico-ipam" + } + } + EOF + ``` + + Since this is the only network we create, it will be used by default by the kubelet. + +9. Verify that Calico started correctly: + + ```shell + calicoctl status + ``` + + should show that Felix (Calico's per-node agent) is running and the there should be a BGP status line for each other node that you've configured and the master. The "Info" column should show "Established": + + ``` + $ calicoctl status + calico-node container is running. Status: Up 15 hours + Running felix version 1.3.0rc5 + + IPv4 BGP status + +---------------+-------------------+-------+----------+-------------+ + | Peer address | Peer type | State | Since | Info | + +---------------+-------------------+-------+----------+-------------+ + | 172.18.203.41 | node-to-node mesh | up | 17:32:26 | Established | + | 172.18.203.42 | node-to-node mesh | up | 17:32:25 | Established | + +---------------+-------------------+-------+----------+-------------+ + + IPv6 BGP status + +--------------+-----------+-------+-------+------+ + | Peer address | Peer type | State | Since | Info | + +--------------+-----------+-------+-------+------+ + +--------------+-----------+-------+-------+------+ + ``` + + If the "Info" column shows "Active" or some other value then Calico is having difficulty connecting to the other host. Check the IP address of the peer is correct and check that Calico is using the correct local IP address (set in the `network-environment` file above). ### Install Kubernetes on the Node -1.) Build & Install Kubernetes binaries +1. Download and Install the kubelet binary: -```shell -# Get the Kubernetes Source -wget https://github.com/kubernetes/kubernetes/releases/download/v1.0.3/kubernetes.tar.gz + ```shell + sudo wget -N -P /usr/bin http://storage.googleapis.com/kubernetes-release/release/v1.1.4/bin/linux/amd64/kubelet + sudo chmod +x /usr/bin/kubelet + ``` -# Untar it -tar -xf kubernetes.tar.gz -tar -xf kubernetes/server/kubernetes-server-linux-amd64.tar.gz -kubernetes/cluster/ubuntu/build.sh +2. Install the `kubelet` systemd unit file: -# Add binaries to /usr/bin -sudo cp -f binaries/minion/* /usr/bin + ```shell + # Download the unit file. + sudo wget -N -P /etc/systemd https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/node/kubelet.service -# Get the iptables based kube-proxy reccomended for this demo -wget https://github.com/projectcalico/calico-kubernetes/releases/download/v0.1.1/kube-proxy -sudo cp kube-proxy /usr/bin/ -sudo chmod +x /usr/bin/kube-proxy -``` + # Enable and start the unit files so that they run on boot + sudo systemctl enable /etc/systemd/kubelet.service + sudo systemctl start kubelet.service + ``` -2.) Install and launch the sample systemd processes settings for launching Kubernetes services +3. Download the `kube-proxy` manifest: -```shell -sudo cp calico-kubernetes-ubuntu-demo-master/node/kube-proxy.service /etc/systemd/ -sudo cp calico-kubernetes-ubuntu-demo-master/node/kube-kubelet.service /etc/systemd/ -sudo systemctl enable /etc/systemd/kube-proxy.service -sudo systemctl enable /etc/systemd/kube-kubelet.service -sudo systemctl start kube-proxy.service -sudo systemctl start kube-kubelet.service -``` + ```shell + wget https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/node/kube-proxy.manifest + ``` -> *You may want to consider checking their status after to ensure everything is running* +4. In that file, replace `` with your master's IP. Then move it into place: + + ```shell + sudo mkdir -p /etc/kubernetes/manifests/ + sudo mv kube-proxy.manifest /etc/kubernetes/manifests/ + ``` + +## Configure kubectl remote access + +To administer your cluster from a separate host (e.g your laptop), you will need the root CA generated earlier, as well as an admin public/private keypair (`ca.pem`, `admin.pem`, `admin-key.pem`). Run the following steps on the machine which you will use to control your cluster. + +1. Download the kubectl binary. + + ```shell + sudo wget -N -P /usr/bin http://storage.googleapis.com/kubernetes-release/release/v1.1.4/bin/linux/amd64/kubectl + sudo chmod +x /usr/bin/kubectl + ``` + +2. Generate the admin public/private keypair. + +3. Export the necessary variables, substituting in correct values for your machine. + + ```shell + # Export the appropriate paths. + export CA_CERT_PATH= + export ADMIN_CERT_PATH= + export ADMIN_KEY_PATH= + + # Export the Master's IP address. + export MASTER_IPV4= + ``` + +4. Configure your host `kubectl` with the admin credentials: + + ```shell + kubectl config set-cluster calico-cluster --server=https://${MASTER_IPV4} --certificate-authority=${CA_CERT_PATH} + kubectl config set-credentials calico-admin --certificate-authority=${CA_CERT_PATH} --client-key=${ADMIN_KEY_PATH} --client-certificate=${ADMIN_CERT_PATH} + kubectl config set-context calico --cluster=calico-cluster --user=calico-admin + kubectl config use-context calico + ``` + +Check your work with `kubectl get nodes`, which should succeed and display the nodes. ## Install the DNS Addon -Most Kubernetes deployments will require the DNS addon for service discovery. For more on DNS service discovery, check [here](https://releases.k8s.io/{{page.githubbranch}}/cluster/addons/dns). +Most Kubernetes deployments will require the DNS addon for service discovery. To install DNS, create the skydns service and replication controller provided. This step makes use of the kubectl configuration made above. -The config repository for this guide comes with manifest files to start the DNS addon. To install DNS, do the following on your Master node. +```shell +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/master/dns/skydns.yaml +``` -Replace `` in `calico-kubernetes-ubuntu-demo-master/dns/skydns-rc.yaml` with your Master's IP address. Then, create `skydns-rc.yaml` and `skydns-svc.yaml` using `kubectl create -f `. +## Install the Kubernetes UI Addon (Optional) + +The Kubernetes UI can be installed using `kubectl` to run the following manifest file. + +```shell +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico-cni/k8s-1.1-docs/samples/kubernetes/master/kube-ui/kube-ui.yaml +``` ## Launch other Services With Calico-Kubernetes -At this point, you have a fully functioning cluster running on kubernetes with a master and 2 nodes networked with Calico. You can now follow any of the [standard documentation](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/) to set up other services on your cluster. +At this point, you have a fully functioning cluster running on Kubernetes with a master and two nodes networked with Calico. You can now follow any of the [standard documentation](https://github.com/kubernetes/kubernetes/tree/{{page.version}}/examples/) to set up other services on your cluster. ## Connectivity to outside the cluster -With this sample configuration, because the containers have private `192.168.0.0/16` IPs, you will need NAT to allow connectivity between containers and the internet. However, in a full datacenter deployment, NAT is not always necessary, since Calico can peer with the border routers over BGP. +Because containers in this guide have private `192.168.0.0/16` IPs, you will need NAT to allow connectivity between containers and the internet. However, in a production data center deployment, NAT is not always necessary, since Calico can peer with the data center's border routers over BGP. ### NAT on the nodes -The simplest method for enabling connectivity from containers to the internet is to use an iptables masquerade rule. This is the standard mechanism [recommended](/docs/admin/networking/#google-compute-engine-gce) in the Kubernetes GCE environment. +The simplest method for enabling connectivity from containers to the internet is to use outgoing NAT on your Kubernetes nodes. -We need to NAT traffic that has a destination outside of the cluster. Internal traffic includes the master/nodes, and the container IP pools. A suitable masquerade chain would follow the pattern below, replacing the following variables: - -- `CONTAINER_SUBNET`: The cluster-wide subnet from which container IPs are chosen. All cbr0 bridge subnets fall within this range. The above example uses `192.168.0.0/16`. -- `KUBERNETES_HOST_SUBNET`: The subnet from which Kubernetes node / master IP addresses have been chosen. -- `HOST_INTERFACE`: The interface on the Kubernetes node which is used for external connectivity. The above example uses `eth0` +Calico can provide outgoing NAT for containers. To enable it, use the following `calicoctl` command: ```shell -sudo iptables -t nat -N KUBE-OUTBOUND-NAT -sudo iptables -t nat -A KUBE-OUTBOUND-NAT -d -o -j RETURN -sudo iptables -t nat -A KUBE-OUTBOUND-NAT -d -o -j RETURN -sudo iptables -t nat -A KUBE-OUTBOUND-NAT -j MASQUERADE -sudo iptables -t nat -A POSTROUTING -j KUBE-OUTBOUND-NAT +ETCD_AUTHORITY= calicoctl pool add --nat-outgoing ``` -This chain should be applied on the master and all nodes. In production, these rules should be persisted, e.g. with `iptables-persistent`. +By default, `` will be `192.168.0.0/16`. You can find out which pools have been configured with the following command: + +```shell +ETCD_AUTHORITY= calicoctl pool show +``` ### NAT at the border router -In a datacenter environment, it is recommended to configure Calico to peer with the border routers over BGP. This means that the container IPs will be routable anywhere in the datacenter, and so NAT is not needed on the nodes (though it may be enabled at the datacenter edge to allow outbound-only internet connectivity). \ No newline at end of file +In a data center environment, it is recommended to configure Calico to peer with the border routers over BGP. This means that the container IPs will be routable anywhere in the data center, and so NAT is not needed on the nodes (though it may be enabled at the data center edge to allow outbound-only internet connectivity). + +The Calico documentation contains more information on how to configure Calico to [peer with existing infrastructure](https://github.com/projectcalico/calico-containers/blob/master/docs/ExternalConnectivity.md). diff --git a/docs/getting-started-guides/ubuntu.md b/docs/getting-started-guides/ubuntu.md index bd6a06b903..1515a55b5d 100644 --- a/docs/getting-started-guides/ubuntu.md +++ b/docs/getting-started-guides/ubuntu.md @@ -6,6 +6,12 @@ in the given examples. You can scale to **any number of nodes** by changing some The original idea was heavily inspired by @jainvipin 's ubuntu single node work, which has been merge into this document. +The scripting referenced here can be used to deploy Kubernetes with +networking based either on Flannel or on a CNI plugin that you supply. +This document is focused on the Flannel case. See +`kubernetes/cluster/ubuntu/config-default.sh` for remarks on how to +use a CNI plugin instead. + [Cloud team from Zhejiang University](https://github.com/ZJU-SEL) will maintain this work. * TOC @@ -14,71 +20,82 @@ work, which has been merge into this document. ## Prerequisites 1. The nodes have installed docker version 1.2+ and bridge-utils to manipulate linux bridge. -2. All machines can communicate with each other. Master node needs to connect the Internet to download the necessary files, while working nodes do not. +2. All machines can communicate with each other. Master node needs to be connected to the +Internet to download the necessary files, while worker nodes do not. 3. These guide is tested OK on Ubuntu 14.04 LTS 64bit server, but it can not work with -Ubuntu 15 which use systemd instead of upstart. We are working around fixing this. -4. Dependencies of this guide: etcd-2.0.12, flannel-0.4.0, k8s-1.0.3, may work with higher versions. +Ubuntu 15 which uses systemd instead of upstart. +4. Dependencies of this guide: etcd-2.2.1, flannel-0.5.5, k8s-1.1.4, may work with higher versions. 5. All the remote servers can be ssh logged in without a password by using key authentication. ## Starting a Cluster -### Download binaries +### Set up working directory -First clone the kubernetes github repo +Clone the kubernetes github repo locally ```shell $ git clone https://github.com/kubernetes/kubernetes.git ``` -Then download all the needed binaries into given directory (cluster/ubuntu/binaries) +#### Configure and start the Kubernetes cluster + +The startup process will first download all the required binaries automatically. +By default etcd version is 2.2.1, flannel version is 0.5.5 and k8s version is 1.1.4. +You can customize your etcd version, flannel version, k8s version by changing corresponding variables +`ETCD_VERSION` , `FLANNEL_VERSION` and `KUBE_VERSION` like following. ```shell -$ cd kubernetes/cluster/ubuntu -$ ./build.sh +$ export KUBE_VERSION=1.0.5 +$ export FLANNEL_VERSION=0.5.0 +$ export ETCD_VERSION=2.2.0 ``` -You can customize your etcd version, flannel version, k8s version by changing corresponding variables -`ETCD_VERSION` , `FLANNEL_VERSION` and `KUBE_VERSION` in build.sh, by default etcd version is 2.0.12, -flannel version is 0.4.0 and k8s version is 1.0.3. +**Note** -Make sure that the involved binaries are located properly in the binaries/master -or binaries/minion directory before you go ahead to the next step . +For users who want to bring up a cluster with k8s version v1.1.1, `controller manager` may fail to start +due to [a known issue](https://github.com/kubernetes/kubernetes/issues/17109). You could raise it +up manually by using following command on the remote master server. Note that +you should do this only after `api-server` is up. Moreover this issue is fixed in v1.1.2 and later. + +```shell +$ sudo service kube-controller-manager start +``` Note that we use flannel here to set up overlay network, yet it's optional. Actually you can build up k8s cluster natively, or use flannel, Open vSwitch or any other SDN tool you like. -#### Configure and start the Kubernetes cluster - An example cluster is listed below: +```shell | IP Address | Role | |-------------|----------| |10.10.103.223| node | |10.10.103.162| node | |10.10.103.250| both master and node| +``` -First configure the cluster information in cluster/ubuntu/config-default.sh, below is a simple sample. +First configure the cluster information in cluster/ubuntu/config-default.sh, following is a simple sample. ```shell export nodes="vcap@10.10.103.250 vcap@10.10.103.162 vcap@10.10.103.223" export role="ai i i" -export NUM_MINIONS=${NUM_MINIONS:-3} +export NUM_NODES=${NUM_NODES:-3} export SERVICE_CLUSTER_IP_RANGE=192.168.3.0/24 export FLANNEL_NET=172.16.0.0/16 ``` -The first variable `nodes` defines all your cluster nodes, MASTER node comes first and +The first variable `nodes` defines all your cluster nodes, master node comes first and separated with blank space like ` ` Then the `role` variable defines the role of above machine in the same order, "ai" stands for machine acts as both master and node, "a" stands for master, "i" stands for node. -The `NUM_MINIONS` variable defines the total number of nodes. +The `NUM_NODES` variable defines the total number of nodes. The `SERVICE_CLUSTER_IP_RANGE` variable defines the kubernetes service IP range. Please make sure that you do have a valid private ip range defined here, because some IaaS provider may reserve private ips. @@ -95,29 +112,32 @@ that conflicts with your own private network range. The `FLANNEL_NET` variable defines the IP range used for flannel overlay network, should not conflict with above `SERVICE_CLUSTER_IP_RANGE`. +You can optionally provide additional Flannel network configuration +through `FLANNEL_OTHER_NET_CONFIG`, as explained in `cluster/ubuntu/config-default.sh`. -**Note:** When deploying, master needs to connect the Internet to download the necessary files. If your machines locate in a private network that need proxy setting to connect the Internet, you can set the config `PROXY_SETTING` in cluster/ubuntu/config-default.sh such as: +**Note:** When deploying, master needs to be connected to the Internet to download the necessary files. +If your machines are located in a private network that need proxy setting to connect the Internet, +you can set the config `PROXY_SETTING` in cluster/ubuntu/config-default.sh such as: + + PROXY_SETTING="http_proxy=http://server:port https_proxy=https://server:port" + +After all the above variables being set correctly, we can use following command in `cluster/` directory to +bring up the whole cluster. ```shell -PROXY_SETTING="http_proxy=http://server:port https_proxy=https://server:port" +$ KUBERNETES_PROVIDER=ubuntu ./kube-up.sh ``` -After all the above variables being set correctly, we can use following command in cluster/ directory to bring up the whole cluster. +The scripts automatically copy binaries and config files to all the machines via `scp` and start kubernetes +service on them. The only thing you need to do is to type the sudo password when promoted. ```shell -KUBERNETES_PROVIDER=ubuntu ./kube-up.sh -``` - -The scripts automatically scp binaries and config files to all the machines and start the k8s service on them. -The only thing you need to do is to type the sudo password when promoted. - -```shell -Deploying minion on machine 10.10.103.223 +Deploying node on machine 10.10.103.223 ... -[sudo] password to copy files and start minion: +[sudo] password to start node: ``` -If all things goes right, you will see the below message from console indicating the k8s is up. +If everything works correctly, you will see the following message from console indicating the k8s cluster is up. ```shell Cluster validation succeeded @@ -125,7 +145,7 @@ Cluster validation succeeded ### Test it out -You can use `kubectl` command to check if the newly created k8s is working correctly. +You can use `kubectl` command to check if the newly created cluster is working correctly. The `kubectl` binary is under the `cluster/ubuntu/binaries` directory. You can make it available via PATH, then you can use the below command smoothly. @@ -139,7 +159,7 @@ NAME LABELS STATUS 10.10.103.250 kubernetes.io/hostname=10.10.103.250 Ready ``` -Also you can run Kubernetes [guest-example](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/guestbook/) to build a redis backend cluster on the k8s． +Also you can run Kubernetes [guest-example](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/examples/guestbook/) to build a redis backend cluster． ### Deploy addons @@ -185,77 +205,79 @@ We are working on these features which we'd like to let everybody know: to eliminate OS-distro differences. 2. Tearing Down scripts: clear and re-create the whole stack by one click. -### Trouble shooting +### Troubleshooting Generally, what this approach does is quite simple: -1. Download and copy binaries and configuration files to proper directories on every node -2. Configure `etcd` using IPs based on input from user -3. Create and start flannel network +1. Download and copy binaries and configuration files to proper directories on every node. +2. Configure `etcd` for master node using IPs based on input from user. +3. Create and start flannel network for worker nodes. -So if you encounter a problem, **check etcd configuration first** - -Please try: +So if you encounter a problem, check etcd configuration of master node first. 1. Check `/var/log/upstart/etcd.log` for suspicious etcd log -2. Check `/etc/default/etcd`, as we do not have much input validation, a right config should be like: +2. You may find following commands useful, the former one to bring down the cluster, while the latter one could start it again. ```shell -ETCD_OPTS="-name infra1 -initial-advertise-peer-urls -listen-peer-urls -initial-cluster-token etcd-cluster-1 -initial-cluster infra1=,infra2=,infra3= -initial-cluster-state new" +$ KUBERNETES_PROVIDER=ubuntu ./kube-down.sh +$ KUBERNETES_PROVIDER=ubuntu ./kube-up.sh ``` -3. You may find following commands useful, the former one to bring down the cluster, while -the latter one could start it again. - -```shell -KUBERNETES_PROVIDER=ubuntu ./kube-down.sh -KUBERNETES_PROVIDER=ubuntu ./kube-up.sh -``` - -4. You can also customize your own settings in `/etc/default/{component_name}`. +3. You can also customize your own settings in `/etc/default/{component_name}` and restart it via +`$ sudo service {component_name} restart`. -### Upgrading a Cluster +## Upgrading a Cluster If you already have a kubernetes cluster, and want to upgrade to a new version, -you can use following command in cluster/ directory to update the whole cluster or a specified node to a new version. +you can use following command in `cluster/` directory to update the whole cluster +or a specified node to a new version. ```shell -KUBERNETES_PROVIDER=ubuntu ./kube-push.sh [-m|-n ] +$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh [-m|-n ] ``` It can be done for all components (by default), master(`-m`) or specified node(`-n`). -If the version is not specified, the script will try to use local binaries.You should ensure all the binaries are well prepared in path `cluster/ubuntu/binaries`. +Upgrading a single node is currently experimental. +If the version is not specified, the script will try to use local binaries. You should ensure all +the binaries are well prepared in the expected directory path cluster/ubuntu/binaries. ```shell $ tree cluster/ubuntu/binaries binaries/ -'��'��'�� kubectl -'��'��'�� master -'��   '��'��'�� etcd -'��   '��'��'�� etcdctl -'��   '��'��'�� flanneld -'��   '��'��'�� kube-apiserver -'��   '��'��'�� kube-controller-manager -'��   '��'��'�� kube-scheduler -'��'��'�� minion - '��'��'�� flanneld - '��'��'�� kubelet - '��'��'�� kube-proxy +├── kubectl +├── master +│   ├── etcd +│   ├── etcdctl +│   ├── flanneld +│   ├── kube-apiserver +│   ├── kube-controller-manager +│   └── kube-scheduler +└── minion + ├── flanneld + ├── kubelet + └── kube-proxy ``` -Upgrading single node is experimental now. You can use following command to get a help. +You can use following command to get a help. ```shell -KUBERNETES_PROVIDER=ubuntu ./kube-push.sh -h +$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh -h ``` -Some examples are as follows: +Here are some examples: * upgrade master to version 1.0.5: `$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh -m 1.0.5` -* upgrade node 10.10.103.223 to version 1.0.5 : `$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh -n 10.10.103.223 1.0.5` +* upgrade node `vcap@10.10.103.223` to version 1.0.5 : `$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh -n 10.10.103.223 1.0.5` * upgrade master and all nodes to version 1.0.5: `$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh 1.0.5` The script will not delete any resources of your cluster, it just replaces the binaries. -You can use `kubectl` command to check if the newly upgraded k8s is working correctly. -For example, use `$ kubectl get nodes` to see if all of your nodes are ready.Or refer to [test-it-out](/docs/getting-started-guides/ubuntu/#test-it-out) \ No newline at end of file + +### Test it out + +You can use the `kubectl` command to check if the newly upgraded kubernetes cluster is working correctly. + +To make sure the version of the upgraded cluster is what you expect, you will find these commands helpful. + +* upgrade all components or master: `$ kubectl version`. Check the *Server Version*. +* upgrade node `vcap@10.10.102.223`: `$ ssh -t vcap@10.10.102.223 'cd /opt/bin && sudo ./kubelet --version'` diff --git a/docs/getting-started-guides/vagrant.md b/docs/getting-started-guides/vagrant.md index f0c63825f1..ce9a18e313 100644 --- a/docs/getting-started-guides/vagrant.md +++ b/docs/getting-started-guides/vagrant.md @@ -8,9 +8,9 @@ Running Kubernetes with Vagrant (and VirtualBox) is an easy way to run/test/deve ### Prerequisites -1. Install latest version >= 1.6.2 of vagrant from http://www.vagrantup.com/downloads.html +1. Install latest version >= 1.7.4 of vagrant from http://www.vagrantup.com/downloads.html 2. Install one of: - 1. Version 4.3.28 of Virtual Box from https://www.virtualbox.org/wiki/Download_Old_Builds_4_3 + 1. The latest version of Virtual Box from https://www.virtualbox.org/wiki/Downloads 2. [VMWare Fusion](https://www.vmware.com/products/fusion/) version 5 or greater as well as the appropriate [Vagrant VMWare Fusion provider](https://www.vagrantup.com/vmware) 3. [VMWare Workstation](https://www.vmware.com/products/workstation/) version 9 or greater as well as the [Vagrant VMWare Workstation provider](https://www.vagrantup.com/vmware) 4. [Parallels Desktop](https://www.parallels.com/products/desktop/) version 9 or greater as well as the [Vagrant Parallels provider](https://parallels.github.io/vagrant-parallels/) @@ -36,11 +36,11 @@ export KUBERNETES_PROVIDER=vagrant The `KUBERNETES_PROVIDER` environment variable tells all of the various cluster management scripts which variant to use. If you forget to set this, the assumption is you are running on Google Compute Engine. -By default, the Vagrant setup will create a single master VM (called kubernetes-master) and one node (called kubernetes-minion-1). Each VM will take 1 GB, so make sure you have at least 2GB to 4GB of free memory (plus appropriate free disk space). +By default, the Vagrant setup will create a single master VM (called kubernetes-master) and one node (called kubernetes-node-1). Each VM will take 1 GB, so make sure you have at least 2GB to 4GB of free memory (plus appropriate free disk space). Vagrant will provision each machine in the cluster with all the necessary components to run Kubernetes. The initial setup can take a few minutes to complete on each machine. -If you installed more than one Vagrant provider, Kubernetes will usually pick the appropriate one. However, you can override which one Kubernetes will use by setting the [`VAGRANT_DEFAULT_PROVIDER`](https://docs.vagrantup.com/v2/providers/default) environment variable: +If you installed more than one Vagrant provider, Kubernetes will usually pick the appropriate one. However, you can override which one Kubernetes will use by setting the [`VAGRANT_DEFAULT_PROVIDER`](https://docs.vagrantup.com/v2/providers/default.html) environment variable: ```shell export VAGRANT_DEFAULT_PROVIDER=parallels @@ -54,14 +54,14 @@ To access the master or any node: ```shell vagrant ssh master -vagrant ssh minion-1 +vagrant ssh node-1 ``` If you are running more than one node, you can access the others by: ```shell -vagrant ssh minion-2 -vagrant ssh minion-3 +vagrant ssh node-2 +vagrant ssh node-3 ``` Each node in the cluster installs the docker daemon and the kubelet. @@ -88,7 +88,7 @@ To view the service status and/or logs on the kubernetes-master: To view the services on any of the nodes: ```shell -[vagrant@kubernetes-master ~] $ vagrant ssh minion-1 +[vagrant@kubernetes-master ~] $ vagrant ssh node-1 [vagrant@kubernetes-master ~] $ sudo su [root@kubernetes-master ~] $ systemctl status kubelet @@ -205,8 +205,8 @@ my-nginx-xql4j 0/1 Pending 0 10s You need to wait for the provisioning to complete, you can monitor the nodes by doing: ```shell -$ vagrant ssh minion-1 -c 'sudo docker images' -kubernetes-minion-1: +$ vagrant ssh node-1 -c 'sudo docker images' +kubernetes-node-1: REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE 96864a7d2df3 26 hours ago 204.4 MB google/cadvisor latest e0575e677c50 13 days ago 12.64 MB @@ -216,8 +216,8 @@ kubernetes-minion-1: Once the docker image for nginx has been downloaded, the container will start and you can list it: ```shell -$ vagrant ssh minion-1 -c 'sudo docker ps' -kubernetes-minion-1: +$ vagrant ssh node-1 -c 'sudo docker ps' +kubernetes-node-1: CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES dbe79bf6e25b nginx:latest "nginx" 21 seconds ago Up 19 seconds k8s--mynginx.8c5b8a3a--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1.etcd--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1--fcfa837f fa0e29c94501 kubernetes/pause:latest "/pause" 8 minutes ago Up 8 minutes 0.0.0.0:8080->80/tcp k8s--net.a90e7ce4--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1.etcd--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1--baf5b21b @@ -236,7 +236,10 @@ my-nginx-xql4j 1/1 Running 0 1m $ ./cluster/kubectl.sh get services NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE -my-nginx 10.0.0.1 80/TCP run=my-nginx 1h + +$ ./cluster/kubectl.sh get replicationcontrollers +CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE +my-nginx my-nginx nginx run=my-nginx 3 1m ``` We did not start any services, hence there are none listed. But we see three replicas displayed properly. @@ -266,6 +269,43 @@ export KUBERNETES_PROVIDER=vagrant ./cluster/kube-up.sh ``` +#### I am getting timeouts when trying to curl the master from my host! + +During provision of the cluster, you may see the following message: + +```shell +Validating node-1 +............. +Waiting for each node to be registered with cloud provider +error: couldn't read version from server: Get https://10.245.1.2/api: dial tcp 10.245.1.2:443: i/o timeout +``` + +Some users have reported VPNs may prevent traffic from being routed to the host machine into the virtual machine network. + +To debug, first verify that the master is binding to the proper IP address: + +``` +$ vagrant ssh master +$ ifconfig | grep eth1 -C 2 +eth1: flags=4163 mtu 1500 inet 10.245.1.2 netmask + 255.255.255.0 broadcast 10.245.1.255 +``` + +Then verify that your host machine has a network connection to a bridge that can serve that address: + +```sh +$ ifconfig | grep 10.245.1 -C 2 + +vboxnet5: flags=4163 mtu 1500 + inet 10.245.1.1 netmask 255.255.255.0 broadcast 10.245.1.255 + inet6 fe80::800:27ff:fe00:5 prefixlen 64 scopeid 0x20 + ether 0a:00:27:00:00:05 txqueuelen 1000 (Ethernet) +``` + +If you do not see a response on your host machine, you will most likely need to connect your host to the virtual network created by the virtualization provider. + +If you do see a network, but are still unable to ping the machine, check if your VPN is blocking the request. + #### I just created the cluster, but I am getting authorization errors! You probably have an incorrect ~/.kubernetes_vagrant_auth file for the cluster you are attempting to contact. @@ -297,14 +337,14 @@ To set up a vagrant cluster for hacking, follow the [vagrant developer guide](ht #### I have brought Vagrant up but the nodes cannot validate! -Log on to one of the nodes (`vagrant ssh minion-1`) and inspect the salt minion log (`sudo cat /var/log/salt/minion`). +Log on to one of the nodes (`vagrant ssh node-1`) and inspect the salt minion log (`sudo cat /var/log/salt/minion`). #### I want to change the number of nodes! -You can control the number of nodes that are instantiated via the environment variable `NUM_MINIONS` on your host machine. If you plan to work with replicas, we strongly encourage you to work with enough nodes to satisfy your largest intended replica size. If you do not plan to work with replicas, you can save some system resources by running with a single node. You do this, by setting `NUM_MINIONS` to 1 like so: +You can control the number of nodes that are instantiated via the environment variable `NUM_NODES` on your host machine. If you plan to work with replicas, we strongly encourage you to work with enough nodes to satisfy your largest intended replica size. If you do not plan to work with replicas, you can save some system resources by running with a single node. You do this, by setting `NUM_NODES` to 1 like so: ```shell -export NUM_MINIONS=1 +export NUM_NODES=1 ``` #### I want my VMs to have more memory! @@ -320,7 +360,7 @@ If you need more granular control, you can set the amount of memory for the mast ```shell export KUBERNETES_MASTER_MEMORY=1536 -export KUBERNETES_MINION_MEMORY=2048 +export KUBERNETES_NODE_MEMORY=2048 ``` #### I ran vagrant suspend and nothing works! @@ -329,8 +369,8 @@ export KUBERNETES_MINION_MEMORY=2048 #### I want vagrant to sync folders via nfs! -You can ensure that vagrant uses nfs to sync folders with virtual machines by setting the KUBERNETES_VAGRANT_USE_NFS environment variable to 'true'. nfs is faster than virtualbox or vmware's 'shared folders' and does not require guest additions. See the [vagrant docs](http://docs.vagrantup.com/v2/synced-folders/nfs) for details on configuring nfs on the host. This setting will have no effect on the libvirt provider, which uses nfs by default. For example: +You can ensure that vagrant uses nfs to sync folders with virtual machines by setting the KUBERNETES_VAGRANT_USE_NFS environment variable to 'true'. nfs is faster than virtualbox or vmware's 'shared folders' and does not require guest additions. See the [vagrant docs](http://docs.vagrantup.com/v2/synced-folders/nfs.html) for details on configuring nfs on the host. This setting will have no effect on the libvirt provider, which uses nfs by default. For example: ```shell export KUBERNETES_VAGRANT_USE_NFS=true -``` \ No newline at end of file +``` diff --git a/docs/getting-started-guides/vsphere.md b/docs/getting-started-guides/vsphere.md index da3eb4b0b4..905719b16c 100644 --- a/docs/getting-started-guides/vsphere.md +++ b/docs/getting-started-guides/vsphere.md @@ -12,13 +12,13 @@ convenient). ### Prerequisites 1. You need administrator credentials to an ESXi machine or vCenter instance. -2. You must have Go (version 1.2 or later) installed: [www.golang.org](http://www.golang.org). +2. You must have Go (see [here](https://github.com/kubernetes/kubernetes/tree/{{page.githubbranch}}/docs/devel/development.md#go-versions) for supported versions) installed: [www.golang.org](http://www.golang.org). 3. You must have your `GOPATH` set up and include `$GOPATH/bin` in your `PATH`. ```shell export GOPATH=$HOME/src/go - mkdir -p $GOPATH - export PATH=$PATH:$GOPATH/bin +mkdir -p $GOPATH +export PATH=$PATH:$GOPATH/bin ``` 4. Install the govc tool to interact with ESXi/vCenter: @@ -31,10 +31,10 @@ go get github.com/vmware/govmomi/govc ### Setup -Download a prebuilt Debian 7.7 VMDK that we'll use as a base image: +Download a prebuilt Debian 8.2 VMDK that we'll use as a base image: ```shell -curl --remote-name-all https://storage.googleapis.com/govmomi/vmdk/2014-11-11/kube.vmdk.gz{,.md5} +curl --remote-name-all https://storage.googleapis.com/govmomi/vmdk/2016-01-08/kube.vmdk.gz{,.md5} md5sum -c kube.vmdk.gz.md5 gzip -d kube.vmdk.gz ``` @@ -42,7 +42,10 @@ gzip -d kube.vmdk.gz Import this VMDK into your vSphere datastore: ```shell -export GOVC_URL='user:pass@hostname' +export GOVC_URL='hostname' # hostname of the vc +export GOVC_USERNAME='username' # username for logging into the vsphere. +export GOVC_PASSWORD='password' # password for the above username +export GOVC_NETWORK='Network Name' # Name of the network the vms should join. Many times it could be "VM Network" export GOVC_INSECURE=1 # If the host above uses a self-signed cert export GOVC_DATASTORE='target datastore' export GOVC_RESOURCE_POOL='resource pool or cluster with access to datastore' @@ -62,12 +65,21 @@ parameters. The guest login for the image that you imported is `kube:kube`. ### Starting a cluster Now, let's continue with deploying Kubernetes. -This process takes about ~10 minutes. +This process takes about ~20-30 minutes depending on your network. + +#### From extracted binary release + +```shell +cd kubernetes +KUBERNETES_PROVIDER=vsphere cluster/kube-up.sh +``` + +#### Build from source ```shell -cd kubernetes # Extracted binary release OR repository root -export KUBERNETES_PROVIDER=vsphere -cluster/kube-up.sh +cd kubernetes +make release +KUBERNETES_PROVIDER=vsphere cluster/kube-up.sh ``` Refer to the top level README and the getting started guide for Google Compute @@ -81,4 +93,4 @@ deployment works just as any other one! The output of `kube-up.sh` displays the IP addresses of the VMs it deploys. You can log into any VM as the `kube` user to poke around and figure out what is going on (find yourself authorized with your SSH key, or use the password -`kube` otherwise). \ No newline at end of file +`kube` otherwise). diff --git a/docs/user-guide/configmap/README.md b/docs/user-guide/configmap/README.md new file mode 100644 index 0000000000..e2f98e70a7 --- /dev/null +++ b/docs/user-guide/configmap/README.md @@ -0,0 +1,117 @@ + + + + + +# ConfigMap example + + + +## Step Zero: Prerequisites + +This example assumes you have a Kubernetes cluster installed and running, and that you have +installed the `kubectl` command line tool somewhere in your path. Please see the [getting +started](../../../docs/getting-started-guides/) for installation instructions for your platform. + +## Step One: Create the ConfigMap + +A ConfigMap contains a set of named strings. + +Use the [`examples/configmap/configmap.yaml`](configmap.yaml) file to create a ConfigMap: + +```console +$ kubectl create -f docs/user-guide/configmap/configmap.yaml +``` + +You can use `kubectl` to see information about the ConfigMap: + +```console +$ kubectl get configmap +NAME DATA +test-secret 2 + +$ kubectl describe configMap test-configmap +Name: test-configmap +Labels: +Annotations: + +Data +==== +data-1: 7 bytes +data-2: 7 bytes +``` + +View the values of the keys with `kubectl get`: + +```console +$ cluster/kubectl.sh get configmaps test-configmap -o yaml +apiVersion: v1 +data: + data-1: value-1 + data-2: value-2 +kind: ConfigMap +metadata: + creationTimestamp: 2016-02-18T20:28:50Z + name: test-configmap + namespace: default + resourceVersion: "1090" + selfLink: /api/v1/namespaces/default/configmaps/test-configmap + uid: 384bd365-d67e-11e5-8cd0-68f728db1985 +``` + +## Step Two: Create a pod that consumes a configMap in environment variables + +Use the [`examples/configmap/env-pod.yaml`](env-pod.yaml) file to create a Pod that consumes the +ConfigMap in environment variables. + +```console +$ kubectl create -f docs/user-guide/configmap/env-pod.yaml +``` + +This pod runs the `env` command to display the environment of the container: + +```console +$ kubectl logs secret-test-pod +KUBE_CONFIG_1=value-1 +KUBE_CONFIG_2=value-2 +``` + +## Step Three: Create a pod that sets the command line using ConfigMap + +Use the [`examples/configmap/command-pod.yaml`](env-pod.yaml) file to create a Pod with a container +whose command is injected with the keys of a ConfigMap + +```console +$ kubectl create -f docs/user-guide/configmap/env-pod.yaml +``` + +This pod runs an `echo` command to display the keys: + +```console +value-1 value-2 +``` + +## Step Four: Create a pod that consumes a configMap in a volume + +Pods can also consume ConfigMaps in volumes. Use the [`examples/configmap/volume-pod.yaml`](volume-pod.yaml) file to create a Pod that consume the ConfigMap in a volume. + +```console +$ kubectl create -f docs/user-guide/configmap/volume-pod.yaml +``` + +This pod runs a `cat` command to print the value of one of the keys in the volume: + +```console +value-1 +``` + + + + + + + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/user-guide/configmap/README.md?pixel)]() + diff --git a/docs/user-guide/configmap/command-pod.yaml b/docs/user-guide/configmap/command-pod.yaml new file mode 100644 index 0000000000..444b4beb66 --- /dev/null +++ b/docs/user-guide/configmap/command-pod.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: config-cmd-test-pod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/busybox + command: [ "/bin/sh", "-c", "echo $(KUBE_CONFIG_1) $(KUBE_CONFIG_2)" ] + env: + - name: KUBE_CONFIG_1 + valueFrom: + configMapKeyRef: + name: test-configmap + key: data-1 + - name: KUBE_CONFIG_2 + valueFrom: + configMapKeyRef: + name: test-configmap + key: data-2 + restartPolicy: Never diff --git a/docs/user-guide/configmap/configmap.yaml b/docs/user-guide/configmap/configmap.yaml new file mode 100644 index 0000000000..38160c473b --- /dev/null +++ b/docs/user-guide/configmap/configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-configmap +data: + data-1: value-1 + data-2: value-2 diff --git a/docs/user-guide/configmap/env-pod.yaml b/docs/user-guide/configmap/env-pod.yaml new file mode 100644 index 0000000000..fe0036e0b2 --- /dev/null +++ b/docs/user-guide/configmap/env-pod.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: config-env-test-pod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/busybox + command: [ "/bin/sh", "-c", "env" ] + env: + - name: KUBE_CONFIG_1 + valueFrom: + configMapKeyRef: + name: test-configmap + key: data-1 + - name: KUBE_CONFIG_2 + valueFrom: + configMapKeyRef: + name: test-configmap + key: data-2 + restartPolicy: Never diff --git a/docs/user-guide/configmap/kubectl/game.properties b/docs/user-guide/configmap/kubectl/game.properties new file mode 100644 index 0000000000..8ba4184c22 --- /dev/null +++ b/docs/user-guide/configmap/kubectl/game.properties @@ -0,0 +1,7 @@ +enemies=aliens +lives=3 +enemies.cheat=true +enemies.cheat.level=noGoodRotten +secret.code.passphrase=UUDDLRLRBABAS +secret.code.allowed=true +secret.code.lives=30 \ No newline at end of file diff --git a/docs/user-guide/configmap/kubectl/ui.properties b/docs/user-guide/configmap/kubectl/ui.properties new file mode 100644 index 0000000000..487bea0347 --- /dev/null +++ b/docs/user-guide/configmap/kubectl/ui.properties @@ -0,0 +1,4 @@ +color.good=purple +color.bad=yellow +allow.textmode=true +how.nice.to.look=fairlyNice diff --git a/docs/user-guide/configmap/redis/redis-config b/docs/user-guide/configmap/redis/redis-config new file mode 100644 index 0000000000..ead340713c --- /dev/null +++ b/docs/user-guide/configmap/redis/redis-config @@ -0,0 +1,2 @@ +maxmemory 2mb +maxmemory-policy allkeys-lru diff --git a/docs/user-guide/configmap/redis/redis-pod.yaml b/docs/user-guide/configmap/redis/redis-pod.yaml new file mode 100644 index 0000000000..259dbf853a --- /dev/null +++ b/docs/user-guide/configmap/redis/redis-pod.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Pod +metadata: + name: redis +spec: + containers: + - name: redis + image: kubernetes/redis:v1 + env: + - name: MASTER + value: "true" + ports: + - containerPort: 6379 + resources: + limits: + cpu: "0.1" + volumeMounts: + - mountPath: /redis-master-data + name: data + - mountPath: /redis-master + name: config + volumes: + - name: data + emptyDir: {} + - name: config + configMap: + name: example-redis-config + items: + - key: redis-config + path: redis.conf diff --git a/docs/user-guide/configmap/volume-pod.yaml b/docs/user-guide/configmap/volume-pod.yaml new file mode 100644 index 0000000000..c34332e976 --- /dev/null +++ b/docs/user-guide/configmap/volume-pod.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: config-volume-test-pod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/busybox + command: [ "/bin/sh", "-c", "cat /etc/config/path/to/special-key" ] + volumeMounts: + - name: config-volume + mountPath: /etc/config + volumes: + - name: config-volume + configMap: + name: test-configmap + items: + - key: data-1 + path: path/to/special-key + restartPolicy: Never \ No newline at end of file diff --git a/docs/user-guide/deployment.yaml b/docs/user-guide/deployment.yaml new file mode 100644 index 0000000000..98e614ceb2 --- /dev/null +++ b/docs/user-guide/deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + name: nginx-deployment +spec: + replicas: 3 + selector: + matchLabels: + name: nginx + template: + metadata: + labels: + name: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 diff --git a/docs/user-guide/horizontal-pod-autoscaling/hpa-php-apache.yaml b/docs/user-guide/horizontal-pod-autoscaling/hpa-php-apache.yaml index 1a2067714d..8aa7804ca1 100644 --- a/docs/user-guide/horizontal-pod-autoscaling/hpa-php-apache.yaml +++ b/docs/user-guide/horizontal-pod-autoscaling/hpa-php-apache.yaml @@ -7,7 +7,7 @@ spec: scaleRef: kind: ReplicationController name: php-apache - namespace: default + subresource: scale minReplicas: 1 maxReplicas: 10 cpuUtilization: diff --git a/docs/user-guide/horizontal-pod-autoscaling/image/Dockerfile b/docs/user-guide/horizontal-pod-autoscaling/image/Dockerfile index 56f2d6252e..1a93ea2ddc 100644 --- a/docs/user-guide/horizontal-pod-autoscaling/image/Dockerfile +++ b/docs/user-guide/horizontal-pod-autoscaling/image/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM php:5-apache ADD index.php /var/www/html/index.php diff --git a/docs/user-guide/liveness/http-liveness.yaml b/docs/user-guide/liveness/http-liveness.yaml index ca8d212ae2..c2c6065450 100644 --- a/docs/user-guide/liveness/http-liveness.yaml +++ b/docs/user-guide/liveness/http-liveness.yaml @@ -13,6 +13,9 @@ spec: httpGet: path: /healthz port: 8080 + httpHeaders: + - name: X-Custom-Header + value: Awesome initialDelaySeconds: 15 timeoutSeconds: 1 name: liveness diff --git a/docs/user-guide/liveness/image/Dockerfile b/docs/user-guide/liveness/image/Dockerfile index d057ecd309..2c5481c63e 100644 --- a/docs/user-guide/liveness/image/Dockerfile +++ b/docs/user-guide/liveness/image/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM scratch ADD server /server diff --git a/docs/user-guide/liveness/image/Makefile b/docs/user-guide/liveness/image/Makefile index 3eb9f0e2fe..f29214f350 100644 --- a/docs/user-guide/liveness/image/Makefile +++ b/docs/user-guide/liveness/image/Makefile @@ -1,3 +1,17 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + all: push server: server.go diff --git a/docs/user-guide/logging-demo/Makefile b/docs/user-guide/logging-demo/Makefile index 810a83f3b7..0df0a18d2e 100644 --- a/docs/user-guide/logging-demo/Makefile +++ b/docs/user-guide/logging-demo/Makefile @@ -1,10 +1,24 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Makefile for launching synthetic logging sources (any platform) # and for reporting the forwarding rules for the # Elasticsearch and Kibana pods for the GCE platform. # For examples of how to observe the ingested logs please # see the appropriate getting started guide e.g. -# Google Cloud Logging: https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/getting-started-guides/logging.md -# With Elasticsearch and Kibana logging: https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/getting-started-guides/logging-elasticsearch.md +# Google Cloud Logging: https://github.com/kubernetes/kubernetes/blob/master/docs/getting-started-guides/logging.md +# With Elasticsearch and Kibana logging: https://github.com/kubernetes/kubernetes/blob/master/docs/getting-started-guides/logging-elasticsearch.md .PHONY: up down logger-up logger-down logger10-up logger10-down diff --git a/docs/user-guide/node-selection/pod-with-node-affinity.yaml b/docs/user-guide/node-selection/pod-with-node-affinity.yaml new file mode 100644 index 0000000000..526f83e4d4 --- /dev/null +++ b/docs/user-guide/node-selection/pod-with-node-affinity.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Pod +metadata: + name: with-labels + annotations: + scheduler.alpha.kubernetes.io/affinity: > + { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.io/e2e-az-name", + "operator": "In", + "values": ["e2e-az1", "e2e-az2"] + } + ] + } + ] + } + } + } + another-annotation-key: another-annotation-value +spec: + containers: + - name: with-labels + image: gcr.io/google_containers/pause:2.0 diff --git a/docs/user-guide/persistent-volumes/simpletest/pod.yaml b/docs/user-guide/persistent-volumes/simpletest/pod.yaml index 3edf2963b3..ccd0045934 100644 --- a/docs/user-guide/persistent-volumes/simpletest/pod.yaml +++ b/docs/user-guide/persistent-volumes/simpletest/pod.yaml @@ -12,7 +12,7 @@ spec: - containerPort: 80 name: "http-server" volumeMounts: - - mountPath: "/var/www/html" + - mountPath: "/usr/share/nginx/html" name: mypd volumes: - name: mypd diff --git a/docs/user-guide/persistent-volumes/volumes/local-01.yaml b/docs/user-guide/persistent-volumes/volumes/local-01.yaml index a465c65149..25acecca93 100644 --- a/docs/user-guide/persistent-volumes/volumes/local-01.yaml +++ b/docs/user-guide/persistent-volumes/volumes/local-01.yaml @@ -10,4 +10,4 @@ spec: accessModes: - ReadWriteOnce hostPath: - path: "/tmp/data01" + path: "/somepath/data01" diff --git a/docs/user-guide/persistent-volumes/volumes/local-02.yaml b/docs/user-guide/persistent-volumes/volumes/local-02.yaml index b946e7105d..e72d2e7d17 100644 --- a/docs/user-guide/persistent-volumes/volumes/local-02.yaml +++ b/docs/user-guide/persistent-volumes/volumes/local-02.yaml @@ -10,5 +10,5 @@ spec: accessModes: - ReadWriteOnce hostPath: - path: "/tmp/data02" + path: "/somepath/data02" persistentVolumeReclaimPolicy: Recycle diff --git a/docs/user-guide/persistent-volumes/volumes/nfs.yaml b/docs/user-guide/persistent-volumes/volumes/nfs.yaml index a6dc2ed250..eae2e7abac 100644 --- a/docs/user-guide/persistent-volumes/volumes/nfs.yaml +++ b/docs/user-guide/persistent-volumes/volumes/nfs.yaml @@ -8,5 +8,5 @@ spec: accessModes: - ReadWriteOnce nfs: - path: /tmp + path: /somepath server: 172.17.0.2 diff --git a/docs/user-guide/replicaset/frontend.yaml b/docs/user-guide/replicaset/frontend.yaml new file mode 100644 index 0000000000..878e10181e --- /dev/null +++ b/docs/user-guide/replicaset/frontend.yaml @@ -0,0 +1,42 @@ +apiVersion: extensions/v1beta1 +kind: ReplicaSet +metadata: + name: frontend + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: guestbook + # tier: frontend +spec: + # this replicas value is default + # modify it according to your case + replicas: 3 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # tier: frontend + template: + metadata: + labels: + app: guestbook + tier: frontend + spec: + containers: + - name: php-redis + image: gcr.io/google_samples/gb-frontend:v3 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access environment variables to find service host + # info, comment out the 'value: dns' line above, and uncomment the + # line below. + # value: env + ports: + - containerPort: 80 diff --git a/docs/user-guide/replicaset/redis-slave.yaml b/docs/user-guide/replicaset/redis-slave.yaml new file mode 100644 index 0000000000..0948371e18 --- /dev/null +++ b/docs/user-guide/replicaset/redis-slave.yaml @@ -0,0 +1,44 @@ +apiVersion: extensions/v1beta1 +kind: ReplicaSet +metadata: + name: redis-slave + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: redis + # role: slave + # tier: backend +spec: + # this replicas value is default + # modify it according to your case + replicas: 2 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # app: guestbook + # role: slave + # tier: backend + template: + metadata: + labels: + app: redis + role: slave + tier: backend + spec: + containers: + - name: slave + image: gcr.io/google_samples/gb-redisslave:v1 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access an environment variable to find the master + # service's host, comment out the 'value: dns' line above, and + # uncomment the line below. + # value: env + ports: + - containerPort: 6379 diff --git a/docs/user-guide/secrets/secret-env-pod.yaml b/docs/user-guide/secrets/secret-env-pod.yaml new file mode 100644 index 0000000000..a5d9c0ff75 --- /dev/null +++ b/docs/user-guide/secrets/secret-env-pod.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: secret-env-pod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/busybox + command: [ "/bin/sh", "-c", "env" ] + env: + - name: MY_SECRET_DATA + valueFrom: + secretKeyRef: + name: test-secret + key: data-1 + restartPolicy: Never diff --git a/docs/user-guide/update-demo/images/kitten/Dockerfile b/docs/user-guide/update-demo/images/kitten/Dockerfile index b053138b35..aef0de58c6 100644 --- a/docs/user-guide/update-demo/images/kitten/Dockerfile +++ b/docs/user-guide/update-demo/images/kitten/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2014 Google Inc. All rights reserved. +# Copyright 2016 The Kubernetes Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docs/user-guide/update-demo/images/nautilus/Dockerfile b/docs/user-guide/update-demo/images/nautilus/Dockerfile index 2904a10791..16280b8215 100644 --- a/docs/user-guide/update-demo/images/nautilus/Dockerfile +++ b/docs/user-guide/update-demo/images/nautilus/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2014 Google Inc. All rights reserved. +# Copyright 2016 The Kubernetes Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.