Update high-availability.md (#8770)
* Update high-availability.md [fix]command should leading * Repair code inside list items, remove "like so" * Modify one code block, change 1-step procedures to not be procedures
This commit is contained in:
Stewart-YU
@@ -55,102 +55,98 @@ For **Option 2**: you can skip to the next step. Any reference to `etcd0`, `etcd
|
|||||||
|
|
||||||
1. Install `cfssl` and `cfssljson`:
|
1. Install `cfssl` and `cfssljson`:
|
||||||
|
|
||||||
```shell
|
```bash
|
||||||
curl -o /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
|
curl -o /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
|
||||||
curl -o /usr/local/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
|
curl -o /usr/local/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
|
||||||
chmod +x /usr/local/bin/cfssl*
|
chmod +x /usr/local/bin/cfssl*
|
||||||
```
|
```
|
||||||
|
|
||||||
1. SSH into `etcd0` and run the following:
|
1. SSH into `etcd0` and run the following:
|
||||||
|
|
||||||
```shell
|
```bash
|
||||||
mkdir -p /etc/kubernetes/pki/etcd
|
mkdir -p /etc/kubernetes/pki/etcd
|
||||||
cd /etc/kubernetes/pki/etcd
|
cd /etc/kubernetes/pki/etcd
|
||||||
```
|
|
||||||
```shell
|
|
||||||
cat >ca-config.json <<EOF
|
|
||||||
{
|
|
||||||
"signing": {
|
|
||||||
"default": {
|
|
||||||
"expiry": "43800h"
|
|
||||||
},
|
|
||||||
"profiles": {
|
|
||||||
"server": {
|
|
||||||
"expiry": "43800h",
|
|
||||||
"usages": [
|
|
||||||
"signing",
|
|
||||||
"key encipherment",
|
|
||||||
"server auth",
|
|
||||||
"client auth"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"client": {
|
|
||||||
"expiry": "43800h",
|
|
||||||
"usages": [
|
|
||||||
"signing",
|
|
||||||
"key encipherment",
|
|
||||||
"client auth"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"peer": {
|
|
||||||
"expiry": "43800h",
|
|
||||||
"usages": [
|
|
||||||
"signing",
|
|
||||||
"key encipherment",
|
|
||||||
"server auth",
|
|
||||||
"client auth"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
```shell
|
|
||||||
cat >ca-csr.json <<EOF
|
|
||||||
{
|
|
||||||
"CN": "etcd",
|
|
||||||
"key": {
|
|
||||||
"algo": "rsa",
|
|
||||||
"size": 2048
|
|
||||||
}
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
{{< note >}}
|
cat >ca-config.json <<EOF
|
||||||
**Optional:** You can modify `ca-csr.json` to add a section for `names`.
|
{
|
||||||
See [the CFSSL wiki](https://github.com/cloudflare/cfssl/wiki/Creating-a-new-CSR) for an example.
|
"signing": {
|
||||||
{{< /note >}}
|
"default": {
|
||||||
|
"expiry": "43800h"
|
||||||
|
},
|
||||||
|
"profiles": {
|
||||||
|
"server": {
|
||||||
|
"expiry": "43800h",
|
||||||
|
"usages": [
|
||||||
|
"signing",
|
||||||
|
"key encipherment",
|
||||||
|
"server auth",
|
||||||
|
"client auth"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"client": {
|
||||||
|
"expiry": "43800h",
|
||||||
|
"usages": [
|
||||||
|
"signing",
|
||||||
|
"key encipherment",
|
||||||
|
"client auth"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"peer": {
|
||||||
|
"expiry": "43800h",
|
||||||
|
"usages": [
|
||||||
|
"signing",
|
||||||
|
"key encipherment",
|
||||||
|
"server auth",
|
||||||
|
"client auth"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
```bash
|
||||||
|
cat >ca-csr.json <<EOF
|
||||||
|
{
|
||||||
|
"CN": "etcd",
|
||||||
|
"key": {
|
||||||
|
"algo": "rsa",
|
||||||
|
"size": 2048
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
1. Next, generate the CA certs like so:
|
{{< note >}}
|
||||||
|
**Optional:** You can modify `ca-csr.json` to add a section for `names`.
|
||||||
|
See [the CFSSL wiki](https://github.com/cloudflare/cfssl/wiki/Creating-a-new-CSR) for an example.
|
||||||
|
{{< /note >}}
|
||||||
|
|
||||||
```shell
|
1. Next, generate the CA certs:
|
||||||
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
|
```bash
|
||||||
```
|
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
|
||||||
|
```
|
||||||
|
|
||||||
### Generate etcd client certs
|
### Generate etcd client certs
|
||||||
|
|
||||||
1. Generate the client certificates.
|
Generate the client certificates. While on `etcd0`, run the following:
|
||||||
|
|
||||||
While on `etcd0`, run the following:
|
```bash
|
||||||
|
cat >client.json <<EOF
|
||||||
|
{
|
||||||
|
"CN": "client",
|
||||||
|
"key": {
|
||||||
|
"algo": "ecdsa",
|
||||||
|
"size": 256
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
```bash
|
||||||
|
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
|
||||||
|
```
|
||||||
|
|
||||||
```shell
|
Both `client.pem` and `client-key.pem` are created.
|
||||||
cat >client.json <<EOF
|
|
||||||
{
|
|
||||||
"CN": "client",
|
|
||||||
"key": {
|
|
||||||
"algo": "ecdsa",
|
|
||||||
"size": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
```shell
|
|
||||||
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
|
|
||||||
```
|
|
||||||
|
|
||||||
This should result in `client.pem` and `client-key.pem` being created.
|
|
||||||
|
|
||||||
### Create SSH access
|
### Create SSH access
|
||||||
|
|
||||||
@@ -158,202 +154,194 @@ In order to copy certs between machines, you must enable SSH access for `scp`.
|
|||||||
|
|
||||||
1. First, open new tabs in your shell for `etcd1` and `etcd2`. Ensure you are SSHed into all three machines and then run the following (it will be a lot quicker if you use tmux syncing - to do this in iTerm enter `cmd+shift+i`):
|
1. First, open new tabs in your shell for `etcd1` and `etcd2`. Ensure you are SSHed into all three machines and then run the following (it will be a lot quicker if you use tmux syncing - to do this in iTerm enter `cmd+shift+i`):
|
||||||
|
|
||||||
```shell
|
```bash
|
||||||
export PEER_NAME=$(hostname)
|
export PEER_NAME=$(hostname)
|
||||||
export PRIVATE_IP=$(ip addr show eth1 | grep -Po 'inet \K[\d.]+')
|
export PRIVATE_IP=$(ip addr show eth1 | grep -Po 'inet \K[\d.]+')
|
||||||
```
|
```
|
||||||
|
|
||||||
Make sure that `eth1` corresponds to the network interface for the IPv4 address of the private network. This might vary depending on your networking setup, so please check by running `echo $PRIVATE_IP` before continuing.
|
Make sure that `eth1` corresponds to the network interface for the IPv4 address of the private network. This might vary depending on your networking setup, so please check by running `echo $PRIVATE_IP` before continuing.
|
||||||
|
|
||||||
1. Next, generate some SSH keys for the boxes:
|
1. Next, generate some SSH keys for the boxes:
|
||||||
|
```bash
|
||||||
|
ssh-keygen -t rsa -b 4096 -C "<email>"
|
||||||
|
```
|
||||||
|
|
||||||
```shell
|
Make sure to replace `<email>` with your email, a placeholder, or an empty string. Keep hitting enter until files exist in `~/.ssh`.
|
||||||
ssh-keygen -t rsa -b 4096 -C "<email>"
|
|
||||||
```
|
|
||||||
|
|
||||||
Make sure to replace `<email>` with your email, a placeholder, or an empty string. Keep hitting enter until files exist in `~/.ssh`.
|
1. Output the contents of the public key file for `etcd1` and `etcd2`:
|
||||||
|
```bash
|
||||||
1. Output the contents of the public key file for `etcd1` and `etcd2`, like so:
|
cat ~/.ssh/id_rsa.pub
|
||||||
|
```
|
||||||
```shell
|
|
||||||
cat ~/.ssh/id_rsa.pub
|
|
||||||
```
|
|
||||||
|
|
||||||
1. Finally, copy the output for each and paste them into `etcd0`'s `~/.ssh/authorized_keys` file. This will permit `etcd1` and `etcd2` to SSH in to the machine.
|
1. Finally, copy the output for each and paste them into `etcd0`'s `~/.ssh/authorized_keys` file. This will permit `etcd1` and `etcd2` to SSH in to the machine.
|
||||||
|
|
||||||
### Generate etcd server and peer certs
|
### Generate etcd server and peer certs
|
||||||
|
|
||||||
1. In order to generate certs, each etcd machine needs the root CA generated by `etcd0`. On `etcd1` and `etcd2`, run the following:
|
1. In order to generate certs, each etcd machine needs the root CA generated by `etcd0`. On `etcd1` and `etcd2`, run the following:
|
||||||
|
```bash
|
||||||
|
mkdir -p /etc/kubernetes/pki/etcd
|
||||||
|
cd /etc/kubernetes/pki/etcd
|
||||||
|
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/ca.pem .
|
||||||
|
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/ca-key.pem .
|
||||||
|
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/client.pem .
|
||||||
|
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/client-key.pem .
|
||||||
|
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/ca-config.json .
|
||||||
|
```
|
||||||
|
|
||||||
```shell
|
Where `<etcd0-ip-address>` corresponds to the public or private IPv4 of `etcd0`.
|
||||||
mkdir -p /etc/kubernetes/pki/etcd
|
|
||||||
cd /etc/kubernetes/pki/etcd
|
|
||||||
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/ca.pem .
|
|
||||||
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/ca-key.pem .
|
|
||||||
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/client.pem .
|
|
||||||
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/client-key.pem .
|
|
||||||
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/ca-config.json .
|
|
||||||
```
|
|
||||||
|
|
||||||
Where `<etcd0-ip-address>` corresponds to the public or private IPv4 of `etcd0`.
|
|
||||||
|
|
||||||
1. Once this is done, run the following on all etcd machines:
|
1. Once this is done, run the following on all etcd machines:
|
||||||
|
```bash
|
||||||
|
cfssl print-defaults csr > config.json
|
||||||
|
sed -i '0,/CN/{s/example\.net/'"$PEER_NAME"'/}' config.json
|
||||||
|
sed -i 's/www\.example\.net/'"$PRIVATE_IP"'/' config.json
|
||||||
|
sed -i 's/example\.net/'"$PEER_NAME"'/' config.json
|
||||||
|
```
|
||||||
|
```bash
|
||||||
|
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server config.json | cfssljson -bare server
|
||||||
|
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer config.json | cfssljson -bare peer
|
||||||
|
```
|
||||||
|
|
||||||
```shell
|
The above will replace the default configuration with your machine's hostname as the peer name, and its IP addresses. Make sure
|
||||||
cfssl print-defaults csr > config.json
|
these are correct before generating the certs. If you found an error, reconfigure `config.json` and re-run the `cfssl` commands.
|
||||||
sed -i '0,/CN/{s/example\.net/'"$PEER_NAME"'/}' config.json
|
|
||||||
sed -i 's/www\.example\.net/'"$PRIVATE_IP"'/' config.json
|
|
||||||
sed -i 's/example\.net/'"$PEER_NAME"'/' config.json
|
|
||||||
|
|
||||||
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server config.json | cfssljson -bare server
|
This results in the following files: `peer.pem`, `peer-key.pem`, `server.pem`, `server-key.pem`.
|
||||||
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer config.json | cfssljson -bare peer
|
|
||||||
```
|
|
||||||
|
|
||||||
The above will replace the default configuration with your machine's hostname as the peer name, and its IP addresses. Make sure
|
|
||||||
these are correct before generating the certs. If you found an error, reconfigure `config.json` and re-run the `cfssl` commands.
|
|
||||||
|
|
||||||
This will result in the following files: `peer.pem`, `peer-key.pem`, `server.pem`, `server-key.pem`.
|
|
||||||
|
|
||||||
### {{< tabs name="etcd_mode" >}}
|
### {{< tabs name="etcd_mode" >}}
|
||||||
{{% tab name="Choose one..." %}}
|
{{% tab name="Choose one..." %}}
|
||||||
Please select one of the tabs to see installation instructions for the respective way to run etcd.
|
Please select one of the tabs to see installation instructions for the respective way to run etcd.
|
||||||
{{% /tab %}}
|
{{% /tab %}}
|
||||||
{{% tab name="systemd" %}}
|
{{% tab name="systemd" %}}
|
||||||
1. First you will install etcd binaries like so:
|
1. First, install etcd binaries:
|
||||||
|
```bash
|
||||||
|
ETCD_VERSION="v3.1.12" curl -sSL https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz | tar -xzv --strip-components=1 -C /usr/local/bin/
|
||||||
|
```
|
||||||
|
|
||||||
```shell
|
It is worth noting that etcd v3.1.12 is the preferred version for Kubernetes v1.10. For other versions of Kubernetes please consult [the changelog](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG.md).
|
||||||
ETCD_VERSION="v3.1.12"; curl -sSL https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz | tar -xzv --strip-components=1 -C /usr/local/bin/
|
|
||||||
```
|
|
||||||
|
|
||||||
It is worth noting that etcd v3.1.12 is the preferred version for Kubernetes v1.10. For other versions of Kubernetes please consult [the changelog](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG.md).
|
Also, please realize that most distributions of Linux already have a version of etcd installed, so you will be replacing the system default.
|
||||||
|
|
||||||
Also, please realize that most distributions of Linux already have a version of etcd installed, so you will be replacing the system default.
|
|
||||||
|
|
||||||
1. Next, generate the environment file that systemd will use:
|
1. Next, generate the environment file that systemd will use:
|
||||||
|
```bash
|
||||||
|
touch /etc/etcd.env
|
||||||
|
echo "PEER_NAME=${PEER_NAME}" >> /etc/etcd.env
|
||||||
|
echo "PRIVATE_IP=${PRIVATE_IP}" >> /etc/etcd.env
|
||||||
|
```
|
||||||
|
|
||||||
```
|
1. Now copy the systemd unit file:
|
||||||
touch /etc/etcd.env
|
```none
|
||||||
echo "PEER_NAME=${PEER_NAME}" >> /etc/etcd.env
|
cat >/etc/systemd/system/etcd.service <<EOF
|
||||||
echo "PRIVATE_IP=${PRIVATE_IP}" >> /etc/etcd.env
|
[Unit]
|
||||||
```
|
Description=etcd
|
||||||
|
Documentation=https://github.com/coreos/etcd
|
||||||
|
Conflicts=etcd.service
|
||||||
|
Conflicts=etcd2.service
|
||||||
|
|
||||||
1. Now copy the systemd unit file like so:
|
[Service]
|
||||||
|
EnvironmentFile=/etc/etcd.env
|
||||||
|
Type=notify
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5s
|
||||||
|
LimitNOFILE=40000
|
||||||
|
TimeoutStartSec=0
|
||||||
|
|
||||||
```shell
|
ExecStart=/usr/local/bin/etcd --name <name> --data-dir /var/lib/etcd --listen-client-urls http://localhost:2379 --advertise-client-urls http://localhost:2379 --listen-peer-urls http://localhost:2380 --initial-advertise-peer-urls http://localhost:2380 --cert-file=/etc/kubernetes/pki/etcd/server.pem --key-file=/etc/kubernetes/pki/etcd/server-key.pem --client-cert-auth --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem --peer-client-cert-auth --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem --initial-cluster <etcd0>=https://<etcd0-ip-address>:2380,<etcd1>=https://<etcd1-ip-address>:2380,<etcd2>=https://<etcd2-ip-address>:2380 --initial-cluster-token my-etcd-token --initial-cluster-state new
|
||||||
cat >/etc/systemd/system/etcd.service <<EOF
|
|
||||||
[Unit]
|
|
||||||
Description=etcd
|
|
||||||
Documentation=https://github.com/coreos/etcd
|
|
||||||
Conflicts=etcd.service
|
|
||||||
Conflicts=etcd2.service
|
|
||||||
|
|
||||||
[Service]
|
[Install]
|
||||||
EnvironmentFile=/etc/etcd.env
|
WantedBy=multi-user.target
|
||||||
Type=notify
|
EOF
|
||||||
Restart=always
|
```
|
||||||
RestartSec=5s
|
|
||||||
LimitNOFILE=40000
|
|
||||||
TimeoutStartSec=0
|
|
||||||
|
|
||||||
ExecStart=/usr/local/bin/etcd --name <name> --data-dir /var/lib/etcd --listen-client-urls http://localhost:2379 --advertise-client-urls http://localhost:2379 --listen-peer-urls http://localhost:2380 --initial-advertise-peer-urls http://localhost:2380 --cert-file=/etc/kubernetes/pki/etcd/server.pem --key-file=/etc/kubernetes/pki/etcd/server-key.pem --client-cert-auth --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem --peer-client-cert-auth --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem --initial-cluster <etcd0>=https://<etcd0-ip-address>:2380,<etcd1>=https://<etcd1-ip-address>:2380,<etcd2>=https://<etcd2-ip-address>:2380 --initial-cluster-token my-etcd-token --initial-cluster-state new
|
Make sure you replace `<etcd0-ip-address>`, `<etcd1-ip-address>` and `<etcd2-ip-address>` with the appropriate IPv4 addresses. Replace `<name>` with the name of this etcd member. Modify the values of `--listen-client-urls`, `--advertise-client-urls`, `--listen-peer-urls` and `--initial-advertise-peer-urls` if needed. Replace `<etcd0>`, `<etcd1>` and `<etcd2>` with real hostnames of each machine. These machines must be able to reach every other using DNS or make sure that records are added to `/etc/hosts`.
|
||||||
|
|
||||||
[Install]
|
1. Finally, launch etcd:
|
||||||
WantedBy=multi-user.target
|
```bash
|
||||||
EOF
|
systemctl daemon-reload
|
||||||
```
|
systemctl start etcd
|
||||||
|
```
|
||||||
Make sure you replace `<etcd0-ip-address>`, `<etcd1-ip-address>` and `<etcd2-ip-address>` with the appropriate IPv4 addresses. Replace `<name>` with the name of this etcd member. Modify the values of `--listen-client-urls`, `--advertise-client-urls`, `--listen-peer-urls` and `--initial-advertise-peer-urls` if needed. Replace `<etcd0>`, `<etcd1>` and `<etcd2>` with real hostnames of each machine. These machines must be able to reach every other using DNS or make sure that records are added to `/etc/hosts`.
|
|
||||||
|
|
||||||
1. Finally, launch etcd like so:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
systemctl daemon-reload
|
|
||||||
systemctl start etcd
|
|
||||||
```
|
|
||||||
|
|
||||||
1. Check that it launched successfully:
|
1. Check that it launched successfully:
|
||||||
|
```bash
|
||||||
|
systemctl status etcd
|
||||||
|
```
|
||||||
|
|
||||||
```shell
|
|
||||||
systemctl status etcd
|
|
||||||
```
|
|
||||||
{{% /tab %}}
|
{{% /tab %}}
|
||||||
{{% tab name="Static Pods" %}}
|
{{% tab name="Static Pods" %}}
|
||||||
**Note**: This is only supported on nodes that have the all dependencies for the kubelet installed. If you are hosting etcd on the master nodes, this has already been set up. If you are hosting etcd on dedicated nodes, you should either use systemd or run the [installation guide](/docs/setup/independent/install-kubeadm/) on each dedicated etcd machine.
|
**Note**: This is only supported on nodes that have the all dependencies for the kubelet installed. If you are hosting etcd on the master nodes, this has already been set up. If you are hosting etcd on dedicated nodes, you should either use systemd or run the [installation guide](/docs/setup/independent/install-kubeadm/) on each dedicated etcd machine.
|
||||||
|
|
||||||
1. The first step is to run the following to generate the manifest file:
|
Run the following to generate the manifest file:
|
||||||
|
|
||||||
```shell
|
<!-- Using indentation instead of code fencing because of https://github.com/russross/blackfriday/issues/239 -->
|
||||||
cat >/etc/kubernetes/manifests/etcd.yaml <<EOF
|
cat >/etc/kubernetes/manifests/etcd.yaml <<EOF
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Pod
|
kind: Pod
|
||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
component: etcd
|
component: etcd
|
||||||
tier: control-plane
|
tier: control-plane
|
||||||
name: <podname>
|
name: <podname>
|
||||||
namespace: kube-system
|
namespace: kube-system
|
||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- command:
|
- command:
|
||||||
- etcd --name <name>
|
- etcd --name <name>
|
||||||
- --data-dir /var/lib/etcd
|
- --data-dir /var/lib/etcd
|
||||||
- --listen-client-urls http://localhost:2379
|
- --listen-client-urls http://localhost:2379
|
||||||
- --advertise-client-urls http://localhost:2379
|
- --advertise-client-urls http://localhost:2379
|
||||||
- --listen-peer-urls http://localhost:2380
|
- --listen-peer-urls http://localhost:2380
|
||||||
- --initial-advertise-peer-urls http://localhost:2380
|
- --initial-advertise-peer-urls http://localhost:2380
|
||||||
- --cert-file=/certs/server.pem
|
- --cert-file=/certs/server.pem
|
||||||
- --key-file=/certs/server-key.pem
|
- --key-file=/certs/server-key.pem
|
||||||
- --client-cert-auth
|
- --client-cert-auth
|
||||||
- --trusted-ca-file=/certs/ca.pem
|
- --trusted-ca-file=/certs/ca.pem
|
||||||
- --peer-cert-file=/certs/peer.pem
|
- --peer-cert-file=/certs/peer.pem
|
||||||
- --peer-key-file=/certs/peer-key.pem
|
- --peer-key-file=/certs/peer-key.pem
|
||||||
- --peer-client-cert-auth
|
- --peer-client-cert-auth
|
||||||
- --peer-trusted-ca-file=/certs/ca.pem
|
- --peer-trusted-ca-file=/certs/ca.pem
|
||||||
- --initial-cluster etcd0=https://<etcd0-ip-address>:2380,etcd1=https://<etcd1-ip-address>:2380,etcd2=https://<etcd2-ip-address>:2380
|
- --initial-cluster etcd0=https://<etcd0-ip-address>:2380,etcd1=https://<etcd1-ip-address>:2380,etcd2=https://<etcd2-ip-address>:2380
|
||||||
- --initial-cluster-token my-etcd-token
|
- --initial-cluster-token my-etcd-token
|
||||||
- --initial-cluster-state new
|
- --initial-cluster-state new
|
||||||
image: k8s.gcr.io/etcd-amd64:3.1.10
|
image: k8s.gcr.io/etcd-amd64:3.1.10
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /health
|
path: /health
|
||||||
port: 2379
|
port: 2379
|
||||||
scheme: HTTP
|
scheme: HTTP
|
||||||
initialDelaySeconds: 15
|
initialDelaySeconds: 15
|
||||||
timeoutSeconds: 15
|
timeoutSeconds: 15
|
||||||
name: etcd
|
name: etcd
|
||||||
env:
|
env:
|
||||||
- name: PUBLIC_IP
|
- name: PUBLIC_IP
|
||||||
valueFrom:
|
valueFrom:
|
||||||
fieldRef:
|
fieldRef:
|
||||||
fieldPath: status.hostIP
|
fieldPath: status.hostIP
|
||||||
- name: PRIVATE_IP
|
- name: PRIVATE_IP
|
||||||
valueFrom:
|
valueFrom:
|
||||||
fieldRef:
|
fieldRef:
|
||||||
fieldPath: status.podIP
|
fieldPath: status.podIP
|
||||||
- name: PEER_NAME
|
- name: PEER_NAME
|
||||||
valueFrom:
|
valueFrom:
|
||||||
fieldRef:
|
fieldRef:
|
||||||
fieldPath: metadata.name
|
fieldPath: metadata.name
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- mountPath: /var/lib/etcd
|
- mountPath: /var/lib/etcd
|
||||||
name: etcd
|
name: etcd
|
||||||
- mountPath: /certs
|
- mountPath: /certs
|
||||||
name: certs
|
name: certs
|
||||||
hostNetwork: true
|
hostNetwork: true
|
||||||
volumes:
|
volumes:
|
||||||
- hostPath:
|
- hostPath:
|
||||||
path: /var/lib/etcd
|
path: /var/lib/etcd
|
||||||
type: DirectoryOrCreate
|
type: DirectoryOrCreate
|
||||||
name: etcd
|
name: etcd
|
||||||
- hostPath:
|
- hostPath:
|
||||||
path: /etc/kubernetes/pki/etcd
|
path: /etc/kubernetes/pki/etcd
|
||||||
name: certs
|
name: certs
|
||||||
EOF
|
EOF
|
||||||
```
|
|
||||||
|
|
||||||
Make sure you replace:
|
Make sure you replace:
|
||||||
* `<podname>` with the name of the node you're running on (e.g. `etcd0`, `etcd1` or `etcd2`)
|
* `<podname>` with the name of the node you're running on (e.g. `etcd0`, `etcd1` or `etcd2`)
|
||||||
* `<etcd0-ip-address>`, `<etcd1-ip-address>` and `<etcd2-ip-address>` with the public IPv4s of the other machines that host etcd.
|
* `<etcd0-ip-address>`, `<etcd1-ip-address>` and `<etcd2-ip-address>` with the public IPv4s of the other machines that host etcd.
|
||||||
{{% /tab %}}
|
{{% /tab %}}
|
||||||
{{< /tabs >}}
|
{{< /tabs >}}
|
||||||
|
|
||||||
@@ -381,22 +369,21 @@ As an example we outline a simple setup based on keepalived. Depending on enviro
|
|||||||
1. Install keepalived, e.g. using your distribution's package manager. The configuration shown here works with version `1.3.5` but is expected to work with may other versions. Make sure to have it enabled (chkconfig, systemd, ...) so that it starts automatically when the respective node comes up.
|
1. Install keepalived, e.g. using your distribution's package manager. The configuration shown here works with version `1.3.5` but is expected to work with may other versions. Make sure to have it enabled (chkconfig, systemd, ...) so that it starts automatically when the respective node comes up.
|
||||||
|
|
||||||
2. Create the following configuration file _/etc/keepalived/keepalived.conf_ on all master nodes:
|
2. Create the following configuration file _/etc/keepalived/keepalived.conf_ on all master nodes:
|
||||||
|
```none
|
||||||
```shell
|
! Configuration File for keepalived
|
||||||
! Configuration File for keepalived
|
global_defs {
|
||||||
global_defs {
|
|
||||||
router_id LVS_DEVEL
|
router_id LVS_DEVEL
|
||||||
}
|
}
|
||||||
|
|
||||||
vrrp_script check_apiserver {
|
vrrp_script check_apiserver {
|
||||||
script "/etc/keepalived/check_apiserver.sh"
|
script "/etc/keepalived/check_apiserver.sh"
|
||||||
interval 3
|
interval 3
|
||||||
weight -2
|
weight -2
|
||||||
fall 10
|
fall 10
|
||||||
rise 2
|
rise 2
|
||||||
}
|
}
|
||||||
|
|
||||||
vrrp_instance VI_1 {
|
vrrp_instance VI_1 {
|
||||||
state <STATE>
|
state <STATE>
|
||||||
interface <INTERFACE>
|
interface <INTERFACE>
|
||||||
virtual_router_id 51
|
virtual_router_id 51
|
||||||
@@ -411,8 +398,8 @@ As an example we outline a simple setup based on keepalived. Depending on enviro
|
|||||||
track_script {
|
track_script {
|
||||||
check_apiserver
|
check_apiserver
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
In the section `vrrp_instance VI_1`, change few lines depending on your setup:
|
In the section `vrrp_instance VI_1`, change few lines depending on your setup:
|
||||||
|
|
||||||
@@ -423,20 +410,19 @@ As an example we outline a simple setup based on keepalived. Depending on enviro
|
|||||||
* `virtual_ipaddresses` should contain the virtual IP for the master nodes.
|
* `virtual_ipaddresses` should contain the virtual IP for the master nodes.
|
||||||
|
|
||||||
3. Install the following health check script to _/etc/keepalived/check_apiserver.sh_ on all master nodes:
|
3. Install the following health check script to _/etc/keepalived/check_apiserver.sh_ on all master nodes:
|
||||||
|
```bash
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
```shell
|
errorExit() {
|
||||||
#!/bin/sh
|
|
||||||
|
|
||||||
errorExit() {
|
|
||||||
echo "*** $*" 1>&2
|
echo "*** $*" 1>&2
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
curl --silent --max-time 2 --insecure https://localhost:6443/ -o /dev/null || errorExit "Error GET https://localhost:6443/"
|
curl --silent --max-time 2 --insecure https://localhost:6443/ -o /dev/null || errorExit "Error GET https://localhost:6443/"
|
||||||
if ip addr | grep -q <VIRTUAL-IP>; then
|
if ip addr | grep -q <VIRTUAL-IP>; then
|
||||||
curl --silent --max-time 2 --insecure https://<VIRTUAL-IP>:6443/ -o /dev/null || errorExit "Error GET https://<VIRTUAL-IP>:6443/"
|
curl --silent --max-time 2 --insecure https://<VIRTUAL-IP>:6443/ -o /dev/null || errorExit "Error GET https://<VIRTUAL-IP>:6443/"
|
||||||
fi
|
fi
|
||||||
```
|
```
|
||||||
|
|
||||||
Replace the `<VIRTUAL-IP>` by your chosen virtual IP.
|
Replace the `<VIRTUAL-IP>` by your chosen virtual IP.
|
||||||
|
|
||||||
@@ -452,55 +438,52 @@ Only follow this step if your etcd is hosted on dedicated nodes (**Option 1**).
|
|||||||
1. Generate SSH keys for each of the master nodes by following the steps in the [create ssh access](#create-ssh-access) section. After doing this, each master will have an SSH key in `~/.ssh/id_rsa.pub` and an entry in `etcd0`'s `~/.ssh/authorized_keys` file.
|
1. Generate SSH keys for each of the master nodes by following the steps in the [create ssh access](#create-ssh-access) section. After doing this, each master will have an SSH key in `~/.ssh/id_rsa.pub` and an entry in `etcd0`'s `~/.ssh/authorized_keys` file.
|
||||||
|
|
||||||
1. Run the following:
|
1. Run the following:
|
||||||
|
```bash
|
||||||
```shell
|
mkdir -p /etc/kubernetes/pki/etcd
|
||||||
mkdir -p /etc/kubernetes/pki/etcd
|
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/ca.pem /etc/kubernetes/pki/etcd
|
||||||
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/ca.pem /etc/kubernetes/pki/etcd
|
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/client.pem /etc/kubernetes/pki/etcd
|
||||||
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/client.pem /etc/kubernetes/pki/etcd
|
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/client-key.pem /etc/kubernetes/pki/etcd
|
||||||
scp root@<etcd0-ip-address>:/etc/kubernetes/pki/etcd/client-key.pem /etc/kubernetes/pki/etcd
|
```
|
||||||
```
|
|
||||||
|
|
||||||
## Run `kubeadm init` on `master0` {#kubeadm-init-master0}
|
## Run `kubeadm init` on `master0` {#kubeadm-init-master0}
|
||||||
|
|
||||||
1. In order for kubeadm to run, you first need to write a configuration file:
|
1. In order for kubeadm to run, you first need to write a configuration file:
|
||||||
|
```bash
|
||||||
|
cat >config.yaml <<EOF
|
||||||
|
apiVersion: kubeadm.k8s.io/v1alpha1
|
||||||
|
kind: MasterConfiguration
|
||||||
|
api:
|
||||||
|
advertiseAddress: <private-ip>
|
||||||
|
etcd:
|
||||||
|
endpoints:
|
||||||
|
- https://<etcd0-ip-address>:2379
|
||||||
|
- https://<etcd1-ip-address>:2379
|
||||||
|
- https://<etcd2-ip-address>:2379
|
||||||
|
caFile: /etc/kubernetes/pki/etcd/ca.pem
|
||||||
|
certFile: /etc/kubernetes/pki/etcd/client.pem
|
||||||
|
keyFile: /etc/kubernetes/pki/etcd/client-key.pem
|
||||||
|
networking:
|
||||||
|
podSubnet: <podCIDR>
|
||||||
|
apiServerCertSANs:
|
||||||
|
- <load-balancer-ip>
|
||||||
|
apiServerExtraArgs:
|
||||||
|
apiserver-count: "3"
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
```shell
|
Ensure that the following placeholders are replaced:
|
||||||
cat >config.yaml <<EOF
|
|
||||||
apiVersion: kubeadm.k8s.io/v1alpha1
|
|
||||||
kind: MasterConfiguration
|
|
||||||
api:
|
|
||||||
advertiseAddress: <private-ip>
|
|
||||||
etcd:
|
|
||||||
endpoints:
|
|
||||||
- https://<etcd0-ip-address>:2379
|
|
||||||
- https://<etcd1-ip-address>:2379
|
|
||||||
- https://<etcd2-ip-address>:2379
|
|
||||||
caFile: /etc/kubernetes/pki/etcd/ca.pem
|
|
||||||
certFile: /etc/kubernetes/pki/etcd/client.pem
|
|
||||||
keyFile: /etc/kubernetes/pki/etcd/client-key.pem
|
|
||||||
networking:
|
|
||||||
podSubnet: <podCIDR>
|
|
||||||
apiServerCertSANs:
|
|
||||||
- <load-balancer-ip>
|
|
||||||
apiServerExtraArgs:
|
|
||||||
apiserver-count: "3"
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
Ensure that the following placeholders are replaced:
|
- `<private-ip>` with the private IPv4 of the master server.
|
||||||
|
- `<etcd0-ip>`, `<etcd1-ip>` and `<etcd2-ip>` with the IP addresses of your three etcd nodes
|
||||||
|
- `<podCIDR>` with your Pod CIDR. Please read the [CNI network section](/docs/setup/independent/create-cluster-kubeadm/#pod-network) of the docs for more information. Some CNI providers do not require a value to be set.
|
||||||
|
- `<load-balancer-ip>` with the virtual IP set up in the load balancer. Please read [setting up a master load balancer](/docs/setup/independent/high-availability/#set-up-master-load-balancer) section of the docs for more information.
|
||||||
|
|
||||||
- `<private-ip>` with the private IPv4 of the master server.
|
**Note:** If you are using Kubernetes 1.9+, you can replace the `apiserver-count: 3` extra argument with `endpoint-reconciler-type: lease`. For more information, see [the documentation](/docs/admin/high-availability/#endpoint-reconciler).
|
||||||
- `<etcd0-ip>`, `<etcd1-ip>` and `<etcd2-ip>` with the IP addresses of your three etcd nodes
|
|
||||||
- `<podCIDR>` with your Pod CIDR. Please read the [CNI network section](/docs/setup/independent/create-cluster-kubeadm/#pod-network) of the docs for more information. Some CNI providers do not require a value to be set.
|
|
||||||
- `<load-balancer-ip>` with the virtual IP set up in the load balancer. Please read [setting up a master load balancer](/docs/setup/independent/high-availability/#set-up-master-load-balancer) section of the docs for more information.
|
|
||||||
|
|
||||||
**Note:** If you are using Kubernetes 1.9+, you can replace the `apiserver-count: 3` extra argument with `endpoint-reconciler-type: lease`. For more information, see [the documentation](/docs/admin/high-availability/#endpoint-reconciler).
|
1. When this is done, run kubeadm:
|
||||||
|
```bash
|
||||||
1. When this is done, run kubeadm like so:
|
kubeadm init --config=config.yaml
|
||||||
|
```
|
||||||
```shell
|
|
||||||
kubeadm init --config=config.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
## Run `kubeadm init` on `master1` and `master2`
|
## Run `kubeadm init` on `master1` and `master2`
|
||||||
|
|
||||||
@@ -510,15 +493,14 @@ Before running kubeadm on the other masters, you need to first copy the K8s CA c
|
|||||||
|
|
||||||
1. Follow the steps in the [create ssh access](#create-ssh-access) section, but instead of adding to `etcd0`'s `authorized_keys` file, add them to `master0`.
|
1. Follow the steps in the [create ssh access](#create-ssh-access) section, but instead of adding to `etcd0`'s `authorized_keys` file, add them to `master0`.
|
||||||
1. Once you've done this, run:
|
1. Once you've done this, run:
|
||||||
|
```bash
|
||||||
```shell
|
scp root@<master0-ip-address>:/etc/kubernetes/pki/* /etc/kubernetes/pki
|
||||||
scp root@<master0-ip-address>:/etc/kubernetes/pki/* /etc/kubernetes/pki
|
rm apiserver.*
|
||||||
rm apiserver.*
|
```
|
||||||
```
|
|
||||||
|
|
||||||
#### Option 2: Copy paste
|
#### Option 2: Copy paste
|
||||||
|
|
||||||
1. Copy the contents of `/etc/kubernetes/pki/ca.crt`, `/etc/kubernetes/pki/ca.key`, `/etc/kubernetes/pki/sa.key` and `/etc/kubernetes/pki/sa.pub` and create these files manually on `master1` and `master2`.
|
Copy the contents of `/etc/kubernetes/pki/ca.crt`, `/etc/kubernetes/pki/ca.key`, `/etc/kubernetes/pki/sa.key` and `/etc/kubernetes/pki/sa.pub` and create these files manually on `master1` and `master2`.
|
||||||
|
|
||||||
When this is done, you can follow the [previous step](#kubeadm-init-master0) to install the control plane with kubeadm.
|
When this is done, you can follow the [previous step](#kubeadm-init-master0) to install the control plane with kubeadm.
|
||||||
|
|
||||||
@@ -539,22 +521,16 @@ Next provision and set up the worker nodes. To do this, you will need to provisi
|
|||||||
## Configure workers
|
## Configure workers
|
||||||
|
|
||||||
1. Reconfigure kube-proxy to access kube-apiserver via the load balancer:
|
1. Reconfigure kube-proxy to access kube-apiserver via the load balancer:
|
||||||
|
```bash
|
||||||
```shell
|
kubectl get configmap -n kube-system kube-proxy -o yaml > kube-proxy-cm.yaml
|
||||||
kubectl get configmap -n kube-system kube-proxy -o yaml > kube-proxy-cm.yaml
|
sed -i 's#server:.*#server: https://<masterLoadBalancerFQDN>:6443#g' kube-proxy-cm.yaml
|
||||||
sed -i 's#server:.*#server: https://<masterLoadBalancerFQDN>:6443#g' kube-proxy-cm.yaml
|
kubectl apply -f kube-proxy-cm.yaml --force
|
||||||
kubectl apply -f kube-proxy-cm.yaml --force
|
# restart all kube-proxy pods to ensure that they load the new configmap
|
||||||
# restart all kube-proxy pods to ensure that they load the new configmap
|
kubectl delete pod -n kube-system -l k8s-app=kube-proxy
|
||||||
kubectl delete pod -n kube-system -l k8s-app=kube-proxy
|
```
|
||||||
```
|
|
||||||
|
|
||||||
1. Reconfigure the kubelet to access kube-apiserver via the load balancer:
|
1. Reconfigure the kubelet to access kube-apiserver via the load balancer:
|
||||||
|
```bash
|
||||||
```shell
|
sudo sed -i 's#server:.*#server: https://<masterLoadBalancerFQDN>:6443#g' /etc/kubernetes/kubelet.conf
|
||||||
sudo sed -i 's#server:.*#server: https://<masterLoadBalancerFQDN>:6443#g' /etc/kubernetes/kubelet.conf
|
sudo systemctl restart kubelet
|
||||||
sudo systemctl restart kubelet
|
```
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user