From b8df3044842e02cd9496c87af796fba1d7bae10f Mon Sep 17 00:00:00 2001 From: Chao Xu Date: Thu, 20 Feb 2020 11:35:35 -0800 Subject: [PATCH 1/2] Introducing concepts about Konnectivity Service. --- .../architecture/master-node-communication.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/content/en/docs/concepts/architecture/master-node-communication.md b/content/en/docs/concepts/architecture/master-node-communication.md index 8a4493e49b..ff536d160b 100644 --- a/content/en/docs/concepts/architecture/master-node-communication.md +++ b/content/en/docs/concepts/architecture/master-node-communication.md @@ -97,13 +97,28 @@ public networks. ### SSH Tunnels -Kubernetes supports SSH tunnels to protect the Master -> Cluster communication +Kubernetes supports SSH tunnels to protect the Master → Cluster communication paths. In this configuration, the apiserver initiates an SSH tunnel to each node in the cluster (connecting to the ssh server listening on port 22) and passes all traffic destined for a kubelet, node, pod, or service through the tunnel. This tunnel ensures that the traffic is not exposed outside of the network in which the nodes are running. -SSH tunnels are currently deprecated so you shouldn't opt to use them unless you know what you are doing. A replacement for this communication channel is being designed. +SSH tunnels are currently deprecated so you shouldn't opt to use them unless you +know what you are doing. The Konnectivity service is a replacement for this +communication channel. + +### Konnectivity service +{{< feature-state for_k8s_version="v1.18" state="beta" >}} + +As a replacement to the SSH tunnels, the Konnectivity service provides TCP +level proxy for the Master → Cluster communication. The Konnectivity consists of +two parts, the Konnectivity server and the Konnectivity agents, running in the +Master network and the Cluster network respectively. The Konnectivity agents +initiate connections to the Konnectivity server and maintain the connections. +All Master → Cluster traffic then goes through these connections. + +See [Konnectivity Service Setup](/docs/tasks/setup-konnectivity/) on how to set +it up in your cluster. {{% /capture %}} From ac1d86457505d9b2be53e121c13559ad4d9612ca Mon Sep 17 00:00:00 2001 From: Chao Xu Date: Fri, 20 Mar 2020 15:31:19 -0700 Subject: [PATCH 2/2] Instructions on how to set up the Konnectivity service. --- .../docs/tasks/setup-konnectivity/_index.md | 5 ++ .../setup-konnectivity/setup-konnectivity.md | 37 ++++++++++ .../egress-selector-configuration.yaml | 21 ++++++ .../konnectivity/konnectivity-agent.yaml | 53 ++++++++++++++ .../admin/konnectivity/konnectivity-rbac.yaml | 24 +++++++ .../konnectivity/konnectivity-server.yaml | 70 +++++++++++++++++++ 6 files changed, 210 insertions(+) create mode 100755 content/en/docs/tasks/setup-konnectivity/_index.md create mode 100644 content/en/docs/tasks/setup-konnectivity/setup-konnectivity.md create mode 100644 content/en/examples/admin/konnectivity/egress-selector-configuration.yaml create mode 100644 content/en/examples/admin/konnectivity/konnectivity-agent.yaml create mode 100644 content/en/examples/admin/konnectivity/konnectivity-rbac.yaml create mode 100644 content/en/examples/admin/konnectivity/konnectivity-server.yaml diff --git a/content/en/docs/tasks/setup-konnectivity/_index.md b/content/en/docs/tasks/setup-konnectivity/_index.md new file mode 100755 index 0000000000..09f254eba0 --- /dev/null +++ b/content/en/docs/tasks/setup-konnectivity/_index.md @@ -0,0 +1,5 @@ +--- +title: "Setup Konnectivity Service" +weight: 20 +--- + diff --git a/content/en/docs/tasks/setup-konnectivity/setup-konnectivity.md b/content/en/docs/tasks/setup-konnectivity/setup-konnectivity.md new file mode 100644 index 0000000000..0fdbd0127d --- /dev/null +++ b/content/en/docs/tasks/setup-konnectivity/setup-konnectivity.md @@ -0,0 +1,37 @@ +--- +title: Setup Konnectivity Service +content_template: templates/task +weight: 110 +--- + +The Konnectivity service provides TCP level proxy for the Master → Cluster +communication. + +You can set it up with the following steps. + +First, you need to configure the API Server to use the Konnectivity service +to direct its network traffic to cluster nodes: +1. Set the `--egress-selector-config-file` flag of the API Server, it is the +path to the API Server egress configuration file. +2. At the path, create a configuration file. For example, + +{{< codenew file="admin/konnectivity/egress-selector-configuration.yaml" >}} + +Next, you need to deploy the Konnectivity service server and agents. +[kubernetes-sigs/apiserver-network-proxy](https://github.com/kubernetes-sigs/apiserver-network-proxy) +is a reference implementation. + +Deploy the Konnectivity server on your master node. The provided yaml assuming +Kubernetes components are deployed as {{< glossary_tooltip text="static pod" +term_id="static-pod" >}} in your cluster. If not , you can deploy it as a +Daemonset to be reliable. + +{{< codenew file="admin/konnectivity/konnectivity-server.yaml" >}} + +Then deploy the Konnectivity agents in your cluster: + +{{< codenew file="admin/konnectivity/konnectivity-agent.yaml" >}} + +Last, if RBAC is enabled in your cluster, create the relevant RBAC rules: + +{{< codenew file="admin/konnectivity/konnectivity-rbac.yaml" >}} diff --git a/content/en/examples/admin/konnectivity/egress-selector-configuration.yaml b/content/en/examples/admin/konnectivity/egress-selector-configuration.yaml new file mode 100644 index 0000000000..6659ff3fbb --- /dev/null +++ b/content/en/examples/admin/konnectivity/egress-selector-configuration.yaml @@ -0,0 +1,21 @@ +apiVersion: apiserver.k8s.io/v1beta1 +kind: EgressSelectorConfiguration +egressSelections: +# Since we want to control the egress traffic to the cluster, we use the +# "cluster" as the name. Other supported values are "etcd", and "master". +- name: cluster + connection: + # This controls the protocol between the API Server and the Konnectivity + # server. Supported values are "GRPC" and "HTTPConnect". There is no + # end user visible difference between the two modes. You need to set the + # Konnectivity server to work in the same mode. + proxyProtocol: GRPC + transport: + # This controls what transport the API Server uses to communicate with the + # Konnectivity server. UDS is recommended if the Konnectivity server + # locates on the same machine as the API Server. You need to configure the + # Konnectivity server to listen on the same UDS socket. + # The other supported transport is "tcp". You will need to set up TLS + # config to secure the TCP transport. + uds: + udsName: /etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket diff --git a/content/en/examples/admin/konnectivity/konnectivity-agent.yaml b/content/en/examples/admin/konnectivity/konnectivity-agent.yaml new file mode 100644 index 0000000000..c3dc71040b --- /dev/null +++ b/content/en/examples/admin/konnectivity/konnectivity-agent.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +# Alternatively, you can deploy the agents as Deployments. It is not necessary +# to have an agent on each node. +kind: DaemonSet +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: konnectivity-agent + namespace: kube-system + name: konnectivity-agent +spec: + selector: + matchLabels: + k8s-app: konnectivity-agent + template: + metadata: + labels: + k8s-app: konnectivity-agent + spec: + priorityClassName: system-cluster-critical + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - image: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent:v0.0.8 + name: konnectivity-agent + command: ["/proxy-agent"] + args: [ + "--logtostderr=true", + "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + # Since the konnectivity server runs with hostNetwork=true, + # this is the IP address of the master machine. + "--proxy-server-host=35.225.206.7", + "--proxy-server-port=8132", + "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token" + ] + volumeMounts: + - mountPath: /var/run/secrets/tokens + name: konnectivity-agent-token + livenessProbe: + httpGet: + port: 8093 + path: /healthz + initialDelaySeconds: 15 + timeoutSeconds: 15 + serviceAccountName: konnectivity-agent + volumes: + - name: konnectivity-agent-token + projected: + sources: + - serviceAccountToken: + path: konnectivity-agent-token + audience: system:konnectivity-server diff --git a/content/en/examples/admin/konnectivity/konnectivity-rbac.yaml b/content/en/examples/admin/konnectivity/konnectivity-rbac.yaml new file mode 100644 index 0000000000..7687f49b77 --- /dev/null +++ b/content/en/examples/admin/konnectivity/konnectivity-rbac.yaml @@ -0,0 +1,24 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:konnectivity-server + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:konnectivity-server +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: konnectivity-agent + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile diff --git a/content/en/examples/admin/konnectivity/konnectivity-server.yaml b/content/en/examples/admin/konnectivity/konnectivity-server.yaml new file mode 100644 index 0000000000..730c26c66a --- /dev/null +++ b/content/en/examples/admin/konnectivity/konnectivity-server.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Pod +metadata: + name: konnectivity-server + namespace: kube-system +spec: + priorityClassName: system-cluster-critical + hostNetwork: true + containers: + - name: konnectivity-server-container + image: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server:v0.0.8 + command: ["/proxy-server"] + args: [ + "--log-file=/var/log/konnectivity-server.log", + "--logtostderr=false", + "--log-file-max-size=0", + # This needs to be consistent with the value set in egressSelectorConfiguration. + "--uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket", + # The following two lines assume the Konnectivity server is + # deployed on the same machine as the apiserver, and the certs and + # key of the API Server are at the specified location. + "--cluster-cert=/etc/srv/kubernetes/pki/apiserver.crt", + "--cluster-key=/etc/srv/kubernetes/pki/apiserver.key", + # This needs to be consistent with the value set in egressSelectorConfiguration. + "--mode=grpc", + "--server-port=0", + "--agent-port=8132", + "--admin-port=8133", + "--agent-namespace=kube-system", + "--agent-service-account=konnectivity-agent", + "--kubeconfig=/etc/srv/kubernetes/konnectivity-server/kubeconfig", + "--authentication-audience=system:konnectivity-server" + ] + livenessProbe: + httpGet: + scheme: HTTP + host: 127.0.0.1 + port: 8133 + path: /healthz + initialDelaySeconds: 30 + timeoutSeconds: 60 + ports: + - name: agentport + containerPort: 8132 + hostPort: 8132 + - name: adminport + containerPort: 8133 + hostPort: 8133 + volumeMounts: + - name: varlogkonnectivityserver + mountPath: /var/log/konnectivity-server.log + readOnly: false + - name: pki + mountPath: /etc/srv/kubernetes/pki + readOnly: true + - name: konnectivity-uds + mountPath: /etc/srv/kubernetes/konnectivity-server + readOnly: false + volumes: + - name: varlogkonnectivityserver + hostPath: + path: /var/log/konnectivity-server.log + type: FileOrCreate + - name: pki + hostPath: + path: /etc/srv/kubernetes/pki + - name: konnectivity-uds + hostPath: + path: /etc/srv/kubernetes/konnectivity-server + type: DirectoryOrCreate