From 8a3244fdd1ab0c9b762b33d28b99f6fee8c1c9d6 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Mon, 26 Oct 2020 13:06:20 -0500 Subject: [PATCH] Dual-stack docs for Kubernetes 1.20 Signed-off-by: Bridget Kromhout Co-authored-by: Tim Bannister Co-authored-by: Lachlan Evenson --- .../services-networking/dual-stack.md | 172 +++++++++++++++--- .../docs/tasks/network/validate-dual-stack.md | 119 +++++++++--- .../networking/dual-stack-default-svc.yaml | 3 +- ...c.yaml => dual-stack-ipfamilies-ipv6.yaml} | 6 +- ...aml => dual-stack-prefer-ipv6-lb-svc.yaml} | 5 +- .../dual-stack-preferred-ipfamilies-svc.yaml | 16 ++ .../networking/dual-stack-preferred-svc.yaml | 13 ++ 7 files changed, 285 insertions(+), 49 deletions(-) rename content/en/examples/service/networking/{dual-stack-ipv4-svc.yaml => dual-stack-ipfamilies-ipv6.yaml} (73%) rename content/en/examples/service/networking/{dual-stack-ipv6-lb-svc.yaml => dual-stack-prefer-ipv6-lb-svc.yaml} (76%) create mode 100644 content/en/examples/service/networking/dual-stack-preferred-ipfamilies-svc.yaml create mode 100644 content/en/examples/service/networking/dual-stack-preferred-svc.yaml diff --git a/content/en/docs/concepts/services-networking/dual-stack.md b/content/en/docs/concepts/services-networking/dual-stack.md index 234e16dc25..2981bffec8 100644 --- a/content/en/docs/concepts/services-networking/dual-stack.md +++ b/content/en/docs/concepts/services-networking/dual-stack.md @@ -3,6 +3,7 @@ reviewers: - lachie83 - khenidak - aramase +- bridgetkromhout title: IPv4/IPv6 dual-stack feature: title: IPv4/IPv6 dual-stack @@ -30,14 +31,17 @@ If you enable IPv4/IPv6 dual-stack networking for your Kubernetes cluster, the c Enabling IPv4/IPv6 dual-stack on your Kubernetes cluster provides the following features: * Dual-stack Pod networking (a single IPv4 and IPv6 address assignment per Pod) - * IPv4 and IPv6 enabled Services (each Service must be for a single address family) + * IPv4 and IPv6 enabled Services * Pod off-cluster egress routing (eg. the Internet) via both IPv4 and IPv6 interfaces ## Prerequisites The following prerequisites are needed in order to utilize IPv4/IPv6 dual-stack Kubernetes clusters: - * Kubernetes 1.16 or later + * Kubernetes 1.20 or later + For information about using dual-stack services with earlier + Kubernetes versions, refer to the documentation for that version + of Kubernetes. * Provider support for dual-stack networking (Cloud provider or otherwise must be able to provide Kubernetes nodes with routable IPv4/IPv6 network interfaces) * A network plugin that supports dual-stack (such as Kubenet or Calico) @@ -68,47 +72,173 @@ An example of an IPv6 CIDR: `fdXY:IJKL:MNOP:15::/64` (this shows the format but ## Services -If your cluster has IPv4/IPv6 dual-stack networking enabled, you can create {{< glossary_tooltip text="Services" term_id="service" >}} with either an IPv4 or an IPv6 address. You can choose the address family for the Service's cluster IP by setting a field, `.spec.ipFamily`, on that Service. -You can only set this field when creating a new Service. Setting the `.spec.ipFamily` field is optional and should only be used if you plan to enable IPv4 and IPv6 {{< glossary_tooltip text="Services" term_id="service" >}} and {{< glossary_tooltip text="Ingresses" term_id="ingress" >}} on your cluster. The configuration of this field not a requirement for [egress](#egress-traffic) traffic. +If your cluster has dual-stack enabled, you can create {{< glossary_tooltip text="Services" term_id="service" >}} which can use IPv4, IPv6, or both. + +The address family of a Service defaults to the address family of the first service cluster IP range (configured via the `--service-cluster-ip-range` flag to the kube-controller-manager). + +When you define a Service you can optionally configure it as dual stack. To specify the behavior you want, you +set the `.spec.ipFamilyPolicy` field to one of the following values: + +* `SingleStack`: Single-stack service. The control plane allocates a cluster IP for the Service, using the first configured service cluster IP range. +* `PreferDualStack`: + * Only used if the cluster has dual-stack enabled. Allocates IPv4 and IPv6 cluster IPs for the Service + * If the cluster does not have dual-stack enabled, this setting follows the same behavior as `SingleStack`. +* `RequireDualStack`: Allocates Service `.spec.ClusterIPs` from both IPv4 and IPv6 address ranges. + * Selects the `.spec.ClusterIP` from the list of `.spec.ClusterIPs` based on the address family of the first element in the `.spec.ipFamilies` array. + * The cluster must have dual-stack networking configured. + +If you would like to define which IP family to use for single stack or define the order of IP families for dual-stack, you can choose the address families by setting an optional field, `.spec.ipFamilies`, on the Service. {{< note >}} -The default address family for your cluster is the address family of the first service cluster IP range configured via the `--service-cluster-ip-range` flag to the kube-controller-manager. +The `.spec.ipFamilies` field is immutable because the `.spec.ClusterIP` cannot be reallocated on a Service that already exists. If you want to change `.spec.ipFamilies`, delete and recreate the Service. {{< /note >}} -You can set `.spec.ipFamily` to either: +You can set `.spec.ipFamilies` to any of the following array values: - * `IPv4`: The API server will assign an IP from a `service-cluster-ip-range` that is `ipv4` - * `IPv6`: The API server will assign an IP from a `service-cluster-ip-range` that is `ipv6` +- `["IPv4"]` +- `["IPv6"]` +- `["IPv4","IPv6"]` (dual stack) +- `["IPv6","IPv4"]` (dual stack) -The following Service specification does not include the `ipFamily` field. Kubernetes will assign an IP address (also known as a "cluster IP") from the first configured `service-cluster-ip-range` to this Service. +The first family you list is used for the legacy `.spec.ClusterIP` field. + +### Dual-stack Service configuration scenarios + +These examples demonstrate the behavior of various dual-stack Service configuration scenarios. + +#### Dual-stack options on new Services + +1. This Service specification does not explicitly define `.spec.ipFamilyPolicy`. When you create this Service, Kubernetes assigns a cluster IP for the Service from the first configured `service-cluster-ip-range` and sets the `.spec.ipFamilyPolicy` to `SingleStack`. ([Services without selectors](/docs/concepts/services-networking/service/#services-without-selectors) and [headless Services](/docs/concepts/services-networking/service/#headless-services) with selectors will behave in this same way.) {{< codenew file="service/networking/dual-stack-default-svc.yaml" >}} -The following Service specification includes the `ipFamily` field. Kubernetes will assign an IPv6 address (also known as a "cluster IP") from the configured `service-cluster-ip-range` to this Service. +1. This Service specification explicitly defines `PreferDualStack` in `.spec.ipFamilyPolicy`. When you create this Service on a dual-stack cluster, Kubernetes assigns both IPv4 and IPv6 addresses for the service. The control plane updates the `.spec` for the Service to record the IP address assignments. The field `.spec.ClusterIPs` is the primary field, and contains both assigned IP addresses; `.spec.ClusterIP` is a secondary field with its value calculated from `.spec.ClusterIPs`. + + * For the `.spec.ClusterIP` field, the control plane records the IP address that is from the same address family as the first service cluster IP range. + * On a single-stack cluster, the `.spec.ClusterIPs` and `.spec.ClusterIP` fields both only list one address. + * On a cluster with dual-stack enabled, specifying `RequireDualStack` in `.spec.ipFamilyPolicy` behaves the same as `PreferDualStack`. -{{< codenew file="service/networking/dual-stack-ipv6-svc.yaml" >}} +{{< codenew file="service/networking/dual-stack-preferred-svc.yaml" >}} -For comparison, the following Service specification will be assigned an IPv4 address (also known as a "cluster IP") from the configured `service-cluster-ip-range` to this Service. +1. This Service specification explicitly defines `IPv6` and `IPv4` in `.spec.ipFamilies` as well as defining `PreferDualStack` in `.spec.ipFamilyPolicy`. When Kubernetes assigns an IPv6 and IPv4 address in `.spec.ClusterIPs`, `.spec.ClusterIP` is set to the IPv6 address because that is the first element in the `.spec.ClusterIPs` array, overriding the default. -{{< codenew file="service/networking/dual-stack-ipv4-svc.yaml" >}} +{{< codenew file="service/networking/dual-stack-preferred-ipfamilies-svc.yaml" >}} -### Type LoadBalancer +#### Dual-stack defaults on existing Services -On cloud providers which support IPv6 enabled external load balancers, setting the `type` field to `LoadBalancer` in additional to setting `ipFamily` field to `IPv6` provisions a cloud load balancer for your Service. +These examples demonstrate the default behavior when dual-stack is newly enabled on a cluster where Services already exist. -## Egress Traffic +1. When dual-stack is enabled on a cluster, existing Services (whether `IPv4` or `IPv6`) are configured by the control plane to set `.spec.ipFamilyPolicy` to `SingleStack` and set `.spec.ipFamilies` to the address family of the existing Service. The existing Service cluster IP will be stored in `.spec.ClusterIPs`. -The use of publicly routable and non-publicly routable IPv6 address blocks is acceptable provided the underlying {{< glossary_tooltip text="CNI" term_id="cni" >}} provider is able to implement the transport. If you have a Pod that uses non-publicly routable IPv6 and want that Pod to reach off-cluster destinations (eg. the public Internet), you must set up IP masquerading for the egress traffic and any replies. The [ip-masq-agent](https://github.com/kubernetes-sigs/ip-masq-agent) is dual-stack aware, so you can use ip-masq-agent for IP masquerading on dual-stack clusters. +{{< codenew file="service/networking/dual-stack-default-svc.yaml" >}} -## Known Issues + You can validate this behavior by using kubectl to inspect an existing service. - * Kubenet forces IPv4,IPv6 positional reporting of IPs (--cluster-cidr) +```shell +kubectl get svc my-service -o yaml +``` +```yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app: MyApp + name: my-service +spec: + clusterIP: 10.0.197.123 + clusterIPs: + - 10.0.197.123 + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: MyApp + type: ClusterIP +status: + loadBalancer: {} +``` +1. When dual-stack is enabled on a cluster, existing [headless Services](/docs/concepts/services-networking/service/#headless-services) with selectors are configured by the control plane to set `.spec.ipFamilyPolicy` to `SingleStack` and set `.spec.ipFamilies` to the address family of the first service cluster IP range (configured via the `--service-cluster-ip-range` flag to the kube-controller-manager) even though `.spec.ClusterIP` is set to `None`. + +{{< codenew file="service/networking/dual-stack-default-svc.yaml" >}} + + You can validate this behavior by using kubectl to inspect an existing headless service with selectors. + +```shell +kubectl get svc my-service -o yaml +``` + +```yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app: MyApp + name: my-service +spec: + clusterIP: None + clusterIPs: + - None + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: MyApp +``` + +#### Switching Services between single-stack and dual-stack + +Services can be changed from single-stack to dual-stack and from dual-stack to single-stack. + +1. To change a Service from single-stack to dual-stack, change `.spec.ipFamilyPolicy` from `SingleStack` to `PreferDualStack` or `RequireDualStack` as desired. When you change this Service from single-stack to dual-stack, Kubernetes assigns the missing address family so that the Service now has IPv4 and IPv6 addresses. + + Edit the Service specification updating the `.spec.ipFamilyPolicy` from `SingleStack` to `PreferDualStack`. + +Before: +```yaml +spec: + ipFamilyPolicy: SingleStack +``` +After: +```yaml +spec: + ipFamilyPolicy: PreferDualStack +``` + +1. To change a Service from dual-stack to single-stack, change `.spec.ipFamilyPolicy` from `PreferDualStack` or `RequireDualStack` to `SingleStack`. When you change this Service from dual-stack to single-stack, Kubernetes retains only the first element in the `.spec.ClusterIPs` array, and sets `.spec.ClusterIP` to that IP address and sets `.spec.ipFamilies` to the address family of `.spec.ClusterIPs`. + +### Headless Services without selector + +For [Headless Services without selectors](/docs/concepts/services-networking/service/#without-selectors) and without `.spec.ipFamilyPolicy` explicitly set, the `.spec.ipFamilyPolicy` field defaults to `RequireDualStack`. + +### Service type LoadBalancer + +To provision a dual-stack load balancer for your Service: + * Set the `.spec.type` field to `LoadBalancer` + * Set `.spec.ipFamilyPolicy` field to `PreferDualStack` or `RequireDualStack` + +{{< note >}} +To use a dual-stack `LoadBalancer` type Service, your cloud provider must support IPv4 and IPv6 load balancers. +{{< /note >}} + +## Egress traffic + +If you want to enable egress traffic in order to reach off-cluster destinations (eg. the public Internet) from a Pod that uses non-publicly routable IPv6 addresses, you need to enable the Pod to use a publicly routed IPv6 address via a mechanism such as transparent proxying or IP masquerading. The [ip-masq-agent](https://github.com/kubernetes-sigs/ip-masq-agent) project supports IP masquerading on dual-stack clusters. + +{{< note >}} +Ensure your {{< glossary_tooltip text="CNI" term_id="cni" >}} provider supports IPv6. +{{< /note >}} ## {{% heading "whatsnext" %}} * [Validate IPv4/IPv6 dual-stack](/docs/tasks/network/validate-dual-stack) networking - - diff --git a/content/en/docs/tasks/network/validate-dual-stack.md b/content/en/docs/tasks/network/validate-dual-stack.md index 305f78db28..5e11cb3057 100644 --- a/content/en/docs/tasks/network/validate-dual-stack.md +++ b/content/en/docs/tasks/network/validate-dual-stack.md @@ -2,7 +2,8 @@ reviewers: - lachie83 - khenidak -min-kubernetes-server-version: v1.16 +- bridgetkromhout +min-kubernetes-server-version: v1.20 title: Validate IPv4/IPv6 dual-stack content_type: task --- @@ -97,31 +98,31 @@ a00:100::4 pod01 ## Validate Services -Create the following Service without the `ipFamily` field set. When this field is not set, the Service gets an IP from the first configured range via `--service-cluster-ip-range` flag on the kube-controller-manager. +Create the following Service that does not explicitly define `.spec.ipFamilyPolicy`. Kubernetes will assign a cluster IP for the Service from the first configured `service-cluster-ip-range` and set the `.spec.ipFamilyPolicy` to `SingleStack`. {{< codenew file="service/networking/dual-stack-default-svc.yaml" >}} -By viewing the YAML for the Service you can observe that the Service has the `ipFamily` field has set to reflect the address family of the first configured range set via `--service-cluster-ip-range` flag on kube-controller-manager. +Use `kubectl` to view the YAML for the Service. ```shell kubectl get svc my-service -o yaml ``` +The Service has `.spec.ipFamilyPolicy` set to `SingleStack` and `.spec.clusterIP` set to an IPv4 address from the first configured range set via `--service-cluster-ip-range` flag on kube-controller-manager. + ```yaml apiVersion: v1 kind: Service metadata: - creationTimestamp: "2019-09-03T20:45:13Z" - labels: - app: MyApp name: my-service namespace: default - resourceVersion: "485836" - selfLink: /api/v1/namespaces/default/services/my-service - uid: b6fa83ef-fe7e-47a3-96a1-ac212fa5b030 spec: - clusterIP: 10.0.29.179 - ipFamily: IPv4 + clusterIP: 10.0.217.164 + clusterIPs: + - 10.0.217.164 + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack ports: - port: 80 protocol: TCP @@ -134,28 +135,100 @@ status: loadBalancer: {} ``` -Create the following Service with the `ipFamily` field set to `IPv6`. +Create the following Service that explicitly defines `IPv6` as the first array element in `.spec.ipFamilies`. Kubernetes will assign a cluster IP for the Service from the IPv6 range configured `service-cluster-ip-range` and set the `.spec.ipFamilyPolicy` to `SingleStack`. -{{< codenew file="service/networking/dual-stack-ipv6-svc.yaml" >}} +{{< codenew file="service/networking/dual-stack-ipfamilies-ipv6.yaml" >}} -Validate that the Service gets a cluster IP address from the IPv6 address block. You may then validate access to the service via the IP and port. +Use `kubectl` to view the YAML for the Service. + +```shell +kubectl get svc my-service -o yaml ``` - kubectl get svc -l app=MyApp -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -my-service ClusterIP fe80:20d::d06b 80/TCP 9s + +The Service has `.spec.ipFamilyPolicy` set to `SingleStack` and `.spec.clusterIP` set to an IPv6 address from the IPv6 range set via `--service-cluster-ip-range` flag on kube-controller-manager. + +```yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app: MyApp + name: my-service +spec: + clusterIP: fd00::5118 + clusterIPs: + - fd00::5118 + ipFamilies: + - IPv6 + ipFamilyPolicy: SingleStack + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: MyApp + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +``` + +Create the following Service that explicitly defines `PreferDualStack` in `.spec.ipFamilyPolicy`. Kubernetes will assign both IPv4 and IPv6 addresses (as this cluster has dual-stack enabled) and select the `.spec.ClusterIP` from the list of `.spec.ClusterIPs` based on the address family of the first element in the `.spec.ipFamilies` array. + +{{< codenew file="service/networking/dual-stack-preferred-svc.yaml" >}} + +{{< note >}} +The `kubectl get svc` command will only show the primary IP in the `CLUSTER-IP` field. + +```shell +kubectl get svc -l app=MyApp + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +my-service ClusterIP 10.0.216.242 80/TCP 5s +``` +{{< /note >}} + +Validate that the Service gets cluster IPs from the IPv4 and IPv6 address blocks using `kubectl describe`. You may then validate access to the service via the IPs and ports. + +```shell +kubectl describe svc -l app=MyApp +``` + +``` +Name: my-service +Namespace: default +Labels: app=MyApp +Annotations: +Selector: app=MyApp +Type: ClusterIP +IP Family Policy: PreferDualStack +IP Families: IPv4,IPv6 +IP: 10.0.216.242 +IPs: 10.0.216.242,fd00::af55 +Port: 80/TCP +TargetPort: 9376/TCP +Endpoints: +Session Affinity: None +Events: ``` ### Create a dual-stack load balanced Service -If the cloud provider supports the provisioning of IPv6 enabled external load balancer, create the following Service with both the `ipFamily` field set to `IPv6` and the `type` field set to `LoadBalancer` +If the cloud provider supports the provisioning of IPv6 enabled external load balancers, create the following Service with `PreferDualStack` in `.spec.ipFamilyPolicy`, `IPv6` as the first element of the `.spec.ipFamilies` array and the `type` field set to `LoadBalancer`. -{{< codenew file="service/networking/dual-stack-ipv6-lb-svc.yaml" >}} +{{< codenew file="service/networking/dual-stack-prefer-ipv6-lb-svc.yaml" >}} + +Check the Service: + +```shell +kubectl get svc -l app=MyApp +``` Validate that the Service receives a `CLUSTER-IP` address from the IPv6 address block along with an `EXTERNAL-IP`. You may then validate access to the service via the IP and port. -``` - kubectl get svc -l app=MyApp -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -my-service ClusterIP fe80:20d::d06b 2001:db8:f100:4002::9d37:c0d7 80:31868/TCP 30s + +```shell +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +my-service LoadBalancer fd00::7ebc 2603:1030:805::5 80:30790/TCP 35s ``` diff --git a/content/en/examples/service/networking/dual-stack-default-svc.yaml b/content/en/examples/service/networking/dual-stack-default-svc.yaml index 00ed87ba19..86eadd5478 100644 --- a/content/en/examples/service/networking/dual-stack-default-svc.yaml +++ b/content/en/examples/service/networking/dual-stack-default-svc.yaml @@ -2,10 +2,11 @@ apiVersion: v1 kind: Service metadata: name: my-service + labels: + app: MyApp spec: selector: app: MyApp ports: - protocol: TCP port: 80 - targetPort: 9376 \ No newline at end of file diff --git a/content/en/examples/service/networking/dual-stack-ipv4-svc.yaml b/content/en/examples/service/networking/dual-stack-ipfamilies-ipv6.yaml similarity index 73% rename from content/en/examples/service/networking/dual-stack-ipv4-svc.yaml rename to content/en/examples/service/networking/dual-stack-ipfamilies-ipv6.yaml index a875f44d6d..7c7239cae6 100644 --- a/content/en/examples/service/networking/dual-stack-ipv4-svc.yaml +++ b/content/en/examples/service/networking/dual-stack-ipfamilies-ipv6.yaml @@ -2,11 +2,13 @@ apiVersion: v1 kind: Service metadata: name: my-service + labels: + app: MyApp spec: - ipFamily: IPv4 + ipFamilies: + - IPv6 selector: app: MyApp ports: - protocol: TCP port: 80 - targetPort: 9376 \ No newline at end of file diff --git a/content/en/examples/service/networking/dual-stack-ipv6-lb-svc.yaml b/content/en/examples/service/networking/dual-stack-prefer-ipv6-lb-svc.yaml similarity index 76% rename from content/en/examples/service/networking/dual-stack-ipv6-lb-svc.yaml rename to content/en/examples/service/networking/dual-stack-prefer-ipv6-lb-svc.yaml index 2586ec9b39..0949a75428 100644 --- a/content/en/examples/service/networking/dual-stack-ipv6-lb-svc.yaml +++ b/content/en/examples/service/networking/dual-stack-prefer-ipv6-lb-svc.yaml @@ -5,11 +5,12 @@ metadata: labels: app: MyApp spec: - ipFamily: IPv6 + ipFamilyPolicy: PreferDualStack + ipFamilies: + - IPv6 type: LoadBalancer selector: app: MyApp ports: - protocol: TCP port: 80 - targetPort: 9376 \ No newline at end of file diff --git a/content/en/examples/service/networking/dual-stack-preferred-ipfamilies-svc.yaml b/content/en/examples/service/networking/dual-stack-preferred-ipfamilies-svc.yaml new file mode 100644 index 0000000000..c31acfec58 --- /dev/null +++ b/content/en/examples/service/networking/dual-stack-preferred-ipfamilies-svc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: my-service + labels: + app: MyApp +spec: + ipFamilyPolicy: PreferDualStack + ipFamilies: + - IPv6 + - IPv4 + selector: + app: MyApp + ports: + - protocol: TCP + port: 80 diff --git a/content/en/examples/service/networking/dual-stack-preferred-svc.yaml b/content/en/examples/service/networking/dual-stack-preferred-svc.yaml new file mode 100644 index 0000000000..8fb5bfa3d3 --- /dev/null +++ b/content/en/examples/service/networking/dual-stack-preferred-svc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: my-service + labels: + app: MyApp +spec: + ipFamilyPolicy: PreferDualStack + selector: + app: MyApp + ports: + - protocol: TCP + port: 80