From 145acad1f71b1fdb85bc47f4dc8af5e9900ff672 Mon Sep 17 00:00:00 2001 From: Qiming Teng Date: Thu, 22 Apr 2021 17:20:33 +0800 Subject: [PATCH 001/175] [zh] Update release notes We still have an old release notes for zh localization. --- content/zh/docs/setup/release/notes.md | 2705 +++++++++++++----------- 1 file changed, 1524 insertions(+), 1181 deletions(-) diff --git a/content/zh/docs/setup/release/notes.md b/content/zh/docs/setup/release/notes.md index 2a4d377368..99eeefa42a 100644 --- a/content/zh/docs/setup/release/notes.md +++ b/content/zh/docs/setup/release/notes.md @@ -1,1372 +1,1715 @@ --- -title: v1.18 发布说明 +title: v1.21 发行说明 weight: 10 card: name: release-notes weight: 20 anchors: - anchor: "#" - title: 当前发行说明 + title: 当前版本的发行说明 - anchor: "#urgent-upgrade-notes" title: 紧急升级说明 --- -# v1.18.0 +# v1.21.0 [Documentation](https://docs.k8s.io) -## Downloads for v1.18.0 +## Downloads for v1.21.0 filename | sha512 hash -------- | ----------- -[kubernetes.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes.tar.gz) | `cd5b86a3947a4f2cea6d857743ab2009be127d782b6f2eb4d37d88918a5e433ad2c7ba34221c34089ba5ba13701f58b657f0711401e51c86f4007cb78744dee7` -[kubernetes-src.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-src.tar.gz) | `fb42cf133355ef18f67c8c4bb555aa1f284906c06e21fa41646e086d34ece774e9d547773f201799c0c703ce48d4d0e62c6ba5b2a4d081e12a339a423e111e52` +[kubernetes.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes.tar.gz) | `19bb76a3fa5ce4b9f043b2a3a77c32365ab1fcb902d8dd6678427fb8be8f49f64a5a03dc46aaef9c7dadee05501cf83412eda46f0edacbb8fc1ed0bf5fb79142` +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-src.tar.gz) | `f942e6d6c10007a6e9ce21e94df597015ae646a7bc3e515caf1a3b79f1354efb9aff59c40f2553a8e3d43fe4a01742241f5af18b69666244906ed11a22e3bc49` ### Client Binaries filename | sha512 hash -------- | ----------- -[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-client-darwin-386.tar.gz) | `26df342ef65745df12fa52931358e7f744111b6fe1e0bddb8c3c6598faf73af997c00c8f9c509efcd7cd7e82a0341a718c08fbd96044bfb58e80d997a6ebd3c2` -[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-client-darwin-amd64.tar.gz) | `803a0fed122ef6b85f7a120b5485723eaade765b7bc8306d0c0da03bd3df15d800699d15ea2270bb7797fa9ce6a81da90e730dc793ea4ed8c0149b63d26eca30` -[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-client-linux-386.tar.gz) | `110844511b70f9f3ebb92c15105e6680a05a562cd83f79ce2d2e25c2dd70f0dbd91cae34433f61364ae1ce4bd573b635f2f632d52de8f72b54acdbc95a15e3f0` -[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-client-linux-amd64.tar.gz) | `594ca3eadc7974ec4d9e4168453e36ca434812167ef8359086cd64d048df525b7bd46424e7cc9c41e65c72bda3117326ba1662d1c9d739567f10f5684fd85bee` -[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-client-linux-arm.tar.gz) | `d3627b763606557a6c9a5766c34198ec00b3a3cd72a55bc2cb47731060d31c4af93543fb53f53791062bb5ace2f15cbaa8592ac29009641e41bd656b0983a079` -[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-client-linux-arm64.tar.gz) | `ba9056eff1452cbdaef699efbf88f74f5309b3f7808d372ebf6918442d0c9fea1653c00b9db3b7626399a460eef9b1fa9e29b827b7784f34561cbc380554e2ea` -[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-client-linux-ppc64le.tar.gz) | `f80fb3769358cb20820ff1a1ce9994de5ed194aabe6c73fb8b8048bffc394d1b926de82c204f0e565d53ffe7562faa87778e97a3ccaaaf770034a992015e3a86` -[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-client-linux-s390x.tar.gz) | `a9b658108b6803d60fa3cd4e76d9e58bf75201017164fe54054b7ccadbb68c4ad7ba7800746940bc518d90475e6c0a96965a26fa50882f4f0e56df404f4ae586` -[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-client-windows-386.tar.gz) | `18adffab5d1be146906fd8531f4eae7153576aac235150ce2da05aee5ae161f6bd527e8dec34ae6131396cd4b3771e0d54ce770c065244ad3175a1afa63c89e1` -[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-client-windows-amd64.tar.gz) | `162396256429cef07154f817de2a6b67635c770311f414e38b1e2db25961443f05d7b8eb1f8da46dec8e31c5d1d2cd45f0c95dad1bc0e12a0a7278a62a0b9a6b` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-client-darwin-amd64.tar.gz) | `be9d1440e418e5253fb8a3d8aba705ca8160746a9bd17325ad626a986b6da9f733af864155a651a32b7bca94b533b8d596005ddbe5248bdeea85db47a1b957ed` +[kubernetes-client-darwin-arm64.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-client-darwin-arm64.tar.gz) | `eed0ddc81d104bb2d41ace13f737c490423d5df4ebddc7376e45c18ed66af35933c9376b912c1c3da105945b04056f6ca0870c156bee8a307cf4189ca5eb1dd1` +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-client-linux-386.tar.gz) | `8a2f30c4434199762f2a96141dab4241c1cce2711bea9ea39cc63c2c5e7d31719ed7f076efac1931604e3a94578d3bbf0cfa454965708c96f3cfb91789868746` +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-client-linux-amd64.tar.gz) | `cd3cfa645fa31de3716f1f63506e31b73d2aa8d37bb558bb3b3e8c151f35b3d74d44e03cbd05be67e380f9a5d015aba460222afdac6677815cd99a85c2325cf0` +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-client-linux-arm.tar.gz) | `936042aa11cea0f6dfd2c30fc5dbe655420b34799bede036b1299a92d6831f589ca10290b73b9c9741560b603ae31e450ad024e273f2b4df5354bfac272691d8` +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-client-linux-arm64.tar.gz) | `42beb75364d7bf4bf526804b8a35bd0ab3e124b712e9d1f45c1b914e6be0166619b30695feb24b3eecef134991dacb9ab3597e788bd9e45cf35addddf20dd7f6` +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-client-linux-ppc64le.tar.gz) | `4baba2ed7046b28370eccc22e2378ae79e3ce58220d6f4f1b6791e8233bec8379e30200bb20b971456b83f2b791ea166fdfcf1ea56908bc1eea03590c0eda468` +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-client-linux-s390x.tar.gz) | `37fa0c4d703aef09ce68c10ef3e7362b0313c8f251ce38eea579cd18fae4023d3d2b70e0f31577cabe6958ab9cfc30e98d25a7c64e69048b423057c3cf728339` +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-client-windows-386.tar.gz) | `6900db36c1e3340edfd6dfd8d720575a904c932d39a8a7fa36401595e971a0235bd42111dbcc1cbb77e7374e47f1380a68c637997c18f96a0d9cdc9f3714c4c9` +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-client-windows-amd64.tar.gz) | `90de67f6f79fc63bcfdf35066e3d84501cc85433265ffad36fd1a7a428a31b446249f0644a1e97495ea8b2a08e6944df6ef30363003750339edaa2aceffe937c` ### Server Binaries filename | sha512 hash -------- | ----------- -[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-server-linux-amd64.tar.gz) | `a92f8d201973d5dfa44a398e95fcf6a7b4feeb1ef879ab3fee1c54370e21f59f725f27a9c09ace8c42c96ac202e297fd458e486c489e05f127a5cade53b8d7c4` -[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-server-linux-arm.tar.gz) | `62fbff3256bc0a83f70244b09149a8d7870d19c2c4b6dee8ca2714fc7388da340876a0f540d2ae9bbd8b81fdedaf4b692c72d2840674db632ba2431d1df1a37d` -[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-server-linux-arm64.tar.gz) | `842910a7013f61a60d670079716b207705750d55a9e4f1f93696d19d39e191644488170ac94d8740f8e3aa3f7f28f61a4347f69d7e93d149c69ac0efcf3688fe` -[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-server-linux-ppc64le.tar.gz) | `95c5b952ac1c4127a5c3b519b664972ee1fb5e8e902551ce71c04e26ad44b39da727909e025614ac1158c258dc60f504b9a354c5ab7583c2ad769717b30b3836` -[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-server-linux-s390x.tar.gz) | `a46522d2119a0fd58074564c1fa95dd8a929a79006b82ba3c4245611da8d2db9fd785c482e1b61a9aa361c5c9a6d73387b0e15e6a7a3d84fffb3f65db3b9deeb` +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-server-linux-amd64.tar.gz) | `3941dcc2309ac19ec185603a79f5a086d8a198f98c04efa23f15a177e5e1f34946ea9392ba9f5d24d0d727839438f067fef1001fc6e88b27b8b01e35bbd962ca` +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-server-linux-arm.tar.gz) | `6507abf6c2ec2b336901dc23269f6c577ec0049b8bad3c9dd6ad63f21aa10f09bfbbfa6e064c2466d250411d3e10f8672791a9e10942e38de7bfbaf7a8bcc9da` +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-server-linux-arm64.tar.gz) | `5abe76f867ca6865344e957bf166b81766c049ec4eb183a8a5580c22a7f8474db1edf90fd901a5833e56128b6825811653a1d27f72fd34ce5b1287a8c10da05c` +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-server-linux-ppc64le.tar.gz) | `62507b182ca25396a285d91241536860e58f54fac937e97cbdf91948c83bb41be97d33277400489bf50e85164d560205540b76e94e5d519892312bdc63df1067` +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-server-linux-s390x.tar.gz) | `04f2a1f7d1388e4a7d7d9f597f872a3da36f26839cfed16aad6df07021c03f4dca1df06b19cfda56df09d1c2d9a13ebd0af40ca1b9b6aecfaf427ab7712d88f3` ### Node Binaries filename | sha512 hash -------- | ----------- -[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-node-linux-amd64.tar.gz) | `f714f80feecb0756410f27efb4cf4a1b5232be0444fbecec9f25cb85a7ccccdcb5be588cddee935294f460046c0726b90f7acc52b20eeb0c46a7200cf10e351a` -[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-node-linux-arm.tar.gz) | `806000b5f6d723e24e2f12d19d1b9b3d16c74b855f51c7063284adf1fcc57a96554a3384f8c05a952c6f6b929a05ed12b69151b1e620c958f74c9600f3db0fcb` -[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-node-linux-arm64.tar.gz) | `c207e9ab60587d135897b5366af79efe9d2833f33401e469b2a4e0d74ecd2cf6bb7d1e5bc18d80737acbe37555707f63dd581ccc6304091c1d98dafdd30130b7` -[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-node-linux-ppc64le.tar.gz) | `a542ed5ed02722af44ef12d1602f363fcd4e93cf704da2ea5d99446382485679626835a40ae2ba47a4a26dce87089516faa54479a1cfdee2229e8e35aa1c17d7` -[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-node-linux-s390x.tar.gz) | `651e0db73ee67869b2ae93cb0574168e4bd7918290fc5662a6b12b708fa628282e3f64be2b816690f5a2d0f4ff8078570f8187e65dee499a876580a7a63d1d19` -[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0/kubernetes-node-windows-amd64.tar.gz) | `d726ed904f9f7fe7e8831df621dc9094b87e767410a129aa675ee08417b662ddec314e165f29ecb777110fbfec0dc2893962b6c71950897ba72baaa7eb6371ed` +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-node-linux-amd64.tar.gz) | `c1831c708109c31b3878e5a9327ea4b9e546504d0b6b00f3d43db78b5dd7d5114d32ac24a9a505f9cadbe61521f0419933348d2cd309ed8cfe3987d9ca8a7e2c` +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-node-linux-arm.tar.gz) | `b68dd5bcfc7f9ce2781952df40c8c3a64c29701beff6ac22f042d6f31d4de220e9200b7e8272ddf608114327770acdaf3cb9a34a0a5206e784bda717ea080e0f` +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-node-linux-arm64.tar.gz) | `7fa84fc500c28774ed25ca34b6f7b208a2bea29d6e8379f84b9f57bd024aa8fe574418cee7ee26edd55310716d43d65ae7b9cbe11e40c995fe2eac7f66bdb423` +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-node-linux-ppc64le.tar.gz) | `a4278b3f8e458e9581e01f0c5ba8443303c987988ee136075a8f2f25515d70ca549fbd2e4d10eefca816c75c381d62d71494bd70c47034ab47f8315bbef4ae37` +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-node-linux-s390x.tar.gz) | `8de2bc6f22f232ff534b45012986eac23893581ccb6c45bd637e40dbe808ce31d5a92375c00dc578bdbadec342b6e5b70c1b9f3d3a7bb26ccfde97d71f9bf84a` +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes-node-windows-amd64.tar.gz) | `b82e94663d330cff7a117f99a7544f27d0bc92b36b5a283b3c23725d5b33e6f15e0ebf784627638f22f2d58c58c0c2b618ddfd226a64ae779693a0861475d355` -## Changelog since v1.17.0 +## Changelog since v1.20.0 -A complete changelog for the release notes is now hosted in a customizable -format at [https://relnotes.k8s.io][1]. Check it out and please give us your -feedback! +# Release notes for v1.21.0-rc.0 -[1]: https://relnotes.k8s.io/?releaseVersions=1.18.0 +[Documentation](https://docs.k8s.io/docs/home) -## What’s New (Major Themes) +# Changelog since v1.20.0 -### Kubernetes Topology Manager Moves to Beta - Align Up! +## What's New (Major Themes) -A beta feature of Kubernetes in release 1.18, the [Topology Manager feature](https://github.com/nolancon/website/blob/f4200307260ea3234540ef13ed80de325e1a7267/content/en/docs/tasks/administer-cluster/topology-manager.md) enables NUMA alignment of CPU and devices (such as SR-IOV VFs) that will allow your workload to run in an environment optimized for low-latency. Prior to the introduction of the Topology Manager, the CPU and Device Manager would make resource allocation decisions independent of each other. This could result in undesirable allocations on multi-socket systems, causing degraded performance on latency critical applications. +### Deprecation of PodSecurityPolicy -### Serverside Apply - Beta 2 +PSP as an admission controller resource is being deprecated. Deployed PodSecurityPolicy's will keep working until version 1.25, their target removal from the codebase. A new feature, with a working title of "PSP replacement policy", is being developed in [KEP-2579](https://features.k8s.io/2579). To learn more, read [PodSecurityPolicy Deprecation: Past, Present, and Future](https://blog.k8s.io/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/). -Server-side Apply was promoted to Beta in 1.16, but is now introducing a second Beta in 1.18. This new version will track and manage changes to fields of all new Kubernetes objects, allowing you to know what changed your resources and when. +### Kubernetes API Reference Documentation -### Extending Ingress with and replacing a deprecated annotation with IngressClass +The API reference is now generated with [`gen-resourcesdocs`](https://github.com/kubernetes-sigs/reference-docs/tree/c96658d89fb21037b7d00d27e6dbbe6b32375837/gen-resourcesdocs) and it is moving to [Kubernetes API](https://docs.k8s.io/reference/kubernetes-api/) -In Kubernetes 1.18, there are two significant additions to Ingress: A new `pathType` field and a new `IngressClass` resource. The `pathType` field allows specifying how paths should be matched. In addition to the default `ImplementationSpecific` type, there are new `Exact` and `Prefix` path types. +### Kustomize Updates in Kubectl -The `IngressClass` resource is used to describe a type of Ingress within a Kubernetes cluster. Ingresses can specify the class they are associated with by using a new `ingressClassName` field on Ingresses. This new resource and field replace the deprecated `kubernetes.io/ingress.class` annotation. +[Kustomize](https://github.com/kubernetes-sigs/kustomize) version in kubectl had a jump from v2.0.3 to [v4.0.5](https://github.com/kubernetes/kubernetes/pull/98946). Kustomize is now treated as a library and future updates will be less sporadic. -### SIG CLI introduces kubectl debug +### Default Container Labels -SIG CLI was debating the need for a debug utility for quite some time already. With the development of [ephemeral containers](/zh/docs/concepts/workloads/pods/ephemeral-containers/), it became more obvious how we can support developers with tooling built on top of `kubectl exec`. The addition of the `kubectl debug` [command](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cli/20190805-kubectl-debug.md) (it is alpha but your feedback is more than welcome), allows developers to easily debug their Pods inside the cluster. We think this addition is invaluable. This command allows one to create a temporary container which runs next to the Pod one is trying to examine, but also attaches to the console for interactive troubleshooting. +Pod with multiple containers can use `kubectl.kubernetes.io/default-container` label to have a container preselected for kubectl commands. More can be read in [KEP-2227](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cli/2227-kubectl-default-container/README.md). -### Introducing Windows CSI support alpha for Kubernetes +### Immutable Secrets and ConfigMaps -With the release of Kubernetes 1.18, an alpha version of CSI Proxy for Windows is getting released. CSI proxy enables non-privileged (pre-approved) containers to perform privileged storage operations on Windows. CSI drivers can now be supported in Windows by leveraging CSI proxy. -SIG Storage made a lot of progress in the 1.18 release. -In particular, the following storage features are moving to GA in Kubernetes 1.18: -- Raw Block Support: Allow volumes to be surfaced as block devices inside containers instead of just mounted filesystems. -- Volume Cloning: Duplicate a PersistentVolumeClaim and underlying storage volume using the Kubernetes API via CSI. -- CSIDriver Kubernetes API Object: Simplifies CSI driver discovery and allows CSI Drivers to customize Kubernetes behavior. +Immutable Secrets and ConfigMaps graduates to GA. This feature allows users to specify that the contents of a particular Secret or ConfigMap is immutable for its object lifetime. For such instances, Kubelet will not watch/poll for changes and therefore reducing apiserver load. -SIG Storage is also introducing the following new storage features as alpha in Kubernetes 1.18: -- Windows CSI Support: Enabling containerized CSI node plugins in Windows via new [CSIProxy](https://github.com/kubernetes-csi/csi-proxy) -- Recursive Volume Ownership OnRootMismatch Option: Add a new “OnRootMismatch” policy that can help shorten the mount time for volumes that require ownership change and have many directories and files. +### Structured Logging in Kubelet -### Other notable announcements +Kubelet has adopted structured logging, thanks to community effort in accomplishing this within the release timeline. Structured logging in the project remains an ongoing effort -- for folks interested in participating, [keep an eye / chime in to the mailing list discussion](https://groups.google.com/g/kubernetes-dev/c/y4WIw-ntUR8). -SIG Network is moving IPv6 to Beta in Kubernetes 1.18, after incrementing significantly the test coverage with new CI jobs. +### Storage Capacity Tracking -NodeLocal DNSCache is an add-on that runs a dnsCache pod as a daemonset to improve clusterDNS performance and reliability. The feature has been in Alpha since 1.13 release. The SIG Network is announcing the GA graduation of Node Local DNSCache [#1351](https://github.com/kubernetes/enhancements/pull/1351) +Traditionally, the Kubernetes scheduler was based on the assumptions that additional persistent storage is available everywhere in the cluster and has infinite capacity. Topology constraints addressed the first point, but up to now pod scheduling was still done without considering that the remaining storage capacity may not be enough to start a new pod. [Storage capacity tracking](https://docs.k8s.io/concepts/storage/storage-capacity/) addresses that by adding an API for a CSI driver to report storage capacity and uses that information in the Kubernetes scheduler when choosing a node for a pod. This feature serves as a stepping stone for supporting dynamic provisioning for local volumes and other volume types that are more capacity constrained. + +### Generic Ephemeral Volumes + +[Generic ephermeral volumes](https://docs.k8s.io/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes) feature allows any existing storage driver that supports dynamic provisioning to be used as an ephemeral volume with the volume’s lifecycle bound to the Pod. It can be used to provide scratch storage that is different from the root disk, for example persistent memory, or a separate local disk on that node. All StorageClass parameters for volume provisioning are supported. All features supported with PersistentVolumeClaims are supported, such as storage capacity tracking, snapshots and restore, and volume resizing. + +### CSI Service Account Token + +CSI Service Account Token feature moves to Beta in 1.21. This feature improves the security posture and allows CSI drivers to receive pods' [bound service account tokens](https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/1205-bound-service-account-tokens/README.md). This feature also provides a knob to re-publish volumes so that short-lived volumes can be refreshed. + +### CSI Health Monitoring + +The CSI health monitoring feature is being released as a second Alpha in Kubernetes 1.21. This feature enables CSI Drivers to share abnormal volume conditions from the underlying storage systems with Kubernetes so that they can be reported as events on PVCs or Pods. This feature serves as a stepping stone towards programmatic detection and resolution of individual volume health issues by Kubernetes. ## Known Issues -No Known Issues Reported +### `TopologyAwareHints` feature falls back to default behavior -## Urgent Upgrade Notes +The feature gate currently falls back to the default behavior in most cases. Enabling the feature gate will add hints to `EndpointSlices`, but functional differences are only observed in non-dual stack kube-proxy implementation. [The fix will be available in coming releases](https://github.com/kubernetes/kubernetes/pull/100804). + +## Urgent Upgrade Notes ### (No, really, you MUST read this before you upgrade) -#### kube-apiserver: -- in an `--encryption-provider-config` config file, an explicit `cacheSize: 0` parameter previously silently defaulted to caching 1000 keys. In Kubernetes 1.18, this now returns a config validation error. To disable caching, you can specify a negative cacheSize value in Kubernetes 1.18+. -- consumers of the 'certificatesigningrequests/approval' API must now have permission to 'approve' CSRs for the specific signer requested by the CSR. More information on the new signerName field and the required authorization can be found at https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests#authorization ([#88246](https://github.com/kubernetes/kubernetes/pull/88246), [@munnerz](https://github.com/munnerz)) [SIG API Machinery, Apps, Auth, CLI, Node and Testing] -- The following features are unconditionally enabled and the corresponding `--feature-gates` flags have been removed: `PodPriority`, `TaintNodesByCondition`, `ResourceQuotaScopeSelectors` and `ScheduleDaemonSetPods` ([#86210](https://github.com/kubernetes/kubernetes/pull/86210), [@draveness](https://github.com/draveness)) [SIG Apps and Scheduling] - -#### kubelet: -- `--enable-cadvisor-endpoints` is now disabled by default. If you need access to the cAdvisor v1 Json API please enable it explicitly in the kubelet command line. Please note that this flag was deprecated in 1.15 and will be removed in 1.19. ([#87440](https://github.com/kubernetes/kubernetes/pull/87440), [@dims](https://github.com/dims)) [SIG Instrumentation, Node and Testing] -- Promote CSIMigrationOpenStack to Beta (off by default since it requires installation of the OpenStack Cinder CSI Driver. The in-tree AWS OpenStack Cinder driver "kubernetes.io/cinder" was deprecated in 1.16 and will be removed in 1.20. Users should enable CSIMigration + CSIMigrationOpenStack features and install the OpenStack Cinder CSI Driver (https://github.com/kubernetes-sigs/cloud-provider-openstack) to avoid disruption to existing Pod and PVC objects at that time. Users should start using the OpenStack Cinder CSI Driver directly for any new volumes. ([#85637](https://github.com/kubernetes/kubernetes/pull/85637), [@dims](https://github.com/dims)) [SIG Cloud Provider] - -#### kubectl: -- `kubectl` and k8s.io/client-go no longer default to a server address of `http://localhost:8080`. If you own one of these legacy clusters, you are *strongly* encouraged to secure your server. If you cannot secure your server, you can set the `$KUBERNETES_MASTER` environment variable to `http://localhost:8080` to continue defaulting the server address. `kubectl` users can also set the server address using the `--server` flag, or in a kubeconfig file specified via `--kubeconfig` or `$KUBECONFIG`. ([#86173](https://github.com/kubernetes/kubernetes/pull/86173), [@soltysh](https://github.com/soltysh)) [SIG API Machinery, CLI and Testing] -- `kubectl run` has removed the previously deprecated generators, along with flags unrelated to creating pods. `kubectl run` now only creates pods. See specific `kubectl create` subcommands to create objects other than pods. -([#87077](https://github.com/kubernetes/kubernetes/pull/87077), [@soltysh](https://github.com/soltysh)) [SIG Architecture, CLI and Testing] -- The deprecated command `kubectl rolling-update` has been removed ([#88057](https://github.com/kubernetes/kubernetes/pull/88057), [@julianvmodesto](https://github.com/julianvmodesto)) [SIG Architecture, CLI and Testing] - -#### client-go: -- Signatures on methods in generated clientsets, dynamic, metadata, and scale clients have been modified to accept `context.Context` as a first argument. Signatures of Create, Update, and Patch methods have been updated to accept CreateOptions, UpdateOptions and PatchOptions respectively. Signatures of Delete and DeleteCollection methods now accept DeleteOptions by value instead of by reference. Generated clientsets with the previous interface have been added in new "deprecated" packages to allow incremental migration to the new APIs. The deprecated packages will be removed in the 1.21 release. A tool is available at http://sigs.k8s.io/clientgofix to rewrite method invocations to the new signatures. - -- The following deprecated metrics are removed, please convert to the corresponding metrics: - - The following replacement metrics are available from v1.14.0: - - `rest_client_request_latency_seconds` -> `rest_client_request_duration_seconds` - - `scheduler_scheduling_latency_seconds` -> `scheduler_scheduling_duration_seconds ` - - `docker_operations` -> `docker_operations_total` - - `docker_operations_latency_microseconds` -> `docker_operations_duration_seconds` - - `docker_operations_errors` -> `docker_operations_errors_total` - - `docker_operations_timeout` -> `docker_operations_timeout_total` - - `network_plugin_operations_latency_microseconds` -> `network_plugin_operations_duration_seconds` - - `kubelet_pod_worker_latency_microseconds` -> `kubelet_pod_worker_duration_seconds` - - `kubelet_pod_start_latency_microseconds` -> `kubelet_pod_start_duration_seconds` - - `kubelet_cgroup_manager_latency_microseconds` -> `kubelet_cgroup_manager_duration_seconds` - - `kubelet_pod_worker_start_latency_microseconds` -> `kubelet_pod_worker_start_duration_seconds` - - `kubelet_pleg_relist_latency_microseconds` -> `kubelet_pleg_relist_duration_seconds` - - `kubelet_pleg_relist_interval_microseconds` -> `kubelet_pleg_relist_interval_seconds` - - `kubelet_eviction_stats_age_microseconds` -> `kubelet_eviction_stats_age_seconds` - - `kubelet_runtime_operations` -> `kubelet_runtime_operations_total` - - `kubelet_runtime_operations_latency_microseconds` -> `kubelet_runtime_operations_duration_seconds` - - `kubelet_runtime_operations_errors` -> `kubelet_runtime_operations_errors_total` - - `kubelet_device_plugin_registration_count` -> `kubelet_device_plugin_registration_total` - - `kubelet_device_plugin_alloc_latency_microseconds` -> `kubelet_device_plugin_alloc_duration_seconds` - - `scheduler_e2e_scheduling_latency_microseconds` -> `scheduler_e2e_scheduling_duration_seconds` - - `scheduler_scheduling_algorithm_latency_microseconds` -> `scheduler_scheduling_algorithm_duration_seconds` - - `scheduler_scheduling_algorithm_predicate_evaluation` -> `scheduler_scheduling_algorithm_predicate_evaluation_seconds` - - `scheduler_scheduling_algorithm_priority_evaluation` -> `scheduler_scheduling_algorithm_priority_evaluation_seconds` - - `scheduler_scheduling_algorithm_preemption_evaluation` -> `scheduler_scheduling_algorithm_preemption_evaluation_seconds` - - `scheduler_binding_latency_microseconds` -> `scheduler_binding_duration_seconds` - - `kubeproxy_sync_proxy_rules_latency_microseconds` -> `kubeproxy_sync_proxy_rules_duration_seconds` - - `apiserver_request_latencies` -> `apiserver_request_duration_seconds` - - `apiserver_dropped_requests` -> `apiserver_dropped_requests_total` - - `etcd_request_latencies_summary` -> `etcd_request_duration_seconds` - - `apiserver_storage_transformation_latencies_microseconds ` -> `apiserver_storage_transformation_duration_seconds` - - `apiserver_storage_data_key_generation_latencies_microseconds` -> `apiserver_storage_data_key_generation_duration_seconds` - - `apiserver_request_count` -> `apiserver_request_total` - - `apiserver_request_latencies_summary` - - The following replacement metrics are available from v1.15.0: - - `apiserver_storage_transformation_failures_total` -> `apiserver_storage_transformation_operations_total` ([#76496](https://github.com/kubernetes/kubernetes/pull/76496), [@danielqsj](https://github.com/danielqsj)) [SIG API Machinery, Cluster Lifecycle, Instrumentation, Network, Node and Scheduling] - +- Kube-proxy's IPVS proxy mode no longer sets the net.ipv4.conf.all.route_localnet sysctl parameter. Nodes upgrading will have net.ipv4.conf.all.route_localnet set to 1 but new nodes will inherit the system default (usually 0). If you relied on any behavior requiring net.ipv4.conf.all.route_localnet, you must set ensure it is enabled as kube-proxy will no longer set it automatically. This change helps to further mitigate CVE-2020-8558. ([#92938](https://github.com/kubernetes/kubernetes/pull/92938), [@lbernail](https://github.com/lbernail)) [SIG Network and Release] + - Kubeadm: during "init" an empty cgroupDriver value in the KubeletConfiguration is now always set to "systemd" unless the user is explicit about it. This requires existing machine setups to configure the container runtime to use the "systemd" driver. Documentation on this topic can be found here: https://kubernetes.io/docs/setup/production-environment/container-runtimes/. When upgrading existing clusters / nodes using "kubeadm upgrade" the old cgroupDriver value is preserved, but in 1.22 this change will also apply to "upgrade". For more information on migrating to the "systemd" driver or remaining on the "cgroupfs" driver see: https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/. ([#99471](https://github.com/kubernetes/kubernetes/pull/99471), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] + - Newly provisioned PVs by EBS plugin will no longer use the deprecated "failure-domain.beta.kubernetes.io/zone" and "failure-domain.beta.kubernetes.io/region" labels. It will use "topology.kubernetes.io/zone" and "topology.kubernetes.io/region" labels instead. ([#99130](https://github.com/kubernetes/kubernetes/pull/99130), [@ayberk](https://github.com/ayberk)) [SIG Cloud Provider, Storage and Testing] + - Newly provisioned PVs by OpenStack Cinder plugin will no longer use the deprecated "failure-domain.beta.kubernetes.io/zone" and "failure-domain.beta.kubernetes.io/region" labels. It will use "topology.kubernetes.io/zone" and "topology.kubernetes.io/region" labels instead. ([#99719](https://github.com/kubernetes/kubernetes/pull/99719), [@jsafrane](https://github.com/jsafrane)) [SIG Cloud Provider and Storage] + - Newly provisioned PVs by gce-pd will no longer have the beta FailureDomain label. gce-pd volume plugin will start to have GA topology label instead. ([#98700](https://github.com/kubernetes/kubernetes/pull/98700), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Cloud Provider, Storage and Testing] + - OpenStack Cinder CSI migration is on by default, Clinder CSI driver must be installed on clusters on OpenStack for Cinder volumes to work. ([#98538](https://github.com/kubernetes/kubernetes/pull/98538), [@dims](https://github.com/dims)) [SIG Storage] + - Remove alpha `CSIMigrationXXComplete` flag and add alpha `InTreePluginXXUnregister` flag. Deprecate `CSIMigrationvSphereComplete` flag and it will be removed in v1.22. ([#98243](https://github.com/kubernetes/kubernetes/pull/98243), [@Jiawei0227](https://github.com/Jiawei0227)) + - Remove storage metrics `storage_operation_errors_total`, since we already have `storage_operation_status_count`.And add new field `status` for `storage_operation_duration_seconds`, so that we can know about all status storage operation latency. ([#98332](https://github.com/kubernetes/kubernetes/pull/98332), [@JornShen](https://github.com/JornShen)) [SIG Instrumentation and Storage] + - The metric `storage_operation_errors_total` is not removed, but is marked deprecated, and the metric `storage_operation_status_count` is marked deprecated. In both cases the `storage_operation_duration_seconds` metric can be used to recover equivalent counts (using `status=fail-unknown` in the case of `storage_operations_errors_total`). ([#99045](https://github.com/kubernetes/kubernetes/pull/99045), [@mattcary](https://github.com/mattcary)) + - `ServiceNodeExclusion`, `NodeDisruptionExclusion` and `LegacyNodeRoleBehavior` features have been promoted to GA. `ServiceNodeExclusion` and `NodeDisruptionExclusion` are now unconditionally enabled, while `LegacyNodeRoleBehavior` is unconditionally disabled. To prevent control plane nodes from being added to load balancers automatically, upgrade users need to add "node.kubernetes.io/exclude-from-external-load-balancers" label to control plane nodes. ([#97543](https://github.com/kubernetes/kubernetes/pull/97543), [@pacoxu](https://github.com/pacoxu)) + ## Changes by Kind ### Deprecation -#### kube-apiserver: -- the following deprecated APIs can no longer be served: - - All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead - - `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead - - `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead - - `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead ([#85903](https://github.com/kubernetes/kubernetes/pull/85903), [@liggitt](https://github.com/liggitt)) [SIG API Machinery, Apps, Cluster Lifecycle, Instrumentation and Testing] - -#### kube-controller-manager: -- Azure service annotation service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset has been deprecated. Its support would be removed in a future release. ([#88462](https://github.com/kubernetes/kubernetes/pull/88462), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] - -#### kubelet: -- The StreamingProxyRedirects feature and `--redirect-container-streaming` flag are deprecated, and will be removed in a future release. The default behavior (proxy streaming requests through the kubelet) will be the only supported option. If you are setting `--redirect-container-streaming=true`, then you must migrate off this configuration. The flag will no longer be able to be enabled starting in v1.20. If you are not setting the flag, no action is necessary. ([#88290](https://github.com/kubernetes/kubernetes/pull/88290), [@tallclair](https://github.com/tallclair)) [SIG API Machinery and Node] -- resource metrics endpoint `/metrics/resource/v1alpha1` as well as all metrics under this endpoint have been deprecated. Please convert to the following metrics emitted by endpoint `/metrics/resource`: - - scrape_error --> scrape_error - - node_cpu_usage_seconds_total --> node_cpu_usage_seconds - - node_memory_working_set_bytes --> node_memory_working_set_bytes - - container_cpu_usage_seconds_total --> container_cpu_usage_seconds - - container_memory_working_set_bytes --> container_memory_working_set_bytes - - scrape_error --> scrape_error - ([#86282](https://github.com/kubernetes/kubernetes/pull/86282), [@RainbowMango](https://github.com/RainbowMango)) [SIG Node] -- In a future release, kubelet will no longer create the CSI NodePublishVolume target directory, in accordance with the CSI specification. CSI drivers may need to be updated accordingly to properly create and process the target path. ([#75535](https://github.com/kubernetes/kubernetes/issues/75535)) [SIG Storage] - -#### kube-proxy: -- `--healthz-port` and `--metrics-port` flags are deprecated, please use `--healthz-bind-address` and `--metrics-bind-address` instead ([#88512](https://github.com/kubernetes/kubernetes/pull/88512), [@SataQiu](https://github.com/SataQiu)) [SIG Network] -- a new `EndpointSliceProxying` feature gate has been added to control the use of EndpointSlices in kube-proxy. The EndpointSlice feature gate that used to control this behavior no longer affects kube-proxy. This feature has been disabled by default. ([#86137](https://github.com/kubernetes/kubernetes/pull/86137), [@robscott](https://github.com/robscott)) - -#### kubeadm: -- command line option "kubelet-version" for `kubeadm upgrade node` has been deprecated and will be removed in a future release. ([#87942](https://github.com/kubernetes/kubernetes/pull/87942), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- deprecate the usage of the experimental flag '--use-api' under the 'kubeadm alpha certs renew' command. ([#88827](https://github.com/kubernetes/kubernetes/pull/88827), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- kube-dns is deprecated and will not be supported in a future version ([#86574](https://github.com/kubernetes/kubernetes/pull/86574), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- the `ClusterStatus` struct present in the kubeadm-config ConfigMap is deprecated and will be removed in a future version. It is going to be maintained by kubeadm until it gets removed. The same information can be found on `etcd` and `kube-apiserver` pod annotations, `kubeadm.kubernetes.io/etcd.advertise-client-urls` and `kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint` respectively. ([#87656](https://github.com/kubernetes/kubernetes/pull/87656), [@ereslibre](https://github.com/ereslibre)) [SIG Cluster Lifecycle] - -#### kubectl: -- the boolean and unset values for the --dry-run flag are deprecated and a value --dry-run=server|client|none will be required in a future version. ([#87580](https://github.com/kubernetes/kubernetes/pull/87580), [@julianvmodesto](https://github.com/julianvmodesto)) [SIG CLI] -- `kubectl apply --server-dry-run` is deprecated and replaced with --dry-run=server ([#87580](https://github.com/kubernetes/kubernetes/pull/87580), [@julianvmodesto](https://github.com/julianvmodesto)) [SIG CLI] - -#### add-ons: -- Remove cluster-monitoring addon ([#85512](https://github.com/kubernetes/kubernetes/pull/85512), [@serathius](https://github.com/serathius)) [SIG Cluster Lifecycle, Instrumentation, Scalability and Testing] - -#### kube-scheduler: -- The `scheduling_duration_seconds` summary metric is deprecated ([#86586](https://github.com/kubernetes/kubernetes/pull/86586), [@xiaoanyunfei](https://github.com/xiaoanyunfei)) [SIG Scheduling] -- The `scheduling_algorithm_predicate_evaluation_seconds` and - `scheduling_algorithm_priority_evaluation_seconds` metrics are deprecated, replaced by `framework_extension_point_duration_seconds[extension_point="Filter"]` and `framework_extension_point_duration_seconds[extension_point="Score"]`. ([#86584](https://github.com/kubernetes/kubernetes/pull/86584), [@xiaoanyunfei](https://github.com/xiaoanyunfei)) [SIG Scheduling] -- `AlwaysCheckAllPredicates` is deprecated in scheduler Policy API. ([#86369](https://github.com/kubernetes/kubernetes/pull/86369), [@Huang-Wei](https://github.com/Huang-Wei)) [SIG Scheduling] - -#### Other deprecations: -- The k8s.io/node-api component is no longer updated. Instead, use the RuntimeClass types located within k8s.io/api, and the generated clients located within k8s.io/client-go ([#87503](https://github.com/kubernetes/kubernetes/pull/87503), [@liggitt](https://github.com/liggitt)) [SIG Node and Release] -- Removed the 'client' label from apiserver_request_total. ([#87669](https://github.com/kubernetes/kubernetes/pull/87669), [@logicalhan](https://github.com/logicalhan)) [SIG API Machinery and Instrumentation] +- Aborting the drain command in a list of nodes will be deprecated. The new behavior will make the drain command go through all nodes even if one or more nodes failed during the drain. For now, users can try such experience by enabling --ignore-errors flag. ([#98203](https://github.com/kubernetes/kubernetes/pull/98203), [@yuzhiquan](https://github.com/yuzhiquan)) +- Delete deprecated `service.beta.kubernetes.io/azure-load-balancer-mixed-protocols` mixed procotol annotation in favor of the MixedProtocolLBService feature ([#97096](https://github.com/kubernetes/kubernetes/pull/97096), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] +- Deprecate the `topologyKeys` field in Service. This capability will be replaced with upcoming work around Topology Aware Subsetting and Service Internal Traffic Policy. ([#96736](https://github.com/kubernetes/kubernetes/pull/96736), [@andrewsykim](https://github.com/andrewsykim)) [SIG Apps] +- Kube-proxy: remove deprecated --cleanup-ipvs flag of kube-proxy, and make --cleanup flag always to flush IPVS ([#97336](https://github.com/kubernetes/kubernetes/pull/97336), [@maaoBit](https://github.com/maaoBit)) [SIG Network] +- Kubeadm: deprecated command "alpha selfhosting pivot" is now removed. ([#97627](https://github.com/kubernetes/kubernetes/pull/97627), [@knight42](https://github.com/knight42)) +- Kubeadm: graduate the command `kubeadm alpha kubeconfig user` to `kubeadm kubeconfig user`. The `kubeadm alpha kubeconfig user` command is deprecated now. ([#97583](https://github.com/kubernetes/kubernetes/pull/97583), [@knight42](https://github.com/knight42)) [SIG Cluster Lifecycle] +- Kubeadm: the "kubeadm alpha certs" command is removed now, please use "kubeadm certs" instead. ([#97706](https://github.com/kubernetes/kubernetes/pull/97706), [@knight42](https://github.com/knight42)) [SIG Cluster Lifecycle] +- Kubeadm: the deprecated kube-dns is no longer supported as an option. If "ClusterConfiguration.dns.type" is set to "kube-dns" kubeadm will now throw an error. ([#99646](https://github.com/kubernetes/kubernetes/pull/99646), [@rajansandeep](https://github.com/rajansandeep)) [SIG Cluster Lifecycle] +- Kubectl: The deprecated `kubectl alpha debug` command is removed. Use `kubectl debug` instead. ([#98111](https://github.com/kubernetes/kubernetes/pull/98111), [@pandaamanda](https://github.com/pandaamanda)) [SIG CLI] +- Official support to build kubernetes with docker-machine / remote docker is removed. This change does not affect building kubernetes with docker locally. ([#97935](https://github.com/kubernetes/kubernetes/pull/97935), [@adeniyistephen](https://github.com/adeniyistephen)) [SIG Release and Testing] +- Remove deprecated `--generator, --replicas, --service-generator, --service-overrides, --schedule` from `kubectl run` + Deprecate `--serviceaccount, --hostport, --requests, --limits` in `kubectl run` ([#99732](https://github.com/kubernetes/kubernetes/pull/99732), [@soltysh](https://github.com/soltysh)) +- Remove the deprecated metrics "scheduling_algorithm_preemption_evaluation_seconds" and "binding_duration_seconds", suggest to use "scheduler_framework_extension_point_duration_seconds" instead. ([#96447](https://github.com/kubernetes/kubernetes/pull/96447), [@chendave](https://github.com/chendave)) [SIG Cluster Lifecycle, Instrumentation, Scheduling and Testing] +- Removing experimental windows container hyper-v support with Docker ([#97141](https://github.com/kubernetes/kubernetes/pull/97141), [@wawa0210](https://github.com/wawa0210)) [SIG Node and Windows] +- Rename metrics `etcd_object_counts` to `apiserver_storage_object_counts` and mark it as stable. The original `etcd_object_counts` metrics name is marked as "Deprecated" and will be removed in the future. ([#99785](https://github.com/kubernetes/kubernetes/pull/99785), [@erain](https://github.com/erain)) [SIG API Machinery, Instrumentation and Testing] +- The GA TokenRequest and TokenRequestProjection feature gates have been removed and are unconditionally enabled. Remove explicit use of those feature gates in CLI invocations. ([#97148](https://github.com/kubernetes/kubernetes/pull/97148), [@wawa0210](https://github.com/wawa0210)) [SIG Node] +- The PodSecurityPolicy API is deprecated in 1.21, and will no longer be served starting in 1.25. ([#97171](https://github.com/kubernetes/kubernetes/pull/97171), [@deads2k](https://github.com/deads2k)) [SIG Auth and CLI] +- The `batch/v2alpha1` CronJob type definitions and clients are deprecated and removed. ([#96987](https://github.com/kubernetes/kubernetes/pull/96987), [@soltysh](https://github.com/soltysh)) [SIG API Machinery, Apps, CLI and Testing] +- The `export` query parameter (inconsistently supported by API resources and deprecated in v1.14) is fully removed. Requests setting this query parameter will now receive a 400 status response. ([#98312](https://github.com/kubernetes/kubernetes/pull/98312), [@deads2k](https://github.com/deads2k)) [SIG API Machinery, Auth and Testing] +- `audit.k8s.io/v1beta1` and `audit.k8s.io/v1alpha1` audit policy configuration and audit events are deprecated in favor of `audit.k8s.io/v1`, available since v1.13. kube-apiserver invocations that specify alpha or beta policy configurations with `--audit-policy-file`, or explicitly request alpha or beta audit events with `--audit-log-version` / `--audit-webhook-version` must update to use `audit.k8s.io/v1` and accept `audit.k8s.io/v1` events prior to v1.24. ([#98858](https://github.com/kubernetes/kubernetes/pull/98858), [@carlory](https://github.com/carlory)) [SIG Auth] +- `discovery.k8s.io/v1beta1` EndpointSlices are deprecated in favor of `discovery.k8s.io/v1`, and will no longer be served in Kubernetes v1.25. ([#100472](https://github.com/kubernetes/kubernetes/pull/100472), [@liggitt](https://github.com/liggitt)) +- `diskformat` storage class parameter for in-tree vSphere volume plugin is deprecated as of v1.21 release. Please consider updating storageclass and remove `diskformat` parameter. vSphere CSI Driver does not support diskformat storageclass parameter. + + vSphere releases less than 67u3 are deprecated as of v1.21. Please consider upgrading vSphere to 67u3 or above. vSphere CSI Driver requires minimum vSphere 67u3. + + VM Hardware version less than 15 is deprecated as of v1.21. Please consider upgrading the Node VM Hardware version to 15 or above. vSphere CSI Driver recommends Node VM's Hardware version set to at least vmx-15. + + Multi vCenter support is deprecated as of v1.21. If you have a Kubernetes cluster spanning across multiple vCenter servers, please consider moving all k8s nodes to a single vCenter Server. vSphere CSI Driver does not support Kubernetes deployment spanning across multiple vCenter servers. + + Support for these deprecations will be available till Kubernetes v1.24. ([#98546](https://github.com/kubernetes/kubernetes/pull/98546), [@divyenpatel](https://github.com/divyenpatel)) ### API Change -#### New API types/versions: -- A new IngressClass resource has been added to enable better Ingress configuration. ([#88509](https://github.com/kubernetes/kubernetes/pull/88509), [@robscott](https://github.com/robscott)) [SIG API Machinery, Apps, CLI, Network, Node and Testing] -- The CSIDriver API has graduated to storage.k8s.io/v1, and is now available for use. ([#84814](https://github.com/kubernetes/kubernetes/pull/84814), [@huffmanca](https://github.com/huffmanca)) [SIG Storage] - -#### New API fields: -- autoscaling/v2beta2 HorizontalPodAutoscaler added a `spec.behavior` field that allows scale behavior to be configured. Behaviors are specified separately for scaling up and down. In each direction a stabilization window can be specified as well as a list of policies and how to select amongst them. Policies can limit the absolute number of pods added or removed, or the percentage of pods added or removed. ([#74525](https://github.com/kubernetes/kubernetes/pull/74525), [@gliush](https://github.com/gliush)) [SIG API Machinery, Apps, Autoscaling and CLI] -- Ingress: - - `spec.ingressClassName` replaces the deprecated `kubernetes.io/ingress.class` annotation, and allows associating an Ingress object with a particular controller. - - path definitions added a `pathType` field to allow indicating how the specified path should be matched against incoming requests. Valid values are `Exact`, `Prefix`, and `ImplementationSpecific` ([#88587](https://github.com/kubernetes/kubernetes/pull/88587), [@cmluciano](https://github.com/cmluciano)) [SIG Apps, Cluster Lifecycle and Network] -- The alpha feature `AnyVolumeDataSource` enables PersistentVolumeClaim objects to use the spec.dataSource field to reference a custom type as a data source ([#88636](https://github.com/kubernetes/kubernetes/pull/88636), [@bswartz](https://github.com/bswartz)) [SIG Apps and Storage] -- The alpha feature `ConfigurableFSGroupPolicy` enables v1 Pods to specify a spec.securityContext.fsGroupChangePolicy policy to control how file permissions are applied to volumes mounted into the pod. ([#88488](https://github.com/kubernetes/kubernetes/pull/88488), [@gnufied](https://github.com/gnufied)) [SIG Storage] -- The alpha feature `ServiceAppProtocol` enables setting an `appProtocol` field in ServicePort and EndpointPort definitions. ([#88503](https://github.com/kubernetes/kubernetes/pull/88503), [@robscott](https://github.com/robscott)) [SIG Apps and Network] -- The alpha feature `ImmutableEphemeralVolumes` enables an `immutable` field in both Secret and ConfigMap objects to mark their contents as immutable. ([#86377](https://github.com/kubernetes/kubernetes/pull/86377), [@wojtek-t](https://github.com/wojtek-t)) [SIG Apps, CLI and Testing] - -#### Other API changes: -- The beta feature `ServerSideApply` enables tracking and managing changed fields for all new objects, which means there will be `managedFields` in `metadata` with the list of managers and their owned fields. -- The alpha feature `ServiceAccountIssuerDiscovery` enables publishing OIDC discovery information and service account token verification keys at `/.well-known/openid-configuration` and `/openid/v1/jwks` endpoints by API servers configured to issue service account tokens. ([#80724](https://github.com/kubernetes/kubernetes/pull/80724), [@cceckman](https://github.com/cceckman)) [SIG API Machinery, Auth, Cluster Lifecycle and Testing] -- CustomResourceDefinition schemas that use `x-kubernetes-list-map-keys` to specify properties that uniquely identify list items must make those properties required or have a default value, to ensure those properties are present for all list items. See https://kubernetes.io/docs/reference/using-api/api-concepts/#merge-strategy for details. ([#88076](https://github.com/kubernetes/kubernetes/pull/88076), [@eloyekunle](https://github.com/eloyekunle)) [SIG API Machinery and Testing] -- CustomResourceDefinition schemas that use `x-kubernetes-list-type: map` or `x-kubernetes-list-type: set` now enable validation that the list items in the corresponding custom resources are unique. ([#84920](https://github.com/kubernetes/kubernetes/pull/84920), [@sttts](https://github.com/sttts)) [SIG API Machinery] - -#### Configuration file changes: - -#### kube-apiserver: -- The `--egress-selector-config-file` configuration file now accepts an apiserver.k8s.io/v1beta1 EgressSelectorConfiguration configuration object, and has been updated to allow specifying HTTP or GRPC connections to the network proxy ([#87179](https://github.com/kubernetes/kubernetes/pull/87179), [@Jefftree](https://github.com/Jefftree)) [SIG API Machinery, Cloud Provider and Cluster Lifecycle] - -#### kube-scheduler: -- A kubescheduler.config.k8s.io/v1alpha2 configuration file version is now accepted, with support for multiple scheduling profiles ([#87628](https://github.com/kubernetes/kubernetes/pull/87628), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling] - - HardPodAffinityWeight moved from a top level ComponentConfig parameter to a PluginConfig parameter of InterPodAffinity Plugin in `kubescheduler.config.k8s.io/v1alpha2` ([#88002](https://github.com/kubernetes/kubernetes/pull/88002), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling and Testing] - - Kube-scheduler can run more than one scheduling profile. Given a pod, the profile is selected by using its `.spec.schedulerName`. ([#88285](https://github.com/kubernetes/kubernetes/pull/88285), [@alculquicondor](https://github.com/alculquicondor)) [SIG Apps, Scheduling and Testing] - - Scheduler Extenders can now be configured in the v1alpha2 component config ([#88768](https://github.com/kubernetes/kubernetes/pull/88768), [@damemi](https://github.com/damemi)) [SIG Release, Scheduling and Testing] - - The PostFilter of scheduler framework is renamed to PreScore in kubescheduler.config.k8s.io/v1alpha2. ([#87751](https://github.com/kubernetes/kubernetes/pull/87751), [@skilxn-go](https://github.com/skilxn-go)) [SIG Scheduling and Testing] - -#### kube-proxy: -- Added kube-proxy flags `--ipvs-tcp-timeout`, `--ipvs-tcpfin-timeout`, `--ipvs-udp-timeout` to configure IPVS connection timeouts. ([#85517](https://github.com/kubernetes/kubernetes/pull/85517), [@andrewsykim](https://github.com/andrewsykim)) [SIG Cluster Lifecycle and Network] -- Added optional `--detect-local-mode` flag to kube-proxy. Valid values are "ClusterCIDR" (default matching previous behavior) and "NodeCIDR" ([#87748](https://github.com/kubernetes/kubernetes/pull/87748), [@satyasm](https://github.com/satyasm)) [SIG Cluster Lifecycle, Network and Scheduling] -- Kube-controller-manager and kube-scheduler expose profiling by default to match the kube-apiserver. Use `--enable-profiling=false` to disable. ([#88663](https://github.com/kubernetes/kubernetes/pull/88663), [@deads2k](https://github.com/deads2k)) [SIG API Machinery, Cloud Provider and Scheduling] -- Kubelet pod resources API now provides the information about active pods only. ([#79409](https://github.com/kubernetes/kubernetes/pull/79409), [@takmatsu](https://github.com/takmatsu)) [SIG Node] -- New flag `--endpointslice-updates-batch-period` in kube-controller-manager can be used to reduce the number of endpointslice updates generated by pod changes. ([#88745](https://github.com/kubernetes/kubernetes/pull/88745), [@mborsz](https://github.com/mborsz)) [SIG API Machinery, Apps and Network] -- New flag `--show-hidden-metrics-for-version` in kube-proxy, kubelet, kube-controller-manager, and kube-scheduler can be used to show all hidden metrics that are deprecated in the previous minor release. ([#85279](https://github.com/kubernetes/kubernetes/pull/85279), [@RainbowMango](https://github.com/RainbowMango)) [SIG Cluster Lifecycle and Network] - -#### Features graduated to beta: - - StartupProbe ([#83437](https://github.com/kubernetes/kubernetes/pull/83437), [@matthyx](https://github.com/matthyx)) [SIG Node, Scalability and Testing] - -#### Features graduated to GA: - - VolumePVCDataSource ([#88686](https://github.com/kubernetes/kubernetes/pull/88686), [@j-griffith](https://github.com/j-griffith)) [SIG Storage] - - TaintBasedEvictions ([#87487](https://github.com/kubernetes/kubernetes/pull/87487), [@skilxn-go](https://github.com/skilxn-go)) [SIG API Machinery, Apps, Node, Scheduling and Testing] - - BlockVolume and CSIBlockVolume ([#88673](https://github.com/kubernetes/kubernetes/pull/88673), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] - - Windows RunAsUserName ([#87790](https://github.com/kubernetes/kubernetes/pull/87790), [@marosset](https://github.com/marosset)) [SIG Apps and Windows] -- The following feature gates are removed, because the associated features were unconditionally enabled in previous releases: CustomResourceValidation, CustomResourceSubresources, CustomResourceWebhookConversion, CustomResourcePublishOpenAPI, CustomResourceDefaulting ([#87475](https://github.com/kubernetes/kubernetes/pull/87475), [@liggitt](https://github.com/liggitt)) [SIG API Machinery] +- 1. PodAffinityTerm includes a namespaceSelector field to allow selecting eligible namespaces based on their labels. + 2. A new CrossNamespacePodAffinity quota scope API that allows restricting which namespaces allowed to use PodAffinityTerm with corss-namespace reference via namespaceSelector or namespaces fields. ([#98582](https://github.com/kubernetes/kubernetes/pull/98582), [@ahg-g](https://github.com/ahg-g)) [SIG API Machinery, Apps, Auth and Testing] +- Add Probe-level terminationGracePeriodSeconds field ([#99375](https://github.com/kubernetes/kubernetes/pull/99375), [@ehashman](https://github.com/ehashman)) [SIG API Machinery, Apps, Node and Testing] +- Added `.spec.completionMode` field to Job, with accepted values `NonIndexed` (default) and `Indexed`. This is an alpha field and is only honored by servers with the `IndexedJob` feature gate enabled. ([#98441](https://github.com/kubernetes/kubernetes/pull/98441), [@alculquicondor](https://github.com/alculquicondor)) [SIG Apps and CLI] +- Adds support for endPort field in NetworkPolicy ([#97058](https://github.com/kubernetes/kubernetes/pull/97058), [@rikatz](https://github.com/rikatz)) [SIG Apps and Network] +- CSIServiceAccountToken graduates to Beta and enabled by default. ([#99298](https://github.com/kubernetes/kubernetes/pull/99298), [@zshihang](https://github.com/zshihang)) +- Cluster admins can now turn off `/debug/pprof` and `/debug/flags/v` endpoint in kubelet by setting `enableProfilingHandler` and `enableDebugFlagsHandler` to `false` in the Kubelet configuration file. Options `enableProfilingHandler` and `enableDebugFlagsHandler` can be set to `true` only when `enableDebuggingHandlers` is also set to `true`. ([#98458](https://github.com/kubernetes/kubernetes/pull/98458), [@SaranBalaji90](https://github.com/SaranBalaji90)) +- DaemonSets accept a MaxSurge integer or percent on their rolling update strategy that will launch the updated pod on nodes and wait for those pods to go ready before marking the old out-of-date pods as deleted. This allows workloads to avoid downtime during upgrades when deployed using DaemonSets. This feature is alpha and is behind the DaemonSetUpdateSurge feature gate. ([#96441](https://github.com/kubernetes/kubernetes/pull/96441), [@smarterclayton](https://github.com/smarterclayton)) [SIG Apps and Testing] +- Enable SPDY pings to keep connections alive, so that `kubectl exec` and `kubectl portforward` won't be interrupted. ([#97083](https://github.com/kubernetes/kubernetes/pull/97083), [@knight42](https://github.com/knight42)) [SIG API Machinery and CLI] +- FieldManager no longer owns fields that get reset before the object is persisted (e.g. "status wiping"). ([#99661](https://github.com/kubernetes/kubernetes/pull/99661), [@kevindelgado](https://github.com/kevindelgado)) [SIG API Machinery, Auth and Testing] +- Fixes server-side apply for APIService resources. ([#98576](https://github.com/kubernetes/kubernetes/pull/98576), [@kevindelgado](https://github.com/kevindelgado)) +- Generic ephemeral volumes are beta. ([#99643](https://github.com/kubernetes/kubernetes/pull/99643), [@pohly](https://github.com/pohly)) [SIG API Machinery, Apps, Auth, CLI, Node, Storage and Testing] +- Hugepages request values are limited to integer multiples of the page size. ([#98515](https://github.com/kubernetes/kubernetes/pull/98515), [@lala123912](https://github.com/lala123912)) [SIG Apps] +- Implement the GetAvailableResources in the podresources API. ([#95734](https://github.com/kubernetes/kubernetes/pull/95734), [@fromanirh](https://github.com/fromanirh)) [SIG Instrumentation, Node and Testing] +- IngressClass resource can now reference a resource in a specific namespace + for implementation-specific configuration (previously only Cluster-level resources were allowed). + This feature can be enabled using the IngressClassNamespacedParams feature gate. ([#99275](https://github.com/kubernetes/kubernetes/pull/99275), [@hbagdi](https://github.com/hbagdi)) +- Jobs API has a new `.spec.suspend` field that can be used to suspend and resume Jobs. This is an alpha field which is only honored by servers with the `SuspendJob` feature gate enabled. ([#98727](https://github.com/kubernetes/kubernetes/pull/98727), [@adtac](https://github.com/adtac)) +- Kubelet Graceful Node Shutdown feature graduates to Beta and enabled by default. ([#99735](https://github.com/kubernetes/kubernetes/pull/99735), [@bobbypage](https://github.com/bobbypage)) +- Kubernetes is now built using go1.15.7 ([#98363](https://github.com/kubernetes/kubernetes/pull/98363), [@cpanato](https://github.com/cpanato)) [SIG Cloud Provider, Instrumentation, Node, Release and Testing] +- Namespace API objects now have a `kubernetes.io/metadata.name` label matching their metadata.name field to allow selecting any namespace by its name using a label selector. ([#96968](https://github.com/kubernetes/kubernetes/pull/96968), [@jayunit100](https://github.com/jayunit100)) [SIG API Machinery, Apps, Cloud Provider, Storage and Testing] +- One new field "InternalTrafficPolicy" in Service is added. + It specifies if the cluster internal traffic should be routed to all endpoints or node-local endpoints only. + "Cluster" routes internal traffic to a Service to all endpoints. + "Local" routes traffic to node-local endpoints only, and traffic is dropped if no node-local endpoints are ready. + The default value is "Cluster". ([#96600](https://github.com/kubernetes/kubernetes/pull/96600), [@maplain](https://github.com/maplain)) [SIG API Machinery, Apps and Network] +- PodDisruptionBudget API objects can now contain conditions in status. ([#98127](https://github.com/kubernetes/kubernetes/pull/98127), [@mortent](https://github.com/mortent)) [SIG API Machinery, Apps, Auth, CLI, Cloud Provider, Cluster Lifecycle and Instrumentation] +- PodSecurityPolicy only stores "generic" as allowed volume type if the GenericEphemeralVolume feature gate is enabled ([#98918](https://github.com/kubernetes/kubernetes/pull/98918), [@pohly](https://github.com/pohly)) [SIG Auth and Security] +- Promote CronJobs to batch/v1 ([#99423](https://github.com/kubernetes/kubernetes/pull/99423), [@soltysh](https://github.com/soltysh)) [SIG API Machinery, Apps, CLI and Testing] +- Promote Immutable Secrets/ConfigMaps feature to Stable. This allows to set `immutable` field in Secret or ConfigMap object to mark their contents as immutable. ([#97615](https://github.com/kubernetes/kubernetes/pull/97615), [@wojtek-t](https://github.com/wojtek-t)) [SIG Apps, Architecture, Node and Testing] +- Remove support for building Kubernetes with bazel. ([#99561](https://github.com/kubernetes/kubernetes/pull/99561), [@BenTheElder](https://github.com/BenTheElder)) [SIG API Machinery, Apps, Architecture, Auth, Autoscaling, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation, Network, Node, Release, Scalability, Scheduling, Storage, Testing and Windows] +- Scheduler extender filter interface now can report unresolvable failed nodes in the new field `FailedAndUnresolvableNodes` of `ExtenderFilterResult` struct. Nodes in this map will be skipped in the preemption phase. ([#92866](https://github.com/kubernetes/kubernetes/pull/92866), [@cofyc](https://github.com/cofyc)) [SIG Scheduling] +- Services can specify loadBalancerClass to use a custom load balancer ([#98277](https://github.com/kubernetes/kubernetes/pull/98277), [@XudongLiuHarold](https://github.com/XudongLiuHarold)) +- Storage capacity tracking (= the CSIStorageCapacity feature) graduates to Beta and enabled by default, storage.k8s.io/v1alpha1/VolumeAttachment and storage.k8s.io/v1alpha1/CSIStorageCapacity objects are deprecated ([#99641](https://github.com/kubernetes/kubernetes/pull/99641), [@pohly](https://github.com/pohly)) +- Support for Indexed Job: a Job that is considered completed when Pods associated to indexes from 0 to (.spec.completions-1) have succeeded. ([#98812](https://github.com/kubernetes/kubernetes/pull/98812), [@alculquicondor](https://github.com/alculquicondor)) [SIG Apps and CLI] +- The BoundServiceAccountTokenVolume feature has been promoted to beta, and enabled by default. + - This changes the tokens provided to containers at `/var/run/secrets/kubernetes.io/serviceaccount/token` to be time-limited, auto-refreshed, and invalidated when the containing pod is deleted. + - Clients should reload the token from disk periodically (once per minute is recommended) to ensure they continue to use a valid token. `k8s.io/client-go` version v11.0.0+ and v0.15.0+ reload tokens automatically. + - By default, injected tokens are given an extended lifetime so they remain valid even after a new refreshed token is provided. The metric `serviceaccount_stale_tokens_total` can be used to monitor for workloads that are depending on the extended lifetime and are continuing to use tokens even after a refreshed token is provided to the container. If that metric indicates no existing workloads are depending on extended lifetimes, injected token lifetime can be shortened to 1 hour by starting `kube-apiserver` with `--service-account-extend-token-expiration=false`. ([#95667](https://github.com/kubernetes/kubernetes/pull/95667), [@zshihang](https://github.com/zshihang)) [SIG API Machinery, Auth, Cluster Lifecycle and Testing] +- The EndpointSlice Controllers are now GA. The `EndpointSliceController` will not populate the `deprecatedTopology` field and will only provide topology information through the `zone` and `nodeName` fields. ([#99870](https://github.com/kubernetes/kubernetes/pull/99870), [@swetharepakula](https://github.com/swetharepakula)) +- The Endpoints controller will now set the `endpoints.kubernetes.io/over-capacity` annotation to "warning" when an Endpoints resource contains more than 1000 addresses. In a future release, the controller will truncate Endpoints that exceed this limit. The EndpointSlice API can be used to support significantly larger number of addresses. ([#99975](https://github.com/kubernetes/kubernetes/pull/99975), [@robscott](https://github.com/robscott)) [SIG Apps and Network] +- The PodDisruptionBudget API has been promoted to policy/v1 with no schema changes. The only functional change is that an empty selector (`{}`) written to a policy/v1 PodDisruptionBudget now selects all pods in the namespace. The behavior of the policy/v1beta1 API remains unchanged. The policy/v1beta1 PodDisruptionBudget API is deprecated and will no longer be served in 1.25+. ([#99290](https://github.com/kubernetes/kubernetes/pull/99290), [@mortent](https://github.com/mortent)) [SIG API Machinery, Apps, Auth, Autoscaling, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation, Scheduling and Testing] +- The `EndpointSlice` API is now GA. The `EndpointSlice` topology field has been removed from the GA API and will be replaced by a new per Endpoint Zone field. If the topology field was previously used, it will be converted into an annotation in the v1 Resource. The `discovery.k8s.io/v1alpha1` API is removed. ([#99662](https://github.com/kubernetes/kubernetes/pull/99662), [@swetharepakula](https://github.com/swetharepakula)) +- The `controller.kubernetes.io/pod-deletion-cost` annotation can be set to offer a hint on the cost of deleting a `Pod` compared to other pods belonging to the same ReplicaSet. Pods with lower deletion cost are deleted first. This is an alpha feature. ([#99163](https://github.com/kubernetes/kubernetes/pull/99163), [@ahg-g](https://github.com/ahg-g)) +- The kube-apiserver now resets `managedFields` that got corrupted by a mutating admission controller. ([#98074](https://github.com/kubernetes/kubernetes/pull/98074), [@kwiesmueller](https://github.com/kwiesmueller)) +- Topology Aware Hints are now available in alpha and can be enabled with the `TopologyAwareHints` feature gate. ([#99522](https://github.com/kubernetes/kubernetes/pull/99522), [@robscott](https://github.com/robscott)) [SIG API Machinery, Apps, Auth, Instrumentation, Network and Testing] +- Users might specify the `kubectl.kubernetes.io/default-exec-container` annotation in a Pod to preselect container for kubectl commands. ([#97099](https://github.com/kubernetes/kubernetes/pull/97099), [@pacoxu](https://github.com/pacoxu)) [SIG CLI] ### Feature -- API request throttling (due to a high rate of requests) is now reported in client-go logs at log level 2. The messages are of the form:`Throttling request took 1.50705208s, request: GET:` The presence of these messages may indicate to the administrator the need to tune the cluster accordingly. ([#87740](https://github.com/kubernetes/kubernetes/pull/87740), [@jennybuckley](https://github.com/jennybuckley)) [SIG API Machinery] -- Add support for mount options to the FC volume plugin ([#87499](https://github.com/kubernetes/kubernetes/pull/87499), [@ejweber](https://github.com/ejweber)) [SIG Storage] -- Added a config-mode flag in azure auth module to enable getting AAD token without spn: prefix in audience claim. When it's not specified, the default behavior doesn't change. ([#87630](https://github.com/kubernetes/kubernetes/pull/87630), [@weinong](https://github.com/weinong)) [SIG API Machinery, Auth, CLI and Cloud Provider] -- Allow for configuration of CoreDNS replica count ([#85837](https://github.com/kubernetes/kubernetes/pull/85837), [@pickledrick](https://github.com/pickledrick)) [SIG Cluster Lifecycle] -- Allow user to specify resource using --filename flag when invoking kubectl exec ([#88460](https://github.com/kubernetes/kubernetes/pull/88460), [@soltysh](https://github.com/soltysh)) [SIG CLI and Testing] -- Apiserver added a new flag --goaway-chance which is the fraction of requests that will be closed gracefully(GOAWAY) to prevent HTTP/2 clients from getting stuck on a single apiserver. ([#88567](https://github.com/kubernetes/kubernetes/pull/88567), [@answer1991](https://github.com/answer1991)) [SIG API Machinery] -- Azure Cloud Provider now supports using Azure network resources (Virtual Network, Load Balancer, Public IP, Route Table, Network Security Group, etc.) in different AAD Tenant and Subscription than those for the Kubernetes cluster. To use the feature, please reference https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/docs/cloud-provider-config.md#host-network-resources-in-different-aad-tenant-and-subscription. ([#88384](https://github.com/kubernetes/kubernetes/pull/88384), [@bowen5](https://github.com/bowen5)) [SIG Cloud Provider] -- Azure VMSS/VMSSVM clients now suppress requests on throttling ([#86740](https://github.com/kubernetes/kubernetes/pull/86740), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Azure cloud provider cache TTL is configurable, list of the azure cloud provider is as following: - - "availabilitySetNodesCacheTTLInSeconds" - - "vmssCacheTTLInSeconds" - - "vmssVirtualMachinesCacheTTLInSeconds" - - "vmCacheTTLInSeconds" - - "loadBalancerCacheTTLInSeconds" - - "nsgCacheTTLInSeconds" - - "routeTableCacheTTLInSeconds" - ([#86266](https://github.com/kubernetes/kubernetes/pull/86266), [@zqingqing1](https://github.com/zqingqing1)) [SIG Cloud Provider] -- Azure global rate limit is switched to per-client. A set of new rate limit configure options are introduced, including routeRateLimit, SubnetsRateLimit, InterfaceRateLimit, RouteTableRateLimit, LoadBalancerRateLimit, PublicIPAddressRateLimit, SecurityGroupRateLimit, VirtualMachineRateLimit, StorageAccountRateLimit, DiskRateLimit, SnapshotRateLimit, VirtualMachineScaleSetRateLimit and VirtualMachineSizeRateLimit. The original rate limit options would be default values for those new client's rate limiter. ([#86515](https://github.com/kubernetes/kubernetes/pull/86515), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Azure network and VM clients now suppress requests on throttling ([#87122](https://github.com/kubernetes/kubernetes/pull/87122), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Azure storage clients now suppress requests on throttling ([#87306](https://github.com/kubernetes/kubernetes/pull/87306), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Azure: add support for single stack IPv6 ([#88448](https://github.com/kubernetes/kubernetes/pull/88448), [@aramase](https://github.com/aramase)) [SIG Cloud Provider] -- DefaultConstraints can be specified for PodTopologySpread Plugin in the scheduler’s ComponentConfig ([#88671](https://github.com/kubernetes/kubernetes/pull/88671), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling] -- DisableAvailabilitySetNodes is added to avoid VM list for VMSS clusters. It should only be used when vmType is "vmss" and all the nodes (including control plane nodes) are VMSS virtual machines. ([#87685](https://github.com/kubernetes/kubernetes/pull/87685), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Elasticsearch supports automatically setting the advertise address ([#85944](https://github.com/kubernetes/kubernetes/pull/85944), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle and Instrumentation] -- EndpointSlices will now be enabled by default. A new `EndpointSliceProxying` feature gate determines if kube-proxy will use EndpointSlices, this is disabled by default. ([#86137](https://github.com/kubernetes/kubernetes/pull/86137), [@robscott](https://github.com/robscott)) [SIG Network] -- Kube-proxy: Added dual-stack IPv4/IPv6 support to the iptables proxier. ([#82462](https://github.com/kubernetes/kubernetes/pull/82462), [@vllry](https://github.com/vllry)) [SIG Network] -- Kubeadm now supports automatic calculations of dual-stack node cidr masks to kube-controller-manager. ([#85609](https://github.com/kubernetes/kubernetes/pull/85609), [@Arvinderpal](https://github.com/Arvinderpal)) [SIG Cluster Lifecycle] -- Kubeadm: add a upgrade health check that deploys a Job ([#81319](https://github.com/kubernetes/kubernetes/pull/81319), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- Kubeadm: add the experimental feature gate PublicKeysECDSA that can be used to create a - cluster with ECDSA certificates from "kubeadm init". Renewal of existing ECDSA certificates is also supported using "kubeadm alpha certs renew", but not switching between the RSA and ECDSA algorithms on the fly or during upgrades. ([#86953](https://github.com/kubernetes/kubernetes/pull/86953), [@rojkov](https://github.com/rojkov)) [SIG API Machinery, Auth and Cluster Lifecycle] -- Kubeadm: implemented structured output of 'kubeadm config images list' command in JSON, YAML, Go template and JsonPath formats ([#86810](https://github.com/kubernetes/kubernetes/pull/86810), [@bart0sh](https://github.com/bart0sh)) [SIG Cluster Lifecycle] -- Kubeadm: on kubeconfig certificate renewal, keep the embedded CA in sync with the one on disk ([#88052](https://github.com/kubernetes/kubernetes/pull/88052), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- Kubeadm: reject a node joining the cluster if a node with the same name already exists ([#81056](https://github.com/kubernetes/kubernetes/pull/81056), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- Kubeadm: support Windows specific kubelet flags in kubeadm-flags.env ([#88287](https://github.com/kubernetes/kubernetes/pull/88287), [@gab-satchi](https://github.com/gab-satchi)) [SIG Cluster Lifecycle and Windows] -- Kubeadm: support automatic retry after failing to pull image ([#86899](https://github.com/kubernetes/kubernetes/pull/86899), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- Kubeadm: upgrade supports fallback to the nearest known etcd version if an unknown k8s version is passed ([#88373](https://github.com/kubernetes/kubernetes/pull/88373), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- Kubectl/drain: add disable-eviction option.Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, and should be used with caution. ([#85571](https://github.com/kubernetes/kubernetes/pull/85571), [@michaelgugino](https://github.com/michaelgugino)) [SIG CLI] -- Kubectl/drain: add skip-wait-for-delete-timeout option. If a pod’s `DeletionTimestamp` is older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip. ([#85577](https://github.com/kubernetes/kubernetes/pull/85577), [@michaelgugino](https://github.com/michaelgugino)) [SIG CLI] -- Option `preConfiguredBackendPoolLoadBalancerTypes` is added to azure cloud provider for the pre-configured load balancers, possible values: `""`, `"internal"`, `"external"`,`"all"` ([#86338](https://github.com/kubernetes/kubernetes/pull/86338), [@gossion](https://github.com/gossion)) [SIG Cloud Provider] -- PodTopologySpread plugin now excludes terminatingPods when making scheduling decisions. ([#87845](https://github.com/kubernetes/kubernetes/pull/87845), [@Huang-Wei](https://github.com/Huang-Wei)) [SIG Scheduling] -- Provider/azure: Network security groups can now be in a separate resource group. ([#87035](https://github.com/kubernetes/kubernetes/pull/87035), [@CecileRobertMichon](https://github.com/CecileRobertMichon)) [SIG Cloud Provider] -- SafeSysctlWhitelist: add net.ipv4.ping_group_range ([#85463](https://github.com/kubernetes/kubernetes/pull/85463), [@AkihiroSuda](https://github.com/AkihiroSuda)) [SIG Auth] -- Scheduler framework permit plugins now run at the end of the scheduling cycle, after reserve plugins. Waiting on permit will remain in the beginning of the binding cycle. ([#88199](https://github.com/kubernetes/kubernetes/pull/88199), [@mateuszlitwin](https://github.com/mateuszlitwin)) [SIG Scheduling] -- Scheduler: Add DefaultBinder plugin ([#87430](https://github.com/kubernetes/kubernetes/pull/87430), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling and Testing] -- Skip default spreading scoring plugin for pods that define TopologySpreadConstraints ([#87566](https://github.com/kubernetes/kubernetes/pull/87566), [@skilxn-go](https://github.com/skilxn-go)) [SIG Scheduling] -- The kubectl --dry-run flag now accepts the values 'client', 'server', and 'none', to support client-side and server-side dry-run strategies. The boolean and unset values for the --dry-run flag are deprecated and a value will be required in a future version. ([#87580](https://github.com/kubernetes/kubernetes/pull/87580), [@julianvmodesto](https://github.com/julianvmodesto)) [SIG CLI] -- Support server-side dry-run in kubectl with --dry-run=server for commands including apply, patch, create, run, annotate, label, set, autoscale, drain, rollout undo, and expose. ([#87714](https://github.com/kubernetes/kubernetes/pull/87714), [@julianvmodesto](https://github.com/julianvmodesto)) [SIG API Machinery, CLI and Testing] -- Add --dry-run=server|client to kubectl delete, taint, replace ([#88292](https://github.com/kubernetes/kubernetes/pull/88292), [@julianvmodesto](https://github.com/julianvmodesto)) [SIG CLI and Testing] -- The feature PodTopologySpread (feature gate `EvenPodsSpread`) has been enabled by default in 1.18. ([#88105](https://github.com/kubernetes/kubernetes/pull/88105), [@Huang-Wei](https://github.com/Huang-Wei)) [SIG Scheduling and Testing] -- The kubelet and the default docker runtime now support running ephemeral containers in the Linux process namespace of a target container. Other container runtimes must implement support for this feature before it will be available for that runtime. ([#84731](https://github.com/kubernetes/kubernetes/pull/84731), [@verb](https://github.com/verb)) [SIG Node] -- The underlying format of the `CPUManager` state file has changed. Upgrades should be seamless, but any third-party tools that rely on reading the previous format need to be updated. ([#84462](https://github.com/kubernetes/kubernetes/pull/84462), [@klueska](https://github.com/klueska)) [SIG Node and Testing] -- Update CNI version to v0.8.5 ([#78819](https://github.com/kubernetes/kubernetes/pull/78819), [@justaugustus](https://github.com/justaugustus)) [SIG API Machinery, Cluster Lifecycle, Network, Release and Testing] -- Webhooks have alpha support for network proxy ([#85870](https://github.com/kubernetes/kubernetes/pull/85870), [@Jefftree](https://github.com/Jefftree)) [SIG API Machinery, Auth and Testing] -- When client certificate files are provided, reload files for new connections, and close connections when a certificate changes. ([#79083](https://github.com/kubernetes/kubernetes/pull/79083), [@jackkleeman](https://github.com/jackkleeman)) [SIG API Machinery, Auth, Node and Testing] -- When deleting objects using kubectl with the --force flag, you are no longer required to also specify --grace-period=0. ([#87776](https://github.com/kubernetes/kubernetes/pull/87776), [@brianpursley](https://github.com/brianpursley)) [SIG CLI] -- Windows nodes on GCE can use virtual TPM-based authentication to the control plane. ([#85466](https://github.com/kubernetes/kubernetes/pull/85466), [@pjh](https://github.com/pjh)) [SIG Cluster Lifecycle] -- You can now pass "--node-ip ::" to kubelet to indicate that it should autodetect an IPv6 address to use as the node's primary address. ([#85850](https://github.com/kubernetes/kubernetes/pull/85850), [@danwinship](https://github.com/danwinship)) [SIG Cloud Provider, Network and Node] -- `kubectl` now contains a `kubectl alpha debug` command. This command allows attaching an ephemeral container to a running pod for the purposes of debugging. ([#88004](https://github.com/kubernetes/kubernetes/pull/88004), [@verb](https://github.com/verb)) [SIG CLI] -- TLS Server Name overrides can now be specified in a kubeconfig file and via --tls-server-name in kubectl ([#88769](https://github.com/kubernetes/kubernetes/pull/88769), [@deads2k](https://github.com/deads2k)) [SIG API Machinery, Auth and CLI] - -#### Metrics: -- Add `rest_client_rate_limiter_duration_seconds` metric to component-base to track client side rate limiter latency in seconds. Broken down by verb and URL. ([#88134](https://github.com/kubernetes/kubernetes/pull/88134), [@jennybuckley](https://github.com/jennybuckley)) [SIG API Machinery, Cluster Lifecycle and Instrumentation] -- Added two client certificate metrics for exec auth: - - `rest_client_certificate_expiration_seconds` a gauge reporting the lifetime of the current client certificate. Reports the time of expiry in seconds since January 1, 1970 UTC. - - `rest_client_certificate_rotation_age` a histogram reporting the age of a just rotated client certificate in seconds. ([#84382](https://github.com/kubernetes/kubernetes/pull/84382), [@sambdavidson](https://github.com/sambdavidson)) [SIG API Machinery, Auth, Cluster Lifecycle and Instrumentation] -- Controller manager serve workqueue metrics ([#87967](https://github.com/kubernetes/kubernetes/pull/87967), [@zhan849](https://github.com/zhan849)) [SIG API Machinery] -- Following metrics have been turned off: - - kubelet_pod_worker_latency_microseconds - - kubelet_pod_start_latency_microseconds - - kubelet_cgroup_manager_latency_microseconds - - kubelet_pod_worker_start_latency_microseconds - - kubelet_pleg_relist_latency_microseconds - - kubelet_pleg_relist_interval_microseconds - - kubelet_eviction_stats_age_microseconds - - kubelet_runtime_operations - - kubelet_runtime_operations_latency_microseconds - - kubelet_runtime_operations_errors - - kubelet_device_plugin_registration_count - - kubelet_device_plugin_alloc_latency_microseconds - - kubelet_docker_operations - - kubelet_docker_operations_latency_microseconds - - kubelet_docker_operations_errors - - kubelet_docker_operations_timeout - - network_plugin_operations_latency_microseconds ([#83841](https://github.com/kubernetes/kubernetes/pull/83841), [@RainbowMango](https://github.com/RainbowMango)) [SIG Network and Node] -- Kube-apiserver metrics will now include request counts, latencies, and response sizes for /healthz, /livez, and /readyz requests. ([#83598](https://github.com/kubernetes/kubernetes/pull/83598), [@jktomer](https://github.com/jktomer)) [SIG API Machinery] -- Kubelet now exports a `server_expiration_renew_failure` and `client_expiration_renew_failure` metric counter if the certificate rotations cannot be performed. ([#84614](https://github.com/kubernetes/kubernetes/pull/84614), [@rphillips](https://github.com/rphillips)) [SIG API Machinery, Auth, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation, Node and Release] -- Kubelet: the metric process_start_time_seconds be marked as with the ALPHA stability level. ([#85446](https://github.com/kubernetes/kubernetes/pull/85446), [@RainbowMango](https://github.com/RainbowMango)) [SIG API Machinery, Cluster Lifecycle, Instrumentation and Node] -- New metric `kubelet_pleg_last_seen_seconds` to aid diagnosis of PLEG not healthy issues. ([#86251](https://github.com/kubernetes/kubernetes/pull/86251), [@bboreham](https://github.com/bboreham)) [SIG Node] - -### Other (Bug, Cleanup or Flake) - -- Fixed a regression with clients prior to 1.15 not being able to update podIP in pod status, or podCIDR in node spec, against >= 1.16 API servers ([#88505](https://github.com/kubernetes/kubernetes/pull/88505), [@liggitt](https://github.com/liggitt)) [SIG Apps and Network] -- Fixed "kubectl describe statefulsets.apps" printing garbage for rolling update partition ([#85846](https://github.com/kubernetes/kubernetes/pull/85846), [@phil9909](https://github.com/phil9909)) [SIG CLI] -- Add a event to PV when filesystem on PV does not match actual filesystem on disk ([#86982](https://github.com/kubernetes/kubernetes/pull/86982), [@gnufied](https://github.com/gnufied)) [SIG Storage] -- Add azure disk WriteAccelerator support ([#87945](https://github.com/kubernetes/kubernetes/pull/87945), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] -- Add delays between goroutines for vm instance update ([#88094](https://github.com/kubernetes/kubernetes/pull/88094), [@aramase](https://github.com/aramase)) [SIG Cloud Provider] -- Add init containers log to cluster dump info. ([#88324](https://github.com/kubernetes/kubernetes/pull/88324), [@zhouya0](https://github.com/zhouya0)) [SIG CLI] -- Addons: elasticsearch discovery supports IPv6 ([#85543](https://github.com/kubernetes/kubernetes/pull/85543), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle and Instrumentation] -- Adds "volume.beta.kubernetes.io/migrated-to" annotation to PV's and PVC's when they are migrated to signal external provisioners to pick up those objects for Provisioning and Deleting. ([#87098](https://github.com/kubernetes/kubernetes/pull/87098), [@davidz627](https://github.com/davidz627)) [SIG Storage] -- All api-server log request lines in a more greppable format. ([#87203](https://github.com/kubernetes/kubernetes/pull/87203), [@lavalamp](https://github.com/lavalamp)) [SIG API Machinery] -- Azure VMSS LoadBalancerBackendAddressPools updating has been improved with sequential-sync + concurrent-async requests. ([#88699](https://github.com/kubernetes/kubernetes/pull/88699), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Azure cloud provider now obtains AAD token who audience claim will not have spn: prefix ([#87590](https://github.com/kubernetes/kubernetes/pull/87590), [@weinong](https://github.com/weinong)) [SIG Cloud Provider] -- AzureFile and CephFS use the new Mount library that prevents logging of sensitive mount options. ([#88684](https://github.com/kubernetes/kubernetes/pull/88684), [@saad-ali](https://github.com/saad-ali)) [SIG Storage] -- Bind dns-horizontal containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83364](https://github.com/kubernetes/kubernetes/pull/83364), [@wawa0210](https://github.com/wawa0210)) [SIG Cluster Lifecycle and Windows] -- Bind kube-dns containers to linux nodes to avoid Windows scheduling ([#83358](https://github.com/kubernetes/kubernetes/pull/83358), [@wawa0210](https://github.com/wawa0210)) [SIG Cluster Lifecycle and Windows] -- Bind metadata-agent containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83363](https://github.com/kubernetes/kubernetes/pull/83363), [@wawa0210](https://github.com/wawa0210)) [SIG Cluster Lifecycle, Instrumentation and Windows] -- Bind metrics-server containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83362](https://github.com/kubernetes/kubernetes/pull/83362), [@wawa0210](https://github.com/wawa0210)) [SIG Cluster Lifecycle, Instrumentation and Windows] -- Bug fixes: Make sure we include latest packages node #351 (@caseydavenport) ([#84163](https://github.com/kubernetes/kubernetes/pull/84163), [@david-tigera](https://github.com/david-tigera)) [SIG Cluster Lifecycle] -- CPU limits are now respected for Windows containers. If a node is over-provisioned, no weighting is used, only limits are respected. ([#86101](https://github.com/kubernetes/kubernetes/pull/86101), [@PatrickLang](https://github.com/PatrickLang)) [SIG Node, Testing and Windows] -- Changed core_pattern on COS nodes to be an absolute path. ([#86329](https://github.com/kubernetes/kubernetes/pull/86329), [@mml](https://github.com/mml)) [SIG Cluster Lifecycle and Node] -- Client-go certificate manager rotation gained the ability to preserve optional intermediate chains accompanying issued certificates ([#88744](https://github.com/kubernetes/kubernetes/pull/88744), [@jackkleeman](https://github.com/jackkleeman)) [SIG API Machinery and Auth] -- Cloud provider config CloudProviderBackoffMode has been removed since it won't be used anymore. ([#88463](https://github.com/kubernetes/kubernetes/pull/88463), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Conformance image now depends on stretch-slim instead of debian-hyperkube-base as that image is being deprecated and removed. ([#88702](https://github.com/kubernetes/kubernetes/pull/88702), [@dims](https://github.com/dims)) [SIG Cluster Lifecycle, Release and Testing] -- Deprecate --generator flag from kubectl create commands ([#88655](https://github.com/kubernetes/kubernetes/pull/88655), [@soltysh](https://github.com/soltysh)) [SIG CLI] -- During initialization phase (preflight), kubeadm now verifies the presence of the conntrack executable ([#85857](https://github.com/kubernetes/kubernetes/pull/85857), [@hnanni](https://github.com/hnanni)) [SIG Cluster Lifecycle] -- EndpointSlice should not contain endpoints for terminating pods ([#89056](https://github.com/kubernetes/kubernetes/pull/89056), [@andrewsykim](https://github.com/andrewsykim)) [SIG Apps and Network] -- Evictions due to pods breaching their ephemeral storage limits are now recorded by the `kubelet_evictions` metric and can be alerted on. ([#87906](https://github.com/kubernetes/kubernetes/pull/87906), [@smarterclayton](https://github.com/smarterclayton)) [SIG Node] -- Filter published OpenAPI schema by making nullable, required fields non-required in order to avoid kubectl to wrongly reject null values. ([#85722](https://github.com/kubernetes/kubernetes/pull/85722), [@sttts](https://github.com/sttts)) [SIG API Machinery] -- Fix /readyz to return error immediately after a shutdown is initiated, before the --shutdown-delay-duration has elapsed. ([#88911](https://github.com/kubernetes/kubernetes/pull/88911), [@tkashem](https://github.com/tkashem)) [SIG API Machinery] -- Fix API Server potential memory leak issue in processing watch request. ([#85410](https://github.com/kubernetes/kubernetes/pull/85410), [@answer1991](https://github.com/answer1991)) [SIG API Machinery] -- Fix EndpointSlice controller race condition and ensure that it handles external changes to EndpointSlices. ([#85703](https://github.com/kubernetes/kubernetes/pull/85703), [@robscott](https://github.com/robscott)) [SIG Apps and Network] -- Fix IPv6 addresses lost issue in pure ipv6 vsphere environment ([#86001](https://github.com/kubernetes/kubernetes/pull/86001), [@hubv](https://github.com/hubv)) [SIG Cloud Provider] -- Fix LoadBalancer rule checking so that no unexpected LoadBalancer updates are made ([#85990](https://github.com/kubernetes/kubernetes/pull/85990), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Fix a bug in kube-proxy that caused it to crash when using load balancers with a different IP family ([#87117](https://github.com/kubernetes/kubernetes/pull/87117), [@aojea](https://github.com/aojea)) [SIG Network] -- Fix a bug in port-forward: named port not working with service ([#85511](https://github.com/kubernetes/kubernetes/pull/85511), [@oke-py](https://github.com/oke-py)) [SIG CLI] -- Fix a bug in the dual-stack IPVS proxier where stale IPv6 endpoints were not being cleaned up ([#87695](https://github.com/kubernetes/kubernetes/pull/87695), [@andrewsykim](https://github.com/andrewsykim)) [SIG Network] -- Fix a bug that orphan revision cannot be adopted and statefulset cannot be synced ([#86801](https://github.com/kubernetes/kubernetes/pull/86801), [@likakuli](https://github.com/likakuli)) [SIG Apps] -- Fix a bug where ExternalTrafficPolicy is not applied to service ExternalIPs. ([#88786](https://github.com/kubernetes/kubernetes/pull/88786), [@freehan](https://github.com/freehan)) [SIG Network] -- Fix a bug where kubenet fails to parse the tc output. ([#83572](https://github.com/kubernetes/kubernetes/pull/83572), [@chendotjs](https://github.com/chendotjs)) [SIG Network] -- Fix a regression in kubenet that prevent pods to obtain ip addresses ([#85993](https://github.com/kubernetes/kubernetes/pull/85993), [@chendotjs](https://github.com/chendotjs)) [SIG Network and Node] -- Fix azure file AuthorizationFailure ([#85475](https://github.com/kubernetes/kubernetes/pull/85475), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] -- Fix bug where EndpointSlice controller would attempt to modify shared objects. ([#85368](https://github.com/kubernetes/kubernetes/pull/85368), [@robscott](https://github.com/robscott)) [SIG API Machinery, Apps and Network] -- Fix handling of aws-load-balancer-security-groups annotation. Security-Groups assigned with this annotation are no longer modified by kubernetes which is the expected behaviour of most users. Also no unnecessary Security-Groups are created anymore if this annotation is used. ([#83446](https://github.com/kubernetes/kubernetes/pull/83446), [@Elias481](https://github.com/Elias481)) [SIG Cloud Provider] -- Fix invalid VMSS updates due to incorrect cache ([#89002](https://github.com/kubernetes/kubernetes/pull/89002), [@ArchangelSDY](https://github.com/ArchangelSDY)) [SIG Cloud Provider] -- Fix isCurrentInstance for Windows by removing the dependency of hostname. ([#89138](https://github.com/kubernetes/kubernetes/pull/89138), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Fix issue #85805 about a resource not found in azure cloud provider when LoadBalancer specified in another resource group. ([#86502](https://github.com/kubernetes/kubernetes/pull/86502), [@levimm](https://github.com/levimm)) [SIG Cloud Provider] -- Fix kubectl annotate error when local=true is set ([#86952](https://github.com/kubernetes/kubernetes/pull/86952), [@zhouya0](https://github.com/zhouya0)) [SIG CLI] -- Fix kubectl create deployment image name ([#86636](https://github.com/kubernetes/kubernetes/pull/86636), [@zhouya0](https://github.com/zhouya0)) [SIG CLI] -- Fix `kubectl drain ignore` daemonsets and others. ([#87361](https://github.com/kubernetes/kubernetes/pull/87361), [@zhouya0](https://github.com/zhouya0)) [SIG CLI] -- Fix missing "apiVersion" for "involvedObject" in Events for Nodes. ([#87537](https://github.com/kubernetes/kubernetes/pull/87537), [@uthark](https://github.com/uthark)) [SIG Apps and Node] -- Fix nil pointer dereference in azure cloud provider ([#85975](https://github.com/kubernetes/kubernetes/pull/85975), [@ldx](https://github.com/ldx)) [SIG Cloud Provider] -- Fix regression in statefulset conversion which prevents applying a statefulset multiple times. ([#87706](https://github.com/kubernetes/kubernetes/pull/87706), [@liggitt](https://github.com/liggitt)) [SIG Apps and Testing] -- Fix route conflicted operations when updating multiple routes together ([#88209](https://github.com/kubernetes/kubernetes/pull/88209), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Fix that prevents repeated fetching of PVC/PV objects by kubelet when processing of pod volumes fails. While this prevents hammering API server in these error scenarios, it means that some errors in processing volume(s) for a pod could now take up to 2-3 minutes before retry. ([#88141](https://github.com/kubernetes/kubernetes/pull/88141), [@tedyu](https://github.com/tedyu)) [SIG Node and Storage] -- Fix the bug PIP's DNS is deleted if no DNS label service annotation isn't set. ([#87246](https://github.com/kubernetes/kubernetes/pull/87246), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] -- Fix control plane hosts rolling upgrade causing thundering herd of LISTs on etcd leading to control plane unavailability. ([#86430](https://github.com/kubernetes/kubernetes/pull/86430), [@wojtek-t](https://github.com/wojtek-t)) [SIG API Machinery, Node and Testing] -- Fix: add azure disk migration support for CSINode ([#88014](https://github.com/kubernetes/kubernetes/pull/88014), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] -- Fix: add non-retriable errors in azure clients ([#87941](https://github.com/kubernetes/kubernetes/pull/87941), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider] -- Fix: add remediation in azure disk attach/detach ([#88444](https://github.com/kubernetes/kubernetes/pull/88444), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider] -- Fix: azure data disk should use same key as os disk by default ([#86351](https://github.com/kubernetes/kubernetes/pull/86351), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider] -- Fix: azure disk could not mounted on Standard_DC4s/DC2s instances ([#86612](https://github.com/kubernetes/kubernetes/pull/86612), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] -- Fix: azure file mount timeout issue ([#88610](https://github.com/kubernetes/kubernetes/pull/88610), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] -- Fix: check disk status before disk azure disk ([#88360](https://github.com/kubernetes/kubernetes/pull/88360), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider] -- Fix: corrupted mount point in csi driver ([#88569](https://github.com/kubernetes/kubernetes/pull/88569), [@andyzhangx](https://github.com/andyzhangx)) [SIG Storage] -- Fix: get azure disk lun timeout issue ([#88158](https://github.com/kubernetes/kubernetes/pull/88158), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] -- Fix: update azure disk max count ([#88201](https://github.com/kubernetes/kubernetes/pull/88201), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] -- Fixed "requested device X but found Y" attach error on AWS. ([#85675](https://github.com/kubernetes/kubernetes/pull/85675), [@jsafrane](https://github.com/jsafrane)) [SIG Cloud Provider and Storage] -- Fixed NetworkPolicy validation that `Except` values are accepted when they are outside the CIDR range. ([#86578](https://github.com/kubernetes/kubernetes/pull/86578), [@tnqn](https://github.com/tnqn)) [SIG Network] -- Fixed a bug in the TopologyManager. Previously, the TopologyManager would only guarantee alignment if container creation was serialized in some way. Alignment is now guaranteed under all scenarios of container creation. ([#87759](https://github.com/kubernetes/kubernetes/pull/87759), [@klueska](https://github.com/klueska)) [SIG Node] -- Fixed a bug which could prevent a provider ID from ever being set for node if an error occurred determining the provider ID when the node was added. ([#87043](https://github.com/kubernetes/kubernetes/pull/87043), [@zjs](https://github.com/zjs)) [SIG Apps and Cloud Provider] -- Fixed a data race in the kubelet image manager that can cause static pod workers to silently stop working. ([#88915](https://github.com/kubernetes/kubernetes/pull/88915), [@roycaihw](https://github.com/roycaihw)) [SIG Node] -- Fixed a panic in the kubelet cleaning up pod volumes ([#86277](https://github.com/kubernetes/kubernetes/pull/86277), [@tedyu](https://github.com/tedyu)) [SIG Storage] -- Fixed a regression where the kubelet would fail to update the ready status of pods. ([#84951](https://github.com/kubernetes/kubernetes/pull/84951), [@tedyu](https://github.com/tedyu)) [SIG Node] -- Fixed an issue that could cause the kubelet to incorrectly run concurrent pod reconciliation loops and crash. ([#89055](https://github.com/kubernetes/kubernetes/pull/89055), [@tedyu](https://github.com/tedyu)) [SIG Node] -- Fixed block CSI volume cleanup after timeouts. ([#88660](https://github.com/kubernetes/kubernetes/pull/88660), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] -- Fixed cleaning of CSI raw block volumes. ([#87978](https://github.com/kubernetes/kubernetes/pull/87978), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] -- Fixed AWS Cloud Provider attempting to delete LoadBalancer security group it didn’t provision, and fixed AWS Cloud Provider creating a default LoadBalancer security group even if annotation `service.beta.kubernetes.io/aws-load-balancer-security-groups` is present because the intended behavior of aws-load-balancer-security-groups is to replace all security groups assigned to the load balancer. ([#84265](https://github.com/kubernetes/kubernetes/pull/84265), [@bhagwat070919](https://github.com/bhagwat070919)) [SIG Cloud Provider] -- Fixed two scheduler metrics (pending_pods and schedule_attempts_total) not being recorded ([#87692](https://github.com/kubernetes/kubernetes/pull/87692), [@everpeace](https://github.com/everpeace)) [SIG Scheduling] -- Fixes an issue with kubelet-reported pod status on deleted/recreated pods. ([#86320](https://github.com/kubernetes/kubernetes/pull/86320), [@liggitt](https://github.com/liggitt)) [SIG Node] -- Fixes conversion error in multi-version custom resources that could cause metadata.generation to increment on no-op patches or updates of a custom resource. ([#88995](https://github.com/kubernetes/kubernetes/pull/88995), [@liggitt](https://github.com/liggitt)) [SIG API Machinery] -- Fixes issue where AAD token obtained by kubectl is incompatible with on-behalf-of flow and oidc. The audience claim before this fix has "spn:" prefix. After this fix, "spn:" prefix is omitted. ([#86412](https://github.com/kubernetes/kubernetes/pull/86412), [@weinong](https://github.com/weinong)) [SIG API Machinery, Auth and Cloud Provider] -- Fixes an issue where you can't attach more than 15 GCE Persistent Disks to c2, n2, m1, m2 machine types. ([#88602](https://github.com/kubernetes/kubernetes/pull/88602), [@yuga711](https://github.com/yuga711)) [SIG Storage] -- Fixes kube-proxy when EndpointSlice feature gate is enabled on Windows. ([#86016](https://github.com/kubernetes/kubernetes/pull/86016), [@robscott](https://github.com/robscott)) [SIG Auth and Network] -- Fixes kubelet crash in client certificate rotation cases ([#88079](https://github.com/kubernetes/kubernetes/pull/88079), [@liggitt](https://github.com/liggitt)) [SIG API Machinery, Auth and Node] -- Fixes service account token admission error in clusters that do not run the service account token controller ([#87029](https://github.com/kubernetes/kubernetes/pull/87029), [@liggitt](https://github.com/liggitt)) [SIG Auth] -- Fixes v1.17.0 regression in --service-cluster-ip-range handling with IPv4 ranges larger than 65536 IP addresses ([#86534](https://github.com/kubernetes/kubernetes/pull/86534), [@liggitt](https://github.com/liggitt)) [SIG Network] -- Fixes wrong validation result of NetworkPolicy PolicyTypes ([#85747](https://github.com/kubernetes/kubernetes/pull/85747), [@tnqn](https://github.com/tnqn)) [SIG Network] -- For subprotocol negotiation, both client and server protocol is required now. ([#86646](https://github.com/kubernetes/kubernetes/pull/86646), [@tedyu](https://github.com/tedyu)) [SIG API Machinery and Node] -- For volumes that allow attaches across multiple nodes, attach and detach operations across different nodes are now executed in parallel. ([#88678](https://github.com/kubernetes/kubernetes/pull/88678), [@verult](https://github.com/verult)) [SIG Storage] -- Garbage collector now can correctly orphan ControllerRevisions when StatefulSets are deleted with orphan propagation policy. ([#84984](https://github.com/kubernetes/kubernetes/pull/84984), [@cofyc](https://github.com/cofyc)) [SIG Apps] -- `Get-kube.sh` uses the gcloud's current local GCP service account for auth when the provider is GCE or GKE instead of the metadata server default ([#88383](https://github.com/kubernetes/kubernetes/pull/88383), [@BenTheElder](https://github.com/BenTheElder)) [SIG Cluster Lifecycle] -- Golang/x/net has been updated to bring in fixes for CVE-2020-9283 ([#88381](https://github.com/kubernetes/kubernetes/pull/88381), [@BenTheElder](https://github.com/BenTheElder)) [SIG API Machinery, CLI, Cloud Provider, Cluster Lifecycle and Instrumentation] -- If a serving certificate’s param specifies a name that is an IP for an SNI certificate, it will have priority for replying to server connections. ([#85308](https://github.com/kubernetes/kubernetes/pull/85308), [@deads2k](https://github.com/deads2k)) [SIG API Machinery] -- Improved yaml parsing performance ([#85458](https://github.com/kubernetes/kubernetes/pull/85458), [@cjcullen](https://github.com/cjcullen)) [SIG API Machinery, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation and Node] -- Improves performance of the node authorizer ([#87696](https://github.com/kubernetes/kubernetes/pull/87696), [@liggitt](https://github.com/liggitt)) [SIG Auth] -- In GKE alpha clusters it will be possible to use the service annotation `cloud.google.com/network-tier: Standard` ([#88487](https://github.com/kubernetes/kubernetes/pull/88487), [@zioproto](https://github.com/zioproto)) [SIG Cloud Provider] -- Includes FSType when describing CSI persistent volumes. ([#85293](https://github.com/kubernetes/kubernetes/pull/85293), [@huffmanca](https://github.com/huffmanca)) [SIG CLI and Storage] -- Iptables/userspace proxy: improve performance by getting local addresses only once per sync loop, instead of for every external IP ([#85617](https://github.com/kubernetes/kubernetes/pull/85617), [@andrewsykim](https://github.com/andrewsykim)) [SIG API Machinery, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation and Network] -- Kube-aggregator: always sets unavailableGauge metric to reflect the current state of a service. ([#87778](https://github.com/kubernetes/kubernetes/pull/87778), [@p0lyn0mial](https://github.com/p0lyn0mial)) [SIG API Machinery] -- Kube-apiserver: fixed a conflict error encountered attempting to delete a pod with gracePeriodSeconds=0 and a resourceVersion precondition ([#85516](https://github.com/kubernetes/kubernetes/pull/85516), [@michaelgugino](https://github.com/michaelgugino)) [SIG API Machinery] -- Kube-proxy no longer modifies shared EndpointSlices. ([#86092](https://github.com/kubernetes/kubernetes/pull/86092), [@robscott](https://github.com/robscott)) [SIG Network] -- Kube-proxy: on dual-stack mode, if it is not able to get the IP Family of an endpoint, logs it with level InfoV(4) instead of Warning, avoiding flooding the logs for endpoints without addresses ([#88934](https://github.com/kubernetes/kubernetes/pull/88934), [@aojea](https://github.com/aojea)) [SIG Network] -- Kubeadm allows to configure single-stack clusters if dual-stack is enabled ([#87453](https://github.com/kubernetes/kubernetes/pull/87453), [@aojea](https://github.com/aojea)) [SIG API Machinery, Cluster Lifecycle and Network] -- Kubeadm now includes CoreDNS version 1.6.7 ([#86260](https://github.com/kubernetes/kubernetes/pull/86260), [@rajansandeep](https://github.com/rajansandeep)) [SIG Cluster Lifecycle] -- Kubeadm upgrades always persist the etcd backup for stacked ([#86861](https://github.com/kubernetes/kubernetes/pull/86861), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- Kubeadm: 'kubeadm alpha kubelet config download' has been removed, please use 'kubeadm upgrade node phase kubelet-config' instead ([#87944](https://github.com/kubernetes/kubernetes/pull/87944), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- Kubeadm: Forward cluster name to the controller-manager arguments ([#85817](https://github.com/kubernetes/kubernetes/pull/85817), [@ereslibre](https://github.com/ereslibre)) [SIG Cluster Lifecycle] -- Kubeadm: add support for the "ci/k8s-master" version label as a replacement for "ci-cross/*", which no longer exists. ([#86609](https://github.com/kubernetes/kubernetes/pull/86609), [@Pensu](https://github.com/Pensu)) [SIG Cluster Lifecycle] -- Kubeadm: apply further improvements to the tentative support for concurrent etcd member join. Fixes a bug where multiple members can receive the same hostname. Increase the etcd client dial timeout and retry timeout for add/remove/... operations. ([#87505](https://github.com/kubernetes/kubernetes/pull/87505), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- Kubeadm: don't write the kubelet environment file on "upgrade apply" ([#85412](https://github.com/kubernetes/kubernetes/pull/85412), [@boluisa](https://github.com/boluisa)) [SIG Cluster Lifecycle] -- Kubeadm: fix potential panic when executing "kubeadm reset" with a corrupted kubelet.conf file ([#86216](https://github.com/kubernetes/kubernetes/pull/86216), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- Kubeadm: fix the bug that 'kubeadm upgrade' hangs in single node cluster ([#88434](https://github.com/kubernetes/kubernetes/pull/88434), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- Kubeadm: make sure images are pre-pulled even if a tag did not change but their contents changed ([#85603](https://github.com/kubernetes/kubernetes/pull/85603), [@bart0sh](https://github.com/bart0sh)) [SIG Cluster Lifecycle] -- Kubeadm: remove 'kubeadm upgrade node config' command since it was deprecated in v1.15, please use 'kubeadm upgrade node phase kubelet-config' instead ([#87975](https://github.com/kubernetes/kubernetes/pull/87975), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- Kubeadm: remove the deprecated CoreDNS feature-gate. It was set to "true" since v1.11 when the feature went GA. In v1.13 it was marked as deprecated and hidden from the CLI. ([#87400](https://github.com/kubernetes/kubernetes/pull/87400), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- Kubeadm: retry `kubeadm-config` ConfigMap creation or mutation if the apiserver is not responding. This will improve resiliency when joining new control plane nodes. ([#85763](https://github.com/kubernetes/kubernetes/pull/85763), [@ereslibre](https://github.com/ereslibre)) [SIG Cluster Lifecycle] -- Kubeadm: tolerate whitespace when validating certificate authority PEM data in kubeconfig files ([#86705](https://github.com/kubernetes/kubernetes/pull/86705), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- Kubeadm: use bind-address option to configure the kube-controller-manager and kube-scheduler http probes ([#86493](https://github.com/kubernetes/kubernetes/pull/86493), [@aojea](https://github.com/aojea)) [SIG Cluster Lifecycle] -- Kubeadm: uses the api-server AdvertiseAddress IP family to choose the etcd endpoint IP family for non external etcd clusters ([#85745](https://github.com/kubernetes/kubernetes/pull/85745), [@aojea](https://github.com/aojea)) [SIG Cluster Lifecycle] -- Kubectl cluster-info dump --output-directory=xxx now generates files with an extension depending on the output format. ([#82070](https://github.com/kubernetes/kubernetes/pull/82070), [@olivierlemasle](https://github.com/olivierlemasle)) [SIG CLI] -- `Kubectl describe ` and `kubectl top pod` will return a message saying `"No resources found"` or `"No resources found in namespace"` if there are no results to display. ([#87527](https://github.com/kubernetes/kubernetes/pull/87527), [@brianpursley](https://github.com/brianpursley)) [SIG CLI] -- `Kubectl drain node --dry-run` will list pods that would be evicted or deleted ([#82660](https://github.com/kubernetes/kubernetes/pull/82660), [@sallyom](https://github.com/sallyom)) [SIG CLI] -- `Kubectl set resources` will no longer return an error if passed an empty change for a resource. `kubectl set subject` will no longer return an error if passed an empty change for a resource. ([#85490](https://github.com/kubernetes/kubernetes/pull/85490), [@sallyom](https://github.com/sallyom)) [SIG CLI] -- Kubelet metrics gathered through metrics-server or prometheus should no longer timeout for Windows nodes running more than 3 pods. ([#87730](https://github.com/kubernetes/kubernetes/pull/87730), [@marosset](https://github.com/marosset)) [SIG Node, Testing and Windows] -- Kubelet metrics have been changed to buckets. For example the `exec/{podNamespace}/{podID}/{containerName}` is now just exec. ([#87913](https://github.com/kubernetes/kubernetes/pull/87913), [@cheftako](https://github.com/cheftako)) [SIG Node] -- Kubelets perform fewer unnecessary pod status update operations on the API server. ([#88591](https://github.com/kubernetes/kubernetes/pull/88591), [@smarterclayton](https://github.com/smarterclayton)) [SIG Node and Scalability] -- Kubernetes will try to acquire the iptables lock every 100 msec during 5 seconds instead of every second. This is especially useful for environments using kube-proxy in iptables mode with a high churn rate of services. ([#85771](https://github.com/kubernetes/kubernetes/pull/85771), [@aojea](https://github.com/aojea)) [SIG Network] -- Limit number of instances in a single update to GCE target pool to 1000. ([#87881](https://github.com/kubernetes/kubernetes/pull/87881), [@wojtek-t](https://github.com/wojtek-t)) [SIG Cloud Provider, Network and Scalability] -- Make Azure clients only retry on specified HTTP status codes ([#88017](https://github.com/kubernetes/kubernetes/pull/88017), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Make error message and service event message more clear ([#86078](https://github.com/kubernetes/kubernetes/pull/86078), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Minimize AWS NLB health check timeout when externalTrafficPolicy set to Local ([#73363](https://github.com/kubernetes/kubernetes/pull/73363), [@kellycampbell](https://github.com/kellycampbell)) [SIG Cloud Provider] -- Pause image contains "Architecture" in non-amd64 images ([#87954](https://github.com/kubernetes/kubernetes/pull/87954), [@BenTheElder](https://github.com/BenTheElder)) [SIG Release] -- Pause image upgraded to 3.2 in kubelet and kubeadm. ([#88173](https://github.com/kubernetes/kubernetes/pull/88173), [@BenTheElder](https://github.com/BenTheElder)) [SIG CLI, Cluster Lifecycle, Node and Testing] -- Plugin/PluginConfig and Policy APIs are mutually exclusive when running the scheduler ([#88864](https://github.com/kubernetes/kubernetes/pull/88864), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling] -- Remove `FilteredNodesStatuses` argument from `PreScore`'s interface. ([#88189](https://github.com/kubernetes/kubernetes/pull/88189), [@skilxn-go](https://github.com/skilxn-go)) [SIG Scheduling and Testing] -- Resolved a performance issue in the node authorizer index maintenance. ([#87693](https://github.com/kubernetes/kubernetes/pull/87693), [@liggitt](https://github.com/liggitt)) [SIG Auth] -- Resolved regression in admission, authentication, and authorization webhook performance in v1.17.0-rc.1 ([#85810](https://github.com/kubernetes/kubernetes/pull/85810), [@liggitt](https://github.com/liggitt)) [SIG API Machinery and Testing] -- Resolves performance regression in `kubectl get all` and in client-go discovery clients constructed using `NewDiscoveryClientForConfig` or `NewDiscoveryClientForConfigOrDie`. ([#86168](https://github.com/kubernetes/kubernetes/pull/86168), [@liggitt](https://github.com/liggitt)) [SIG API Machinery] -- Reverted a kubectl azure auth module change where oidc claim spn: prefix was omitted resulting a breaking behavior with existing Azure AD OIDC enabled api-server ([#87507](https://github.com/kubernetes/kubernetes/pull/87507), [@weinong](https://github.com/weinong)) [SIG API Machinery, Auth and Cloud Provider] -- Shared informers are now more reliable in the face of network disruption. ([#86015](https://github.com/kubernetes/kubernetes/pull/86015), [@squeed](https://github.com/squeed)) [SIG API Machinery] -- Specifying PluginConfig for the same plugin more than once fails scheduler startup. - Specifying extenders and configuring .ignoredResources for the NodeResourcesFit plugin fails ([#88870](https://github.com/kubernetes/kubernetes/pull/88870), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling] -- Terminating a restartPolicy=Never pod no longer has a chance to report the pod succeeded when it actually failed. ([#88440](https://github.com/kubernetes/kubernetes/pull/88440), [@smarterclayton](https://github.com/smarterclayton)) [SIG Node and Testing] -- The CSR signing cert/key pairs will be reloaded from disk like the kube-apiserver cert/key pairs ([#86816](https://github.com/kubernetes/kubernetes/pull/86816), [@deads2k](https://github.com/deads2k)) [SIG API Machinery, Apps and Auth] -- The EventRecorder from k8s.io/client-go/tools/events will now create events in the default namespace (instead of kube-system) when the related object does not have it set. ([#88815](https://github.com/kubernetes/kubernetes/pull/88815), [@enj](https://github.com/enj)) [SIG API Machinery] -- The audit event sourceIPs list will now always end with the IP that sent the request directly to the API server. ([#87167](https://github.com/kubernetes/kubernetes/pull/87167), [@tallclair](https://github.com/tallclair)) [SIG API Machinery and Auth] -- The sample-apiserver aggregated conformance test has updated to use the Kubernetes v1.17.0 sample apiserver ([#84735](https://github.com/kubernetes/kubernetes/pull/84735), [@liggitt](https://github.com/liggitt)) [SIG API Machinery, Architecture, CLI and Testing] -- To reduce chances of throttling, VM cache is set to nil when Azure node provisioning state is deleting ([#87635](https://github.com/kubernetes/kubernetes/pull/87635), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- VMSS cache is added so that less chances of VMSS GET throttling ([#85885](https://github.com/kubernetes/kubernetes/pull/85885), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] -- Wait for kubelet & kube-proxy to be ready on Windows node within 10s ([#85228](https://github.com/kubernetes/kubernetes/pull/85228), [@YangLu1031](https://github.com/YangLu1031)) [SIG Cluster Lifecycle] -- `kubectl apply -f --prune -n ` should prune all resources not defined in the file in the cli specified namespace. ([#85613](https://github.com/kubernetes/kubernetes/pull/85613), [@MartinKaburu](https://github.com/MartinKaburu)) [SIG CLI] -- `kubectl create clusterrolebinding` creates rbac.authorization.k8s.io/v1 object ([#85889](https://github.com/kubernetes/kubernetes/pull/85889), [@oke-py](https://github.com/oke-py)) [SIG CLI] -- `kubectl diff` now returns 1 only on diff finding changes, and >1 on kubectl errors. The "exit status code 1" message has also been muted. ([#87437](https://github.com/kubernetes/kubernetes/pull/87437), [@apelisse](https://github.com/apelisse)) [SIG CLI and Testing] - -## Dependencies - -- Update Calico to v3.8.4 ([#84163](https://github.com/kubernetes/kubernetes/pull/84163), [@david-tigera](https://github.com/david-tigera))[SIG Cluster Lifecycle] -- Update aws-sdk-go dependency to v1.28.2 ([#87253](https://github.com/kubernetes/kubernetes/pull/87253), [@SaranBalaji90](https://github.com/SaranBalaji90))[SIG API Machinery and Cloud Provider] -- Update CNI version to v0.8.5 ([#78819](https://github.com/kubernetes/kubernetes/pull/78819), [@justaugustus](https://github.com/justaugustus))[SIG Release, Testing, Network, Cluster Lifecycle and API Machinery] -- Update cri-tools to v1.17.0 ([#86305](https://github.com/kubernetes/kubernetes/pull/86305), [@saschagrunert](https://github.com/saschagrunert))[SIG Release and Cluster Lifecycle] -- Pause image upgraded to 3.2 in kubelet and kubeadm ([#88173](https://github.com/kubernetes/kubernetes/pull/88173), [@BenTheElder](https://github.com/BenTheElder))[SIG CLI, Node, Testing and Cluster Lifecycle] -- Update CoreDNS version to 1.6.7 in kubeadm ([#86260](https://github.com/kubernetes/kubernetes/pull/86260), [@rajansandeep](https://github.com/rajansandeep))[SIG Cluster Lifecycle] -- Update golang.org/x/crypto to fix CVE-2020-9283 ([#8838](https://github.com/kubernetes/kubernetes/pull/88381), [@BenTheElder](https://github.com/BenTheElder))[SIG CLI, Instrumentation, API Machinery, CLuster Lifecycle and Cloud Provider] -- Update Go to 1.13.8 ([#87648](https://github.com/kubernetes/kubernetes/pull/87648), [@ialidzhikov](https://github.com/ialidzhikov))[SIG Release and Testing] -- Update Cluster-Autoscaler to 1.18.0 ([#89095](https://github.com/kubernetes/kubernetes/pull/89095), [@losipiuk](https://github.com/losipiuk))[SIG Autoscaling and Cluster Lifecycle] - - - -# v1.18.0-rc.1 - -[Documentation](https://docs.k8s.io) - -## Downloads for v1.18.0-rc.1 - -filename | sha512 hash --------- | ----------- -[kubernetes.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes.tar.gz) | `c17231d5de2e0677e8af8259baa11a388625821c79b86362049f2edb366404d6f4b4587b8f13ccbceeb2f32c6a9fe98607f779c0f3e1caec438f002e3a2c8c21` -[kubernetes-src.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-src.tar.gz) | `e84ffad57c301f5d6e90f916b996d5abb0c987928c3ca6b1565f7b042588f839b994ca12c43fc36f0ffb63f9fabc15110eb08be253b8939f49cd951e956da618` - -### Client Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-client-darwin-386.tar.gz) | `1aea99923d492436b3eb91aaecffac94e5d0aa2b38a0930d266fda85c665bbc4569745c409aa302247df3b578ce60324e7a489eb26240e97d4e65a67428ea3d1` -[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-client-darwin-amd64.tar.gz) | `07fa7340a959740bd52b83ff44438bbd988e235277dad1e43f125f08ac85230a24a3b755f4e4c8645743444fa2b66a3602fc445d7da6d2fc3770e8c21ba24b33` -[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-client-linux-386.tar.gz) | `48cebd26448fdd47aa36257baa4c716a98fda055bbf6a05230f2a3fe3c1b99b4e483668661415392190f3eebb9cb6e15c784626b48bb2541d93a37902f0e3974` -[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-client-linux-amd64.tar.gz) | `c3a5fedf263f07a07f59c01fea6c63c1e0b76ee8dc67c45b6c134255c28ed69171ccc2f91b6a45d6a8ec5570a0a7562e24c33b9d7b0d1a864f4dc04b178b3c04` -[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-client-linux-arm.tar.gz) | `a6b11a55bd38583bbaac14931a6862f8ce6493afe30947ba29e5556654a571593358278df59412bbeb6888fa127e9ae4c0047a9d46cb59394995010796df6b14` -[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-client-linux-arm64.tar.gz) | `9e15331ac8010154a9b64f5488969fc8ee2f21059639896cb84c5cf4f05f4c9d1d8970cb6f9831de6b34013848227c1972c12a698d07aac1ecc056e972fe6f79` -[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-client-linux-ppc64le.tar.gz) | `f828fe6252678de9d4822e482f5873309ae9139b2db87298ab3273ce45d38aa07b6b9b42b76c140705f27ba71e101d58b43e59ac7259d7c08dc647ea809e207c` -[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-client-linux-s390x.tar.gz) | `19da4b45f0666c063934af616f3e7ed3caa99d4ee1e46d53efadc7a8a4d38e43a36ced7249acd7ad3dcc4b4f60d8451b4f7ec7727e478ee2fadd14d353228bce` -[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-client-windows-386.tar.gz) | `775c9afb6cb3e7c4ba53e9f48a5df2cf207234a33059bd74448bc9f177dd120fb3f9c58ab45048a566326acc43bc8a67e886e10ef99f20780c8f63bb17426ebd` -[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-client-windows-amd64.tar.gz) | `208d2595a5b57ac97aac75b4a2a6130f0c937f781a030bde1a432daf4bc51f2fa523fca2eb84c38798489c4b536ee90aad22f7be8477985d9691d51ad8e1c4dc` - -### Server Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-server-linux-amd64.tar.gz) | `dcf832eae04f9f52ff473754ef5cfe697b35f4dc1a282622c94fa10943c8c35f4a8777a0c58c7de871c3c428c8973bf72d6bcd8751416d4c682125268b8fcefe` -[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-server-linux-arm.tar.gz) | `a04e34bea28eb1c8b492e8b1dd3c0dd87ebee71a7dbbef72be10a335e553361af7e48296e504f9844496b04e66350871114d20cfac3f3b49550d8be60f324ba3` -[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-server-linux-arm64.tar.gz) | `a6af086b07a8c2e498f32b43e6511bf6a5e6baf358c572c6910c8df17cd6cae94f562f459714fcead1595767cb14c7f639c5735f1411173bbd38d5604c082a77` -[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-server-linux-ppc64le.tar.gz) | `5a960ef5ba0c255f587f2ac0b028cd03136dc91e4efc5d1becab46417852e5524d18572b6f66259531ec6fea997da3c4d162ac153a9439672154375053fec6c7` -[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-server-linux-s390x.tar.gz) | `0f32c7d9b14bc238b9a5764d8f00edc4d3bf36bcf06b340b81061424e6070768962425194a8c2025c3a7ffb97b1de551d3ad23d1591ae34dd4e3ba25ab364c33` - -### Node Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-node-linux-amd64.tar.gz) | `27d8955d535d14f3f4dca501fd27e4f06fad84c6da878ea5332a5c83b6955667f6f731bfacaf5a3a23c09f14caa400f9bee927a0f269f5374de7f79cd1919b3b` -[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-node-linux-arm.tar.gz) | `0d56eccad63ba608335988e90b377fe8ae978b177dc836cdb803a5c99d99e8f3399a666d9477ca9cfe5964944993e85c416aec10a99323e3246141efc0b1cc9e` -[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-node-linux-arm64.tar.gz) | `79bb9be66f9e892d866b28e5cc838245818edb9706981fab6ccbff493181b341c1fcf6fe5d2342120a112eb93af413f5ba191cfba1ab4c4a8b0546a5ad8ec220` -[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-node-linux-ppc64le.tar.gz) | `3e9e2c6f9a2747d828069511dce8b4034c773c2d122f005f4508e22518055c1e055268d9d86773bbd26fbd2d887d783f408142c6c2f56ab2f2365236fd4d2635` -[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-node-linux-s390x.tar.gz) | `4f96e018c336fa13bb6df6f7217fe46a2b5c47f806f786499c429604ccba2ebe558503ab2c72f63250aa25b61dae2d166e4b80ae10f6ab37d714f87c1dcf6691` -[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-rc.1/kubernetes-node-windows-amd64.tar.gz) | `ab110d76d506746af345e5897ef4f6993d5f53ac818ba69a334f3641047351aa63bfb3582841a9afca51dd0baff8b9010077d9c8ec85d2d69e4172b8d4b338b0` - -## Changelog since v1.18.0-beta.2 - -## Changes by Kind - -### API Change - -- Removes ConfigMap as suggestion for IngressClass parameters ([#89093](https://github.com/kubernetes/kubernetes/pull/89093), [@robscott](https://github.com/robscott)) [SIG Network] - -### Other (Bug, Cleanup or Flake) - -- EndpointSlice should not contain endpoints for terminating pods ([#89056](https://github.com/kubernetes/kubernetes/pull/89056), [@andrewsykim](https://github.com/andrewsykim)) [SIG Apps and Network] -- Fix a bug where ExternalTrafficPolicy is not applied to service ExternalIPs. ([#88786](https://github.com/kubernetes/kubernetes/pull/88786), [@freehan](https://github.com/freehan)) [SIG Network] -- Fix invalid VMSS updates due to incorrect cache ([#89002](https://github.com/kubernetes/kubernetes/pull/89002), [@ArchangelSDY](https://github.com/ArchangelSDY)) [SIG Cloud Provider] -- Fix isCurrentInstance for Windows by removing the dependency of hostname. ([#89138](https://github.com/kubernetes/kubernetes/pull/89138), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Fixed a data race in kubelet image manager that can cause static pod workers to silently stop working. ([#88915](https://github.com/kubernetes/kubernetes/pull/88915), [@roycaihw](https://github.com/roycaihw)) [SIG Node] -- Fixed an issue that could cause the kubelet to incorrectly run concurrent pod reconciliation loops and crash. ([#89055](https://github.com/kubernetes/kubernetes/pull/89055), [@tedyu](https://github.com/tedyu)) [SIG Node] -- Kube-proxy: on dual-stack mode, if it is not able to get the IP Family of an endpoint, logs it with level InfoV(4) instead of Warning, avoiding flooding the logs for endpoints without addresses ([#88934](https://github.com/kubernetes/kubernetes/pull/88934), [@aojea](https://github.com/aojea)) [SIG Network] -- Update Cluster Autoscaler to 1.18.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.18.0 ([#89095](https://github.com/kubernetes/kubernetes/pull/89095), [@losipiuk](https://github.com/losipiuk)) [SIG Autoscaling and Cluster Lifecycle] - - -# v1.18.0-beta.2 - -[Documentation](https://docs.k8s.io) - -## Downloads for v1.18.0-beta.2 - -filename | sha512 hash --------- | ----------- -[kubernetes.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes.tar.gz) | `3017430ca17f8a3523669b4a02c39cedfc6c48b07281bc0a67a9fbe9d76547b76f09529172cc01984765353a6134a43733b7315e0dff370bba2635dd2a6289af` -[kubernetes-src.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-src.tar.gz) | `c5fd60601380a99efff4458b1c9cf4dc02195f6f756b36e590e54dff68f7064daf32cf63980dddee13ef9dec7a60ad4eeb47a288083fdbbeeef4bc038384e9ea` - -### Client Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-client-darwin-386.tar.gz) | `7e49ede167b9271d4171e477fa21d267b2fb35f80869337d5b323198dc12f71b61441975bf925ad6e6cd7b61cbf6372d386417dc1e5c9b3c87ae651021c37237` -[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-client-darwin-amd64.tar.gz) | `3f5cdf0e85eee7d0773e0ae2df1c61329dea90e0da92b02dae1ffd101008dc4bade1c4951fc09f0cad306f0bcb7d16da8654334ddee43d5015913cc4ac8f3eda` -[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-client-linux-386.tar.gz) | `b67b41c11bfecb88017c33feee21735c56f24cf6f7851b63c752495fc0fb563cd417a67a81f46bca091f74dc00fca1f296e483d2e3dfe2004ea4b42e252d30b9` -[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-client-linux-amd64.tar.gz) | `1fef2197cb80003e3a5c26f05e889af9d85fbbc23e27747944d2997ace4bfa28f3670b13c08f5e26b7e274176b4e2df89c1162aebd8b9506e63b39b311b2d405` -[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-client-linux-arm.tar.gz) | `84e5f4d9776490219ee94a84adccd5dfc7c0362eb330709771afcde95ec83f03d96fe7399eec218e47af0a1e6445e24d95e6f9c66c0882ef8233a09ff2022420` -[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-client-linux-arm64.tar.gz) | `ba613b114e0cca32fa21a3d10f845aa2f215d3af54e775f917ff93919f7dd7075efe254e4047a85a1f4b817fc2bd78006c2e8873885f1208cbc02db99e2e2e25` -[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-client-linux-ppc64le.tar.gz) | `502a6938d8c4bbe04abbd19b59919d86765058ff72334848be4012cec493e0e7027c6cd950cf501367ac2026eea9f518110cb72d1c792322b396fc2f73d23217` -[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-client-linux-s390x.tar.gz) | `c24700e0ed2ef5c1d2dd282d638c88d90392ae90ea420837b39fd8e1cfc19525017325ccda71d8472fdaea174762208c09e1bba9bbc77c89deef6fac5e847ba2` -[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-client-windows-386.tar.gz) | `0d4c5a741b052f790c8b0923c9586ee9906225e51cf4dc8a56fc303d4d61bb5bf77fba9e65151dec7be854ff31da8fc2dcd3214563e1b4b9951e6af4aa643da4` -[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-client-windows-amd64.tar.gz) | `841ef2e306c0c9593f04d9528ee019bf3b667761227d9afc1d6ca8bf1aa5631dc25f5fe13ff329c4bf0c816b971fd0dec808f879721e0f3bf51ce49772b38010` - -### Server Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-server-linux-amd64.tar.gz) | `b373df2e6ef55215e712315a5508e85a39126bd81b7b93c6b6305238919a88c740077828a6f19bcd97141951048ef7a19806ef6b1c3e1772dbc45715c5fcb3af` -[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-server-linux-arm.tar.gz) | `b8103cb743c23076ce8dd7c2da01c8dd5a542fbac8480e82dc673139c8ee5ec4495ca33695e7a18dd36412cf1e18ed84c8de05042525ddd8e869fbdfa2766569` -[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-server-linux-arm64.tar.gz) | `8f8f05cf64fb9c8d80cdcb4935b2d3e3edc48bdd303231ae12f93e3f4d979237490744a11e24ba7f52dbb017ca321a8e31624dcffa391b8afda3d02078767fa0` -[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-server-linux-ppc64le.tar.gz) | `b313b911c46f2ec129537407af3f165f238e48caeb4b9e530783ffa3659304a544ed02bef8ece715c279373b9fb2c781bd4475560e02c4b98a6d79837bc81938` -[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-server-linux-s390x.tar.gz) | `a1b6b06571141f507b12e5ef98efb88f4b6b9aba924722b2a74f11278d29a2972ab8290608360151d124608e6e24da0eb3516d484cb5fa12ff2987562f15964a` - -### Node Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-node-linux-amd64.tar.gz) | `20e02ca327543cddb2568ead3d5de164cbfb2914ab6416106d906bf12fcfbc4e55b13bea4d6a515e8feab038e2c929d72c4d6909dfd7881ba69fd1e8c772ab99` -[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-node-linux-arm.tar.gz) | `ecd817ef05d6284f9c6592b84b0a48ea31cf4487030c9fb36518474b2a33dad11b9c852774682e60e4e8b074e6bea7016584ca281dddbe2994da5eaf909025c0` -[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-node-linux-arm64.tar.gz) | `0020d32b7908ffd5055c8b26a8b3033e4702f89efcfffe3f6fcdb8a9921fa8eaaed4193c85597c24afd8c523662454f233521bb7055841a54c182521217ccc9d` -[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-node-linux-ppc64le.tar.gz) | `e065411d66d486e7793449c1b2f5a412510b913bf7f4e728c0a20e275642b7668957050dc266952cdff09acc391369ae6ac5230184db89af6823ba400745f2fc` -[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-node-linux-s390x.tar.gz) | `082ee90413beaaea41d6cbe9a18f7d783a95852607f3b94190e0ca12aacdd97d87e233b87117871bfb7d0a4b6302fbc7688549492a9bc50a2f43a5452504d3ce` -[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.2/kubernetes-node-windows-amd64.tar.gz) | `fb5aca0cc36be703f9d4033eababd581bac5de8399c50594db087a99ed4cb56e4920e960eb81d0132d696d094729254eeda2a5c0cb6e65e3abca6c8d61da579e` - -## Changelog since v1.18.0-beta.1 - -## Urgent Upgrade Notes - -### (No, really, you MUST read this before you upgrade) - -- `kubectl` no longer defaults to `http://localhost:8080`. If you own one of these legacy clusters, you are *strongly- encouraged to secure your server. If you cannot secure your server, you can set `KUBERNETES_MASTER` if you were relying on that behavior and you're a client-go user. Set `--server`, `--kubeconfig` or `KUBECONFIG` to make it work in `kubectl`. ([#86173](https://github.com/kubernetes/kubernetes/pull/86173), [@soltysh](https://github.com/soltysh)) [SIG API Machinery, CLI and Testing] - -## Changes by Kind - -### Deprecation - -- AlgorithmSource is removed from v1alpha2 Scheduler ComponentConfig ([#87999](https://github.com/kubernetes/kubernetes/pull/87999), [@damemi](https://github.com/damemi)) [SIG Scheduling] -- Kube-proxy: deprecate `--healthz-port` and `--metrics-port` flag, please use `--healthz-bind-address` and `--metrics-bind-address` instead ([#88512](https://github.com/kubernetes/kubernetes/pull/88512), [@SataQiu](https://github.com/SataQiu)) [SIG Network] -- Kubeadm: deprecate the usage of the experimental flag '--use-api' under the 'kubeadm alpha certs renew' command. ([#88827](https://github.com/kubernetes/kubernetes/pull/88827), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] - -### API Change - -- A new IngressClass resource has been added to enable better Ingress configuration. ([#88509](https://github.com/kubernetes/kubernetes/pull/88509), [@robscott](https://github.com/robscott)) [SIG API Machinery, Apps, CLI, Network, Node and Testing] -- Added GenericPVCDataSource feature gate to enable using arbitrary custom resources as the data source for a PVC. ([#88636](https://github.com/kubernetes/kubernetes/pull/88636), [@bswartz](https://github.com/bswartz)) [SIG Apps and Storage] -- Allow user to specify fsgroup permission change policy for pods ([#88488](https://github.com/kubernetes/kubernetes/pull/88488), [@gnufied](https://github.com/gnufied)) [SIG Apps and Storage] -- BlockVolume and CSIBlockVolume features are now GA. ([#88673](https://github.com/kubernetes/kubernetes/pull/88673), [@jsafrane](https://github.com/jsafrane)) [SIG Apps, Node and Storage] -- CustomResourceDefinition schemas that use `x-kubernetes-list-map-keys` to specify properties that uniquely identify list items must make those properties required or have a default value, to ensure those properties are present for all list items. See https://kubernetes.io/docs/reference/using-api/api-concepts/#merge-strategy for details. ([#88076](https://github.com/kubernetes/kubernetes/pull/88076), [@eloyekunle](https://github.com/eloyekunle)) [SIG API Machinery and Testing] -- Fixes a regression with clients prior to 1.15 not being able to update podIP in pod status, or podCIDR in node spec, against >= 1.16 API servers ([#88505](https://github.com/kubernetes/kubernetes/pull/88505), [@liggitt](https://github.com/liggitt)) [SIG Apps and Network] -- Ingress: Add Exact and Prefix maching to Ingress PathTypes ([#88587](https://github.com/kubernetes/kubernetes/pull/88587), [@cmluciano](https://github.com/cmluciano)) [SIG Apps, Cluster Lifecycle and Network] -- Ingress: Add alternate backends via TypedLocalObjectReference ([#88775](https://github.com/kubernetes/kubernetes/pull/88775), [@cmluciano](https://github.com/cmluciano)) [SIG Apps and Network] -- Ingress: allow wildcard hosts in IngressRule ([#88858](https://github.com/kubernetes/kubernetes/pull/88858), [@cmluciano](https://github.com/cmluciano)) [SIG Network] -- Kube-controller-manager and kube-scheduler expose profiling by default to match the kube-apiserver. Use `--enable-profiling=false` to disable. ([#88663](https://github.com/kubernetes/kubernetes/pull/88663), [@deads2k](https://github.com/deads2k)) [SIG API Machinery, Cloud Provider and Scheduling] -- Move TaintBasedEvictions feature gates to GA ([#87487](https://github.com/kubernetes/kubernetes/pull/87487), [@skilxn-go](https://github.com/skilxn-go)) [SIG API Machinery, Apps, Node, Scheduling and Testing] -- New flag --endpointslice-updates-batch-period in kube-controller-manager can be used to reduce number of endpointslice updates generated by pod changes. ([#88745](https://github.com/kubernetes/kubernetes/pull/88745), [@mborsz](https://github.com/mborsz)) [SIG API Machinery, Apps and Network] -- Scheduler Extenders can now be configured in the v1alpha2 component config ([#88768](https://github.com/kubernetes/kubernetes/pull/88768), [@damemi](https://github.com/damemi)) [SIG Release, Scheduling and Testing] -- The apiserver/v1alph1#EgressSelectorConfiguration API is now beta. ([#88502](https://github.com/kubernetes/kubernetes/pull/88502), [@caesarxuchao](https://github.com/caesarxuchao)) [SIG API Machinery] -- The storage.k8s.io/CSIDriver has moved to GA, and is now available for use. ([#84814](https://github.com/kubernetes/kubernetes/pull/84814), [@huffmanca](https://github.com/huffmanca)) [SIG API Machinery, Apps, Auth, Node, Scheduling, Storage and Testing] -- VolumePVCDataSource moves to GA in 1.18 release ([#88686](https://github.com/kubernetes/kubernetes/pull/88686), [@j-griffith](https://github.com/j-griffith)) [SIG Apps, CLI and Cluster Lifecycle] - -### Feature - -- Add `rest_client_rate_limiter_duration_seconds` metric to component-base to track client side rate limiter latency in seconds. Broken down by verb and URL. ([#88134](https://github.com/kubernetes/kubernetes/pull/88134), [@jennybuckley](https://github.com/jennybuckley)) [SIG API Machinery, Cluster Lifecycle and Instrumentation] -- Allow user to specify resource using --filename flag when invoking kubectl exec ([#88460](https://github.com/kubernetes/kubernetes/pull/88460), [@soltysh](https://github.com/soltysh)) [SIG CLI and Testing] -- Apiserver add a new flag --goaway-chance which is the fraction of requests that will be closed gracefully(GOAWAY) to prevent HTTP/2 clients from getting stuck on a single apiserver. - After the connection closed(received GOAWAY), the client's other in-flight requests won't be affected, and the client will reconnect. - The flag min value is 0 (off), max is .02 (1/50 requests); .001 (1/1000) is a recommended starting point. - Clusters with single apiservers, or which don't use a load balancer, should NOT enable this. ([#88567](https://github.com/kubernetes/kubernetes/pull/88567), [@answer1991](https://github.com/answer1991)) [SIG API Machinery] -- Azure: add support for single stack IPv6 ([#88448](https://github.com/kubernetes/kubernetes/pull/88448), [@aramase](https://github.com/aramase)) [SIG Cloud Provider] -- DefaultConstraints can be specified for the PodTopologySpread plugin in the component config ([#88671](https://github.com/kubernetes/kubernetes/pull/88671), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling] -- Kubeadm: support Windows specific kubelet flags in kubeadm-flags.env ([#88287](https://github.com/kubernetes/kubernetes/pull/88287), [@gab-satchi](https://github.com/gab-satchi)) [SIG Cluster Lifecycle and Windows] -- Kubectl cluster-info dump changed to only display a message telling you the location where the output was written when the output is not standard output. ([#88765](https://github.com/kubernetes/kubernetes/pull/88765), [@brianpursley](https://github.com/brianpursley)) [SIG CLI] -- Print NotReady when pod is not ready based on its conditions. ([#88240](https://github.com/kubernetes/kubernetes/pull/88240), [@soltysh](https://github.com/soltysh)) [SIG CLI] -- Scheduler Extender API is now located under k8s.io/kube-scheduler/extender ([#88540](https://github.com/kubernetes/kubernetes/pull/88540), [@damemi](https://github.com/damemi)) [SIG Release, Scheduling and Testing] -- Signatures on scale client methods have been modified to accept `context.Context` as a first argument. Signatures of Get, Update, and Patch methods have been updated to accept GetOptions, UpdateOptions and PatchOptions respectively. ([#88599](https://github.com/kubernetes/kubernetes/pull/88599), [@julianvmodesto](https://github.com/julianvmodesto)) [SIG API Machinery, Apps, Autoscaling and CLI] -- Signatures on the dynamic client methods have been modified to accept `context.Context` as a first argument. Signatures of Delete and DeleteCollection methods now accept DeleteOptions by value instead of by reference. ([#88906](https://github.com/kubernetes/kubernetes/pull/88906), [@liggitt](https://github.com/liggitt)) [SIG API Machinery, Apps, CLI, Cluster Lifecycle, Storage and Testing] -- Signatures on the metadata client methods have been modified to accept `context.Context` as a first argument. Signatures of Delete and DeleteCollection methods now accept DeleteOptions by value instead of by reference. ([#88910](https://github.com/kubernetes/kubernetes/pull/88910), [@liggitt](https://github.com/liggitt)) [SIG API Machinery, Apps and Testing] -- Webhooks will have alpha support for network proxy ([#85870](https://github.com/kubernetes/kubernetes/pull/85870), [@Jefftree](https://github.com/Jefftree)) [SIG API Machinery, Auth and Testing] -- When client certificate files are provided, reload files for new connections, and close connections when a certificate changes. ([#79083](https://github.com/kubernetes/kubernetes/pull/79083), [@jackkleeman](https://github.com/jackkleeman)) [SIG API Machinery, Auth, Node and Testing] -- When deleting objects using kubectl with the --force flag, you are no longer required to also specify --grace-period=0. ([#87776](https://github.com/kubernetes/kubernetes/pull/87776), [@brianpursley](https://github.com/brianpursley)) [SIG CLI] -- `kubectl` now contains a `kubectl alpha debug` command. This command allows attaching an ephemeral container to a running pod for the purposes of debugging. ([#88004](https://github.com/kubernetes/kubernetes/pull/88004), [@verb](https://github.com/verb)) [SIG CLI] +- A client-go metric, rest_client_exec_plugin_call_total, has been added to track total calls to client-go credential plugins. ([#98892](https://github.com/kubernetes/kubernetes/pull/98892), [@ankeesler](https://github.com/ankeesler)) [SIG API Machinery, Auth, Cluster Lifecycle and Instrumentation] +- A new histogram metric to track the time it took to delete a job by the `TTLAfterFinished` controller ([#98676](https://github.com/kubernetes/kubernetes/pull/98676), [@ahg-g](https://github.com/ahg-g)) +- AWS cloud provider supports auto-discovering subnets without any `kubernetes.io/cluster/` tags. It also supports additional service annotation `service.beta.kubernetes.io/aws-load-balancer-subnets` to manually configure the subnets. ([#97431](https://github.com/kubernetes/kubernetes/pull/97431), [@kishorj](https://github.com/kishorj)) +- Aborting the drain command in a list of nodes will be deprecated. The new behavior will make the drain command go through all nodes even if one or more nodes failed during the drain. For now, users can try such experience by enabling --ignore-errors flag. ([#98203](https://github.com/kubernetes/kubernetes/pull/98203), [@yuzhiquan](https://github.com/yuzhiquan)) +- Add --permit-address-sharing flag to `kube-apiserver` to listen with `SO_REUSEADDR`. While allowing to listen on wildcard IPs like 0.0.0.0 and specific IPs in parallel, it avoids waiting for the kernel to release socket in `TIME_WAIT` state, and hence, considerably reducing `kube-apiserver` restart times under certain conditions. ([#93861](https://github.com/kubernetes/kubernetes/pull/93861), [@sttts](https://github.com/sttts)) +- Add `csi_operations_seconds` metric on kubelet that exposes CSI operations duration and status for node CSI operations. ([#98979](https://github.com/kubernetes/kubernetes/pull/98979), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Instrumentation and Storage] +- Add `migrated` field into `storage_operation_duration_seconds` metric ([#99050](https://github.com/kubernetes/kubernetes/pull/99050), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Apps, Instrumentation and Storage] +- Add flag --lease-reuse-duration-seconds for kube-apiserver to config etcd lease reuse duration. ([#97009](https://github.com/kubernetes/kubernetes/pull/97009), [@lingsamuel](https://github.com/lingsamuel)) [SIG API Machinery and Scalability] +- Add metric etcd_lease_object_counts for kube-apiserver to observe max objects attached to a single etcd lease. ([#97480](https://github.com/kubernetes/kubernetes/pull/97480), [@lingsamuel](https://github.com/lingsamuel)) [SIG API Machinery, Instrumentation and Scalability] +- Add support to generate client-side binaries for new darwin/arm64 platform ([#97743](https://github.com/kubernetes/kubernetes/pull/97743), [@dims](https://github.com/dims)) [SIG Release and Testing] +- Added `ephemeral_volume_controller_create[_failures]_total` counters to kube-controller-manager metrics ([#99115](https://github.com/kubernetes/kubernetes/pull/99115), [@pohly](https://github.com/pohly)) [SIG API Machinery, Apps, Cluster Lifecycle, Instrumentation and Storage] +- Added support for installing `arm64` node artifacts. ([#99242](https://github.com/kubernetes/kubernetes/pull/99242), [@liu-cong](https://github.com/liu-cong)) +- Adds alpha feature `VolumeCapacityPriority` which makes the scheduler prioritize nodes based on the best matching size of statically provisioned PVs across multiple topologies. ([#96347](https://github.com/kubernetes/kubernetes/pull/96347), [@cofyc](https://github.com/cofyc)) [SIG Apps, Network, Scheduling, Storage and Testing] +- Adds the ability to pass --strict-transport-security-directives to the kube-apiserver to set the HSTS header appropriately. Be sure you understand the consequences to browsers before setting this field. ([#96502](https://github.com/kubernetes/kubernetes/pull/96502), [@249043822](https://github.com/249043822)) [SIG Auth] +- Adds two new metrics to cronjobs, a histogram to track the time difference when a job is created and the expected time when it should be created, as well as a gauge for the missed schedules of a cronjob ([#99341](https://github.com/kubernetes/kubernetes/pull/99341), [@alaypatel07](https://github.com/alaypatel07)) +- Alpha implementation of Kubectl Command Headers: SIG CLI KEP 859 enabled when KUBECTL_COMMAND_HEADERS environment variable set on the client command line. ([#98952](https://github.com/kubernetes/kubernetes/pull/98952), [@seans3](https://github.com/seans3)) +- Base-images: Update to debian-iptables:buster-v1.4.0 + - Uses iptables 1.8.5 + - base-images: Update to debian-base:buster-v1.3.0 + - cluster/images/etcd: Build etcd:3.4.13-2 image + - Uses debian-base:buster-v1.3.0 ([#98401](https://github.com/kubernetes/kubernetes/pull/98401), [@pacoxu](https://github.com/pacoxu)) [SIG Testing] +- CRIContainerLogRotation graduates to GA and unconditionally enabled. ([#99651](https://github.com/kubernetes/kubernetes/pull/99651), [@umohnani8](https://github.com/umohnani8)) +- Component owner can configure the allowlist of metric label with flag '--allow-metric-labels'. ([#99385](https://github.com/kubernetes/kubernetes/pull/99385), [@YoyinZyc](https://github.com/YoyinZyc)) [SIG API Machinery, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation and Release] +- Component owner can configure the allowlist of metric label with flag '--allow-metric-labels'. ([#99738](https://github.com/kubernetes/kubernetes/pull/99738), [@YoyinZyc](https://github.com/YoyinZyc)) [SIG API Machinery, Cluster Lifecycle and Instrumentation] +- EmptyDir memory backed volumes are sized as the the minimum of pod allocatable memory on a host and an optional explicit user provided value. ([#100319](https://github.com/kubernetes/kubernetes/pull/100319), [@derekwaynecarr](https://github.com/derekwaynecarr)) [SIG Node] +- Enables Kubelet to check volume condition and log events to corresponding pods. ([#99284](https://github.com/kubernetes/kubernetes/pull/99284), [@fengzixu](https://github.com/fengzixu)) [SIG Apps, Instrumentation, Node and Storage] +- EndpointSliceNodeName graduates to GA and thus will be unconditionally enabled -- NodeName will always be available in the v1beta1 API. ([#99746](https://github.com/kubernetes/kubernetes/pull/99746), [@swetharepakula](https://github.com/swetharepakula)) +- Export `NewDebuggingRoundTripper` function and `DebugLevel` options in the k8s.io/client-go/transport package. ([#98324](https://github.com/kubernetes/kubernetes/pull/98324), [@atosatto](https://github.com/atosatto)) +- Kube-proxy iptables: new metric sync_proxy_rules_iptables_total that exposes the number of rules programmed per table in each iteration ([#99653](https://github.com/kubernetes/kubernetes/pull/99653), [@aojea](https://github.com/aojea)) [SIG Instrumentation and Network] +- Kube-scheduler now logs plugin scoring summaries at --v=4 ([#99411](https://github.com/kubernetes/kubernetes/pull/99411), [@damemi](https://github.com/damemi)) [SIG Scheduling] +- Kubeadm now includes CoreDNS v1.8.0. ([#96429](https://github.com/kubernetes/kubernetes/pull/96429), [@rajansandeep](https://github.com/rajansandeep)) [SIG Cluster Lifecycle] +- Kubeadm: IPv6DualStack feature gate graduates to Beta and enabled by default ([#99294](https://github.com/kubernetes/kubernetes/pull/99294), [@pacoxu](https://github.com/pacoxu)) +- Kubeadm: a warning to user as ipv6 site-local is deprecated ([#99574](https://github.com/kubernetes/kubernetes/pull/99574), [@pacoxu](https://github.com/pacoxu)) [SIG Cluster Lifecycle and Network] +- Kubeadm: add support for certificate chain validation. When using kubeadm in external CA mode, this allows an intermediate CA to be used to sign the certificates. The intermediate CA certificate must be appended to each signed certificate for this to work correctly. ([#97266](https://github.com/kubernetes/kubernetes/pull/97266), [@robbiemcmichael](https://github.com/robbiemcmichael)) [SIG Cluster Lifecycle] +- Kubeadm: amend the node kernel validation to treat CGROUP_PIDS, FAIR_GROUP_SCHED as required and CFS_BANDWIDTH, CGROUP_HUGETLB as optional ([#96378](https://github.com/kubernetes/kubernetes/pull/96378), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle and Node] +- Kubeadm: apply the "node.kubernetes.io/exclude-from-external-load-balancers" label on control plane nodes during "init", "join" and "upgrade" to preserve backwards compatibility with the lagacy LB mode where nodes labeled as "master" where excluded. To opt-out you can remove the label from a node. See #97543 and the linked KEP for more details. ([#98269](https://github.com/kubernetes/kubernetes/pull/98269), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] +- Kubeadm: if the user has customized their image repository via the kubeadm configuration, pass the custom pause image repository and tag to the kubelet via --pod-infra-container-image not only for Docker but for all container runtimes. This flag tells the kubelet that it should not garbage collect the image. ([#99476](https://github.com/kubernetes/kubernetes/pull/99476), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] +- Kubeadm: perform pre-flight validation on host/node name upon `kubeadm init` and `kubeadm join`, showing warnings on non-compliant names ([#99194](https://github.com/kubernetes/kubernetes/pull/99194), [@pacoxu](https://github.com/pacoxu)) +- Kubectl version changed to write a warning message to stderr if the client and server version difference exceeds the supported version skew of +/-1 minor version. ([#98250](https://github.com/kubernetes/kubernetes/pull/98250), [@brianpursley](https://github.com/brianpursley)) [SIG CLI] +- Kubectl: Add `--use-protocol-buffers` flag to kubectl top pods and nodes. ([#96655](https://github.com/kubernetes/kubernetes/pull/96655), [@serathius](https://github.com/serathius)) +- Kubectl: `kubectl get` will omit managed fields by default now. Users could set `--show-managed-fields` to true to show managedFields when the output format is either `json` or `yaml`. ([#96878](https://github.com/kubernetes/kubernetes/pull/96878), [@knight42](https://github.com/knight42)) [SIG CLI and Testing] +- Kubectl: a Pod can be preselected as default container using `kubectl.kubernetes.io/default-container` annotation ([#99833](https://github.com/kubernetes/kubernetes/pull/99833), [@mengjiao-liu](https://github.com/mengjiao-liu)) +- Kubectl: add bash-completion for comma separated list on `kubectl get` ([#98301](https://github.com/kubernetes/kubernetes/pull/98301), [@phil9909](https://github.com/phil9909)) +- Kubernetes is now built using go1.15.8 ([#98834](https://github.com/kubernetes/kubernetes/pull/98834), [@cpanato](https://github.com/cpanato)) [SIG Cloud Provider, Instrumentation, Release and Testing] +- Kubernetes is now built with Golang 1.16 ([#98572](https://github.com/kubernetes/kubernetes/pull/98572), [@justaugustus](https://github.com/justaugustus)) [SIG API Machinery, Auth, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation, Node, Release and Testing] +- Kubernetes is now built with Golang 1.16.1 ([#100106](https://github.com/kubernetes/kubernetes/pull/100106), [@justaugustus](https://github.com/justaugustus)) [SIG Cloud Provider, Instrumentation, Release and Testing] +- Metrics can now be disabled explicitly via a command line flag (i.e. '--disabled-metrics=metric1,metric2') ([#99217](https://github.com/kubernetes/kubernetes/pull/99217), [@logicalhan](https://github.com/logicalhan)) +- New admission controller `DenyServiceExternalIPs` is available. Clusters which do not *need* the Service `externalIPs` feature should enable this controller and be more secure. ([#97395](https://github.com/kubernetes/kubernetes/pull/97395), [@thockin](https://github.com/thockin)) +- Overall, enable the feature of `PreferNominatedNode` will improve the performance of scheduling where preemption might frequently happen, but in theory, enable the feature of `PreferNominatedNode`, the pod might not be scheduled to the best candidate node in the cluster. ([#93179](https://github.com/kubernetes/kubernetes/pull/93179), [@chendave](https://github.com/chendave)) [SIG Scheduling and Testing] +- Persistent Volumes formatted with the btrfs filesystem will now automatically resize when expanded. ([#99361](https://github.com/kubernetes/kubernetes/pull/99361), [@Novex](https://github.com/Novex)) [SIG Storage] +- Port the devicemanager to Windows node to allow device plugins like directx ([#93285](https://github.com/kubernetes/kubernetes/pull/93285), [@aarnaud](https://github.com/aarnaud)) [SIG Node, Testing and Windows] +- Removes cAdvisor JSON metrics (/stats/container, /stats//, /stats////) from the kubelet. ([#99236](https://github.com/kubernetes/kubernetes/pull/99236), [@pacoxu](https://github.com/pacoxu)) +- Rename metrics `etcd_object_counts` to `apiserver_storage_object_counts` and mark it as stable. The original `etcd_object_counts` metrics name is marked as "Deprecated" and will be removed in the future. ([#99785](https://github.com/kubernetes/kubernetes/pull/99785), [@erain](https://github.com/erain)) [SIG API Machinery, Instrumentation and Testing] +- Sysctls graduates to General Availability and thus unconditionally enabled. ([#99158](https://github.com/kubernetes/kubernetes/pull/99158), [@wgahnagl](https://github.com/wgahnagl)) +- The Kubernetes pause image manifest list now contains an image for Windows Server 20H2. ([#97322](https://github.com/kubernetes/kubernetes/pull/97322), [@claudiubelu](https://github.com/claudiubelu)) [SIG Windows] +- The NodeAffinity plugin implements the PreFilter extension, offering enhanced performance for Filter. ([#99213](https://github.com/kubernetes/kubernetes/pull/99213), [@AliceZhang2016](https://github.com/AliceZhang2016)) [SIG Scheduling] +- The `CronJobControllerV2` feature flag graduates to Beta and set to be enabled by default. ([#98878](https://github.com/kubernetes/kubernetes/pull/98878), [@soltysh](https://github.com/soltysh)) +- The `EndpointSlice` mirroring controller mirrors endpoints annotations and labels to the generated endpoint slices, it also ensures that updates on any of these fields are mirrored. + The well-known annotation `endpoints.kubernetes.io/last-change-trigger-time` is skipped and not mirrored. ([#98116](https://github.com/kubernetes/kubernetes/pull/98116), [@aojea](https://github.com/aojea)) +- The `RunAsGroup` feature has been promoted to GA in this release. ([#94641](https://github.com/kubernetes/kubernetes/pull/94641), [@krmayankk](https://github.com/krmayankk)) [SIG Auth and Node] +- The `ServiceAccountIssuerDiscovery` feature has graduated to GA, and is unconditionally enabled. The `ServiceAccountIssuerDiscovery` feature-gate will be removed in 1.22. ([#98553](https://github.com/kubernetes/kubernetes/pull/98553), [@mtaufen](https://github.com/mtaufen)) [SIG API Machinery, Auth and Testing] +- The `TTLAfterFinished` feature flag is now beta and enabled by default ([#98678](https://github.com/kubernetes/kubernetes/pull/98678), [@ahg-g](https://github.com/ahg-g)) +- The apimachinery util/net function used to detect the bind address `ResolveBindAddress()` takes into consideration global IP addresses on loopback interfaces when 1) the host has default routes, or 2) there are no global IPs on those interfaces in order to support more complex network scenarios like BGP Unnumbered RFC 5549 ([#95790](https://github.com/kubernetes/kubernetes/pull/95790), [@aojea](https://github.com/aojea)) [SIG Network] +- The feature gate `RootCAConfigMap` graduated to GA in v1.21 and therefore will be unconditionally enabled. This flag will be removed in v1.22 release. ([#98033](https://github.com/kubernetes/kubernetes/pull/98033), [@zshihang](https://github.com/zshihang)) +- The pause image upgraded to `v3.4.1` in kubelet and kubeadm for both Linux and Windows. ([#98205](https://github.com/kubernetes/kubernetes/pull/98205), [@pacoxu](https://github.com/pacoxu)) +- Update pause container to run as pseudo user and group `65535:65535`. This implies the release of version 3.5 of the container images. ([#97963](https://github.com/kubernetes/kubernetes/pull/97963), [@saschagrunert](https://github.com/saschagrunert)) [SIG CLI, Cloud Provider, Cluster Lifecycle, Node, Release, Security and Testing] +- Update the latest validated version of Docker to 20.10 ([#98977](https://github.com/kubernetes/kubernetes/pull/98977), [@neolit123](https://github.com/neolit123)) [SIG CLI, Cluster Lifecycle and Node] +- Upgrade node local dns to 1.17.0 for better IPv6 support ([#99749](https://github.com/kubernetes/kubernetes/pull/99749), [@pacoxu](https://github.com/pacoxu)) [SIG Cloud Provider and Network] +- Upgrades `IPv6Dualstack` to `Beta` and turns it on by default. New clusters or existing clusters are not be affected until an actor starts adding secondary Pods and service CIDRS CLI flags as described here: [IPv4/IPv6 Dual-stack](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/563-dual-stack) ([#98969](https://github.com/kubernetes/kubernetes/pull/98969), [@khenidak](https://github.com/khenidak)) +- Users might specify the `kubectl.kubernetes.io/default-container` annotation in a Pod to preselect container for kubectl commands. ([#99581](https://github.com/kubernetes/kubernetes/pull/99581), [@mengjiao-liu](https://github.com/mengjiao-liu)) [SIG CLI] +- When downscaling ReplicaSets, ready and creation timestamps are compared in a logarithmic scale. ([#99212](https://github.com/kubernetes/kubernetes/pull/99212), [@damemi](https://github.com/damemi)) [SIG Apps and Testing] +- When the kubelet is watching a ConfigMap or Secret purely in the context of setting environment variables + for containers, only hold that watch for a defined duration before cancelling it. This change reduces the CPU + and memory usage of the kube-apiserver in large clusters. ([#99393](https://github.com/kubernetes/kubernetes/pull/99393), [@chenyw1990](https://github.com/chenyw1990)) [SIG API Machinery, Node and Testing] +- WindowsEndpointSliceProxying feature gate has graduated to beta and is enabled by default. This means kube-proxy will read from EndpointSlices instead of Endpoints on Windows by default. ([#99794](https://github.com/kubernetes/kubernetes/pull/99794), [@robscott](https://github.com/robscott)) [SIG Network] +- `kubectl wait` ensures that observedGeneration >= generation to prevent stale state reporting. An example scenario can be found on CRD updates. ([#97408](https://github.com/kubernetes/kubernetes/pull/97408), [@KnicKnic](https://github.com/KnicKnic)) ### Documentation -- Update Japanese translation for kubectl help ([#86837](https://github.com/kubernetes/kubernetes/pull/86837), [@inductor](https://github.com/inductor)) [SIG CLI and Docs] -- `kubectl plugin` now prints a note how to install krew ([#88577](https://github.com/kubernetes/kubernetes/pull/88577), [@corneliusweig](https://github.com/corneliusweig)) [SIG CLI] +- Azure file migration graduates to beta, with CSIMigrationAzureFile flag off by default + as it requires installation of AzureFile CSI Driver. Users should enable CSIMigration and + CSIMigrationAzureFile features and install the [AzureFile CSI Driver](https://github.com/kubernetes-sigs/azurefile-csi-driver) + to avoid disruption to existing Pod and PVC objects at that time. Azure File CSI driver does not support using same persistent + volume with different fsgroups. When CSI migration is enabled for azurefile driver, such case is not supported. + (there is a case we support where volume is mounted with 0777 and then it readable/writable by everyone) ([#96293](https://github.com/kubernetes/kubernetes/pull/96293), [@andyzhangx](https://github.com/andyzhangx)) +- Official support to build kubernetes with docker-machine / remote docker is removed. This change does not affect building kubernetes with docker locally. ([#97935](https://github.com/kubernetes/kubernetes/pull/97935), [@adeniyistephen](https://github.com/adeniyistephen)) [SIG Release and Testing] +- Set kubelet option `--volume-stats-agg-period` to negative value to disable volume calculations. ([#96675](https://github.com/kubernetes/kubernetes/pull/96675), [@pacoxu](https://github.com/pacoxu)) [SIG Node] -### Other (Bug, Cleanup or Flake) +### Failing Test -- Azure VMSS LoadBalancerBackendAddressPools updating has been improved with squential-sync + concurrent-async requests. ([#88699](https://github.com/kubernetes/kubernetes/pull/88699), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- AzureFile and CephFS use new Mount library that prevents logging of sensitive mount options. ([#88684](https://github.com/kubernetes/kubernetes/pull/88684), [@saad-ali](https://github.com/saad-ali)) [SIG API Machinery, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation and Storage] -- Build: Enable kube-cross image-building on K8s Infra ([#88562](https://github.com/kubernetes/kubernetes/pull/88562), [@justaugustus](https://github.com/justaugustus)) [SIG Release and Testing] -- Client-go certificate manager rotation gained the ability to preserve optional intermediate chains accompanying issued certificates ([#88744](https://github.com/kubernetes/kubernetes/pull/88744), [@jackkleeman](https://github.com/jackkleeman)) [SIG API Machinery and Auth] -- Conformance image now depends on stretch-slim instead of debian-hyperkube-base as that image is being deprecated and removed. ([#88702](https://github.com/kubernetes/kubernetes/pull/88702), [@dims](https://github.com/dims)) [SIG Cluster Lifecycle, Release and Testing] -- Deprecate --generator flag from kubectl create commands ([#88655](https://github.com/kubernetes/kubernetes/pull/88655), [@soltysh](https://github.com/soltysh)) [SIG CLI] -- FIX: prevent apiserver from panicking when failing to load audit webhook config file ([#88879](https://github.com/kubernetes/kubernetes/pull/88879), [@JoshVanL](https://github.com/JoshVanL)) [SIG API Machinery and Auth] -- Fix /readyz to return error immediately after a shutdown is initiated, before the --shutdown-delay-duration has elapsed. ([#88911](https://github.com/kubernetes/kubernetes/pull/88911), [@tkashem](https://github.com/tkashem)) [SIG API Machinery] -- Fix a bug where kubenet fails to parse the tc output. ([#83572](https://github.com/kubernetes/kubernetes/pull/83572), [@chendotjs](https://github.com/chendotjs)) [SIG Network] -- Fix describe ingress annotations not sorted. ([#88394](https://github.com/kubernetes/kubernetes/pull/88394), [@zhouya0](https://github.com/zhouya0)) [SIG CLI] -- Fix handling of aws-load-balancer-security-groups annotation. Security-Groups assigned with this annotation are no longer modified by kubernetes which is the expected behaviour of most users. Also no unnecessary Security-Groups are created anymore if this annotation is used. ([#83446](https://github.com/kubernetes/kubernetes/pull/83446), [@Elias481](https://github.com/Elias481)) [SIG Cloud Provider] -- Fix kubectl create deployment image name ([#86636](https://github.com/kubernetes/kubernetes/pull/86636), [@zhouya0](https://github.com/zhouya0)) [SIG CLI] -- Fix missing "apiVersion" for "involvedObject" in Events for Nodes. ([#87537](https://github.com/kubernetes/kubernetes/pull/87537), [@uthark](https://github.com/uthark)) [SIG Apps and Node] -- Fix that prevents repeated fetching of PVC/PV objects by kubelet when processing of pod volumes fails. While this prevents hammering API server in these error scenarios, it means that some errors in processing volume(s) for a pod could now take up to 2-3 minutes before retry. ([#88141](https://github.com/kubernetes/kubernetes/pull/88141), [@tedyu](https://github.com/tedyu)) [SIG Node and Storage] -- Fix: azure file mount timeout issue ([#88610](https://github.com/kubernetes/kubernetes/pull/88610), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] -- Fix: corrupted mount point in csi driver ([#88569](https://github.com/kubernetes/kubernetes/pull/88569), [@andyzhangx](https://github.com/andyzhangx)) [SIG Storage] -- Fixed a bug in the TopologyManager. Previously, the TopologyManager would only guarantee alignment if container creation was serialized in some way. Alignment is now guaranteed under all scenarios of container creation. ([#87759](https://github.com/kubernetes/kubernetes/pull/87759), [@klueska](https://github.com/klueska)) [SIG Node] -- Fixed block CSI volume cleanup after timeouts. ([#88660](https://github.com/kubernetes/kubernetes/pull/88660), [@jsafrane](https://github.com/jsafrane)) [SIG Node and Storage] -- Fixes issue where you can't attach more than 15 GCE Persistent Disks to c2, n2, m1, m2 machine types. ([#88602](https://github.com/kubernetes/kubernetes/pull/88602), [@yuga711](https://github.com/yuga711)) [SIG Storage] -- For volumes that allow attaches across multiple nodes, attach and detach operations across different nodes are now executed in parallel. ([#88678](https://github.com/kubernetes/kubernetes/pull/88678), [@verult](https://github.com/verult)) [SIG Apps, Node and Storage] -- Hide kubectl.kubernetes.io/last-applied-configuration in describe command ([#88758](https://github.com/kubernetes/kubernetes/pull/88758), [@soltysh](https://github.com/soltysh)) [SIG Auth and CLI] -- In GKE alpha clusters it will be possible to use the service annotation `cloud.google.com/network-tier: Standard` ([#88487](https://github.com/kubernetes/kubernetes/pull/88487), [@zioproto](https://github.com/zioproto)) [SIG Cloud Provider] -- Kubelets perform fewer unnecessary pod status update operations on the API server. ([#88591](https://github.com/kubernetes/kubernetes/pull/88591), [@smarterclayton](https://github.com/smarterclayton)) [SIG Node and Scalability] -- Plugin/PluginConfig and Policy APIs are mutually exclusive when running the scheduler ([#88864](https://github.com/kubernetes/kubernetes/pull/88864), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling] -- Specifying PluginConfig for the same plugin more than once fails scheduler startup. - - Specifying extenders and configuring .ignoredResources for the NodeResourcesFit plugin fails ([#88870](https://github.com/kubernetes/kubernetes/pull/88870), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling] -- Support TLS Server Name overrides in kubeconfig file and via --tls-server-name in kubectl ([#88769](https://github.com/kubernetes/kubernetes/pull/88769), [@deads2k](https://github.com/deads2k)) [SIG API Machinery, Auth and CLI] -- Terminating a restartPolicy=Never pod no longer has a chance to report the pod succeeded when it actually failed. ([#88440](https://github.com/kubernetes/kubernetes/pull/88440), [@smarterclayton](https://github.com/smarterclayton)) [SIG Node and Testing] -- The EventRecorder from k8s.io/client-go/tools/events will now create events in the default namespace (instead of kube-system) when the related object does not have it set. ([#88815](https://github.com/kubernetes/kubernetes/pull/88815), [@enj](https://github.com/enj)) [SIG API Machinery] -- The audit event sourceIPs list will now always end with the IP that sent the request directly to the API server. ([#87167](https://github.com/kubernetes/kubernetes/pull/87167), [@tallclair](https://github.com/tallclair)) [SIG API Machinery and Auth] -- Update to use golang 1.13.8 ([#87648](https://github.com/kubernetes/kubernetes/pull/87648), [@ialidzhikov](https://github.com/ialidzhikov)) [SIG Release and Testing] -- Validate kube-proxy flags --ipvs-tcp-timeout, --ipvs-tcpfin-timeout, --ipvs-udp-timeout ([#88657](https://github.com/kubernetes/kubernetes/pull/88657), [@chendotjs](https://github.com/chendotjs)) [SIG Network] +- Escape the special characters like `[`, `]` and ` ` that exist in vsphere windows path ([#98830](https://github.com/kubernetes/kubernetes/pull/98830), [@liyanhui1228](https://github.com/liyanhui1228)) [SIG Storage and Windows] +- Kube-proxy: fix a bug on UDP `NodePort` Services where stale connection tracking entries may blackhole the traffic directed to the `NodePort` ([#98305](https://github.com/kubernetes/kubernetes/pull/98305), [@aojea](https://github.com/aojea)) +- Kubelet: fixes a bug in the HostPort dockershim implementation that caused the conformance test "HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" to fail. ([#98755](https://github.com/kubernetes/kubernetes/pull/98755), [@aojea](https://github.com/aojea)) [SIG Cloud Provider, Network and Node] + +### Bug or Regression + +- AcceleratorStats will be available in the Summary API of kubelet when cri_stats_provider is used. ([#96873](https://github.com/kubernetes/kubernetes/pull/96873), [@ruiwen-zhao](https://github.com/ruiwen-zhao)) [SIG Node] +- All data is no longer automatically deleted when a failure is detected during creation of the volume data file on a CSI volume. Now only the data file and volume path is removed. ([#96021](https://github.com/kubernetes/kubernetes/pull/96021), [@huffmanca](https://github.com/huffmanca)) +- Clean ReplicaSet by revision instead of creation timestamp in deployment controller ([#97407](https://github.com/kubernetes/kubernetes/pull/97407), [@waynepeking348](https://github.com/waynepeking348)) [SIG Apps] +- Cleanup subnet in frontend IP configs to prevent huge subnet request bodies in some scenarios. ([#98133](https://github.com/kubernetes/kubernetes/pull/98133), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] +- Client-go exec credential plugins will pass stdin only when interactive terminal is detected on stdin. This fixes a bug where previously it was checking if **stdout** is an interactive terminal. ([#99654](https://github.com/kubernetes/kubernetes/pull/99654), [@ankeesler](https://github.com/ankeesler)) +- Cloud-controller-manager: routes controller should not depend on --allocate-node-cidrs ([#97029](https://github.com/kubernetes/kubernetes/pull/97029), [@andrewsykim](https://github.com/andrewsykim)) [SIG Cloud Provider and Testing] +- Cluster Autoscaler version bump to v1.20.0 ([#97011](https://github.com/kubernetes/kubernetes/pull/97011), [@towca](https://github.com/towca)) +- Creating a PVC with DataSource should fail for non-CSI plugins. ([#97086](https://github.com/kubernetes/kubernetes/pull/97086), [@xing-yang](https://github.com/xing-yang)) [SIG Apps and Storage] +- EndpointSlice controller is now less likely to emit FailedToUpdateEndpointSlices events. ([#99345](https://github.com/kubernetes/kubernetes/pull/99345), [@robscott](https://github.com/robscott)) [SIG Apps and Network] +- EndpointSlice controllers are less likely to create duplicate EndpointSlices. ([#100103](https://github.com/kubernetes/kubernetes/pull/100103), [@robscott](https://github.com/robscott)) [SIG Apps and Network] +- EndpointSliceMirroring controller is now less likely to emit FailedToUpdateEndpointSlices events. ([#99756](https://github.com/kubernetes/kubernetes/pull/99756), [@robscott](https://github.com/robscott)) [SIG Apps and Network] +- Ensure all vSphere nodes are are tracked by volume attach-detach controller ([#96689](https://github.com/kubernetes/kubernetes/pull/96689), [@gnufied](https://github.com/gnufied)) +- Ensure empty string annotations are copied over in rollbacks. ([#94858](https://github.com/kubernetes/kubernetes/pull/94858), [@waynepeking348](https://github.com/waynepeking348)) +- Ensure only one LoadBalancer rule is created when HA mode is enabled ([#99825](https://github.com/kubernetes/kubernetes/pull/99825), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] +- Ensure that client-go's EventBroadcaster is safe (non-racy) during shutdown. ([#95664](https://github.com/kubernetes/kubernetes/pull/95664), [@DirectXMan12](https://github.com/DirectXMan12)) [SIG API Machinery] +- Explicitly pass `KUBE_BUILD_CONFORMANCE=y` in `package-tarballs` to reenable building the conformance tarballs. ([#100571](https://github.com/kubernetes/kubernetes/pull/100571), [@puerco](https://github.com/puerco)) +- Fix Azure file migration e2e test failure when CSIMigration is turned on. ([#97877](https://github.com/kubernetes/kubernetes/pull/97877), [@andyzhangx](https://github.com/andyzhangx)) +- Fix CSI-migrated inline EBS volumes failing to mount if their volumeID is prefixed by aws:// ([#96821](https://github.com/kubernetes/kubernetes/pull/96821), [@wongma7](https://github.com/wongma7)) [SIG Storage] +- Fix CVE-2020-8555 for Gluster client connections. ([#97922](https://github.com/kubernetes/kubernetes/pull/97922), [@liggitt](https://github.com/liggitt)) [SIG Storage] +- Fix NPE in ephemeral storage eviction ([#98261](https://github.com/kubernetes/kubernetes/pull/98261), [@wzshiming](https://github.com/wzshiming)) [SIG Node] +- Fix PermissionDenied issue on SMB mount for Windows ([#99550](https://github.com/kubernetes/kubernetes/pull/99550), [@andyzhangx](https://github.com/andyzhangx)) +- Fix bug that would let the Horizontal Pod Autoscaler scale down despite at least one metric being unavailable/invalid ([#99514](https://github.com/kubernetes/kubernetes/pull/99514), [@mikkeloscar](https://github.com/mikkeloscar)) [SIG Apps and Autoscaling] +- Fix cgroup handling for systemd with cgroup v2 ([#98365](https://github.com/kubernetes/kubernetes/pull/98365), [@odinuge](https://github.com/odinuge)) [SIG Node] +- Fix counting error in service/nodeport/loadbalancer quota check ([#97451](https://github.com/kubernetes/kubernetes/pull/97451), [@pacoxu](https://github.com/pacoxu)) [SIG API Machinery, Network and Testing] +- Fix errors when accessing Windows container stats for Dockershim ([#98510](https://github.com/kubernetes/kubernetes/pull/98510), [@jsturtevant](https://github.com/jsturtevant)) [SIG Node and Windows] +- Fix kube-proxy container image architecture for non amd64 images. ([#98526](https://github.com/kubernetes/kubernetes/pull/98526), [@saschagrunert](https://github.com/saschagrunert)) +- Fix missing cadvisor machine metrics. ([#97006](https://github.com/kubernetes/kubernetes/pull/97006), [@lingsamuel](https://github.com/lingsamuel)) [SIG Node] +- Fix nil VMSS name when setting service to auto mode ([#97366](https://github.com/kubernetes/kubernetes/pull/97366), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] +- Fix privileged config of Pod Sandbox which was previously ignored. ([#96877](https://github.com/kubernetes/kubernetes/pull/96877), [@xeniumlee](https://github.com/xeniumlee)) +- Fix the panic when kubelet registers if a node object already exists with no Status.Capacity or Status.Allocatable ([#95269](https://github.com/kubernetes/kubernetes/pull/95269), [@SataQiu](https://github.com/SataQiu)) [SIG Node] +- Fix the regression with the slow pods termination. Before this fix pods may take an additional time to terminate - up to one minute. Reversing the change that ensured that CNI resources cleaned up when the pod is removed on API server. ([#97980](https://github.com/kubernetes/kubernetes/pull/97980), [@SergeyKanzhelev](https://github.com/SergeyKanzhelev)) [SIG Node] +- Fix to recover CSI volumes from certain dangling attachments ([#96617](https://github.com/kubernetes/kubernetes/pull/96617), [@yuga711](https://github.com/yuga711)) [SIG Apps and Storage] +- Fix: azure file latency issue for metadata-heavy workloads ([#97082](https://github.com/kubernetes/kubernetes/pull/97082), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] +- Fixed Cinder volume IDs on OpenStack Train ([#96673](https://github.com/kubernetes/kubernetes/pull/96673), [@jsafrane](https://github.com/jsafrane)) [SIG Cloud Provider] +- Fixed FibreChannel volume plugin corrupting filesystems on detach of multipath volumes. ([#97013](https://github.com/kubernetes/kubernetes/pull/97013), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] +- Fixed a bug in kubelet that will saturate CPU utilization after containerd got restarted. ([#97174](https://github.com/kubernetes/kubernetes/pull/97174), [@hanlins](https://github.com/hanlins)) [SIG Node] +- Fixed a bug that causes smaller number of conntrack-max being used under CPU static policy. (#99225, @xh4n3) ([#99613](https://github.com/kubernetes/kubernetes/pull/99613), [@xh4n3](https://github.com/xh4n3)) [SIG Network] +- Fixed a bug that on k8s nodes, when the policy of INPUT chain in filter table is not ACCEPT, healthcheck nodeport would not work. + Added iptables rules to allow healthcheck nodeport traffic. ([#97824](https://github.com/kubernetes/kubernetes/pull/97824), [@hanlins](https://github.com/hanlins)) [SIG Network] +- Fixed a bug that the kubelet cannot start on BtrfS. ([#98042](https://github.com/kubernetes/kubernetes/pull/98042), [@gjkim42](https://github.com/gjkim42)) [SIG Node] +- Fixed a race condition on API server startup ensuring previously created webhook configurations are effective before the first write request is admitted. ([#95783](https://github.com/kubernetes/kubernetes/pull/95783), [@roycaihw](https://github.com/roycaihw)) [SIG API Machinery] +- Fixed an issue with garbage collection failing to clean up namespaced children of an object also referenced incorrectly by cluster-scoped children ([#98068](https://github.com/kubernetes/kubernetes/pull/98068), [@liggitt](https://github.com/liggitt)) [SIG API Machinery and Apps] +- Fixed authentication_duration_seconds metric scope. Previously, it included whole apiserver request duration which yields inaccurate results. ([#99944](https://github.com/kubernetes/kubernetes/pull/99944), [@marseel](https://github.com/marseel)) +- Fixed bug in CPUManager with race on container map access ([#97427](https://github.com/kubernetes/kubernetes/pull/97427), [@klueska](https://github.com/klueska)) [SIG Node] +- Fixed bug that caused cAdvisor to incorrectly detect single-socket multi-NUMA topology. ([#99315](https://github.com/kubernetes/kubernetes/pull/99315), [@iwankgb](https://github.com/iwankgb)) [SIG Node] +- Fixed cleanup of block devices when /var/lib/kubelet is a symlink. ([#96889](https://github.com/kubernetes/kubernetes/pull/96889), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] +- Fixed no effect namespace when exposing deployment with --dry-run=client. ([#97492](https://github.com/kubernetes/kubernetes/pull/97492), [@masap](https://github.com/masap)) [SIG CLI] +- Fixed provisioning of Cinder volumes migrated to CSI when StorageClass with AllowedTopologies was used. ([#98311](https://github.com/kubernetes/kubernetes/pull/98311), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] +- Fixes a bug of identifying the correct containerd process. ([#97888](https://github.com/kubernetes/kubernetes/pull/97888), [@pacoxu](https://github.com/pacoxu)) +- Fixes add-on manager leader election to use leases instead of endpoints, similar to what kube-controller-manager does in 1.20 ([#98968](https://github.com/kubernetes/kubernetes/pull/98968), [@liggitt](https://github.com/liggitt)) +- Fixes connection errors when using `--volume-host-cidr-denylist` or `--volume-host-allow-local-loopback` ([#98436](https://github.com/kubernetes/kubernetes/pull/98436), [@liggitt](https://github.com/liggitt)) [SIG Network and Storage] +- Fixes problem where invalid selector on `PodDisruptionBudget` leads to a nil pointer dereference that causes the Controller manager to crash loop. ([#98750](https://github.com/kubernetes/kubernetes/pull/98750), [@mortent](https://github.com/mortent)) +- Fixes spurious errors about IPv6 in `kube-proxy` logs on nodes with IPv6 disabled. ([#99127](https://github.com/kubernetes/kubernetes/pull/99127), [@danwinship](https://github.com/danwinship)) +- Fixing a bug where a failed node may not have the NoExecute taint set correctly ([#96876](https://github.com/kubernetes/kubernetes/pull/96876), [@howieyuen](https://github.com/howieyuen)) [SIG Apps and Node] +- GCE Internal LoadBalancer sync loop will now release the ILB IP address upon sync failure. An error in ILB forwarding rule creation will no longer leak IP addresses. ([#97740](https://github.com/kubernetes/kubernetes/pull/97740), [@prameshj](https://github.com/prameshj)) [SIG Cloud Provider and Network] +- Ignore update pod with no new images in alwaysPullImages admission controller ([#96668](https://github.com/kubernetes/kubernetes/pull/96668), [@pacoxu](https://github.com/pacoxu)) [SIG Apps, Auth and Node] +- Improve speed of vSphere PV provisioning and reduce number of API calls ([#100054](https://github.com/kubernetes/kubernetes/pull/100054), [@gnufied](https://github.com/gnufied)) [SIG Cloud Provider and Storage] +- KUBECTL_EXTERNAL_DIFF now accepts equal sign for additional parameters. ([#98158](https://github.com/kubernetes/kubernetes/pull/98158), [@dougsland](https://github.com/dougsland)) [SIG CLI] +- Kube-apiserver: an update of a pod with a generic ephemeral volume dropped that volume if the feature had been disabled since creating the pod with such a volume ([#99446](https://github.com/kubernetes/kubernetes/pull/99446), [@pohly](https://github.com/pohly)) [SIG Apps, Node and Storage] +- Kube-proxy: remove deprecated --cleanup-ipvs flag of kube-proxy, and make --cleanup flag always to flush IPVS ([#97336](https://github.com/kubernetes/kubernetes/pull/97336), [@maaoBit](https://github.com/maaoBit)) [SIG Network] +- Kubeadm installs etcd v3.4.13 when creating cluster v1.19 ([#97244](https://github.com/kubernetes/kubernetes/pull/97244), [@pacoxu](https://github.com/pacoxu)) +- Kubeadm: Fixes a kubeadm upgrade bug that could cause a custom CoreDNS configuration to be replaced with the default. ([#97016](https://github.com/kubernetes/kubernetes/pull/97016), [@rajansandeep](https://github.com/rajansandeep)) [SIG Cluster Lifecycle] +- Kubeadm: Some text in the `kubeadm upgrade plan` output has changed. If you have scripts or other automation that parses this output, please review these changes and update your scripts to account for the new output. ([#98728](https://github.com/kubernetes/kubernetes/pull/98728), [@stmcginnis](https://github.com/stmcginnis)) [SIG Cluster Lifecycle] +- Kubeadm: fix a bug in the host memory detection code on 32bit Linux platforms ([#97403](https://github.com/kubernetes/kubernetes/pull/97403), [@abelbarrera15](https://github.com/abelbarrera15)) [SIG Cluster Lifecycle] +- Kubeadm: fix a bug where "kubeadm join" would not properly handle missing names for existing etcd members. ([#97372](https://github.com/kubernetes/kubernetes/pull/97372), [@ihgann](https://github.com/ihgann)) [SIG Cluster Lifecycle] +- Kubeadm: fix a bug where "kubeadm upgrade" commands can fail if CoreDNS v1.8.0 is installed. ([#97919](https://github.com/kubernetes/kubernetes/pull/97919), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] +- Kubeadm: fix a bug where external credentials in an existing admin.conf prevented the CA certificate to be written in the cluster-info ConfigMap. ([#98882](https://github.com/kubernetes/kubernetes/pull/98882), [@kvaps](https://github.com/kvaps)) [SIG Cluster Lifecycle] +- Kubeadm: get k8s CI version markers from k8s infra bucket ([#98836](https://github.com/kubernetes/kubernetes/pull/98836), [@hasheddan](https://github.com/hasheddan)) [SIG Cluster Lifecycle and Release] +- Kubeadm: skip validating pod subnet against node-cidr-mask when allocate-node-cidrs is set to be false ([#98984](https://github.com/kubernetes/kubernetes/pull/98984), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] +- Kubectl logs: `--ignore-errors` is now honored by all containers, maintaining consistency with parallelConsumeRequest behavior. ([#97686](https://github.com/kubernetes/kubernetes/pull/97686), [@wzshiming](https://github.com/wzshiming)) +- Kubectl-convert: Fix `no kind "Ingress" is registered for version` error ([#97754](https://github.com/kubernetes/kubernetes/pull/97754), [@wzshiming](https://github.com/wzshiming)) +- Kubectl: Fixed panic when describing an ingress backend without an API Group ([#100505](https://github.com/kubernetes/kubernetes/pull/100505), [@lauchokyip](https://github.com/lauchokyip)) [SIG CLI] +- Kubelet now cleans up orphaned volume directories automatically ([#95301](https://github.com/kubernetes/kubernetes/pull/95301), [@lorenz](https://github.com/lorenz)) [SIG Node and Storage] +- Kubelet.exe on Windows now checks that the process running as administrator and the executing user account is listed in the built-in administrators group. This is the equivalent to checking the process is running as uid 0. ([#96616](https://github.com/kubernetes/kubernetes/pull/96616), [@perithompson](https://github.com/perithompson)) [SIG Node and Windows] +- Kubelet: Fix kubelet from panic after getting the wrong signal ([#98200](https://github.com/kubernetes/kubernetes/pull/98200), [@wzshiming](https://github.com/wzshiming)) [SIG Node] +- Kubelet: Fix repeatedly acquiring the inhibit lock ([#98088](https://github.com/kubernetes/kubernetes/pull/98088), [@wzshiming](https://github.com/wzshiming)) [SIG Node] +- Kubelet: Fixed the bug of getting the number of cpu when the number of cpu logical processors is more than 64 in windows ([#97378](https://github.com/kubernetes/kubernetes/pull/97378), [@hwdef](https://github.com/hwdef)) [SIG Node and Windows] +- Limits lease to have 1000 maximum attached objects. ([#98257](https://github.com/kubernetes/kubernetes/pull/98257), [@lingsamuel](https://github.com/lingsamuel)) +- Mitigate CVE-2020-8555 for kube-up using GCE by preventing local loopback folume hosts. ([#97934](https://github.com/kubernetes/kubernetes/pull/97934), [@mattcary](https://github.com/mattcary)) [SIG Cloud Provider and Storage] +- On single-stack configured (IPv4 or IPv6, but not both) clusters, Services which are both headless (no clusterIP) and selectorless (empty or undefined selector) will report `ipFamilyPolicy RequireDualStack` and will have entries in `ipFamilies[]` for both IPv4 and IPv6. This is a change from alpha, but does not have any impact on the manually-specified Endpoints and EndpointSlices for the Service. ([#99555](https://github.com/kubernetes/kubernetes/pull/99555), [@thockin](https://github.com/thockin)) [SIG Apps and Network] +- Performance regression #97685 has been fixed. ([#97860](https://github.com/kubernetes/kubernetes/pull/97860), [@MikeSpreitzer](https://github.com/MikeSpreitzer)) [SIG API Machinery] +- Pod Log stats for windows now reports metrics ([#99221](https://github.com/kubernetes/kubernetes/pull/99221), [@jsturtevant](https://github.com/jsturtevant)) [SIG Node, Storage, Testing and Windows] +- Pod status updates faster when reacting on probe results. The first readiness probe will be called faster when startup probes succeeded, which will make Pod status as ready faster. ([#98376](https://github.com/kubernetes/kubernetes/pull/98376), [@matthyx](https://github.com/matthyx)) +- Readjust `kubelet_containers_per_pod_count` buckets to only show metrics greater than 1. ([#98169](https://github.com/kubernetes/kubernetes/pull/98169), [@wawa0210](https://github.com/wawa0210)) +- Remove CSI topology from migrated in-tree gcepd volume. ([#97823](https://github.com/kubernetes/kubernetes/pull/97823), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Cloud Provider and Storage] +- Requests with invalid timeout parameters in the request URL now appear in the audit log correctly. ([#96901](https://github.com/kubernetes/kubernetes/pull/96901), [@tkashem](https://github.com/tkashem)) [SIG API Machinery and Testing] +- Resolve a "concurrent map read and map write" crashing error in the kubelet ([#95111](https://github.com/kubernetes/kubernetes/pull/95111), [@choury](https://github.com/choury)) [SIG Node] +- Resolves spurious `Failed to list *v1.Secret` or `Failed to list *v1.ConfigMap` messages in kubelet logs. ([#99538](https://github.com/kubernetes/kubernetes/pull/99538), [@liggitt](https://github.com/liggitt)) [SIG Auth and Node] +- ResourceQuota of an entity now inclusively calculate Pod overhead ([#99600](https://github.com/kubernetes/kubernetes/pull/99600), [@gjkim42](https://github.com/gjkim42)) +- Return zero time (midnight on Jan. 1, 1970) instead of negative number when reporting startedAt and finishedAt of the not started or a running Pod when using `dockershim` as a runtime. ([#99585](https://github.com/kubernetes/kubernetes/pull/99585), [@Iceber](https://github.com/Iceber)) +- Reverts breaking change to inline AzureFile volumes; referenced secrets are now searched for in the same namespace as the pod as in previous releases. ([#100563](https://github.com/kubernetes/kubernetes/pull/100563), [@msau42](https://github.com/msau42)) +- Scores from InterPodAffinity have stronger differentiation. ([#98096](https://github.com/kubernetes/kubernetes/pull/98096), [@leileiwan](https://github.com/leileiwan)) [SIG Scheduling] +- Specifying the KUBE_TEST_REPO environment variable when e2e tests are executed will instruct the test infrastructure to load that image from a location within the specified repo, using a predefined pattern. ([#93510](https://github.com/kubernetes/kubernetes/pull/93510), [@smarterclayton](https://github.com/smarterclayton)) [SIG Testing] +- Static pods will be deleted gracefully. ([#98103](https://github.com/kubernetes/kubernetes/pull/98103), [@gjkim42](https://github.com/gjkim42)) [SIG Node] +- Sync node status during kubelet node shutdown. + Adds an pod admission handler that rejects new pods when the node is in progress of shutting down. ([#98005](https://github.com/kubernetes/kubernetes/pull/98005), [@wzshiming](https://github.com/wzshiming)) [SIG Node] +- The calculation of pod UIDs for static pods has changed to ensure each static pod gets a unique value - this will cause all static pod containers to be recreated/restarted if an in-place kubelet upgrade from 1.20 to 1.21 is performed. Note that draining pods before upgrading the kubelet across minor versions is the supported upgrade path. ([#87461](https://github.com/kubernetes/kubernetes/pull/87461), [@bboreham](https://github.com/bboreham)) [SIG Node] +- The maximum number of ports allowed in EndpointSlices has been increased from 100 to 20,000 ([#99795](https://github.com/kubernetes/kubernetes/pull/99795), [@robscott](https://github.com/robscott)) [SIG Network] +- Truncates a message if it hits the `NoteLengthLimit` when the scheduler records an event for the pod that indicates the pod has failed to schedule. ([#98715](https://github.com/kubernetes/kubernetes/pull/98715), [@carlory](https://github.com/carlory)) +- Updated k8s.gcr.io/ingress-gce-404-server-with-metrics-amd64 to a version that serves /metrics endpoint on a non-default port. ([#97621](https://github.com/kubernetes/kubernetes/pull/97621), [@vbannai](https://github.com/vbannai)) [SIG Cloud Provider] +- Updates the commands ` + - kubectl kustomize {arg} + - kubectl apply -k {arg} + `to use same code as kustomize CLI [v4.0.5](https://github.com/kubernetes-sigs/kustomize/releases/tag/kustomize%2Fv4.0.5) ([#98946](https://github.com/kubernetes/kubernetes/pull/98946), [@monopole](https://github.com/monopole)) +- Use force unmount for NFS volumes if regular mount fails after 1 minute timeout ([#96844](https://github.com/kubernetes/kubernetes/pull/96844), [@gnufied](https://github.com/gnufied)) [SIG Storage] +- Use network.Interface.VirtualMachine.ID to get the binded VM + Skip standalone VM when reconciling LoadBalancer ([#97635](https://github.com/kubernetes/kubernetes/pull/97635), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] +- Using exec auth plugins with kubectl no longer results in warnings about constructing many client instances from the same exec auth config. ([#97857](https://github.com/kubernetes/kubernetes/pull/97857), [@liggitt](https://github.com/liggitt)) [SIG API Machinery and Auth] +- When a CNI plugin returns dual-stack pod IPs, kubelet will now try to respect the + "primary IP family" of the cluster by picking a primary pod IP of the same family + as the (primary) node IP, rather than assuming that the CNI plugin returned the IPs + in the order the administrator wanted (since some CNI plugins don't allow + configuring this). ([#97979](https://github.com/kubernetes/kubernetes/pull/97979), [@danwinship](https://github.com/danwinship)) [SIG Network and Node] +- When dynamically provisioning Azure File volumes for a premium account, the requested size will be set to 100GB if the request is initially lower than this value to accommodate Azure File requirements. ([#99122](https://github.com/kubernetes/kubernetes/pull/99122), [@huffmanca](https://github.com/huffmanca)) [SIG Cloud Provider and Storage] +- When using `Containerd` on Windows, the `C:\Windows\System32\drivers\etc\hosts` file will now be managed by kubelet. ([#83730](https://github.com/kubernetes/kubernetes/pull/83730), [@claudiubelu](https://github.com/claudiubelu)) +- `VolumeBindingArgs` now allow `BindTimeoutSeconds` to be set as zero, while the value zero indicates no waiting for the checking of volume binding operation. ([#99835](https://github.com/kubernetes/kubernetes/pull/99835), [@chendave](https://github.com/chendave)) [SIG Scheduling and Storage] +- `kubectl exec` and `kubectl attach` now honor the `--quiet` flag which suppresses output from the local binary that could be confused by a script with the remote command output (all non-failure output is hidden). In addition, print inline with exec and attach the list of alternate containers when we default to the first spec.container. ([#99004](https://github.com/kubernetes/kubernetes/pull/99004), [@smarterclayton](https://github.com/smarterclayton)) [SIG CLI] + +### Other (Cleanup or Flake) + +- APIs for kubelet annotations and labels from `k8s.io/kubernetes/pkg/kubelet/apis` are now moved under `k8s.io/kubelet/pkg/apis/` ([#98931](https://github.com/kubernetes/kubernetes/pull/98931), [@michaelbeaumont](https://github.com/michaelbeaumont)) +- Apiserver_request_duration_seconds is promoted to stable status. ([#99925](https://github.com/kubernetes/kubernetes/pull/99925), [@logicalhan](https://github.com/logicalhan)) [SIG API Machinery, Instrumentation and Testing] +- Bump github.com/Azure/go-autorest/autorest to v0.11.12 ([#97033](https://github.com/kubernetes/kubernetes/pull/97033), [@patrickshan](https://github.com/patrickshan)) [SIG API Machinery, CLI, Cloud Provider and Cluster Lifecycle] +- Clients required to use go1.15.8+ or go1.16+ if kube-apiserver has the goaway feature enabled to avoid unexpected data race condition. ([#98809](https://github.com/kubernetes/kubernetes/pull/98809), [@answer1991](https://github.com/answer1991)) +- Delete deprecated `service.beta.kubernetes.io/azure-load-balancer-mixed-protocols` mixed procotol annotation in favor of the MixedProtocolLBService feature ([#97096](https://github.com/kubernetes/kubernetes/pull/97096), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] +- EndpointSlice generation is now incremented when labels change. ([#99750](https://github.com/kubernetes/kubernetes/pull/99750), [@robscott](https://github.com/robscott)) [SIG Network] +- Featuregate AllowInsecureBackendProxy graduates to GA and unconditionally enabled. ([#99658](https://github.com/kubernetes/kubernetes/pull/99658), [@deads2k](https://github.com/deads2k)) +- Increase timeout for pod lifecycle test to reach pod status=ready ([#96691](https://github.com/kubernetes/kubernetes/pull/96691), [@hh](https://github.com/hh)) +- Increased `CSINodeIDMaxLength` from 128 bytes to 192 bytes. ([#98753](https://github.com/kubernetes/kubernetes/pull/98753), [@Jiawei0227](https://github.com/Jiawei0227)) +- Kube-apiserver: The OIDC authenticator no longer waits 10 seconds before attempting to fetch the metadata required to verify tokens. ([#97693](https://github.com/kubernetes/kubernetes/pull/97693), [@enj](https://github.com/enj)) [SIG API Machinery and Auth] +- Kube-proxy: Traffic from the cluster directed to ExternalIPs is always sent directly to the Service. ([#96296](https://github.com/kubernetes/kubernetes/pull/96296), [@aojea](https://github.com/aojea)) [SIG Network and Testing] +- Kubeadm: change the default image repository for CI images from 'gcr.io/kubernetes-ci-images' to 'gcr.io/k8s-staging-ci-images' ([#97087](https://github.com/kubernetes/kubernetes/pull/97087), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] +- Kubectl: The deprecated `kubectl alpha debug` command is removed. Use `kubectl debug` instead. ([#98111](https://github.com/kubernetes/kubernetes/pull/98111), [@pandaamanda](https://github.com/pandaamanda)) [SIG CLI] +- Kubelet command line flags related to dockershim are now showing deprecation message as they will be removed along with dockershim in future release. ([#98730](https://github.com/kubernetes/kubernetes/pull/98730), [@dims](https://github.com/dims)) +- Official support to build kubernetes with docker-machine / remote docker is removed. This change does not affect building kubernetes with docker locally. ([#97618](https://github.com/kubernetes/kubernetes/pull/97618), [@jherrera123](https://github.com/jherrera123)) [SIG Release and Testing] +- Process start time on Windows now uses current process information ([#97491](https://github.com/kubernetes/kubernetes/pull/97491), [@jsturtevant](https://github.com/jsturtevant)) [SIG API Machinery, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation and Windows] +- Resolves flakes in the Ingress conformance tests due to conflicts with controllers updating the Ingress object ([#98430](https://github.com/kubernetes/kubernetes/pull/98430), [@liggitt](https://github.com/liggitt)) [SIG Network and Testing] +- The `AttachVolumeLimit` feature gate (GA since v1.17) has been removed and now unconditionally enabled. ([#96539](https://github.com/kubernetes/kubernetes/pull/96539), [@ialidzhikov](https://github.com/ialidzhikov)) +- The `CSINodeInfo` feature gate that is GA since v1.17 is unconditionally enabled, and can no longer be specified via the `--feature-gates` argument. ([#96561](https://github.com/kubernetes/kubernetes/pull/96561), [@ialidzhikov](https://github.com/ialidzhikov)) [SIG Apps, Auth, Scheduling, Storage and Testing] +- The `apiserver_request_total` metric is promoted to stable status and no longer has a content-type dimensions, so any alerts/charts which presume the existence of this will fail. This is however, unlikely to be the case since it was effectively an unbounded dimension in the first place. ([#99788](https://github.com/kubernetes/kubernetes/pull/99788), [@logicalhan](https://github.com/logicalhan)) +- The default delegating authorization options now allow unauthenticated access to healthz, readyz, and livez. A system:masters user connecting to an authz delegator will not perform an authz check. ([#98325](https://github.com/kubernetes/kubernetes/pull/98325), [@deads2k](https://github.com/deads2k)) [SIG API Machinery, Auth, Cloud Provider and Scheduling] +- The deprecated feature gates `CSIDriverRegistry`, `BlockVolume` and `CSIBlockVolume` are now unconditionally enabled and can no longer be specified in component invocations. ([#98021](https://github.com/kubernetes/kubernetes/pull/98021), [@gavinfish](https://github.com/gavinfish)) [SIG Storage] +- The deprecated feature gates `RotateKubeletClientCertificate`, `AttachVolumeLimit`, `VolumePVCDataSource` and `EvenPodsSpread` are now unconditionally enabled and can no longer be specified in component invocations. ([#97306](https://github.com/kubernetes/kubernetes/pull/97306), [@gavinfish](https://github.com/gavinfish)) [SIG Node, Scheduling and Storage] +- The e2e suite can be instructed not to wait for pods in kube-system to be ready or for all nodes to be ready by passing `--allowed-not-ready-nodes=-1` when invoking the e2e.test program. This allows callers to run subsets of the e2e suite in scenarios other than perfectly healthy clusters. ([#98781](https://github.com/kubernetes/kubernetes/pull/98781), [@smarterclayton](https://github.com/smarterclayton)) [SIG Testing] +- The feature gates `WindowsGMSA` and `WindowsRunAsUserName` that are GA since v1.18 are now removed. ([#96531](https://github.com/kubernetes/kubernetes/pull/96531), [@ialidzhikov](https://github.com/ialidzhikov)) [SIG Node and Windows] +- The new `-gce-zones` flag on the `e2e.test` binary instructs tests that check for information about how the cluster interacts with the cloud to limit their queries to the provided zone list. If not specified, the current behavior of asking the cloud provider for all available zones in multi zone clusters is preserved. ([#98787](https://github.com/kubernetes/kubernetes/pull/98787), [@smarterclayton](https://github.com/smarterclayton)) [SIG API Machinery, Cluster Lifecycle and Testing] +- Update cri-tools to [v1.20.0](https://github.com/kubernetes-sigs/cri-tools/releases/tag/v1.20.0) ([#97967](https://github.com/kubernetes/kubernetes/pull/97967), [@rajibmitra](https://github.com/rajibmitra)) [SIG Cloud Provider] +- Windows nodes on GCE will take longer to start due to dependencies installed at node creation time. ([#98284](https://github.com/kubernetes/kubernetes/pull/98284), [@pjh](https://github.com/pjh)) [SIG Cloud Provider] +- `apiserver_storage_objects` (a newer version of `etcd_object_counts`) is promoted and marked as stable. ([#100082](https://github.com/kubernetes/kubernetes/pull/100082), [@logicalhan](https://github.com/logicalhan)) + +### Uncategorized + +- GCE L4 Loadbalancers now handle > 5 ports in service spec correctly. ([#99595](https://github.com/kubernetes/kubernetes/pull/99595), [@prameshj](https://github.com/prameshj)) [SIG Cloud Provider] +- The DownwardAPIHugePages feature is beta. Users may use the feature if all workers in their cluster are min 1.20 version. The feature will be enabled by default in all installations in 1.22. ([#99610](https://github.com/kubernetes/kubernetes/pull/99610), [@derekwaynecarr](https://github.com/derekwaynecarr)) [SIG Node] + +## Dependencies + +### Added +- github.com/go-errors/errors: [v1.0.1](https://github.com/go-errors/errors/tree/v1.0.1) +- github.com/gobuffalo/here: [v0.6.0](https://github.com/gobuffalo/here/tree/v0.6.0) +- github.com/google/shlex: [e7afc7f](https://github.com/google/shlex/tree/e7afc7f) +- github.com/markbates/pkger: [v0.17.1](https://github.com/markbates/pkger/tree/v0.17.1) +- github.com/moby/spdystream: [v0.2.0](https://github.com/moby/spdystream/tree/v0.2.0) +- github.com/monochromegane/go-gitignore: [205db1a](https://github.com/monochromegane/go-gitignore/tree/205db1a) +- github.com/niemeyer/pretty: [a10e7ca](https://github.com/niemeyer/pretty/tree/a10e7ca) +- github.com/xlab/treeprint: [a009c39](https://github.com/xlab/treeprint/tree/a009c39) +- go.starlark.net: 8dd3e2e +- golang.org/x/term: 6a3ed07 +- sigs.k8s.io/kustomize/api: v0.8.5 +- sigs.k8s.io/kustomize/cmd/config: v0.9.7 +- sigs.k8s.io/kustomize/kustomize/v4: v4.0.5 +- sigs.k8s.io/kustomize/kyaml: v0.10.15 + +### Changed +- dmitri.shuralyov.com/gpu/mtl: 666a987 → 28db891 +- github.com/Azure/go-autorest/autorest: [v0.11.1 → v0.11.12](https://github.com/Azure/go-autorest/autorest/compare/v0.11.1...v0.11.12) +- github.com/NYTimes/gziphandler: [56545f4 → v1.1.1](https://github.com/NYTimes/gziphandler/compare/56545f4...v1.1.1) +- github.com/cilium/ebpf: [1c8d4c9 → v0.2.0](https://github.com/cilium/ebpf/compare/1c8d4c9...v0.2.0) +- github.com/container-storage-interface/spec: [v1.2.0 → v1.3.0](https://github.com/container-storage-interface/spec/compare/v1.2.0...v1.3.0) +- github.com/containerd/console: [v1.0.0 → v1.0.1](https://github.com/containerd/console/compare/v1.0.0...v1.0.1) +- github.com/containerd/containerd: [v1.4.1 → v1.4.4](https://github.com/containerd/containerd/compare/v1.4.1...v1.4.4) +- github.com/coredns/corefile-migration: [v1.0.10 → v1.0.11](https://github.com/coredns/corefile-migration/compare/v1.0.10...v1.0.11) +- github.com/creack/pty: [v1.1.7 → v1.1.11](https://github.com/creack/pty/compare/v1.1.7...v1.1.11) +- github.com/docker/docker: [bd33bbf → v20.10.2+incompatible](https://github.com/docker/docker/compare/bd33bbf...v20.10.2) +- github.com/go-logr/logr: [v0.2.0 → v0.4.0](https://github.com/go-logr/logr/compare/v0.2.0...v0.4.0) +- github.com/go-openapi/spec: [v0.19.3 → v0.19.5](https://github.com/go-openapi/spec/compare/v0.19.3...v0.19.5) +- github.com/go-openapi/strfmt: [v0.19.3 → v0.19.5](https://github.com/go-openapi/strfmt/compare/v0.19.3...v0.19.5) +- github.com/go-openapi/validate: [v0.19.5 → v0.19.8](https://github.com/go-openapi/validate/compare/v0.19.5...v0.19.8) +- github.com/gogo/protobuf: [v1.3.1 → v1.3.2](https://github.com/gogo/protobuf/compare/v1.3.1...v1.3.2) +- github.com/golang/mock: [v1.4.1 → v1.4.4](https://github.com/golang/mock/compare/v1.4.1...v1.4.4) +- github.com/google/cadvisor: [v0.38.5 → v0.39.0](https://github.com/google/cadvisor/compare/v0.38.5...v0.39.0) +- github.com/heketi/heketi: [c2e2a4a → v10.2.0+incompatible](https://github.com/heketi/heketi/compare/c2e2a4a...v10.2.0) +- github.com/kisielk/errcheck: [v1.2.0 → v1.5.0](https://github.com/kisielk/errcheck/compare/v1.2.0...v1.5.0) +- github.com/konsorten/go-windows-terminal-sequences: [v1.0.3 → v1.0.2](https://github.com/konsorten/go-windows-terminal-sequences/compare/v1.0.3...v1.0.2) +- github.com/kr/text: [v0.1.0 → v0.2.0](https://github.com/kr/text/compare/v0.1.0...v0.2.0) +- github.com/mattn/go-runewidth: [v0.0.2 → v0.0.7](https://github.com/mattn/go-runewidth/compare/v0.0.2...v0.0.7) +- github.com/miekg/dns: [v1.1.4 → v1.1.35](https://github.com/miekg/dns/compare/v1.1.4...v1.1.35) +- github.com/moby/sys/mountinfo: [v0.1.3 → v0.4.0](https://github.com/moby/sys/mountinfo/compare/v0.1.3...v0.4.0) +- github.com/moby/term: [672ec06 → df9cb8a](https://github.com/moby/term/compare/672ec06...df9cb8a) +- github.com/mrunalp/fileutils: [abd8a0e → v0.5.0](https://github.com/mrunalp/fileutils/compare/abd8a0e...v0.5.0) +- github.com/olekukonko/tablewriter: [a0225b3 → v0.0.4](https://github.com/olekukonko/tablewriter/compare/a0225b3...v0.0.4) +- github.com/opencontainers/runc: [v1.0.0-rc92 → v1.0.0-rc93](https://github.com/opencontainers/runc/compare/v1.0.0-rc92...v1.0.0-rc93) +- github.com/opencontainers/runtime-spec: [4d89ac9 → e6143ca](https://github.com/opencontainers/runtime-spec/compare/4d89ac9...e6143ca) +- github.com/opencontainers/selinux: [v1.6.0 → v1.8.0](https://github.com/opencontainers/selinux/compare/v1.6.0...v1.8.0) +- github.com/sergi/go-diff: [v1.0.0 → v1.1.0](https://github.com/sergi/go-diff/compare/v1.0.0...v1.1.0) +- github.com/sirupsen/logrus: [v1.6.0 → v1.7.0](https://github.com/sirupsen/logrus/compare/v1.6.0...v1.7.0) +- github.com/syndtr/gocapability: [d983527 → 42c35b4](https://github.com/syndtr/gocapability/compare/d983527...42c35b4) +- github.com/willf/bitset: [d5bec33 → v1.1.11](https://github.com/willf/bitset/compare/d5bec33...v1.1.11) +- github.com/yuin/goldmark: [v1.1.27 → v1.2.1](https://github.com/yuin/goldmark/compare/v1.1.27...v1.2.1) +- golang.org/x/crypto: 7f63de1 → 5ea612d +- golang.org/x/exp: 6cc2880 → 85be41e +- golang.org/x/mobile: d2bd2a2 → e6ae53a +- golang.org/x/mod: v0.3.0 → ce943fd +- golang.org/x/net: 69a7880 → 3d97a24 +- golang.org/x/sync: cd5d95a → 67f06af +- golang.org/x/sys: 5cba982 → a50acf3 +- golang.org/x/time: 3af7569 → f8bda1e +- golang.org/x/tools: c1934b7 → v0.1.0 +- gopkg.in/check.v1: 41f04d3 → 8fa4692 +- gopkg.in/yaml.v2: v2.2.8 → v2.4.0 +- gotest.tools/v3: v3.0.2 → v3.0.3 +- k8s.io/gengo: 83324d8 → b6c5ce2 +- k8s.io/klog/v2: v2.4.0 → v2.8.0 +- k8s.io/kube-openapi: d219536 → 591a79e +- k8s.io/system-validators: v1.2.0 → v1.4.0 +- sigs.k8s.io/apiserver-network-proxy/konnectivity-client: v0.0.14 → v0.0.15 +- sigs.k8s.io/structured-merge-diff/v4: v4.0.2 → v4.1.0 + +### Removed +- github.com/codegangsta/negroni: [v1.0.0](https://github.com/codegangsta/negroni/tree/v1.0.0) +- github.com/docker/spdystream: [449fdfc](https://github.com/docker/spdystream/tree/449fdfc) +- github.com/golangplus/bytes: [45c989f](https://github.com/golangplus/bytes/tree/45c989f) +- github.com/golangplus/fmt: [2a5d6d7](https://github.com/golangplus/fmt/tree/2a5d6d7) +- github.com/gorilla/context: [v1.1.1](https://github.com/gorilla/context/tree/v1.1.1) +- github.com/kr/pty: [v1.1.5](https://github.com/kr/pty/tree/v1.1.5) +- rsc.io/quote/v3: v3.1.0 +- rsc.io/sampler: v1.3.0 +- sigs.k8s.io/kustomize: v2.0.3+incompatible -# v1.18.0-beta.1 +## Dependencies -[Documentation](https://docs.k8s.io) +### Added +- github.com/go-errors/errors: [v1.0.1](https://github.com/go-errors/errors/tree/v1.0.1) +- github.com/gobuffalo/here: [v0.6.0](https://github.com/gobuffalo/here/tree/v0.6.0) +- github.com/google/shlex: [e7afc7f](https://github.com/google/shlex/tree/e7afc7f) +- github.com/markbates/pkger: [v0.17.1](https://github.com/markbates/pkger/tree/v0.17.1) +- github.com/moby/spdystream: [v0.2.0](https://github.com/moby/spdystream/tree/v0.2.0) +- github.com/monochromegane/go-gitignore: [205db1a](https://github.com/monochromegane/go-gitignore/tree/205db1a) +- github.com/niemeyer/pretty: [a10e7ca](https://github.com/niemeyer/pretty/tree/a10e7ca) +- github.com/xlab/treeprint: [a009c39](https://github.com/xlab/treeprint/tree/a009c39) +- go.starlark.net: 8dd3e2e +- golang.org/x/term: 6a3ed07 +- sigs.k8s.io/kustomize/api: v0.8.5 +- sigs.k8s.io/kustomize/cmd/config: v0.9.7 +- sigs.k8s.io/kustomize/kustomize/v4: v4.0.5 +- sigs.k8s.io/kustomize/kyaml: v0.10.15 -## Downloads for v1.18.0-beta.1 +### Changed +- dmitri.shuralyov.com/gpu/mtl: 666a987 → 28db891 +- github.com/Azure/go-autorest/autorest: [v0.11.1 → v0.11.12](https://github.com/Azure/go-autorest/autorest/compare/v0.11.1...v0.11.12) +- github.com/NYTimes/gziphandler: [56545f4 → v1.1.1](https://github.com/NYTimes/gziphandler/compare/56545f4...v1.1.1) +- github.com/cilium/ebpf: [1c8d4c9 → v0.2.0](https://github.com/cilium/ebpf/compare/1c8d4c9...v0.2.0) +- github.com/container-storage-interface/spec: [v1.2.0 → v1.3.0](https://github.com/container-storage-interface/spec/compare/v1.2.0...v1.3.0) +- github.com/containerd/console: [v1.0.0 → v1.0.1](https://github.com/containerd/console/compare/v1.0.0...v1.0.1) +- github.com/containerd/containerd: [v1.4.1 → v1.4.4](https://github.com/containerd/containerd/compare/v1.4.1...v1.4.4) +- github.com/coredns/corefile-migration: [v1.0.10 → v1.0.11](https://github.com/coredns/corefile-migration/compare/v1.0.10...v1.0.11) +- github.com/creack/pty: [v1.1.7 → v1.1.11](https://github.com/creack/pty/compare/v1.1.7...v1.1.11) +- github.com/docker/docker: [bd33bbf → v20.10.2+incompatible](https://github.com/docker/docker/compare/bd33bbf...v20.10.2) +- github.com/go-logr/logr: [v0.2.0 → v0.4.0](https://github.com/go-logr/logr/compare/v0.2.0...v0.4.0) +- github.com/go-openapi/spec: [v0.19.3 → v0.19.5](https://github.com/go-openapi/spec/compare/v0.19.3...v0.19.5) +- github.com/go-openapi/strfmt: [v0.19.3 → v0.19.5](https://github.com/go-openapi/strfmt/compare/v0.19.3...v0.19.5) +- github.com/go-openapi/validate: [v0.19.5 → v0.19.8](https://github.com/go-openapi/validate/compare/v0.19.5...v0.19.8) +- github.com/gogo/protobuf: [v1.3.1 → v1.3.2](https://github.com/gogo/protobuf/compare/v1.3.1...v1.3.2) +- github.com/golang/mock: [v1.4.1 → v1.4.4](https://github.com/golang/mock/compare/v1.4.1...v1.4.4) +- github.com/google/cadvisor: [v0.38.5 → v0.39.0](https://github.com/google/cadvisor/compare/v0.38.5...v0.39.0) +- github.com/heketi/heketi: [c2e2a4a → v10.2.0+incompatible](https://github.com/heketi/heketi/compare/c2e2a4a...v10.2.0) +- github.com/kisielk/errcheck: [v1.2.0 → v1.5.0](https://github.com/kisielk/errcheck/compare/v1.2.0...v1.5.0) +- github.com/konsorten/go-windows-terminal-sequences: [v1.0.3 → v1.0.2](https://github.com/konsorten/go-windows-terminal-sequences/compare/v1.0.3...v1.0.2) +- github.com/kr/text: [v0.1.0 → v0.2.0](https://github.com/kr/text/compare/v0.1.0...v0.2.0) +- github.com/mattn/go-runewidth: [v0.0.2 → v0.0.7](https://github.com/mattn/go-runewidth/compare/v0.0.2...v0.0.7) +- github.com/miekg/dns: [v1.1.4 → v1.1.35](https://github.com/miekg/dns/compare/v1.1.4...v1.1.35) +- github.com/moby/sys/mountinfo: [v0.1.3 → v0.4.0](https://github.com/moby/sys/mountinfo/compare/v0.1.3...v0.4.0) +- github.com/moby/term: [672ec06 → df9cb8a](https://github.com/moby/term/compare/672ec06...df9cb8a) +- github.com/mrunalp/fileutils: [abd8a0e → v0.5.0](https://github.com/mrunalp/fileutils/compare/abd8a0e...v0.5.0) +- github.com/olekukonko/tablewriter: [a0225b3 → v0.0.4](https://github.com/olekukonko/tablewriter/compare/a0225b3...v0.0.4) +- github.com/opencontainers/runc: [v1.0.0-rc92 → v1.0.0-rc93](https://github.com/opencontainers/runc/compare/v1.0.0-rc92...v1.0.0-rc93) +- github.com/opencontainers/runtime-spec: [4d89ac9 → e6143ca](https://github.com/opencontainers/runtime-spec/compare/4d89ac9...e6143ca) +- github.com/opencontainers/selinux: [v1.6.0 → v1.8.0](https://github.com/opencontainers/selinux/compare/v1.6.0...v1.8.0) +- github.com/sergi/go-diff: [v1.0.0 → v1.1.0](https://github.com/sergi/go-diff/compare/v1.0.0...v1.1.0) +- github.com/sirupsen/logrus: [v1.6.0 → v1.7.0](https://github.com/sirupsen/logrus/compare/v1.6.0...v1.7.0) +- github.com/syndtr/gocapability: [d983527 → 42c35b4](https://github.com/syndtr/gocapability/compare/d983527...42c35b4) +- github.com/willf/bitset: [d5bec33 → v1.1.11](https://github.com/willf/bitset/compare/d5bec33...v1.1.11) +- github.com/yuin/goldmark: [v1.1.27 → v1.2.1](https://github.com/yuin/goldmark/compare/v1.1.27...v1.2.1) +- golang.org/x/crypto: 7f63de1 → 5ea612d +- golang.org/x/exp: 6cc2880 → 85be41e +- golang.org/x/mobile: d2bd2a2 → e6ae53a +- golang.org/x/mod: v0.3.0 → ce943fd +- golang.org/x/net: 69a7880 → 3d97a24 +- golang.org/x/sync: cd5d95a → 67f06af +- golang.org/x/sys: 5cba982 → a50acf3 +- golang.org/x/time: 3af7569 → f8bda1e +- golang.org/x/tools: c1934b7 → v0.1.0 +- gopkg.in/check.v1: 41f04d3 → 8fa4692 +- gopkg.in/yaml.v2: v2.2.8 → v2.4.0 +- gotest.tools/v3: v3.0.2 → v3.0.3 +- k8s.io/gengo: 83324d8 → b6c5ce2 +- k8s.io/klog/v2: v2.4.0 → v2.8.0 +- k8s.io/kube-openapi: d219536 → 591a79e +- k8s.io/system-validators: v1.2.0 → v1.4.0 +- sigs.k8s.io/apiserver-network-proxy/konnectivity-client: v0.0.14 → v0.0.15 +- sigs.k8s.io/structured-merge-diff/v4: v4.0.2 → v4.1.0 + +### Removed +- github.com/codegangsta/negroni: [v1.0.0](https://github.com/codegangsta/negroni/tree/v1.0.0) +- github.com/docker/spdystream: [449fdfc](https://github.com/docker/spdystream/tree/449fdfc) +- github.com/golangplus/bytes: [45c989f](https://github.com/golangplus/bytes/tree/45c989f) +- github.com/golangplus/fmt: [2a5d6d7](https://github.com/golangplus/fmt/tree/2a5d6d7) +- github.com/gorilla/context: [v1.1.1](https://github.com/gorilla/context/tree/v1.1.1) +- github.com/kr/pty: [v1.1.5](https://github.com/kr/pty/tree/v1.1.5) +- rsc.io/quote/v3: v3.1.0 +- rsc.io/sampler: v1.3.0 +- sigs.k8s.io/kustomize: v2.0.3+incompatible + + + +# v1.21.0-rc.0 + + +## Downloads for v1.21.0-rc.0 + +### Source Code filename | sha512 hash -------- | ----------- -[kubernetes.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes.tar.gz) | `7c182ca905b3a31871c01ab5fdaf46f074547536c7975e069ff230af0d402dfc0346958b1d084bd2c108582ffc407484e6a15a1cd93e9affbe34b6e99409ef1f` -[kubernetes-src.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-src.tar.gz) | `d104b8c792b1517bd730787678c71c8ee3b259de81449192a49a1c6e37a6576d28f69b05c2019cc4a4c40ddeb4d60b80138323df3f85db8682caabf28e67c2de` +[kubernetes.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes.tar.gz) | ef53a41955d6f8a8d2a94636af98b55d633fb8a5081517559039e019b3dd65c9d10d4e7fa297ab88a7865d772f3eecf72e7b0eeba5e87accb4000c91da33e148 +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-src.tar.gz) | 9335a01b50d351776d3b8d00c07a5233844c51d307e361fa7e55a0620c1cb8b699e43eacf45ae9cafd8cbc44752e6987450c528a5bede8204706b7673000b5fc -### Client Binaries +### Client binaries filename | sha512 hash -------- | ----------- -[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-client-darwin-386.tar.gz) | `bc337bb8f200a789be4b97ce99b9d7be78d35ebd64746307c28339dc4628f56d9903e0818c0888aaa9364357a528d1ac6fd34f74377000f292ec502fbea3837e` -[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-client-darwin-amd64.tar.gz) | `38dfa5e0b0cfff39942c913a6bcb2ad8868ec43457d35cffba08217bb6e7531720e0731f8588505f4c81193ce5ec0e5fe6870031cf1403fbbde193acf7e53540` -[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-client-linux-386.tar.gz) | `8e63ec7ce29c69241120c037372c6c779e3f16253eabd612c7cbe6aa89326f5160eb5798004d723c5cd72d458811e98dac3574842eb6a57b2798ecd2bbe5bcf9` -[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-client-linux-amd64.tar.gz) | `c1be9f184a7c3f896a785c41cd6ece9d90d8cb9b1f6088bdfb5557d8856c55e455f6688f5f54c2114396d5ae7adc0361e34ebf8e9c498d0187bd785646ccc1d0` -[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-client-linux-arm.tar.gz) | `8eab02453cfd9e847632a774a0e0cf3a33c7619fb4ced7f1840e1f71444e8719b1c8e8cbfdd1f20bb909f3abe39cdcac74f14cb9c878c656d35871b7c37c7cbe` -[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-client-linux-arm64.tar.gz) | `f7df0ec02d2e7e63278d5386e8153cfe2b691b864f17b6452cc824a5f328d688976c975b076e60f1c6b3c859e93e477134fbccc53bb49d9e846fb038b34eee48` -[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-client-linux-ppc64le.tar.gz) | `36dd5b10addca678a518e6d052c9d6edf473e3f87388a2f03f714c93c5fbfe99ace16cf3b382a531be20a8fe6f4160f8d891800dd2cff5f23c9ca12c2f4a151b` -[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-client-linux-s390x.tar.gz) | `5bdbb44b996ab4ccf3a383780270f5cfdbf174982c300723c8bddf0a48ae5e459476031c1d51b9d30ffd621d0a126c18a5de132ef1d92fca2f3e477665ea10cc` -[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-client-windows-386.tar.gz) | `5dea3d4c4e91ef889850143b361974250e99a3c526f5efee23ff9ccdcd2ceca4a2247e7c4f236bdfa77d2150157da5d676ac9c3ba26cf3a2f1e06d8827556f77` -[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-client-windows-amd64.tar.gz) | `db298e698391368703e6aea7f4345aec5a4b8c69f9d8ff6c99fb5804a6cea16d295fb01e70fe943ade3d4ce9200a081ad40da21bd331317ec9213f69b4d6c48f` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-client-darwin-amd64.tar.gz) | 964135e43234cee275c452f5f06fb6d2bcd3cff3211a0d50fa35fff1cc4446bc5a0ac5125405dadcfb6596cb152afe29fabf7aad5b35b100e1288db890b70f8e +[kubernetes-client-darwin-arm64.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-client-darwin-arm64.tar.gz) | 50d782abaa4ded5e706b3192d87effa953ceabbd7d91e3d48b0c1fa2206a1963a909c14b923560f5d09cac2c7392edc5f38a13fbf1e9a40bc94e3afe8de10622 +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-client-linux-386.tar.gz) | 72af5562f24184a2d7c27f95fa260470da979fbdcacce39a372f8f3add2991d7af8bc78f4e1dbe7a0f97e3f559b149b72a51491d3b13008da81872ee50f02f37 +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-client-linux-amd64.tar.gz) | 1eddb8f6b51e005bc6f7b519d036cbe3d2f6d97dbf7d212dd933fb56354c29f222d050519115a9bcf94555aef095db7cf763469e47bb4ae3c6c07f97edf437cb +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-client-linux-arm.tar.gz) | 670f8ca60ea3cf0bb3262a772715e0ea735fccda6a92f3186299361dc455b304ae177d4017e0b67bbfa4a95e36f4cc3f7eb335e2a5130c93ac3fba2aff4519bf +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-client-linux-arm64.tar.gz) | a69a47907cff138ba393d8c87044fd95d97f3ca8f35d301b50742e2801ad7c229d99d6667971091f65825eb51854d585be0dd7421670110b1aa567e67e7ab4b3 +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-client-linux-ppc64le.tar.gz) | b929feade94b71c81908abdcd4343b1e1e20098fd65e10d4d02585ad649d292d06f52c7ddc349efa188ce5b093e703c7aa9582c6ae5a69699adb87bbf5350243 +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-client-linux-s390x.tar.gz) | 899d1470e412282cf289d8e24806d1a08c62ec0151f345ae3c9e497cc7bc0feab76498de4dd897d6adcdfa0c422e6b1a37e25d928669030f53457fd69d6e7df7 +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-client-windows-386.tar.gz) | 9f0bc90a269eabd06fe4f637b5172a3a6a7d3de26de0d66504c2e1f2093083c584ea39031db6075a7da7a86b98c48bed25aa88d4ac09060b38692c6a5b637078 +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-client-windows-amd64.tar.gz) | 05c8cc10188a1294b0d51d052942742a9b26411a08ec73494bf0e728a8a167e0a7863bdfc8864e76a371b584380098381805341e18b4b283b5d0cf298d5f7c7c -### Server Binaries +### Server binaries filename | sha512 hash -------- | ----------- -[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-server-linux-amd64.tar.gz) | `c6284929dd5940e750b48db72ffbc09f73c5ec31ab3db283babb8e4e07cd8cbb27642f592009caae4717981c0db82c16312849ef4cbafe76acc4264c7d5864ac` -[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-server-linux-arm.tar.gz) | `6fc9552cf082c54cc0833b19876117c87ba7feb5a12c7e57f71b52208daf03eaef3ca56bd22b7bce2d6e81b5a23537cf6f5497a6eaa356c0aab1d3de26c309f9` -[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-server-linux-arm64.tar.gz) | `b794b9c399e548949b5bfb2fe71123e86c2034847b2c99aca34b6de718a35355bbecdae9dc2a81c49e3c82fb4b5862526a3f63c2862b438895e12c5ea884f22e` -[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-server-linux-ppc64le.tar.gz) | `fddaed7a54f97046a91c29534645811c6346e973e22950b2607b8c119c2377e9ec2d32144f81626078cdaeca673129cc4016c1a3dbd3d43674aa777089fb56ac` -[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-server-linux-s390x.tar.gz) | `65951a534bb55069c7419f41cbcdfe2fae31541d8a3f9eca11fc2489addf281c5ad2d13719212657da0be5b898f22b57ac39446d99072872fbacb0a7d59a4f74` +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-server-linux-amd64.tar.gz) | 355f278728ef7ac7eb2f5568c99c1429543c6302bbd0ed3bd0378c08116075e56ae850a49241313f078e2392702672ec6c9b70c8d97b4f2f5f4bee36828a63ba +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-server-linux-arm.tar.gz) | 9ac02c2825e2fd4e92f0c0f67180c67c24e32841ccbabc82284bf6293727ffecfae65e8a42b527c2a7ca482752384928eb65c2a1706144ae7819a6b3a1ab291c +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-server-linux-arm64.tar.gz) | eb412453da03c82a9248412c8ccf4d4baa1fbfa81edd8d4f81d28969b40a3727e18934accc68f643d253446c58ffd2623292402495480b3d4b2a837b5318b957 +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-server-linux-ppc64le.tar.gz) | 07da2812c35bbc427ee5b4a0b601c3ae271e0d50ab0dd4c5c25399f43506fa2a187642eb9d4d2085df7b90264d48ea2f31088af87d9efa7eb2e87f91e1fdbde4 +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-server-linux-s390x.tar.gz) | 3b79442a3d6e389c4ff105922a8e49994c0b6c088d2c501bd8c78d9f9e814902f5bb72c8f9c89380b750fda9b3a336759b9b68f11d70bef4f0e984564a95c29e -### Node Binaries +### Node binaries filename | sha512 hash -------- | ----------- -[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-node-linux-amd64.tar.gz) | `992059efb5cae7ed0ef55820368d854bad1c6d13a70366162cd3b5111ce24c371c7c87ded2012f055e08b2ff1b4ef506e1f4e065daa3ac474fef50b5efa4fb07` -[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-node-linux-arm.tar.gz) | `c63ae0f8add5821ad267774314b8c8c1ffe3b785872bf278e721fd5dfdad1a5db1d4db3720bea0a36bf10d9c6dd93e247560162c0eac6e1b743246f587d3b27a` -[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-node-linux-arm64.tar.gz) | `47adb9ddf6eaf8f475b89f59ee16fbd5df183149a11ad1574eaa645b47a6d58aec2ca70ba857ce9f1a5793d44cf7a61ebc6874793bb685edaf19410f4f76fd13` -[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-node-linux-ppc64le.tar.gz) | `a3bc4a165567c7b76a3e45ab7b102d6eb3ecf373eb048173f921a4964cf9be8891d0d5b8dafbd88c3af7b0e21ef3d41c1e540c3347ddd84b929b3a3d02ceb7b2` -[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-node-linux-s390x.tar.gz) | `109ddf37c748f69584c829db57107c3518defe005c11fcd2a1471845c15aae0a3c89aafdd734229f4069ed18856cc650c80436684e1bdc43cfee3149b0324746` -[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-beta.1/kubernetes-node-windows-amd64.tar.gz) | `a3a75d2696ad3136476ad7d811e8eabaff5111b90e592695e651d6111f819ebf0165b8b7f5adc05afb5f7f01d1e5fb64876cb696e492feb20a477a5800382b7a` +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-node-linux-amd64.tar.gz) | f12edf1faf5f07de1ebc5a8626601c12927902e10aca3f11e398637382fdf55365dbd9a0ef38858553fb7569495ae2cf68f155dd2e49b85b27d76fb599bb92e4 +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-node-linux-arm.tar.gz) | 4fba8fc4e2102f07fb778aab597ec7231ea65c35e1aa618fe98b707b64a931237bd842c173e9120326e4d9deb983bb3917176762bba2212612bbc09d6e2105c4 +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-node-linux-arm64.tar.gz) | a2e1be5459a8346839970faf4e7ebdb8ab9f3273e02babf1f3199b06bdb67434a2d18fcd1628cf1b989756e99d8dad6624a455b9db11d50f51f509f4df5c27da +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-node-linux-ppc64le.tar.gz) | 16d2c1cc295474fc49fe9a827ddd73e81bdd6b76af7074987b90250023f99b6d70bf474e204c7d556802111984fcb3a330740b150bdc7970d0e3634eb94a1665 +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-node-linux-s390x.tar.gz) | 9dc6faa6cd007b13dfce703f3e271f80adcc4e029c90a4a9b4f2f143b9756f2893f8af3d7c2cf813f2bd6731cffd87d15d4229456c1685939f65bf467820ec6e +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-rc.0/kubernetes-node-windows-amd64.tar.gz) | f8bac2974c9142bfb80cd5eadeda79f79f27b78899a4e6e71809b795c708824ba442be83fdbadb98e01c3823dd8350776358258a205e851ed045572923cacba7 -## Changelog since v1.18.0-beta.0 +## Changelog since v1.21.0-beta.1 ## Urgent Upgrade Notes ### (No, really, you MUST read this before you upgrade) -- The StreamingProxyRedirects feature and `--redirect-container-streaming` flag are deprecated, and will be removed in a future release. The default behavior (proxy streaming requests through the kubelet) will be the only supported option. - If you are setting `--redirect-container-streaming=true`, then you must migrate off this configuration. The flag will no longer be able to be enabled starting in v1.20. If you are not setting the flag, no action is necessary. ([#88290](https://github.com/kubernetes/kubernetes/pull/88290), [@tallclair](https://github.com/tallclair)) [SIG API Machinery and Node] + - Migrated pkg/kubelet/cm/cpuset/cpuset.go to structured logging. Exit code changed from 255 to 1. ([#100007](https://github.com/kubernetes/kubernetes/pull/100007), [@utsavoza](https://github.com/utsavoza)) [SIG Instrumentation and Node] + +## Changes by Kind -- Yes. - - Feature Name: Support using network resources (VNet, LB, IP, etc.) in different AAD Tenant and Subscription than those for the cluster. - - Changes in Pull Request: - - 1. Add properties `networkResourceTenantID` and `networkResourceSubscriptionID` in cloud provider auth config section, which indicates the location of network resources. - 2. Add function `GetMultiTenantServicePrincipalToken` to fetch multi-tenant service principal token, which will be used by Azure VM/VMSS Clients in this feature. - 3. Add function `GetNetworkResourceServicePrincipalToken` to fetch network resource service principal token, which will be used by Azure Network Resource (Load Balancer, Public IP, Route Table, Network Security Group and their sub level resources) Clients in this feature. - 4. Related unit tests. - - None. - - User Documentation: In PR https://github.com/kubernetes-sigs/cloud-provider-azure/pull/301 ([#88384](https://github.com/kubernetes/kubernetes/pull/88384), [@bowen5](https://github.com/bowen5)) [SIG Cloud Provider] +### API Change +- Add Probe-level terminationGracePeriodSeconds field ([#99375](https://github.com/kubernetes/kubernetes/pull/99375), [@ehashman](https://github.com/ehashman)) [SIG API Machinery, Apps, Node and Testing] +- CSIServiceAccountToken is Beta now ([#99298](https://github.com/kubernetes/kubernetes/pull/99298), [@zshihang](https://github.com/zshihang)) [SIG Auth, Storage and Testing] +- Discovery.k8s.io/v1beta1 EndpointSlices are deprecated in favor of discovery.k8s.io/v1, and will no longer be served in Kubernetes v1.25. ([#100472](https://github.com/kubernetes/kubernetes/pull/100472), [@liggitt](https://github.com/liggitt)) [SIG Network] +- FieldManager no longer owns fields that get reset before the object is persisted (e.g. "status wiping"). ([#99661](https://github.com/kubernetes/kubernetes/pull/99661), [@kevindelgado](https://github.com/kevindelgado)) [SIG API Machinery, Auth and Testing] +- Generic ephemeral volumes are beta. ([#99643](https://github.com/kubernetes/kubernetes/pull/99643), [@pohly](https://github.com/pohly)) [SIG API Machinery, Apps, Auth, CLI, Node, Storage and Testing] +- Implement the GetAvailableResources in the podresources API. ([#95734](https://github.com/kubernetes/kubernetes/pull/95734), [@fromanirh](https://github.com/fromanirh)) [SIG Instrumentation, Node and Testing] +- The Endpoints controller will now set the `endpoints.kubernetes.io/over-capacity` annotation to "warning" when an Endpoints resource contains more than 1000 addresses. In a future release, the controller will truncate Endpoints that exceed this limit. The EndpointSlice API can be used to support significantly larger number of addresses. ([#99975](https://github.com/kubernetes/kubernetes/pull/99975), [@robscott](https://github.com/robscott)) [SIG Apps and Network] +- The PodDisruptionBudget API has been promoted to policy/v1 with no schema changes. The only functional change is that an empty selector (`{}`) written to a policy/v1 PodDisruptionBudget now selects all pods in the namespace. The behavior of the policy/v1beta1 API remains unchanged. The policy/v1beta1 PodDisruptionBudget API is deprecated and will no longer be served in 1.25+. ([#99290](https://github.com/kubernetes/kubernetes/pull/99290), [@mortent](https://github.com/mortent)) [SIG API Machinery, Apps, Auth, Autoscaling, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation, Scheduling and Testing] +- Topology Aware Hints are now available in alpha and can be enabled with the `TopologyAwareHints` feature gate. ([#99522](https://github.com/kubernetes/kubernetes/pull/99522), [@robscott](https://github.com/robscott)) [SIG API Machinery, Apps, Auth, Instrumentation, Network and Testing] + +### Feature + +- Add e2e test to validate performance metrics of volume lifecycle operations ([#94334](https://github.com/kubernetes/kubernetes/pull/94334), [@RaunakShah](https://github.com/RaunakShah)) [SIG Storage and Testing] +- EmptyDir memory backed volumes are sized as the the minimum of pod allocatable memory on a host and an optional explicit user provided value. ([#100319](https://github.com/kubernetes/kubernetes/pull/100319), [@derekwaynecarr](https://github.com/derekwaynecarr)) [SIG Node] +- Enables Kubelet to check volume condition and log events to corresponding pods. ([#99284](https://github.com/kubernetes/kubernetes/pull/99284), [@fengzixu](https://github.com/fengzixu)) [SIG Apps, Instrumentation, Node and Storage] +- Introduce a churn operator to scheduler perf testing framework. ([#98900](https://github.com/kubernetes/kubernetes/pull/98900), [@Huang-Wei](https://github.com/Huang-Wei)) [SIG Scheduling and Testing] +- Kubernetes is now built with Golang 1.16.1 ([#100106](https://github.com/kubernetes/kubernetes/pull/100106), [@justaugustus](https://github.com/justaugustus)) [SIG Cloud Provider, Instrumentation, Release and Testing] +- Migrated pkg/kubelet/cm/devicemanager to structured logging ([#99976](https://github.com/kubernetes/kubernetes/pull/99976), [@knabben](https://github.com/knabben)) [SIG Instrumentation and Node] +- Migrated pkg/kubelet/cm/memorymanager to structured logging ([#99974](https://github.com/kubernetes/kubernetes/pull/99974), [@knabben](https://github.com/knabben)) [SIG Instrumentation and Node] +- Migrated pkg/kubelet/cm/topologymanager to structure logging ([#99969](https://github.com/kubernetes/kubernetes/pull/99969), [@knabben](https://github.com/knabben)) [SIG Instrumentation and Node] +- Rename metrics `etcd_object_counts` to `apiserver_storage_object_counts` and mark it as stable. The original `etcd_object_counts` metrics name is marked as "Deprecated" and will be removed in the future. ([#99785](https://github.com/kubernetes/kubernetes/pull/99785), [@erain](https://github.com/erain)) [SIG API Machinery, Instrumentation and Testing] +- Update pause container to run as pseudo user and group `65535:65535`. This implies the release of version 3.5 of the container images. ([#97963](https://github.com/kubernetes/kubernetes/pull/97963), [@saschagrunert](https://github.com/saschagrunert)) [SIG CLI, Cloud Provider, Cluster Lifecycle, Node, Release, Security and Testing] +- Users might specify the `kubectl.kubernetes.io/default-exec-container` annotation in a Pod to preselect container for kubectl commands. ([#99833](https://github.com/kubernetes/kubernetes/pull/99833), [@mengjiao-liu](https://github.com/mengjiao-liu)) [SIG CLI] + +### Bug or Regression + +- Add ability to skip OpenAPI handler installation to the GenericAPIServer ([#100341](https://github.com/kubernetes/kubernetes/pull/100341), [@kevindelgado](https://github.com/kevindelgado)) [SIG API Machinery] +- Count pod overhead against an entity's ResourceQuota ([#99600](https://github.com/kubernetes/kubernetes/pull/99600), [@gjkim42](https://github.com/gjkim42)) [SIG API Machinery and Node] +- EndpointSlice controllers are less likely to create duplicate EndpointSlices. ([#100103](https://github.com/kubernetes/kubernetes/pull/100103), [@robscott](https://github.com/robscott)) [SIG Apps and Network] +- Ensure only one LoadBalancer rule is created when HA mode is enabled ([#99825](https://github.com/kubernetes/kubernetes/pull/99825), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] +- Fixed a race condition on API server startup ensuring previously created webhook configurations are effective before the first write request is admitted. ([#95783](https://github.com/kubernetes/kubernetes/pull/95783), [@roycaihw](https://github.com/roycaihw)) [SIG API Machinery] +- Fixed authentication_duration_seconds metric. Previously it included whole apiserver request duration. ([#99944](https://github.com/kubernetes/kubernetes/pull/99944), [@marseel](https://github.com/marseel)) [SIG API Machinery, Instrumentation and Scalability] +- Fixes issue where inline AzueFile secrets could not be accessed from the pod's namespace. ([#100563](https://github.com/kubernetes/kubernetes/pull/100563), [@msau42](https://github.com/msau42)) [SIG Storage] +- Improve speed of vSphere PV provisioning and reduce number of API calls ([#100054](https://github.com/kubernetes/kubernetes/pull/100054), [@gnufied](https://github.com/gnufied)) [SIG Cloud Provider and Storage] +- Kubectl: Fixed panic when describing an ingress backend without an API Group ([#100505](https://github.com/kubernetes/kubernetes/pull/100505), [@lauchokyip](https://github.com/lauchokyip)) [SIG CLI] +- Kubectl: fix case of age column in describe node (#96963, @bl-ue) ([#96963](https://github.com/kubernetes/kubernetes/pull/96963), [@bl-ue](https://github.com/bl-ue)) [SIG CLI] +- Kubelet.exe on Windows now checks that the process running as administrator and the executing user account is listed in the built-in administrators group. This is the equivalent to checking the process is running as uid 0. ([#96616](https://github.com/kubernetes/kubernetes/pull/96616), [@perithompson](https://github.com/perithompson)) [SIG Node and Windows] +- Kubelet: Fixed the bug of getting the number of cpu when the number of cpu logical processors is more than 64 in windows ([#97378](https://github.com/kubernetes/kubernetes/pull/97378), [@hwdef](https://github.com/hwdef)) [SIG Node and Windows] +- Pass `KUBE_BUILD_CONFORMANCE=y` to the package-tarballs to reenable building the conformance tarballs. ([#100571](https://github.com/kubernetes/kubernetes/pull/100571), [@puerco](https://github.com/puerco)) [SIG Release] +- Pod Log stats for windows now reports metrics ([#99221](https://github.com/kubernetes/kubernetes/pull/99221), [@jsturtevant](https://github.com/jsturtevant)) [SIG Node, Storage, Testing and Windows] + +### Other (Cleanup or Flake) + +- A new storage E2E testsuite covers CSIStorageCapacity publishing if a driver opts into the test. ([#100537](https://github.com/kubernetes/kubernetes/pull/100537), [@pohly](https://github.com/pohly)) [SIG Storage and Testing] +- Convert cmd/kubelet/app/server.go to structured logging ([#98334](https://github.com/kubernetes/kubernetes/pull/98334), [@wawa0210](https://github.com/wawa0210)) [SIG Node] +- If kube-apiserver enabled goaway feature, clients required golang 1.15.8 or 1.16+ version to avoid un-expected data race issue. ([#98809](https://github.com/kubernetes/kubernetes/pull/98809), [@answer1991](https://github.com/answer1991)) [SIG API Machinery] +- Increased CSINodeIDMaxLength from 128 bytes to 192 bytes. ([#98753](https://github.com/kubernetes/kubernetes/pull/98753), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Apps and Storage] +- Migrate `pkg/kubelet/pluginmanager` to structured logging ([#99885](https://github.com/kubernetes/kubernetes/pull/99885), [@qingwave](https://github.com/qingwave)) [SIG Node] +- Migrate `pkg/kubelet/preemption/preemption.go` and `pkg/kubelet/logs/container_log_manager.go` to structured logging ([#99848](https://github.com/kubernetes/kubernetes/pull/99848), [@qingwave](https://github.com/qingwave)) [SIG Node] +- Migrate `pkg/kubelet/(cri)` to structured logging ([#99006](https://github.com/kubernetes/kubernetes/pull/99006), [@yangjunmyfm192085](https://github.com/yangjunmyfm192085)) [SIG Node] +- Migrate `pkg/kubelet/(node, pod)` to structured logging ([#98847](https://github.com/kubernetes/kubernetes/pull/98847), [@yangjunmyfm192085](https://github.com/yangjunmyfm192085)) [SIG Node] +- Migrate `pkg/kubelet/(volume,container)` to structured logging ([#98850](https://github.com/kubernetes/kubernetes/pull/98850), [@yangjunmyfm192085](https://github.com/yangjunmyfm192085)) [SIG Node] +- Migrate `pkg/kubelet/kubelet_node_status.go` to structured logging ([#98154](https://github.com/kubernetes/kubernetes/pull/98154), [@yangjunmyfm192085](https://github.com/yangjunmyfm192085)) [SIG Node and Release] +- Migrate `pkg/kubelet/lifecycle,oom` to structured logging ([#99479](https://github.com/kubernetes/kubernetes/pull/99479), [@mengjiao-liu](https://github.com/mengjiao-liu)) [SIG Instrumentation and Node] +- Migrate cmd/kubelet/+ pkg/kubelet/cadvisor/cadvisor_linux.go + pkg/kubelet/cri/remote/util/util_unix.go + pkg/kubelet/images/image_manager.go to structured logging ([#99994](https://github.com/kubernetes/kubernetes/pull/99994), [@AfrouzMashayekhi](https://github.com/AfrouzMashayekhi)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/cm/container_manager_linux.go and pkg/kubelet/cm/container_manager_stub.go to structured logging ([#100001](https://github.com/kubernetes/kubernetes/pull/100001), [@shiyajuan123](https://github.com/shiyajuan123)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/cm/cpumanage/{topology/togit pology.go, policy_none.go, cpu_assignment.go} to structured logging ([#100163](https://github.com/kubernetes/kubernetes/pull/100163), [@lala123912](https://github.com/lala123912)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/cm/cpumanager/state to structured logging ([#99563](https://github.com/kubernetes/kubernetes/pull/99563), [@jmguzik](https://github.com/jmguzik)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/config to structured logging ([#100002](https://github.com/kubernetes/kubernetes/pull/100002), [@AfrouzMashayekhi](https://github.com/AfrouzMashayekhi)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/kubelet.go to structured logging ([#99861](https://github.com/kubernetes/kubernetes/pull/99861), [@navidshaikh](https://github.com/navidshaikh)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/kubeletconfig to structured logging ([#100265](https://github.com/kubernetes/kubernetes/pull/100265), [@ehashman](https://github.com/ehashman)) [SIG Node] +- Migrate pkg/kubelet/kuberuntime to structured logging ([#99970](https://github.com/kubernetes/kubernetes/pull/99970), [@krzysiekg](https://github.com/krzysiekg)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/prober to structured logging ([#99830](https://github.com/kubernetes/kubernetes/pull/99830), [@krzysiekg](https://github.com/krzysiekg)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/winstats to structured logging ([#99855](https://github.com/kubernetes/kubernetes/pull/99855), [@hexxdump](https://github.com/hexxdump)) [SIG Instrumentation and Node] +- Migrate probe log messages to structured logging ([#97093](https://github.com/kubernetes/kubernetes/pull/97093), [@aldudko](https://github.com/aldudko)) [SIG Instrumentation and Node] +- Migrate remaining kubelet files to structured logging ([#100196](https://github.com/kubernetes/kubernetes/pull/100196), [@ehashman](https://github.com/ehashman)) [SIG Instrumentation and Node] +- `apiserver_storage_objects` (a newer version of `etcd_object_counts) is promoted and marked as stable. ([#100082](https://github.com/kubernetes/kubernetes/pull/100082), [@logicalhan](https://github.com/logicalhan)) [SIG API Machinery, Instrumentation and Testing] + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- github.com/cilium/ebpf: [1c8d4c9 → v0.2.0](https://github.com/cilium/ebpf/compare/1c8d4c9...v0.2.0) +- github.com/containerd/console: [v1.0.0 → v1.0.1](https://github.com/containerd/console/compare/v1.0.0...v1.0.1) +- github.com/containerd/containerd: [v1.4.1 → v1.4.4](https://github.com/containerd/containerd/compare/v1.4.1...v1.4.4) +- github.com/creack/pty: [v1.1.9 → v1.1.11](https://github.com/creack/pty/compare/v1.1.9...v1.1.11) +- github.com/docker/docker: [bd33bbf → v20.10.2+incompatible](https://github.com/docker/docker/compare/bd33bbf...v20.10.2) +- github.com/google/cadvisor: [v0.38.8 → v0.39.0](https://github.com/google/cadvisor/compare/v0.38.8...v0.39.0) +- github.com/konsorten/go-windows-terminal-sequences: [v1.0.3 → v1.0.2](https://github.com/konsorten/go-windows-terminal-sequences/compare/v1.0.3...v1.0.2) +- github.com/moby/sys/mountinfo: [v0.1.3 → v0.4.0](https://github.com/moby/sys/mountinfo/compare/v0.1.3...v0.4.0) +- github.com/moby/term: [672ec06 → df9cb8a](https://github.com/moby/term/compare/672ec06...df9cb8a) +- github.com/mrunalp/fileutils: [abd8a0e → v0.5.0](https://github.com/mrunalp/fileutils/compare/abd8a0e...v0.5.0) +- github.com/opencontainers/runc: [v1.0.0-rc92 → v1.0.0-rc93](https://github.com/opencontainers/runc/compare/v1.0.0-rc92...v1.0.0-rc93) +- github.com/opencontainers/runtime-spec: [4d89ac9 → e6143ca](https://github.com/opencontainers/runtime-spec/compare/4d89ac9...e6143ca) +- github.com/opencontainers/selinux: [v1.6.0 → v1.8.0](https://github.com/opencontainers/selinux/compare/v1.6.0...v1.8.0) +- github.com/sirupsen/logrus: [v1.6.0 → v1.7.0](https://github.com/sirupsen/logrus/compare/v1.6.0...v1.7.0) +- github.com/syndtr/gocapability: [d983527 → 42c35b4](https://github.com/syndtr/gocapability/compare/d983527...42c35b4) +- github.com/willf/bitset: [d5bec33 → v1.1.11](https://github.com/willf/bitset/compare/d5bec33...v1.1.11) +- gotest.tools/v3: v3.0.2 → v3.0.3 +- k8s.io/klog/v2: v2.5.0 → v2.8.0 +- sigs.k8s.io/structured-merge-diff/v4: v4.0.3 → v4.1.0 + +### Removed +_Nothing has changed._ + + + +# v1.21.0-beta.1 + + +## Downloads for v1.21.0-beta.1 + +### Source Code + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes.tar.gz) | c9f4f25242e319e5d90f49d26f239a930aad69677c0f3c2387c56bb13482648a26ed234be2bfe2352508f35010e3eb6d3b127c31a9f24fa1e53ac99c38520fe4 +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-src.tar.gz) | 255357db8fa160cab2187658906b674a8b0d9b9a5b5f688cc7b69dc124f5da00362c6cc18ae9b80f7ddb3da6f64c2ab2f12fb9b63a4e063c7366a5375b175cda + +### Client binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-client-darwin-amd64.tar.gz) | 02efd389c8126456416fd2c7ea25c3cc30f612649ad91f631f068d6c0e5e539484d3763cb9a8645ad6b8077e4fcd1552a659d7516ebc4ce6828cf823b65c3016 +[kubernetes-client-darwin-arm64.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-client-darwin-arm64.tar.gz) | ac90dcd1699d1d7ff9c8342d481f6d0d97ccdc3ec501a56dc7c9e1898a8f77f712bf66942d304bfe581b5494f13e3efa211865de88f89749780e9e26e673dbdb +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-client-linux-386.tar.gz) | cce5fb84cc7a1ee664f89d8ad3064307c51c044e9ddd2ae5a004939b69d3b3ef6f29acc5782e27d0c8f0d6d3d9c96e922f5d1b99d210ca3e754666d775df9f0c +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-client-linux-amd64.tar.gz) | 2e93bbd2e60ad7cd8fe495115e96c55b1dc8facd100a827ef9c197a732679b60cceb9ea7bf92a1f5e328c3b8adfa8d3922cbc5d8370e374f3381b83f5b877b4f +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-client-linux-arm.tar.gz) | 23f03b6a8fa9decce9b89a2c1bd3dae6d0b2f9e533e35a79e2c5a29326a165259677594ae83c877219a21bdb95557a284e55f4eec12954742794579c89a7d7e5 +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-client-linux-arm64.tar.gz) | 3acf3101b46568b0ded6b90f13df0e918870d6812dc1a584903ddb8ba146484a204b9e442f863df47c7d4dab043fd9f7294c5510d3eb09004993d6d3b1e9e13c +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-client-linux-ppc64le.tar.gz) | f749198df69577f62872d3096138a1b8969ec6b1636eb68eb56640bf33cf5f97a11df4363462749a1c0dc3ccbb8ae76c5d66864bf1c5cf7e52599caaf498e504 +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-client-linux-s390x.tar.gz) | 3f6c0189d59fca22cdded3a02c672ef703d17e6ab0831e173a870e14ccec436c142600e9fc35b403571b6906f2be8d18d38d33330f7caada971bbe1187b388f6 +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-client-windows-386.tar.gz) | 03d92371c425cf331c80807c0ac56f953be304fc6719057258a363d527d186d610e1d4b4d401b34128062983265c2e21f2d2389231aa66a6f5787eee78142cf6 +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-client-windows-amd64.tar.gz) | 489ece0c886a025ca3a25d28518637a5a824ea6544e7ef8778321036f13c8909a978ad4ceca966cec1e1cda99f25ca78bfd37460d1231c77436d216d43c872ad + +### Server binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-server-linux-amd64.tar.gz) | 2e95cb31d5afcb6842c41d25b7d0c18dd7e65693b2d93c8aa44e5275f9c6201e1a67685c7a8ddefa334babb04cb559d26e39b6a18497695a07dc270568cae108 +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-server-linux-arm.tar.gz) | 2927e82b98404c077196ce3968f3afd51a7576aa56d516019bd3976771c0213ba01e78da5b77478528e770da0d334e9457995fafb98820ed68b2ee34beb68856 +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-server-linux-arm64.tar.gz) | e0f7aea3ea598214a9817bc04949389cb7e4e7b9503141a590ef48c0b681fe44a4243ebc6280752fa41aa1093149b3ee1bcef7664edb746097a342281825430b +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-server-linux-ppc64le.tar.gz) | c011f7eb01294e9ba5d5ced719068466f88ed595dcb8d554a36a4dd5118fb6b3d6bafe8bf89aa2d42988e69793ed777ba77b8876c6ec74f898a43cfce1f61bf4 +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-server-linux-s390x.tar.gz) | 15f6683e7f16caab7eebead2b7c15799460abbf035a43de0b75f96b0be19908f58add98a777a0cca916230d60cf6bfe3fee92b9dcff50274b1e37c243c157969 + +### Node binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-node-linux-amd64.tar.gz) | ed58679561197110f366b9109f7afd62c227bfc271918ccf3eea203bb2ab6428eb5db4dd6c965f202a8a636f66da199470269b863815809b99d53d2fa47af2ea +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-node-linux-arm.tar.gz) | 7e6c7f1957fcdecec8fef689c5019edbc0d0c11d22dafbfef0a07121d10d8f6273644f73511bd06a9a88b04d81a940bd6645ffb5711422af64af547a45c76273 +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-node-linux-arm64.tar.gz) | a3618f29967e7a1574917a67f0296e65780321eda484b99aa32bfd4dc9b35acdefce33da952ac52dfb509fbac5bf700cf177431fad2ab4adcab0544538939faa +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-node-linux-ppc64le.tar.gz) | 326d3eb521b41bdf489912177f70b8cdd7cd828bb9b3d847ed3694eb27e457f24e0a88b8e51b726eee39800a3c5a40c1b30e3a8ec4a34d8041b3d8ef05d1b749 +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-node-linux-s390x.tar.gz) | 022d05ebaa66a0332c4fe18cdaf23d14c2c7e4d1f2af7f27baaf1eb042e6890dc3434b4ac8ba58c35d590717956f8c3458112685aff4938b94b18e263c3f4256 +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.1/kubernetes-node-windows-amd64.tar.gz) | fa691ed93f07af6bc1cf57e20a30580d6c528f88e5fea3c14f39c1820969dc5a0eb476c5b87b288593d0c086c4dd93aff6165082393283c3f46c210f9bb66d61 + +## Changelog since v1.21.0-beta.0 + +## Urgent Upgrade Notes + +### (No, really, you MUST read this before you upgrade) + + - Kubeadm: during "init" an empty cgroupDriver value in the KubeletConfiguration is now always set to "systemd" unless the user is explicit about it. This requires existing machine setups to configure the container runtime to use the "systemd" driver. Documentation on this topic can be found here: https://kubernetes.io/docs/setup/production-environment/container-runtimes/. When upgrading existing clusters / nodes using "kubeadm upgrade" the old cgroupDriver value is preserved, but in 1.22 this change will also apply to "upgrade". For more information on migrating to the "systemd" driver or remaining on the "cgroupfs" driver see: https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/. ([#99471](https://github.com/kubernetes/kubernetes/pull/99471), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] + - Migrate `pkg/kubelet/(dockershim, network)` to structured logging + Exit code changed from 255 to 1 ([#98939](https://github.com/kubernetes/kubernetes/pull/98939), [@yangjunmyfm192085](https://github.com/yangjunmyfm192085)) [SIG Network and Node] + - Migrate `pkg/kubelet/certificate` to structured logging + Exit code changed from 255 to 1 ([#98993](https://github.com/kubernetes/kubernetes/pull/98993), [@SataQiu](https://github.com/SataQiu)) [SIG Auth and Node] + - Newly provisioned PVs by EBS plugin will no longer use the deprecated "failure-domain.beta.kubernetes.io/zone" and "failure-domain.beta.kubernetes.io/region" labels. It will use "topology.kubernetes.io/zone" and "topology.kubernetes.io/region" labels instead. ([#99130](https://github.com/kubernetes/kubernetes/pull/99130), [@ayberk](https://github.com/ayberk)) [SIG Cloud Provider, Storage and Testing] + - Newly provisioned PVs by OpenStack Cinder plugin will no longer use the deprecated "failure-domain.beta.kubernetes.io/zone" and "failure-domain.beta.kubernetes.io/region" labels. It will use "topology.kubernetes.io/zone" and "topology.kubernetes.io/region" labels instead. ([#99719](https://github.com/kubernetes/kubernetes/pull/99719), [@jsafrane](https://github.com/jsafrane)) [SIG Cloud Provider and Storage] + - OpenStack Cinder CSI migration is on by default, Clinder CSI driver must be installed on clusters on OpenStack for Cinder volumes to work. ([#98538](https://github.com/kubernetes/kubernetes/pull/98538), [@dims](https://github.com/dims)) [SIG Storage] + - Package pkg/kubelet/server migrated to structured logging + Exit code changed from 255 to 1 ([#99838](https://github.com/kubernetes/kubernetes/pull/99838), [@adisky](https://github.com/adisky)) [SIG Node] + - Pkg/kubelet/kuberuntime/kuberuntime_manager.go migrated to structured logging + Exit code changed from 255 to 1 ([#99841](https://github.com/kubernetes/kubernetes/pull/99841), [@adisky](https://github.com/adisky)) [SIG Instrumentation and Node] + ## Changes by Kind ### Deprecation -- Azure service annotation service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset has been deprecated. Its support would be removed in a future release. ([#88462](https://github.com/kubernetes/kubernetes/pull/88462), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] +- Kubeadm: the deprecated kube-dns is no longer supported as an option. If "ClusterConfiguration.dns.type" is set to "kube-dns" kubeadm will now throw an error. ([#99646](https://github.com/kubernetes/kubernetes/pull/99646), [@rajansandeep](https://github.com/rajansandeep)) [SIG Cluster Lifecycle] +- Remove deprecated --generator --replicas --service-generator --service-overrides --schedule from kubectl run + Deprecate --serviceaccount --hostport --requests --limits in kubectl run ([#99732](https://github.com/kubernetes/kubernetes/pull/99732), [@soltysh](https://github.com/soltysh)) [SIG CLI and Testing] +- `audit.k8s.io/v1beta1` and `audit.k8s.io/v1alpha1` audit policy configuration and audit events are deprecated in favor of `audit.k8s.io/v1`, available since v1.13. kube-apiserver invocations that specify alpha or beta policy configurations with `--audit-policy-file`, or explicitly request alpha or beta audit events with `--audit-log-version` / `--audit-webhook-version` must update to use `audit.k8s.io/v1` and accept `audit.k8s.io/v1` events prior to v1.24. ([#98858](https://github.com/kubernetes/kubernetes/pull/98858), [@carlory](https://github.com/carlory)) [SIG Auth] +- `diskformat` stroage class parameter for in-tree vSphere volume plugin is deprecated as of v1.21 release. Please consider updating storageclass and remove `diskformat` parameter. vSphere CSI Driver does not support diskformat storageclass parameter. + + vSphere releases less than 67u3 are deprecated as of v1.21. Please consider upgrading vSphere to 67u3 or above. vSphere CSI Driver requires minimum vSphere 67u3. + + VM Hardware version less than 15 is deprecated as of v1.21. Please consider upgrading the Node VM Hardware version to 15 or above. vSphere CSI Driver recommends Node VM's Hardware version set to at least vmx-15. + + Multi vCenter support is deprecated as of v1.21. If you have a Kubernetes cluster spanning across multiple vCenter servers, please consider moving all k8s nodes to a single vCenter Server. vSphere CSI Driver does not support Kubernetes deployment spanning across multiple vCenter servers. + + Support for these deprecations will be available till Kubernetes v1.24. ([#98546](https://github.com/kubernetes/kubernetes/pull/98546), [@divyenpatel](https://github.com/divyenpatel)) [SIG Cloud Provider and Storage] ### API Change -- API additions to apiserver types ([#87179](https://github.com/kubernetes/kubernetes/pull/87179), [@Jefftree](https://github.com/Jefftree)) [SIG API Machinery, Cloud Provider and Cluster Lifecycle] -- Add Scheduling Profiles to kubescheduler.config.k8s.io/v1alpha2 ([#88087](https://github.com/kubernetes/kubernetes/pull/88087), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling and Testing] -- Added support for multiple sizes huge pages on a container level ([#84051](https://github.com/kubernetes/kubernetes/pull/84051), [@bart0sh](https://github.com/bart0sh)) [SIG Apps, Node and Storage] -- AppProtocol is a new field on Service and Endpoints resources, enabled with the ServiceAppProtocol feature gate. ([#88503](https://github.com/kubernetes/kubernetes/pull/88503), [@robscott](https://github.com/robscott)) [SIG Apps and Network] -- Fixed missing validation of uniqueness of list items in lists with `x-kubernetes-list-type: map` or x-kubernetes-list-type: set` in CustomResources. ([#84920](https://github.com/kubernetes/kubernetes/pull/84920), [@sttts](https://github.com/sttts)) [SIG API Machinery] -- Introduces optional --detect-local flag to kube-proxy. - Currently the only supported value is "cluster-cidr", - which is the default if not specified. ([#87748](https://github.com/kubernetes/kubernetes/pull/87748), [@satyasm](https://github.com/satyasm)) [SIG Cluster Lifecycle, Network and Scheduling] -- Kube-scheduler can run more than one scheduling profile. Given a pod, the profile is selected by using its `.spec.SchedulerName`. ([#88285](https://github.com/kubernetes/kubernetes/pull/88285), [@alculquicondor](https://github.com/alculquicondor)) [SIG Apps, Scheduling and Testing] -- Moving Windows RunAsUserName feature to GA ([#87790](https://github.com/kubernetes/kubernetes/pull/87790), [@marosset](https://github.com/marosset)) [SIG Apps and Windows] +- 1. PodAffinityTerm includes a namespaceSelector field to allow selecting eligible namespaces based on their labels. + 2. A new CrossNamespacePodAffinity quota scope API that allows restricting which namespaces allowed to use PodAffinityTerm with corss-namespace reference via namespaceSelector or namespaces fields. ([#98582](https://github.com/kubernetes/kubernetes/pull/98582), [@ahg-g](https://github.com/ahg-g)) [SIG API Machinery, Apps, Auth and Testing] +- Add a default metadata name labels for selecting any namespace by its name. ([#96968](https://github.com/kubernetes/kubernetes/pull/96968), [@jayunit100](https://github.com/jayunit100)) [SIG API Machinery, Apps, Cloud Provider, Storage and Testing] +- Added `.spec.completionMode` field to Job, with accepted values `NonIndexed` (default) and `Indexed` ([#98441](https://github.com/kubernetes/kubernetes/pull/98441), [@alculquicondor](https://github.com/alculquicondor)) [SIG Apps and CLI] +- Clarified NetworkPolicy policyTypes documentation ([#97216](https://github.com/kubernetes/kubernetes/pull/97216), [@joejulian](https://github.com/joejulian)) [SIG Network] +- DaemonSets accept a MaxSurge integer or percent on their rolling update strategy that will launch the updated pod on nodes and wait for those pods to go ready before marking the old out-of-date pods as deleted. This allows workloads to avoid downtime during upgrades when deployed using DaemonSets. This feature is alpha and is behind the DaemonSetUpdateSurge feature gate. ([#96441](https://github.com/kubernetes/kubernetes/pull/96441), [@smarterclayton](https://github.com/smarterclayton)) [SIG Apps and Testing] +- EndpointSlice API is now GA. The EndpointSlice topology field has been removed from the GA API and will be replaced by a new per Endpoint Zone field. If the topology field was previously used, it will be converted into an annotation in the v1 Resource. The discovery.k8s.io/v1alpha1 API is removed. ([#99662](https://github.com/kubernetes/kubernetes/pull/99662), [@swetharepakula](https://github.com/swetharepakula)) [SIG API Machinery, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation, Network and Testing] +- EndpointSlice Controllers are now GA. The EndpointSlice Controller will not populate the `deprecatedTopology` field and will only provide topology information through the `zone` and `nodeName` fields. ([#99870](https://github.com/kubernetes/kubernetes/pull/99870), [@swetharepakula](https://github.com/swetharepakula)) [SIG API Machinery, Apps, Auth, Network and Testing] +- IngressClass resource can now reference a resource in a specific namespace + for implementation-specific configuration(previously only Cluster-level resources were allowed). + This feature can be enabled using the IngressClassNamespacedParams feature gate. ([#99275](https://github.com/kubernetes/kubernetes/pull/99275), [@hbagdi](https://github.com/hbagdi)) [SIG API Machinery, CLI and Network] +- Introduce conditions for PodDisruptionBudget ([#98127](https://github.com/kubernetes/kubernetes/pull/98127), [@mortent](https://github.com/mortent)) [SIG API Machinery, Apps, Auth, CLI, Cloud Provider, Cluster Lifecycle and Instrumentation] +- Jobs API has a new .spec.suspend field that can be used to suspend and resume Jobs ([#98727](https://github.com/kubernetes/kubernetes/pull/98727), [@adtac](https://github.com/adtac)) [SIG API Machinery, Apps, Node, Scheduling and Testing] +- Kubelet Graceful Node Shutdown feature is now beta. ([#99735](https://github.com/kubernetes/kubernetes/pull/99735), [@bobbypage](https://github.com/bobbypage)) [SIG Node] +- Limit the quest value of hugepage to integer multiple of page size. ([#98515](https://github.com/kubernetes/kubernetes/pull/98515), [@lala123912](https://github.com/lala123912)) [SIG Apps] +- One new field "InternalTrafficPolicy" in Service is added. + It specifies if the cluster internal traffic should be routed to all endpoints or node-local endpoints only. + "Cluster" routes internal traffic to a Service to all endpoints. + "Local" routes traffic to node-local endpoints only, and traffic is dropped if no node-local endpoints are ready. + The default value is "Cluster". ([#96600](https://github.com/kubernetes/kubernetes/pull/96600), [@maplain](https://github.com/maplain)) [SIG API Machinery, Apps and Network] +- PodSecurityPolicy only stores "generic" as allowed volume type if the GenericEphemeralVolume feature gate is enabled ([#98918](https://github.com/kubernetes/kubernetes/pull/98918), [@pohly](https://github.com/pohly)) [SIG Auth and Security] +- Promote CronJobs to batch/v1 ([#99423](https://github.com/kubernetes/kubernetes/pull/99423), [@soltysh](https://github.com/soltysh)) [SIG API Machinery, Apps, CLI and Testing] +- Remove support for building Kubernetes with bazel. ([#99561](https://github.com/kubernetes/kubernetes/pull/99561), [@BenTheElder](https://github.com/BenTheElder)) [SIG API Machinery, Apps, Architecture, Auth, Autoscaling, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation, Network, Node, Release, Scalability, Scheduling, Storage, Testing and Windows] +- Setting loadBalancerClass in load balancer type of service is available with this PR. + Users who want to use a custom load balancer can specify loadBalancerClass to achieve it. ([#98277](https://github.com/kubernetes/kubernetes/pull/98277), [@XudongLiuHarold](https://github.com/XudongLiuHarold)) [SIG API Machinery, Apps, Cloud Provider and Network] +- Storage capacity tracking (= the CSIStorageCapacity feature) is beta, storage.k8s.io/v1alpha1/VolumeAttachment and storage.k8s.io/v1alpha1/CSIStorageCapacity objects are deprecated ([#99641](https://github.com/kubernetes/kubernetes/pull/99641), [@pohly](https://github.com/pohly)) [SIG API Machinery, Apps, Auth, Scheduling, Storage and Testing] +- Support for Indexed Job: a Job that is considered completed when Pods associated to indexes from 0 to (.spec.completions-1) have succeeded. ([#98812](https://github.com/kubernetes/kubernetes/pull/98812), [@alculquicondor](https://github.com/alculquicondor)) [SIG Apps and CLI] +- The apiserver now resets managedFields that got corrupted by a mutating admission controller. ([#98074](https://github.com/kubernetes/kubernetes/pull/98074), [@kwiesmueller](https://github.com/kwiesmueller)) [SIG API Machinery and Testing] +- `controller.kubernetes.io/pod-deletion-cost` annotation can be set to offer a hint on the cost of deleting a pod compared to other pods belonging to the same ReplicaSet. Pods with lower deletion cost are deleted first. This is an alpha feature. ([#99163](https://github.com/kubernetes/kubernetes/pull/99163), [@ahg-g](https://github.com/ahg-g)) [SIG Apps] ### Feature -- Add --dry-run to kubectl delete, taint, replace ([#88292](https://github.com/kubernetes/kubernetes/pull/88292), [@julianvmodesto](https://github.com/julianvmodesto)) [SIG CLI and Testing] -- Add huge page stats to Allocated resources in "kubectl describe node" ([#80605](https://github.com/kubernetes/kubernetes/pull/80605), [@odinuge](https://github.com/odinuge)) [SIG CLI] -- Kubeadm: The ClusterStatus struct present in the kubeadm-config ConfigMap is deprecated and will be removed on a future version. It is going to be maintained by kubeadm until it gets removed. The same information can be found on `etcd` and `kube-apiserver` pod annotations, `kubeadm.kubernetes.io/etcd.advertise-client-urls` and `kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint` respectively. ([#87656](https://github.com/kubernetes/kubernetes/pull/87656), [@ereslibre](https://github.com/ereslibre)) [SIG Cluster Lifecycle] -- Kubeadm: add the experimental feature gate PublicKeysECDSA that can be used to create a - cluster with ECDSA certificates from "kubeadm init". Renewal of existing ECDSA certificates is - also supported using "kubeadm alpha certs renew", but not switching between the RSA and - ECDSA algorithms on the fly or during upgrades. ([#86953](https://github.com/kubernetes/kubernetes/pull/86953), [@rojkov](https://github.com/rojkov)) [SIG API Machinery, Auth and Cluster Lifecycle] -- Kubeadm: on kubeconfig certificate renewal, keep the embedded CA in sync with the one on disk ([#88052](https://github.com/kubernetes/kubernetes/pull/88052), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- Kubeadm: upgrade supports fallback to the nearest known etcd version if an unknown k8s version is passed ([#88373](https://github.com/kubernetes/kubernetes/pull/88373), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- New flag `--show-hidden-metrics-for-version` in kube-scheduler can be used to show all hidden metrics that deprecated in the previous minor release. ([#84913](https://github.com/kubernetes/kubernetes/pull/84913), [@serathius](https://github.com/serathius)) [SIG Instrumentation and Scheduling] -- Scheduler framework permit plugins now run at the end of the scheduling cycle, after reserve plugins. Waiting on permit will remain in the beginning of the binding cycle. ([#88199](https://github.com/kubernetes/kubernetes/pull/88199), [@mateuszlitwin](https://github.com/mateuszlitwin)) [SIG Scheduling] -- The kubelet and the default docker runtime now support running ephemeral containers in the Linux process namespace of a target container. Other container runtimes must implement this feature before it will be available in that runtime. ([#84731](https://github.com/kubernetes/kubernetes/pull/84731), [@verb](https://github.com/verb)) [SIG Node] +- A client-go metric, rest_client_exec_plugin_call_total, has been added to track total calls to client-go credential plugins. ([#98892](https://github.com/kubernetes/kubernetes/pull/98892), [@ankeesler](https://github.com/ankeesler)) [SIG API Machinery, Auth, Cluster Lifecycle and Instrumentation] +- Add --use-protocol-buffers flag to kubectl top pods and nodes ([#96655](https://github.com/kubernetes/kubernetes/pull/96655), [@serathius](https://github.com/serathius)) [SIG CLI] +- Add support to generate client-side binaries for new darwin/arm64 platform ([#97743](https://github.com/kubernetes/kubernetes/pull/97743), [@dims](https://github.com/dims)) [SIG Release and Testing] +- Added `ephemeral_volume_controller_create[_failures]_total` counters to kube-controller-manager metrics ([#99115](https://github.com/kubernetes/kubernetes/pull/99115), [@pohly](https://github.com/pohly)) [SIG API Machinery, Apps, Cluster Lifecycle, Instrumentation and Storage] +- Adds alpha feature `VolumeCapacityPriority` which makes the scheduler prioritize nodes based on the best matching size of statically provisioned PVs across multiple topologies. ([#96347](https://github.com/kubernetes/kubernetes/pull/96347), [@cofyc](https://github.com/cofyc)) [SIG Apps, Network, Scheduling, Storage and Testing] +- Adds two new metrics to cronjobs, a histogram to track the time difference when a job is created and the expected time when it should be created, and a gauge for the missed schedules of a cronjob ([#99341](https://github.com/kubernetes/kubernetes/pull/99341), [@alaypatel07](https://github.com/alaypatel07)) [SIG Apps and Instrumentation] +- Alpha implementation of Kubectl Command Headers: SIG CLI KEP 859 enabled when KUBECTL_COMMAND_HEADERS environment variable set on the client command line. + - To enable: export KUBECTL_COMMAND_HEADERS=1; kubectl ... ([#98952](https://github.com/kubernetes/kubernetes/pull/98952), [@seans3](https://github.com/seans3)) [SIG API Machinery and CLI] +- Component owner can configure the allowlist of metric label with flag '--allow-metric-labels'. ([#99738](https://github.com/kubernetes/kubernetes/pull/99738), [@YoyinZyc](https://github.com/YoyinZyc)) [SIG API Machinery, Cluster Lifecycle and Instrumentation] +- Disruption controller only sends one event per PodDisruptionBudget if scale can't be computed ([#98128](https://github.com/kubernetes/kubernetes/pull/98128), [@mortent](https://github.com/mortent)) [SIG Apps] +- EndpointSliceNodeName will always be enabled, so NodeName will always be available in the v1beta1 API. ([#99746](https://github.com/kubernetes/kubernetes/pull/99746), [@swetharepakula](https://github.com/swetharepakula)) [SIG Apps and Network] +- Graduate CRIContainerLogRotation feature gate to GA. ([#99651](https://github.com/kubernetes/kubernetes/pull/99651), [@umohnani8](https://github.com/umohnani8)) [SIG Node and Testing] +- Kube-proxy iptables: new metric sync_proxy_rules_iptables_total that exposes the number of rules programmed per table in each iteration ([#99653](https://github.com/kubernetes/kubernetes/pull/99653), [@aojea](https://github.com/aojea)) [SIG Instrumentation and Network] +- Kube-scheduler now logs plugin scoring summaries at --v=4 ([#99411](https://github.com/kubernetes/kubernetes/pull/99411), [@damemi](https://github.com/damemi)) [SIG Scheduling] +- Kubeadm: a warning to user as ipv6 site-local is deprecated ([#99574](https://github.com/kubernetes/kubernetes/pull/99574), [@pacoxu](https://github.com/pacoxu)) [SIG Cluster Lifecycle and Network] +- Kubeadm: apply the "node.kubernetes.io/exclude-from-external-load-balancers" label on control plane nodes during "init", "join" and "upgrade" to preserve backwards compatibility with the lagacy LB mode where nodes labeled as "master" where excluded. To opt-out you can remove the label from a node. See #97543 and the linked KEP for more details. ([#98269](https://github.com/kubernetes/kubernetes/pull/98269), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] +- Kubeadm: if the user has customized their image repository via the kubeadm configuration, pass the custom pause image repository and tag to the kubelet via --pod-infra-container-image not only for Docker but for all container runtimes. This flag tells the kubelet that it should not garbage collect the image. ([#99476](https://github.com/kubernetes/kubernetes/pull/99476), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] +- Kubeadm: promote IPv6DualStack feature gate to Beta ([#99294](https://github.com/kubernetes/kubernetes/pull/99294), [@pacoxu](https://github.com/pacoxu)) [SIG Cluster Lifecycle] +- Kubectl version changed to write a warning message to stderr if the client and server version difference exceeds the supported version skew of +/-1 minor version. ([#98250](https://github.com/kubernetes/kubernetes/pull/98250), [@brianpursley](https://github.com/brianpursley)) [SIG CLI] +- Kubernetes is now built with Golang 1.16 ([#98572](https://github.com/kubernetes/kubernetes/pull/98572), [@justaugustus](https://github.com/justaugustus)) [SIG API Machinery, Auth, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation, Node, Release and Testing] +- Persistent Volumes formatted with the btrfs filesystem will now automatically resize when expanded. ([#99361](https://github.com/kubernetes/kubernetes/pull/99361), [@Novex](https://github.com/Novex)) [SIG Storage] +- Remove cAdvisor json metrics api collected by Kubelet ([#99236](https://github.com/kubernetes/kubernetes/pull/99236), [@pacoxu](https://github.com/pacoxu)) [SIG Node] +- Sysctls is now GA and locked to default ([#99158](https://github.com/kubernetes/kubernetes/pull/99158), [@wgahnagl](https://github.com/wgahnagl)) [SIG Node] +- The NodeAffinity plugin implements the PreFilter extension, offering enhanced performance for Filter. ([#99213](https://github.com/kubernetes/kubernetes/pull/99213), [@AliceZhang2016](https://github.com/AliceZhang2016)) [SIG Scheduling] +- The endpointslice mirroring controller mirrors endpoints annotations and labels to the generated endpoint slices, it also ensures that updates on any of these fields are mirrored. + The well-known annotation endpoints.kubernetes.io/last-change-trigger-time is skipped and not mirrored. ([#98116](https://github.com/kubernetes/kubernetes/pull/98116), [@aojea](https://github.com/aojea)) [SIG Apps, Network and Testing] +- Update the latest validated version of Docker to 20.10 ([#98977](https://github.com/kubernetes/kubernetes/pull/98977), [@neolit123](https://github.com/neolit123)) [SIG CLI, Cluster Lifecycle and Node] +- Upgrade node local dns to 1.17.0 for better IPv6 support ([#99749](https://github.com/kubernetes/kubernetes/pull/99749), [@pacoxu](https://github.com/pacoxu)) [SIG Cloud Provider and Network] +- Users might specify the `kubectl.kubernetes.io/default-exec-container` annotation in a Pod to preselect container for kubectl commands. ([#99581](https://github.com/kubernetes/kubernetes/pull/99581), [@mengjiao-liu](https://github.com/mengjiao-liu)) [SIG CLI] +- When downscaling ReplicaSets, ready and creation timestamps are compared in a logarithmic scale. ([#99212](https://github.com/kubernetes/kubernetes/pull/99212), [@damemi](https://github.com/damemi)) [SIG Apps and Testing] +- When the kubelet is watching a ConfigMap or Secret purely in the context of setting environment variables + for containers, only hold that watch for a defined duration before cancelling it. This change reduces the CPU + and memory usage of the kube-apiserver in large clusters. ([#99393](https://github.com/kubernetes/kubernetes/pull/99393), [@chenyw1990](https://github.com/chenyw1990)) [SIG API Machinery, Node and Testing] +- WindowsEndpointSliceProxying feature gate has graduated to beta and is enabled by default. This means kube-proxy will read from EndpointSlices instead of Endpoints on Windows by default. ([#99794](https://github.com/kubernetes/kubernetes/pull/99794), [@robscott](https://github.com/robscott)) [SIG Network] -### Other (Bug, Cleanup or Flake) +### Bug or Regression -- Add delays between goroutines for vm instance update ([#88094](https://github.com/kubernetes/kubernetes/pull/88094), [@aramase](https://github.com/aramase)) [SIG Cloud Provider] -- Add init containers log to cluster dump info. ([#88324](https://github.com/kubernetes/kubernetes/pull/88324), [@zhouya0](https://github.com/zhouya0)) [SIG CLI] -- CPU limits are now respected for Windows containers. If a node is over-provisioned, no weighting is used - only limits are respected. ([#86101](https://github.com/kubernetes/kubernetes/pull/86101), [@PatrickLang](https://github.com/PatrickLang)) [SIG Node, Testing and Windows] -- Cloud provider config CloudProviderBackoffMode has been removed since it won't be used anymore. ([#88463](https://github.com/kubernetes/kubernetes/pull/88463), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Evictions due to pods breaching their ephemeral storage limits are now recorded by the `kubelet_evictions` metric and can be alerted on. ([#87906](https://github.com/kubernetes/kubernetes/pull/87906), [@smarterclayton](https://github.com/smarterclayton)) [SIG Node] -- Fix: add remediation in azure disk attach/detach ([#88444](https://github.com/kubernetes/kubernetes/pull/88444), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider] -- Fix: check disk status before disk azure disk ([#88360](https://github.com/kubernetes/kubernetes/pull/88360), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider] -- Fixed cleaning of CSI raw block volumes. ([#87978](https://github.com/kubernetes/kubernetes/pull/87978), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] -- Get-kube.sh uses the gcloud's current local GCP service account for auth when the provider is GCE or GKE instead of the metadata server default ([#88383](https://github.com/kubernetes/kubernetes/pull/88383), [@BenTheElder](https://github.com/BenTheElder)) [SIG Cluster Lifecycle] -- Golang/x/net has been updated to bring in fixes for CVE-2020-9283 ([#88381](https://github.com/kubernetes/kubernetes/pull/88381), [@BenTheElder](https://github.com/BenTheElder)) [SIG API Machinery, CLI, Cloud Provider, Cluster Lifecycle and Instrumentation] -- Kubeadm now includes CoreDNS version 1.6.7 ([#86260](https://github.com/kubernetes/kubernetes/pull/86260), [@rajansandeep](https://github.com/rajansandeep)) [SIG Cluster Lifecycle] -- Kubeadm: fix the bug that 'kubeadm upgrade' hangs in single node cluster ([#88434](https://github.com/kubernetes/kubernetes/pull/88434), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- Optimize kubectl version help info ([#88313](https://github.com/kubernetes/kubernetes/pull/88313), [@zhouya0](https://github.com/zhouya0)) [SIG CLI] -- Removes the deprecated command `kubectl rolling-update` ([#88057](https://github.com/kubernetes/kubernetes/pull/88057), [@julianvmodesto](https://github.com/julianvmodesto)) [SIG Architecture, CLI and Testing] +- Creating a PVC with DataSource should fail for non-CSI plugins. ([#97086](https://github.com/kubernetes/kubernetes/pull/97086), [@xing-yang](https://github.com/xing-yang)) [SIG Apps and Storage] +- EndpointSlice controller is now less likely to emit FailedToUpdateEndpointSlices events. ([#99345](https://github.com/kubernetes/kubernetes/pull/99345), [@robscott](https://github.com/robscott)) [SIG Apps and Network] +- EndpointSliceMirroring controller is now less likely to emit FailedToUpdateEndpointSlices events. ([#99756](https://github.com/kubernetes/kubernetes/pull/99756), [@robscott](https://github.com/robscott)) [SIG Apps and Network] +- Fix --ignore-errors does not take effect if multiple logs are printed and unfollowed ([#97686](https://github.com/kubernetes/kubernetes/pull/97686), [@wzshiming](https://github.com/wzshiming)) [SIG CLI] +- Fix bug that would let the Horizontal Pod Autoscaler scale down despite at least one metric being unavailable/invalid ([#99514](https://github.com/kubernetes/kubernetes/pull/99514), [@mikkeloscar](https://github.com/mikkeloscar)) [SIG Apps and Autoscaling] +- Fix cgroup handling for systemd with cgroup v2 ([#98365](https://github.com/kubernetes/kubernetes/pull/98365), [@odinuge](https://github.com/odinuge)) [SIG Node] +- Fix smb mount PermissionDenied issue on Windows ([#99550](https://github.com/kubernetes/kubernetes/pull/99550), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider, Storage and Windows] +- Fixed a bug that causes smaller number of conntrack-max being used under CPU static policy. (#99225, @xh4n3) ([#99613](https://github.com/kubernetes/kubernetes/pull/99613), [@xh4n3](https://github.com/xh4n3)) [SIG Network] +- Fixed bug that caused cAdvisor to incorrectly detect single-socket multi-NUMA topology. ([#99315](https://github.com/kubernetes/kubernetes/pull/99315), [@iwankgb](https://github.com/iwankgb)) [SIG Node] +- Fixes add-on manager leader election ([#98968](https://github.com/kubernetes/kubernetes/pull/98968), [@liggitt](https://github.com/liggitt)) [SIG Cloud Provider] +- Improved update time of pod statuses following new probe results. ([#98376](https://github.com/kubernetes/kubernetes/pull/98376), [@matthyx](https://github.com/matthyx)) [SIG Node and Testing] +- Kube-apiserver: an update of a pod with a generic ephemeral volume dropped that volume if the feature had been disabled since creating the pod with such a volume ([#99446](https://github.com/kubernetes/kubernetes/pull/99446), [@pohly](https://github.com/pohly)) [SIG Apps, Node and Storage] +- Kubeadm: skip validating pod subnet against node-cidr-mask when allocate-node-cidrs is set to be false ([#98984](https://github.com/kubernetes/kubernetes/pull/98984), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] +- On single-stack configured (IPv4 or IPv6, but not both) clusters, Services which are both headless (no clusterIP) and selectorless (empty or undefined selector) will report `ipFamilyPolicy RequireDualStack` and will have entries in `ipFamilies[]` for both IPv4 and IPv6. This is a change from alpha, but does not have any impact on the manually-specified Endpoints and EndpointSlices for the Service. ([#99555](https://github.com/kubernetes/kubernetes/pull/99555), [@thockin](https://github.com/thockin)) [SIG Apps and Network] +- Resolves spurious `Failed to list *v1.Secret` or `Failed to list *v1.ConfigMap` messages in kubelet logs. ([#99538](https://github.com/kubernetes/kubernetes/pull/99538), [@liggitt](https://github.com/liggitt)) [SIG Auth and Node] +- Return zero time (midnight on Jan. 1, 1970) instead of negative number when reporting startedAt and finishedAt of the not started or a running Pod when using dockershim as a runtime. ([#99585](https://github.com/kubernetes/kubernetes/pull/99585), [@Iceber](https://github.com/Iceber)) [SIG Node] +- Stdin is now only passed to client-go exec credential plugins when it is detected to be an interactive terminal. Previously, it was passed to client-go exec plugins when **stdout*- was detected to be an interactive terminal. ([#99654](https://github.com/kubernetes/kubernetes/pull/99654), [@ankeesler](https://github.com/ankeesler)) [SIG API Machinery and Auth] +- The maximum number of ports allowed in EndpointSlices has been increased from 100 to 20,000 ([#99795](https://github.com/kubernetes/kubernetes/pull/99795), [@robscott](https://github.com/robscott)) [SIG Network] +- Updates the commands + - kubectl kustomize {arg} + - kubectl apply -k {arg} + to use same code as kustomize CLI v4.0.5 + - [v4.0.5]: https://github.com/kubernetes-sigs/kustomize/releases/tag/kustomize%2Fv4.0.5 ([#98946](https://github.com/kubernetes/kubernetes/pull/98946), [@monopole](https://github.com/monopole)) [SIG API Machinery, Architecture, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation, Node and Storage] +- When a CNI plugin returns dual-stack pod IPs, kubelet will now try to respect the + "primary IP family" of the cluster by picking a primary pod IP of the same family + as the (primary) node IP, rather than assuming that the CNI plugin returned the IPs + in the order the administrator wanted (since some CNI plugins don't allow + configuring this). ([#97979](https://github.com/kubernetes/kubernetes/pull/97979), [@danwinship](https://github.com/danwinship)) [SIG Network and Node] +- When using Containerd on Windows, the "C:\Windows\System32\drivers\etc\hosts" file will now be managed by kubelet. ([#83730](https://github.com/kubernetes/kubernetes/pull/83730), [@claudiubelu](https://github.com/claudiubelu)) [SIG Node and Windows] +- `VolumeBindingArgs` now allow `BindTimeoutSeconds` to be set as zero, while the value zero indicates no waiting for the checking of volume binding operation. ([#99835](https://github.com/kubernetes/kubernetes/pull/99835), [@chendave](https://github.com/chendave)) [SIG Scheduling and Storage] +- `kubectl exec` and `kubectl attach` now honor the `--quiet` flag which suppresses output from the local binary that could be confused by a script with the remote command output (all non-failure output is hidden). In addition, print inline with exec and attach the list of alternate containers when we default to the first spec.container. ([#99004](https://github.com/kubernetes/kubernetes/pull/99004), [@smarterclayton](https://github.com/smarterclayton)) [SIG CLI] + +### Other (Cleanup or Flake) + +- Apiserver_request_duration_seconds is promoted to stable status. ([#99925](https://github.com/kubernetes/kubernetes/pull/99925), [@logicalhan](https://github.com/logicalhan)) [SIG API Machinery, Instrumentation and Testing] +- Apiserver_request_total is promoted to stable status and no longer has a content-type dimensions, so any alerts/charts which presume the existence of this will fail. This is however, unlikely to be the case since it was effectively an unbounded dimension in the first place. ([#99788](https://github.com/kubernetes/kubernetes/pull/99788), [@logicalhan](https://github.com/logicalhan)) [SIG API Machinery, Instrumentation and Testing] +- EndpointSlice generation is now incremented when labels change. ([#99750](https://github.com/kubernetes/kubernetes/pull/99750), [@robscott](https://github.com/robscott)) [SIG Network] +- Featuregate AllowInsecureBackendProxy is promoted to GA ([#99658](https://github.com/kubernetes/kubernetes/pull/99658), [@deads2k](https://github.com/deads2k)) [SIG API Machinery] +- Migrate `pkg/kubelet/(eviction)` to structured logging ([#99032](https://github.com/kubernetes/kubernetes/pull/99032), [@yangjunmyfm192085](https://github.com/yangjunmyfm192085)) [SIG Node] +- Migrate deployment controller log messages to structured logging ([#97507](https://github.com/kubernetes/kubernetes/pull/97507), [@aldudko](https://github.com/aldudko)) [SIG Apps] +- Migrate pkg/kubelet/cloudresource to structured logging ([#98999](https://github.com/kubernetes/kubernetes/pull/98999), [@sladyn98](https://github.com/sladyn98)) [SIG Node] +- Migrate pkg/kubelet/cri/remote logs to structured logging ([#98589](https://github.com/kubernetes/kubernetes/pull/98589), [@chenyw1990](https://github.com/chenyw1990)) [SIG Node] +- Migrate pkg/kubelet/kuberuntime/kuberuntime_container.go logs to structured logging ([#96973](https://github.com/kubernetes/kubernetes/pull/96973), [@chenyw1990](https://github.com/chenyw1990)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/status to structured logging ([#99836](https://github.com/kubernetes/kubernetes/pull/99836), [@navidshaikh](https://github.com/navidshaikh)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/token to structured logging ([#99264](https://github.com/kubernetes/kubernetes/pull/99264), [@palnabarun](https://github.com/palnabarun)) [SIG Auth, Instrumentation and Node] +- Migrate pkg/kubelet/util to structured logging ([#99823](https://github.com/kubernetes/kubernetes/pull/99823), [@navidshaikh](https://github.com/navidshaikh)) [SIG Instrumentation and Node] +- Migrate proxy/userspace/proxier.go logs to structured logging ([#97837](https://github.com/kubernetes/kubernetes/pull/97837), [@JornShen](https://github.com/JornShen)) [SIG Network] +- Migrate some kubelet/metrics log messages to structured logging ([#98627](https://github.com/kubernetes/kubernetes/pull/98627), [@jialaijun](https://github.com/jialaijun)) [SIG Instrumentation and Node] +- Process start time on Windows now uses current process information ([#97491](https://github.com/kubernetes/kubernetes/pull/97491), [@jsturtevant](https://github.com/jsturtevant)) [SIG API Machinery, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation and Windows] + +### Uncategorized + +- Migrate pkg/kubelet/stats to structured logging ([#99607](https://github.com/kubernetes/kubernetes/pull/99607), [@krzysiekg](https://github.com/krzysiekg)) [SIG Node] +- The DownwardAPIHugePages feature is beta. Users may use the feature if all workers in their cluster are min 1.20 version. The feature will be enabled by default in all installations in 1.22. ([#99610](https://github.com/kubernetes/kubernetes/pull/99610), [@derekwaynecarr](https://github.com/derekwaynecarr)) [SIG Node] + +## Dependencies + +### Added +- github.com/go-errors/errors: [v1.0.1](https://github.com/go-errors/errors/tree/v1.0.1) +- github.com/gobuffalo/here: [v0.6.0](https://github.com/gobuffalo/here/tree/v0.6.0) +- github.com/google/shlex: [e7afc7f](https://github.com/google/shlex/tree/e7afc7f) +- github.com/markbates/pkger: [v0.17.1](https://github.com/markbates/pkger/tree/v0.17.1) +- github.com/monochromegane/go-gitignore: [205db1a](https://github.com/monochromegane/go-gitignore/tree/205db1a) +- github.com/niemeyer/pretty: [a10e7ca](https://github.com/niemeyer/pretty/tree/a10e7ca) +- github.com/xlab/treeprint: [a009c39](https://github.com/xlab/treeprint/tree/a009c39) +- go.starlark.net: 8dd3e2e +- golang.org/x/term: 6a3ed07 +- sigs.k8s.io/kustomize/api: v0.8.5 +- sigs.k8s.io/kustomize/cmd/config: v0.9.7 +- sigs.k8s.io/kustomize/kustomize/v4: v4.0.5 +- sigs.k8s.io/kustomize/kyaml: v0.10.15 + +### Changed +- dmitri.shuralyov.com/gpu/mtl: 666a987 → 28db891 +- github.com/creack/pty: [v1.1.7 → v1.1.9](https://github.com/creack/pty/compare/v1.1.7...v1.1.9) +- github.com/go-openapi/spec: [v0.19.3 → v0.19.5](https://github.com/go-openapi/spec/compare/v0.19.3...v0.19.5) +- github.com/go-openapi/strfmt: [v0.19.3 → v0.19.5](https://github.com/go-openapi/strfmt/compare/v0.19.3...v0.19.5) +- github.com/go-openapi/validate: [v0.19.5 → v0.19.8](https://github.com/go-openapi/validate/compare/v0.19.5...v0.19.8) +- github.com/google/cadvisor: [v0.38.7 → v0.38.8](https://github.com/google/cadvisor/compare/v0.38.7...v0.38.8) +- github.com/kr/text: [v0.1.0 → v0.2.0](https://github.com/kr/text/compare/v0.1.0...v0.2.0) +- github.com/mattn/go-runewidth: [v0.0.2 → v0.0.7](https://github.com/mattn/go-runewidth/compare/v0.0.2...v0.0.7) +- github.com/olekukonko/tablewriter: [a0225b3 → v0.0.4](https://github.com/olekukonko/tablewriter/compare/a0225b3...v0.0.4) +- github.com/sergi/go-diff: [v1.0.0 → v1.1.0](https://github.com/sergi/go-diff/compare/v1.0.0...v1.1.0) +- golang.org/x/crypto: 7f63de1 → 5ea612d +- golang.org/x/exp: 6cc2880 → 85be41e +- golang.org/x/mobile: d2bd2a2 → e6ae53a +- golang.org/x/mod: v0.3.0 → ce943fd +- golang.org/x/net: 69a7880 → 3d97a24 +- golang.org/x/sys: 5cba982 → a50acf3 +- golang.org/x/time: 3af7569 → f8bda1e +- golang.org/x/tools: 113979e → v0.1.0 +- gopkg.in/check.v1: 41f04d3 → 8fa4692 +- gopkg.in/yaml.v2: v2.2.8 → v2.4.0 +- k8s.io/kube-openapi: d219536 → 591a79e +- k8s.io/system-validators: v1.3.0 → v1.4.0 + +### Removed +- github.com/codegangsta/negroni: [v1.0.0](https://github.com/codegangsta/negroni/tree/v1.0.0) +- github.com/golangplus/bytes: [45c989f](https://github.com/golangplus/bytes/tree/45c989f) +- github.com/golangplus/fmt: [2a5d6d7](https://github.com/golangplus/fmt/tree/2a5d6d7) +- github.com/gorilla/context: [v1.1.1](https://github.com/gorilla/context/tree/v1.1.1) +- github.com/kr/pty: [v1.1.5](https://github.com/kr/pty/tree/v1.1.5) +- sigs.k8s.io/kustomize: v2.0.3+incompatible -# v1.18.0-alpha.5 -[Documentation](https://docs.k8s.io) +# v1.21.0-beta.0 -## Downloads for v1.18.0-alpha.5 + +## Downloads for v1.21.0-beta.0 + +### Source Code filename | sha512 hash -------- | ----------- -[kubernetes.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes.tar.gz) | `6452cac2b80721e9f577cb117c29b9ac6858812b4275c2becbf74312566f7d016e8b34019bd1bf7615131b191613bf9b973e40ad9ac8f6de9007d41ef2d7fd70` -[kubernetes-src.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-src.tar.gz) | `e41d9d4dd6910a42990051fcdca4bf5d3999df46375abd27ffc56aae9b455ae984872302d590da6aa85bba6079334fb5fe511596b415ee79843dee1c61c137da` +[kubernetes.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes.tar.gz) | 69b73a03b70b0ed006e9fef3f5b9bc68f0eb8dc40db6cc04777c03a2cb83a008c783012ca186b1c48357fb192403dbcf6960f120924785e2076e215b9012d546 +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-src.tar.gz) | 9620fb6d37634271bdd423c09f33f3bd29e74298aa82c47dffc8cb6bd2ff44fa8987a53c53bc529db4ca96ec41503aa81cc8d0c3ac106f3b06c4720de933a8e6 -### Client Binaries +### Client binaries filename | sha512 hash -------- | ----------- -[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-client-darwin-386.tar.gz) | `5c95935863492b31d4aaa6be93260088dafea27663eb91edca980ca3a8485310e60441bc9050d4d577e9c3f7ffd96db516db8d64321124cec1b712e957c9fe1c` -[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-client-darwin-amd64.tar.gz) | `868faa578b3738604d8be62fae599ccc556799f1ce54807f1fe72599f20f8a1f98ad8152fac14a08a463322530b696d375253ba3653325e74b587df6e0510da3` -[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-client-linux-386.tar.gz) | `76a89d1d30b476b47f8fb808e342f89608e5c1c1787c4c06f2d7e763f9482e2ae8b31e6ad26541972e2b9a3a7c28327e3150cdd355e8b8d8b050a801bbf08d49` -[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-client-linux-amd64.tar.gz) | `07ad96a09b44d1c707d7c68312c5d69b101a3424bf1e6e9400b2e7a3fba78df04302985d473ddd640d8f3f0257be34110dbe1304b9565dd9d7a4639b7b7b85fd` -[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-client-linux-arm.tar.gz) | `c04fed9fa370a75c1b8e18b2be0821943bb9befcc784d14762ea3278e73600332a9b324d5eeaa1801d20ad6be07a553c41dcf4fa7ab3eadd0730ab043d687c8c` -[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-client-linux-arm64.tar.gz) | `4199147dea9954333df26d34248a1cb7b02ebbd6380ffcd42d9f9ed5fdabae45a59215474dab3c11436c82e60bd27cbd03b3dde288bf611cd3e78b87c783c6a9` -[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-client-linux-ppc64le.tar.gz) | `4f6d4d61d1c52d3253ca19031ebcd4bad06d19b68bbaaab5c8e8c590774faea4a5ceab1f05f2706b61780927e1467815b3479342c84d45df965aba78414727c4` -[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-client-linux-s390x.tar.gz) | `e2a454151ae5dd891230fb516a3f73f73ab97832db66fd3d12e7f1657a569f58a9fe2654d50ddd7d8ec88a5ff5094199323a4c6d7d44dcf7edb06cca11dd4de1` -[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-client-windows-386.tar.gz) | `14b262ba3b71c41f545db2a017cf1746075ada5745a858d2a62bc9df7c5dc10607220375db85e2c4cb85307b09709e58bc66a407488e0961191e3249dc7742b0` -[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-client-windows-amd64.tar.gz) | `26353c294755a917216664364b524982b7f5fc6aa832ce90134bb178df8a78604963c68873f121ea5f2626ff615bdbf2ffe54e00578739cde6df42ffae034732` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-client-darwin-amd64.tar.gz) | 2a6f3fcd6b571f5ccde56b91e6e179a01899244be496dae16a2a16e0405c9437b75c6dc853b56f9a4876a7c0a60ec624ccd28400bf8fb960258263172f6860ba +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-client-linux-386.tar.gz) | 78fe9ad9f9a9bc043293327223f0038a2c087ca65e87187a6dcae7a24aef9565fe498d295a4639b0b90524469a04930022fcecd815d0afc742eb87ddd8eb7ef5 +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-client-linux-amd64.tar.gz) | c025f5e5bd132355e7dd1296cf2ec752264e7f754c4d95fc34b076bd75bef2f571d30872bcb3d138ce95c592111353d275a80eb31f82c07000874b4c56282dbd +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-client-linux-arm.tar.gz) | 9975cd2f08fbc202575fb15ba6fc51dab23155ca4d294ebb48516a81efa51f58bab3a87d41c865103756189b554c020371d729ad42880ba788f25047ffc46910 +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-client-linux-arm64.tar.gz) | 56a6836e24471e42e9d9a8488453f2d55598d70c8aca0a307d5116139c930c25c469fd0d1ab5060fbe88dad75a9b5209a08dc11d644af5f3ebebfbcb6c16266c +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-client-linux-ppc64le.tar.gz) | b6a6cc9baad0ad85ed079ee80e6d6acc905095cfb440998bbc0f553b94fa80077bd58b8692754de477517663d51161705e6e89a1b6d04aa74819800db3517722 +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-client-linux-s390x.tar.gz) | 7b743481b340f510bf9ae28ea8ea91150aa1e8c37fe104b66d7b3aff62f5e6db3c590d2c13d14dbb5c928de31c7613372def2496075853611d10d6b5fa5b60bd +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-client-windows-386.tar.gz) | df06c7a524ce84c1f8d7836aa960c550c88dbca0ec4854df4dd0a85b3c84b8ecbc41b54e8c4669ce28ac670659ff0fad795deb1bc539f3c3b3aa885381265f5a +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-client-windows-amd64.tar.gz) | 4568497b684564f2a94fbea6cbfd778b891231470d9a6956c3b7a3268643d13b855c0fc5ebea5f769300cc0c7719c2c331c387f468816f182f63e515adeaa7a0 -### Server Binaries +### Server binaries filename | sha512 hash -------- | ----------- -[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-server-linux-amd64.tar.gz) | `ba77e0e7c610f59647c1b2601f82752964a0f54b7ad609a89b00fcfd553d0f0249f6662becbabaa755bb769b36a2000779f08022c40fb8cc61440337481317a1` -[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-server-linux-arm.tar.gz) | `45e87b3e844ea26958b0b489e8c9b90900a3253000850f5ff9e87ffdcafba72ab8fd17b5ba092051a58a4bc277912c047a85940ec7f093dff6f9e8bf6fed3b42` -[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-server-linux-arm64.tar.gz) | `155e136e3124ead69c594eead3398d6cfdbb8f823c324880e8a7bbd1b570b05d13a77a69abd0a6758cfcc7923971cc6da4d3e0c1680fd519b632803ece00d5ce` -[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-server-linux-ppc64le.tar.gz) | `3fa0fb8221da19ad9d03278961172b7fa29a618b30abfa55e7243bb937dede8df56658acf02e6b61e7274fbc9395e237f49c62f2a83017eca2a69f67af31c01c` -[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-server-linux-s390x.tar.gz) | `db3199c3d7ba0b326d71dc8b80f50b195e79e662f71386a3b2976d47d13d7b0136887cc21df6f53e70a3d733da6eac7bbbf3bab2df8a1909a3cee4b44c32dd0b` +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-server-linux-amd64.tar.gz) | 42883cca2d312153baf693fc6024a295359a421e74fd70eefc927413be4e0353debe634e7cca6b9a8f7d8a0cee3717e03ba5d29a306e93139b1c2f3027535a6d +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-server-linux-arm.tar.gz) | e0042215e84c769ba4fc4d159ccf67b2c4a26206bfffb0ec5152723dc813ff9c1426aa0e9b963d7bfa2efb266ca43561b596b459152882ebb42102ccf60bd8eb +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-server-linux-arm64.tar.gz) | bfad29d43e14152cb9bc7c4df6aa77929c6eca64a294bb832215bdba9fa0ee2195a2b709c0267dc7426bb371b547ee80bb8461a8c678c9bffa0819aa7db96289 +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-server-linux-ppc64le.tar.gz) | ca67674c01c6cebdc8160c85b449eab1a23bb0557418665246e0208543fa2eaaf97679685c7b49bee3a4300904c0399c3d762ae34dc3e279fd69ce792c4b07ff +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-server-linux-s390x.tar.gz) | 285352b628ec754b01b8ad4ef1427223a142d58ebcb46f6861df14d68643133b32330460b213b1ba5bc5362ff2b6dacd8e0c2d20cce6e760fa1954af8a60df8b -### Node Binaries +### Node binaries filename | sha512 hash -------- | ----------- -[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-node-linux-amd64.tar.gz) | `addcdfbad7f12647e6babb8eadf853a374605c8f18bf63f416fa4d3bf1b903aa206679d840433206423a984bb925e7983366edcdf777cf5daef6ef88e53d6dfa` -[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-node-linux-arm.tar.gz) | `b2ac54e0396e153523d116a2aaa32c919d6243931e0104cd47a23f546d710e7abdaa9eae92d978ce63c92041e63a9b56f5dd8fd06c812a7018a10ecac440f768` -[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-node-linux-arm64.tar.gz) | `7aab36f2735cba805e4fd109831a1af0f586a88db3f07581b6dc2a2aab90076b22c96b490b4f6461a8fb690bf78948b6d514274f0d6fb0664081de2d44dc48e1` -[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-node-linux-ppc64le.tar.gz) | `a579936f07ebf86f69f297ac50ba4c34caf2c0b903f73190eb581c78382b05ef36d41ade5bfd25d7b1b658cfcbee3d7125702a18e7480f9b09a62733a512a18a` -[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-node-linux-s390x.tar.gz) | `58fa0359ddd48835192fab1136a2b9b45d1927b04411502c269cda07cb8a8106536973fb4c7fedf1d41893a524c9fe2e21078fdf27bfbeed778273d024f14449` -[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.5/kubernetes-node-windows-amd64.tar.gz) | `9086c03cd92b440686cea6d8c4e48045cc46a43ab92ae0e70350b3f51804b9e2aaae7178142306768bae00d9ef6dd938167972bfa90b12223540093f735a45db` +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-node-linux-amd64.tar.gz) | d92d9b30e7e44134a0cd9db4c01924d365991ea16b3131200b02a82cff89c8701f618cd90e7f1c65427bd4bb5f78b10d540b2262de2c143b401fa44e5b25627b +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-node-linux-arm.tar.gz) | 551092f23c27fdea4bb2d0547f6075892534892a96fc2be7786f82b58c93bffdb5e1c20f8f11beb8bed46c24f36d4c18ec5ac9755435489efa28e6ae775739bd +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-node-linux-arm64.tar.gz) | 26ae7f4163e527349b8818ee38b9ee062314ab417f307afa49c146df8f5a2bd689509b128bd4a1efd3896fd89571149a9955ada91f8ca0c2f599cd863d613c86 +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-node-linux-ppc64le.tar.gz) | 821fa953f6cebc69d2d481e489f3e90899813d20e2eefbabbcadd019d004108e7540f741fabe60e8e7c6adbb1053ac97898bbdddec3ca19f34a71aa3312e0d4e +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-node-linux-s390x.tar.gz) | 22197d4f66205d5aa9de83dfddcc4f2bb3195fd7067cdb5c21e61dbeae217bc112fb7ecff8a539579b60ad92298c2b4c87b9b7c7e6ec1ee1ffa0c6e4bc4412c1 +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-beta.0/kubernetes-node-windows-amd64.tar.gz) | 7e22e0d9603562a04dee16a513579f06b1ff6354d97d669bd68f8777ec7f89f6ef027fb23ab0445d7bba0bb689352f0cc748ce90e3f597c6ebe495464a96b860 -## Changelog since v1.18.0-alpha.3 +## Changelog since v1.21.0-alpha.3 + +## Urgent Upgrade Notes + +### (No, really, you MUST read this before you upgrade) + + - The metric `storage_operation_errors_total` is not removed, but is marked deprecated, and the metric `storage_operation_status_count` is marked deprecated. In both cases the storage_operation_duration_seconds metric can be used to recover equivalent counts (using `status=fail-unknown` in the case of `storage_operations_errors_total`). ([#99045](https://github.com/kubernetes/kubernetes/pull/99045), [@mattcary](https://github.com/mattcary)) [SIG Instrumentation and Storage] + +## Changes by Kind ### Deprecation -- Kubeadm: command line option "kubelet-version" for `kubeadm upgrade node` has been deprecated and will be removed in a future release. ([#87942](https://github.com/kubernetes/kubernetes/pull/87942), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] +- The `batch/v2alpha1` CronJob type definitions and clients are deprecated and removed. ([#96987](https://github.com/kubernetes/kubernetes/pull/96987), [@soltysh](https://github.com/soltysh)) [SIG API Machinery, Apps, CLI and Testing] ### API Change -- Kubelet podresources API now provides the information about active pods only. ([#79409](https://github.com/kubernetes/kubernetes/pull/79409), [@takmatsu](https://github.com/takmatsu)) [SIG Node] -- Remove deprecated fields from .leaderElection in kubescheduler.config.k8s.io/v1alpha2 ([#87904](https://github.com/kubernetes/kubernetes/pull/87904), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling] -- Signatures on generated clientset methods have been modified to accept `context.Context` as a first argument. Signatures of generated Create, Update, and Patch methods have been updated to accept CreateOptions, UpdateOptions and PatchOptions respectively. Clientsets that with the previous interface have been added in new "deprecated" packages to allow incremental migration to the new APIs. The deprecated packages will be removed in the 1.21 release. ([#87299](https://github.com/kubernetes/kubernetes/pull/87299), [@mikedanese](https://github.com/mikedanese)) [SIG API Machinery, Apps, Auth, Autoscaling, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation, Network, Node, Scheduling, Storage, Testing and Windows] -- The k8s.io/node-api component is no longer updated. Instead, use the RuntimeClass types located within k8s.io/api, and the generated clients located within k8s.io/client-go ([#87503](https://github.com/kubernetes/kubernetes/pull/87503), [@liggitt](https://github.com/liggitt)) [SIG Node and Release] +- Cluster admins can now turn off /debug/pprof and /debug/flags/v endpoint in kubelet by setting enableProfilingHandler and enableDebugFlagsHandler to false in their kubelet configuration file. enableProfilingHandler and enableDebugFlagsHandler can be set to true only when enableDebuggingHandlers is also set to true. ([#98458](https://github.com/kubernetes/kubernetes/pull/98458), [@SaranBalaji90](https://github.com/SaranBalaji90)) [SIG Node] +- The BoundServiceAccountTokenVolume feature has been promoted to beta, and enabled by default. + - This changes the tokens provided to containers at `/var/run/secrets/kubernetes.io/serviceaccount/token` to be time-limited, auto-refreshed, and invalidated when the containing pod is deleted. + - Clients should reload the token from disk periodically (once per minute is recommended) to ensure they continue to use a valid token. `k8s.io/client-go` version v11.0.0+ and v0.15.0+ reload tokens automatically. + - By default, injected tokens are given an extended lifetime so they remain valid even after a new refreshed token is provided. The metric `serviceaccount_stale_tokens_total` can be used to monitor for workloads that are depending on the extended lifetime and are continuing to use tokens even after a refreshed token is provided to the container. If that metric indicates no existing workloads are depending on extended lifetimes, injected token lifetime can be shortened to 1 hour by starting `kube-apiserver` with `--service-account-extend-token-expiration=false`. ([#95667](https://github.com/kubernetes/kubernetes/pull/95667), [@zshihang](https://github.com/zshihang)) [SIG API Machinery, Auth, Cluster Lifecycle and Testing] ### Feature -- Add indexer for storage cacher ([#85445](https://github.com/kubernetes/kubernetes/pull/85445), [@shaloulcy](https://github.com/shaloulcy)) [SIG API Machinery] -- Add support for mount options to the FC volume plugin ([#87499](https://github.com/kubernetes/kubernetes/pull/87499), [@ejweber](https://github.com/ejweber)) [SIG Storage] -- Added a config-mode flag in azure auth module to enable getting AAD token without spn: prefix in audience claim. When it's not specified, the default behavior doesn't change. ([#87630](https://github.com/kubernetes/kubernetes/pull/87630), [@weinong](https://github.com/weinong)) [SIG API Machinery, Auth, CLI and Cloud Provider] -- Introduced BackoffManager interface for backoff management ([#87829](https://github.com/kubernetes/kubernetes/pull/87829), [@zhan849](https://github.com/zhan849)) [SIG API Machinery] -- PodTopologySpread plugin now excludes terminatingPods when making scheduling decisions. ([#87845](https://github.com/kubernetes/kubernetes/pull/87845), [@Huang-Wei](https://github.com/Huang-Wei)) [SIG Scheduling] -- Promote CSIMigrationOpenStack to Beta (off by default since it requires installation of the OpenStack Cinder CSI Driver) - The in-tree AWS OpenStack Cinder "kubernetes.io/cinder" was already deprecated a while ago and will be removed in 1.20. Users should enable CSIMigration + CSIMigrationOpenStack features and install the OpenStack Cinder CSI Driver (https://github.com/kubernetes-sigs/cloud-provider-openstack) to avoid disruption to existing Pod and PVC objects at that time. - Users should start using the OpenStack Cinder CSI Driver directly for any new volumes. ([#85637](https://github.com/kubernetes/kubernetes/pull/85637), [@dims](https://github.com/dims)) [SIG Cloud Provider] +- A new histogram metric to track the time it took to delete a job by the ttl-after-finished controller ([#98676](https://github.com/kubernetes/kubernetes/pull/98676), [@ahg-g](https://github.com/ahg-g)) [SIG Apps and Instrumentation] +- AWS cloudprovider supports auto-discovering subnets without any kubernetes.io/cluster/ tags. It also supports additional service annotation service.beta.kubernetes.io/aws-load-balancer-subnets to manually configure the subnets. ([#97431](https://github.com/kubernetes/kubernetes/pull/97431), [@kishorj](https://github.com/kishorj)) [SIG Cloud Provider] +- Add --permit-address-sharing flag to kube-apiserver to listen with SO_REUSEADDR. While allowing to listen on wildcard IPs like 0.0.0.0 and specific IPs in parallel, it avoid waiting for the kernel to release socket in TIME_WAIT state, and hence, considably reducing kube-apiserver restart times under certain conditions. ([#93861](https://github.com/kubernetes/kubernetes/pull/93861), [@sttts](https://github.com/sttts)) [SIG API Machinery] +- Add `csi_operations_seconds` metric on kubelet that exposes CSI operations duration and status for node CSI operations. ([#98979](https://github.com/kubernetes/kubernetes/pull/98979), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Instrumentation and Storage] +- Add `migrated` field into `storage_operation_duration_seconds` metric ([#99050](https://github.com/kubernetes/kubernetes/pull/99050), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Apps, Instrumentation and Storage] +- Add bash-completion for comma separated list on `kubectl get` ([#98301](https://github.com/kubernetes/kubernetes/pull/98301), [@phil9909](https://github.com/phil9909)) [SIG CLI] +- Added support for installing arm64 node artifacts. ([#99242](https://github.com/kubernetes/kubernetes/pull/99242), [@liu-cong](https://github.com/liu-cong)) [SIG Cloud Provider] +- Feature gate RootCAConfigMap is graduated to GA in 1.21 and will be removed in 1.22. ([#98033](https://github.com/kubernetes/kubernetes/pull/98033), [@zshihang](https://github.com/zshihang)) [SIG API Machinery and Auth] +- Kubeadm: during "init" and "join" perform preflight validation on the host / node name and throw warnings if a name is not compliant ([#99194](https://github.com/kubernetes/kubernetes/pull/99194), [@pacoxu](https://github.com/pacoxu)) [SIG Cluster Lifecycle] +- Kubectl: `kubectl get` will omit managed fields by default now. Users could set `--show-managed-fields` to true to show managedFields when the output format is either `json` or `yaml`. ([#96878](https://github.com/kubernetes/kubernetes/pull/96878), [@knight42](https://github.com/knight42)) [SIG CLI and Testing] +- Metrics can now be disabled explicitly via a command line flag (i.e. '--disabled-metrics=bad_metric1,bad_metric2') ([#99217](https://github.com/kubernetes/kubernetes/pull/99217), [@logicalhan](https://github.com/logicalhan)) [SIG API Machinery, Cluster Lifecycle and Instrumentation] +- TTLAfterFinished is now beta and enabled by default ([#98678](https://github.com/kubernetes/kubernetes/pull/98678), [@ahg-g](https://github.com/ahg-g)) [SIG Apps and Auth] +- The `RunAsGroup` feature has been promoted to GA in this release. ([#94641](https://github.com/kubernetes/kubernetes/pull/94641), [@krmayankk](https://github.com/krmayankk)) [SIG Auth and Node] +- Turn CronJobControllerV2 on by default. ([#98878](https://github.com/kubernetes/kubernetes/pull/98878), [@soltysh](https://github.com/soltysh)) [SIG Apps] +- UDP protocol support for Agnhost connect subcommand ([#98639](https://github.com/kubernetes/kubernetes/pull/98639), [@knabben](https://github.com/knabben)) [SIG Testing] +- Upgrades `IPv6Dualstack` to `Beta` and turns it on by default. Clusters new and existing will not be affected until user starting adding secondary pod and service cidrs cli flags as described here: https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/563-dual-stack ([#98969](https://github.com/kubernetes/kubernetes/pull/98969), [@khenidak](https://github.com/khenidak)) [SIG API Machinery, Apps, Cloud Provider, Network and Node] -### Design +### Documentation -- The scheduler Permit extension point doesn't return a boolean value in its Allow() and Reject() functions. ([#87936](https://github.com/kubernetes/kubernetes/pull/87936), [@Huang-Wei](https://github.com/Huang-Wei)) [SIG Scheduling] +- Fix ALPHA stability level reference link ([#98641](https://github.com/kubernetes/kubernetes/pull/98641), [@Jeffwan](https://github.com/Jeffwan)) [SIG Auth, Cloud Provider, Instrumentation and Storage] -### Other (Bug, Cleanup or Flake) +### Failing Test -- Adds "volume.beta.kubernetes.io/migrated-to" annotation to PV's and PVC's when they are migrated to signal external provisioners to pick up those objects for Provisioning and Deleting. ([#87098](https://github.com/kubernetes/kubernetes/pull/87098), [@davidz627](https://github.com/davidz627)) [SIG Apps and Storage] -- Fix a bug in the dual-stack IPVS proxier where stale IPv6 endpoints were not being cleaned up ([#87695](https://github.com/kubernetes/kubernetes/pull/87695), [@andrewsykim](https://github.com/andrewsykim)) [SIG Network] -- Fix kubectl drain ignore daemonsets and others. ([#87361](https://github.com/kubernetes/kubernetes/pull/87361), [@zhouya0](https://github.com/zhouya0)) [SIG CLI] -- Fix: add azure disk migration support for CSINode ([#88014](https://github.com/kubernetes/kubernetes/pull/88014), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] -- Fix: add non-retriable errors in azure clients ([#87941](https://github.com/kubernetes/kubernetes/pull/87941), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider] -- Fixed NetworkPolicy validation that Except values are accepted when they are outside the CIDR range. ([#86578](https://github.com/kubernetes/kubernetes/pull/86578), [@tnqn](https://github.com/tnqn)) [SIG Network] -- Improves performance of the node authorizer ([#87696](https://github.com/kubernetes/kubernetes/pull/87696), [@liggitt](https://github.com/liggitt)) [SIG Auth] -- Iptables/userspace proxy: improve performance by getting local addresses only once per sync loop, instead of for every external IP ([#85617](https://github.com/kubernetes/kubernetes/pull/85617), [@andrewsykim](https://github.com/andrewsykim)) [SIG API Machinery, CLI, Cloud Provider, Cluster Lifecycle, Instrumentation and Network] -- Kube-aggregator: always sets unavailableGauge metric to reflect the current state of a service. ([#87778](https://github.com/kubernetes/kubernetes/pull/87778), [@p0lyn0mial](https://github.com/p0lyn0mial)) [SIG API Machinery] -- Kubeadm allows to configure single-stack clusters if dual-stack is enabled ([#87453](https://github.com/kubernetes/kubernetes/pull/87453), [@aojea](https://github.com/aojea)) [SIG API Machinery, Cluster Lifecycle and Network] -- Kubeadm: 'kubeadm alpha kubelet config download' has been removed, please use 'kubeadm upgrade node phase kubelet-config' instead ([#87944](https://github.com/kubernetes/kubernetes/pull/87944), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- Kubeadm: remove 'kubeadm upgrade node config' command since it was deprecated in v1.15, please use 'kubeadm upgrade node phase kubelet-config' instead ([#87975](https://github.com/kubernetes/kubernetes/pull/87975), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] -- Kubectl describe and kubectl top pod will return a message saying "No resources found" or "No resources found in namespace" if there are no results to display. ([#87527](https://github.com/kubernetes/kubernetes/pull/87527), [@brianpursley](https://github.com/brianpursley)) [SIG CLI] -- Kubelet metrics gathered through metrics-server or prometheus should no longer timeout for Windows nodes running more than 3 pods. ([#87730](https://github.com/kubernetes/kubernetes/pull/87730), [@marosset](https://github.com/marosset)) [SIG Node, Testing and Windows] -- Kubelet metrics have been changed to buckets. - For example the exec/{podNamespace}/{podID}/{containerName} is now just exec. ([#87913](https://github.com/kubernetes/kubernetes/pull/87913), [@cheftako](https://github.com/cheftako)) [SIG Node] -- Limit number of instances in a single update to GCE target pool to 1000. ([#87881](https://github.com/kubernetes/kubernetes/pull/87881), [@wojtek-t](https://github.com/wojtek-t)) [SIG Cloud Provider, Network and Scalability] -- Make Azure clients only retry on specified HTTP status codes ([#88017](https://github.com/kubernetes/kubernetes/pull/88017), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Pause image contains "Architecture" in non-amd64 images ([#87954](https://github.com/kubernetes/kubernetes/pull/87954), [@BenTheElder](https://github.com/BenTheElder)) [SIG Release] -- Pods that are considered for preemption and haven't started don't produce an error log. ([#87900](https://github.com/kubernetes/kubernetes/pull/87900), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling] -- Prevent error message from being displayed when running kubectl plugin list and your path includes an empty string ([#87633](https://github.com/kubernetes/kubernetes/pull/87633), [@brianpursley](https://github.com/brianpursley)) [SIG CLI] -- `kubectl create clusterrolebinding` creates rbac.authorization.k8s.io/v1 object ([#85889](https://github.com/kubernetes/kubernetes/pull/85889), [@oke-py](https://github.com/oke-py)) [SIG CLI] +- Escape the special characters like `[`, `]` and ` ` that exist in vsphere windows path ([#98830](https://github.com/kubernetes/kubernetes/pull/98830), [@liyanhui1228](https://github.com/liyanhui1228)) [SIG Storage and Windows] +- Kube-proxy: fix a bug on UDP NodePort Services where stale conntrack entries may blackhole the traffic directed to the NodePort. ([#98305](https://github.com/kubernetes/kubernetes/pull/98305), [@aojea](https://github.com/aojea)) [SIG Network] -# v1.18.0-alpha.4 +### Bug or Regression -[Documentation](https://docs.k8s.io) +- Add missing --kube-api-content-type in kubemark hollow template ([#98911](https://github.com/kubernetes/kubernetes/pull/98911), [@Jeffwan](https://github.com/Jeffwan)) [SIG Scalability and Testing] +- Avoid duplicate error messages when runing kubectl edit quota ([#98201](https://github.com/kubernetes/kubernetes/pull/98201), [@pacoxu](https://github.com/pacoxu)) [SIG API Machinery and Apps] +- Cleanup subnet in frontend IP configs to prevent huge subnet request bodies in some scenarios. ([#98133](https://github.com/kubernetes/kubernetes/pull/98133), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] +- Fix errors when accessing Windows container stats for Dockershim ([#98510](https://github.com/kubernetes/kubernetes/pull/98510), [@jsturtevant](https://github.com/jsturtevant)) [SIG Node and Windows] +- Fixes spurious errors about IPv6 in kube-proxy logs on nodes with IPv6 disabled. ([#99127](https://github.com/kubernetes/kubernetes/pull/99127), [@danwinship](https://github.com/danwinship)) [SIG Network and Node] +- In the method that ensures that the docker and containerd are in the correct containers with the proper OOM score set up, fixed the bug of identifying containerd process. ([#97888](https://github.com/kubernetes/kubernetes/pull/97888), [@pacoxu](https://github.com/pacoxu)) [SIG Node] +- Kubelet now cleans up orphaned volume directories automatically ([#95301](https://github.com/kubernetes/kubernetes/pull/95301), [@lorenz](https://github.com/lorenz)) [SIG Node and Storage] +- When dynamically provisioning Azure File volumes for a premium account, the requested size will be set to 100GB if the request is initially lower than this value to accommodate Azure File requirements. ([#99122](https://github.com/kubernetes/kubernetes/pull/99122), [@huffmanca](https://github.com/huffmanca)) [SIG Cloud Provider and Storage] -## Important note about manual tag +### Other (Cleanup or Flake) -Due to a [tagging bug in our Release Engineering tooling](https://github.com/kubernetes/release/issues/1080) during `v1.18.0-alpha.3`, we needed to push a manual tag (`v1.18.0-alpha.4`). +- APIs for kubelet annotations and labels from k8s.io/kubernetes/pkg/kubelet/apis are now available under k8s.io/kubelet/pkg/apis/ ([#98931](https://github.com/kubernetes/kubernetes/pull/98931), [@michaelbeaumont](https://github.com/michaelbeaumont)) [SIG Apps, Auth and Node] +- Migrate `pkg/kubelet/(pod, pleg)` to structured logging ([#98990](https://github.com/kubernetes/kubernetes/pull/98990), [@gjkim42](https://github.com/gjkim42)) [SIG Instrumentation and Node] +- Migrate pkg/kubelet/nodestatus to structured logging ([#99001](https://github.com/kubernetes/kubernetes/pull/99001), [@QiWang19](https://github.com/QiWang19)) [SIG Node] +- Migrate pkg/kubelet/server logs to structured logging ([#98643](https://github.com/kubernetes/kubernetes/pull/98643), [@chenyw1990](https://github.com/chenyw1990)) [SIG Node] +- Migrate proxy/winkernel/proxier.go logs to structured logging ([#98001](https://github.com/kubernetes/kubernetes/pull/98001), [@JornShen](https://github.com/JornShen)) [SIG Network and Windows] +- Migrate scheduling_queue.go to structured logging ([#98358](https://github.com/kubernetes/kubernetes/pull/98358), [@tanjing2020](https://github.com/tanjing2020)) [SIG Scheduling] +- Several flags related to the deprecated dockershim which are present in the kubelet command line are now deprecated. ([#98730](https://github.com/kubernetes/kubernetes/pull/98730), [@dims](https://github.com/dims)) [SIG Node] +- The deprecated feature gates `CSIDriverRegistry`, `BlockVolume` and `CSIBlockVolume` are now unconditionally enabled and can no longer be specified in component invocations. ([#98021](https://github.com/kubernetes/kubernetes/pull/98021), [@gavinfish](https://github.com/gavinfish)) [SIG Storage] -**No binaries have been produced or will be provided for `v1.18.0-alpha.4`.** +## Dependencies -The changelog for `v1.18.0-alpha.4` is included as part of the [changelog since v1.18.0-alpha.3][#changelog-since-v1180-alpha3] section. +### Added +_Nothing has changed._ -# v1.18.0-alpha.3 +### Changed +- sigs.k8s.io/structured-merge-diff/v4: v4.0.2 → v4.0.3 -[Documentation](https://docs.k8s.io) +### Removed +_Nothing has changed._ -## Downloads for v1.18.0-alpha.3 + + +# v1.21.0-alpha.3 + + +## Downloads for v1.21.0-alpha.3 + +### Source Code filename | sha512 hash -------- | ----------- -[kubernetes.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes.tar.gz) | `60bf3bfc23b428f53fd853bac18a4a905b980fcc0bacd35ccd6357a89cfc26e47de60975ea6b712e65980e6b9df82a22331152d9f08ed4dba44558ba23a422d4` -[kubernetes-src.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-src.tar.gz) | `8adf1016565a7c93713ab6fa4293c2d13b4f6e4e1ec4dcba60bd71e218b4dbe9ef5eb7dbb469006743f498fc7ddeb21865cd12bec041af60b1c0edce8b7aecd5` +[kubernetes.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes.tar.gz) | 704ec916a1dbd134c54184d2652671f80ae09274f9d23dbbed312944ebeccbc173e2e6b6949b38bdbbfdaf8aa032844deead5efeda1b3150f9751386d9184bc8 +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-src.tar.gz) | 57db9e7560cfc9c10e7059cb5faf9c4bd5eb8f9b7964f44f000a417021cf80873184b774e7c66c80d4aba84c14080c6bc335618db3d2e5f276436ae065e25408 -### Client Binaries +### Client binaries filename | sha512 hash -------- | ----------- -[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-client-darwin-386.tar.gz) | `abb32e894e8280c772e96227b574da81cd1eac374b8d29158b7f222ed550087c65482eef4a9817dfb5f2baf0d9b85fcdfa8feced0fbc1aacced7296853b57e1f` -[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-client-darwin-amd64.tar.gz) | `5e4b1a993264e256ec1656305de7c306094cae9781af8f1382df4ce4eed48ce030827fde1a5e757d4ad57233d52075c9e4e93a69efbdc1102e4ba810705ccddc` -[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-client-linux-386.tar.gz) | `68da39c2ae101d2b38f6137ceda07eb0c2124794982a62ef483245dbffb0611c1441ca085fa3127e7a9977f45646788832a783544ff06954114548ea0e526e46` -[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-client-linux-amd64.tar.gz) | `dc236ffa8ad426620e50181419e9bebe3c161e953dbfb8a019f61b11286e1eb950b40d7cc03423bdf3e6974973bcded51300f98b55570c29732fa492dcde761d` -[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-client-linux-arm.tar.gz) | `ab0a8bd6dc31ea160b731593cdc490b3cc03668b1141cf95310bd7060dcaf55c7ee9842e0acae81063fdacb043c3552ccdd12a94afd71d5310b3ce056fdaa06c` -[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-client-linux-arm64.tar.gz) | `159ea083c601710d0d6aea423eeb346c99ffaf2abd137d35a53e87a07f5caf12fca8790925f3196f67b768fa92a024f83b50325dbca9ccd4dde6c59acdce3509` -[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-client-linux-ppc64le.tar.gz) | `16b0459adfa26575d13be49ab53ac7f0ffd05e184e4e13d2dfbfe725d46bb8ac891e1fd8aebe36ecd419781d4cc5cf3bd2aaaf5263cf283724618c4012408f40` -[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-client-linux-s390x.tar.gz) | `d5aa1f5d89168995d2797eb839a04ce32560f405b38c1c0baaa0e313e4771ae7bb3b28e22433ad5897d36aadf95f73eb69d8d411d31c4115b6b0adf5fe041f85` -[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-client-windows-386.tar.gz) | `374e16a1e52009be88c94786f80174d82dff66399bf294c9bee18a2159c42251c5debef1109a92570799148b08024960c6c50b8299a93fd66ebef94f198f34e9` -[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-client-windows-amd64.tar.gz) | `5a94c1068c19271f810b994adad8e62fae03b3d4473c7c9e6d056995ff7757ea61dd8d140c9267dd41e48808876673ce117826d35a3c1bb5652752f11a044d57` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-client-darwin-amd64.tar.gz) | e2706efda92d5cf4f8b69503bb2f7703a8754407eff7f199bb77847838070e720e5f572126c14daa4c0c03b59bb1a63c1dfdeb6e936a40eff1d5497e871e3409 +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-client-linux-386.tar.gz) | 007bb23c576356ed0890bdfd25a0f98d552599e0ffec19fb982591183c7c1f216d8a3ffa3abf15216be12ae5c4b91fdcd48a7306a2d26b007b86a6abd553fc61 +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-client-linux-amd64.tar.gz) | 39504b0c610348beba60e8866fff265bad58034f74504951cd894c151a248db718d10f77ebc83f2c38b2d517f8513a46325b38889eefa261ca6dbffeceba50ff +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-client-linux-arm.tar.gz) | 30bc2c40d0c759365422ad1651a6fb35909be771f463c5b971caf401f9209525d05256ab70c807e88628dd357c2896745eecf13eda0b748464da97d0a5ef2066 +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-client-linux-arm64.tar.gz) | 085cdf574dc8fd33ece667130b8c45830b522a07860e03a2384283b1adea73a9652ef3dfaa566e69ee00aea1a6461608814b3ce7a3f703e4a934304f7ae12f97 +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-client-linux-ppc64le.tar.gz) | b34b845037d83ea7b3e2d80a9ede4f889b71b17b93b1445f0d936a36e98c13ed6ada125630a68d9243a5fcd311ee37cdcc0c05da484da8488ea5060bc529dbfc +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-client-linux-s390x.tar.gz) | c4758adc7a404b776556efaa79655db2a70777c562145d6ea6887f3335988367a0c2fcd4383e469340f2a768b22e786951de212805ca1cb91104d41c21e0c9ce +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-client-windows-386.tar.gz) | f51edc79702bbd1d9cb3a672852a405e11b20feeab64c5411a7e85c9af304960663eb6b23ef96e0f8c44a722fecf58cb6d700ea2c42c05b3269d8efd5ad803f2 +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-client-windows-amd64.tar.gz) | 6a3507ce4ac40a0dc7e4720538863fa15f8faf025085a032f34b8fa0f6fa4e8c26849baf649b5b32829b9182e04f82721b13950d31cf218c35be6bf1c05d6abf -### Server Binaries +### Server binaries filename | sha512 hash -------- | ----------- -[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-server-linux-amd64.tar.gz) | `a677bec81f0eba75114b92ff955bac74512b47e53959d56a685dae5edd527283d91485b1e86ad74ef389c5405863badf7eb22e2f0c9a568a4d0cb495c6a5c32f` -[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-server-linux-arm.tar.gz) | `2fb696f86ff13ebeb5f3cf2b254bf41303644c5ea84a292782eac6123550702655284d957676d382698c091358e5c7fe73f32803699c19be7138d6530fe413b6` -[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-server-linux-arm64.tar.gz) | `738e95da9cfb8f1309479078098de1c38cef5e1dd5ee1129b77651a936a412b7cd0cf15e652afc7421219646a98846ab31694970432e48dea9c9cafa03aa59cf` -[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-server-linux-ppc64le.tar.gz) | `7a85bfcbb2aa636df60c41879e96e788742ecd72040cb0db2a93418439c125218c58a4cfa96d01b0296c295793e94c544e87c2d98d50b49bc4cb06b41f874376` -[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-server-linux-s390x.tar.gz) | `1f1cdb2efa3e7cac857203d8845df2fdaa5cf1f20df764efffff29371945ec58f6deeba06f8fbf70b96faf81b0c955bf4cb84e30f9516cb2cc1ed27c2d2185a6` +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-server-linux-amd64.tar.gz) | 19181d162dfb0b30236e2bf1111000e037eece87c037ca2b24622ca94cb88db86aa4da4ca533522518b209bc9983bbfd6b880a7898e0da96b33f3f6c4690539b +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-server-linux-arm.tar.gz) | 42a02f9e08a78ad5da6e5fa1ab12bf1e3c967c472fdbdadbd8746586da74dc8093682ba8513ff2a5301393c47ee9021b860e88ada56b13da386ef485708e46ca +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-server-linux-arm64.tar.gz) | 3c8ba8eb02f70061689bd7fab7813542005efe2edc6cfc6b7aecd03ffedf0b81819ad91d69fff588e83023d595eefbfe636aa55e1856add8733bf42fff3c748f +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-server-linux-ppc64le.tar.gz) | cd9e6537450411c39a06fd0b5819db3d16b668d403fb3627ec32c0e32dd1c4860e942934578ca0e1d1b8e6f21f450ff81e37e0cd46ff5c5faf7847ab074aefc5 +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-server-linux-s390x.tar.gz) | ada3f65e53bc0e0c0229694dd48c425388089d6d77111a62476d1b08f6ad1d8ab3d60b9ed7d95ac1b42c2c6be8dc0618f40679717160769743c43583d8452362 -### Node Binaries +### Node binaries filename | sha512 hash -------- | ----------- -[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-node-linux-amd64.tar.gz) | `4ccfced3f5ba4adfa58f4a9d1b2c5bdb3e89f9203ab0e27d11eb1c325ac323ebe63c015d2c9d070b233f5d1da76cab5349da3528511c1cd243e66edc9af381c4` -[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-node-linux-arm.tar.gz) | `d695a69d18449062e4c129e54ec8384c573955f8108f4b78adc2ec929719f2196b995469c728dd6656c63c44cda24315543939f85131ebc773cfe0de689df55b` -[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-node-linux-arm64.tar.gz) | `21df1da88c89000abc22f97e482c3aaa5ce53ec9628d83dda2e04a1d86c4d53be46c03ed6f1f211df3ee5071bce39d944ff7716b5b6ada3b9c4821d368b0a898` -[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-node-linux-ppc64le.tar.gz) | `ff77e3aacb6ed9d89baed92ef542c8b5cec83151b6421948583cf608bca3b779dce41fc6852961e00225d5e1502f6a634bfa61a36efa90e1aee90dedb787c2d2` -[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-node-linux-s390x.tar.gz) | `57d75b7977ec1a0f6e7ed96a304dbb3b8664910f42ca19aab319a9ec33535ff5901dfca4abcb33bf5741cde6d152acd89a5f8178f0efe1dc24430e0c1af5b98f` -[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.3/kubernetes-node-windows-amd64.tar.gz) | `63fdbb71773cfd73a914c498e69bb9eea3fc314366c99ffb8bd42ec5b4dae807682c83c1eb5cfb1e2feb4d11d9e49cc85ba644e954241320a835798be7653d61` +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-node-linux-amd64.tar.gz) | ae0fec6aa59e49624b55d9a11c12fdf717ddfe04bdfd4f69965d03004a34e52ee4a3e83f7b61d0c6a86f43b72c99f3decb195b39ae529ef30526d18ec5f58f83 +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-node-linux-arm.tar.gz) | 9a48c140ab53b7ed8ecec6903988a1a474efc16d2538e5974bc9a12f0c9190be78c4f9e326bf4e982d0b7045a80b99dd0fda7e9b650663be5b89bfd991596746 +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-node-linux-arm64.tar.gz) | 6912adbc9300344bea470d6435f7b387bfce59767078c11728ce59faf47cd3f72b41b9604fcc5cda45e9816fe939fbe2fb33e52a773e6ff2dfa9a615b4df6141 +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-node-linux-ppc64le.tar.gz) | d66dccfe3e6ed6d81567c70703f15375a53992b3a5e2814b98c32e581b861ad95912e03ed2562415d087624c008038bb4a816611fa255442ae752968ea15856b +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-node-linux-s390x.tar.gz) | ad8c69a28f1fbafa3f1cb54909bfd3fc22b104bed63d7ca2b296208c9d43eb5f2943a0ff267da4c185186cdd9f7f77b315cd7f5f1bf9858c0bf42eceb9ac3c58 +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.3/kubernetes-node-windows-amd64.tar.gz) | 91d723aa848a9cb028f5bcb41090ca346fb973961521d025c4399164de2c8029b57ca2c4daca560d3c782c05265d2eb0edb0abcce6f23d3efbecf2316a54d650 -## Changelog since v1.18.0-alpha.2 +## Changelog since v1.21.0-alpha.2 + +## Urgent Upgrade Notes + +### (No, really, you MUST read this before you upgrade) + + - Newly provisioned PVs by gce-pd will no longer have the beta FailureDomain label. gce-pd volume plugin will start to have GA topology label instead. ([#98700](https://github.com/kubernetes/kubernetes/pull/98700), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Cloud Provider, Storage and Testing] + - Remove alpha CSIMigrationXXComplete flag and add alpha InTreePluginXXUnregister flag. Deprecate CSIMigrationvSphereComplete flag and it will be removed in 1.22. ([#98243](https://github.com/kubernetes/kubernetes/pull/98243), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Node and Storage] + +## Changes by Kind + +### API Change + +- Adds support for portRange / EndPort in Network Policy ([#97058](https://github.com/kubernetes/kubernetes/pull/97058), [@rikatz](https://github.com/rikatz)) [SIG Apps and Network] +- Fixes using server-side apply with APIService resources ([#98576](https://github.com/kubernetes/kubernetes/pull/98576), [@kevindelgado](https://github.com/kevindelgado)) [SIG API Machinery, Apps and Testing] +- Kubernetes is now built using go1.15.7 ([#98363](https://github.com/kubernetes/kubernetes/pull/98363), [@cpanato](https://github.com/cpanato)) [SIG Cloud Provider, Instrumentation, Node, Release and Testing] +- Scheduler extender filter interface now can report unresolvable failed nodes in the new field `FailedAndUnresolvableNodes` of `ExtenderFilterResult` struct. Nodes in this map will be skipped in the preemption phase. ([#92866](https://github.com/kubernetes/kubernetes/pull/92866), [@cofyc](https://github.com/cofyc)) [SIG Scheduling] + +### Feature + +- A lease can only attach up to 10k objects. ([#98257](https://github.com/kubernetes/kubernetes/pull/98257), [@lingsamuel](https://github.com/lingsamuel)) [SIG API Machinery] +- Add ignore-errors flag for drain, support none-break drain in group ([#98203](https://github.com/kubernetes/kubernetes/pull/98203), [@yuzhiquan](https://github.com/yuzhiquan)) [SIG CLI] +- Base-images: Update to debian-iptables:buster-v1.4.0 + - Uses iptables 1.8.5 + - base-images: Update to debian-base:buster-v1.3.0 + - cluster/images/etcd: Build etcd:3.4.13-2 image + - Uses debian-base:buster-v1.3.0 ([#98401](https://github.com/kubernetes/kubernetes/pull/98401), [@pacoxu](https://github.com/pacoxu)) [SIG Testing] +- Export NewDebuggingRoundTripper function and DebugLevel options in the k8s.io/client-go/transport package. ([#98324](https://github.com/kubernetes/kubernetes/pull/98324), [@atosatto](https://github.com/atosatto)) [SIG API Machinery] +- Kubectl wait ensures that observedGeneration >= generation if applicable ([#97408](https://github.com/kubernetes/kubernetes/pull/97408), [@KnicKnic](https://github.com/KnicKnic)) [SIG CLI] +- Kubernetes is now built using go1.15.8 ([#98834](https://github.com/kubernetes/kubernetes/pull/98834), [@cpanato](https://github.com/cpanato)) [SIG Cloud Provider, Instrumentation, Release and Testing] +- New admission controller "denyserviceexternalips" is available. Clusters which do not *need- the Service "externalIPs" feature should enable this controller and be more secure. ([#97395](https://github.com/kubernetes/kubernetes/pull/97395), [@thockin](https://github.com/thockin)) [SIG API Machinery] +- Overall, enable the feature of `PreferNominatedNode` will improve the performance of scheduling where preemption might frequently happen, but in theory, enable the feature of `PreferNominatedNode`, the pod might not be scheduled to the best candidate node in the cluster. ([#93179](https://github.com/kubernetes/kubernetes/pull/93179), [@chendave](https://github.com/chendave)) [SIG Scheduling and Testing] +- Pause image upgraded to 3.4.1 in kubelet and kubeadm for both Linux and Windows. ([#98205](https://github.com/kubernetes/kubernetes/pull/98205), [@pacoxu](https://github.com/pacoxu)) [SIG CLI, Cloud Provider, Cluster Lifecycle, Node, Testing and Windows] +- The `ServiceAccountIssuerDiscovery` feature has graduated to GA, and is unconditionally enabled. The `ServiceAccountIssuerDiscovery` feature-gate will be removed in 1.22. ([#98553](https://github.com/kubernetes/kubernetes/pull/98553), [@mtaufen](https://github.com/mtaufen)) [SIG API Machinery, Auth and Testing] + +### Documentation + +- Feat: azure file migration go beta in 1.21. Feature gates CSIMigration to Beta (on by default) and CSIMigrationAzureFile to Beta (off by default since it requires installation of the AzureFile CSI Driver) + The in-tree AzureFile plugin "kubernetes.io/azure-file" is now deprecated and will be removed in 1.23. Users should enable CSIMigration + CSIMigrationAzureFile features and install the AzureFile CSI Driver (https://github.com/kubernetes-sigs/azurefile-csi-driver) to avoid disruption to existing Pod and PVC objects at that time. + Users should start using the AzureFile CSI Driver directly for any new volumes. ([#96293](https://github.com/kubernetes/kubernetes/pull/96293), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider] + +### Failing Test + +- Kubelet: the HostPort implementation in dockershim was not taking into consideration the HostIP field, causing that the same HostPort can not be used with different IP addresses. + This bug causes the conformance test "HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" to fail. ([#98755](https://github.com/kubernetes/kubernetes/pull/98755), [@aojea](https://github.com/aojea)) [SIG Cloud Provider, Network and Node] + +### Bug or Regression + +- Fix NPE in ephemeral storage eviction ([#98261](https://github.com/kubernetes/kubernetes/pull/98261), [@wzshiming](https://github.com/wzshiming)) [SIG Node] +- Fixed a bug that on k8s nodes, when the policy of INPUT chain in filter table is not ACCEPT, healthcheck nodeport would not work. + Added iptables rules to allow healthcheck nodeport traffic. ([#97824](https://github.com/kubernetes/kubernetes/pull/97824), [@hanlins](https://github.com/hanlins)) [SIG Network] +- Fixed kube-proxy container image architecture for non amd64 images. ([#98526](https://github.com/kubernetes/kubernetes/pull/98526), [@saschagrunert](https://github.com/saschagrunert)) [SIG API Machinery, Release and Testing] +- Fixed provisioning of Cinder volumes migrated to CSI when StorageClass with AllowedTopologies was used. ([#98311](https://github.com/kubernetes/kubernetes/pull/98311), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] +- Fixes a panic in the disruption budget controller for PDB objects with invalid selectors ([#98750](https://github.com/kubernetes/kubernetes/pull/98750), [@mortent](https://github.com/mortent)) [SIG Apps] +- Fixes connection errors when using `--volume-host-cidr-denylist` or `--volume-host-allow-local-loopback` ([#98436](https://github.com/kubernetes/kubernetes/pull/98436), [@liggitt](https://github.com/liggitt)) [SIG Network and Storage] +- If the user specifies an invalid timeout in the request URL, the request will be aborted with an HTTP 400. + - in cases where the client specifies a timeout in the request URL, the overall request deadline is shortened now since the deadline is setup as soon as the request is received by the apiserver. ([#96901](https://github.com/kubernetes/kubernetes/pull/96901), [@tkashem](https://github.com/tkashem)) [SIG API Machinery and Testing] +- Kubeadm: Some text in the `kubeadm upgrade plan` output has changed. If you have scripts or other automation that parses this output, please review these changes and update your scripts to account for the new output. ([#98728](https://github.com/kubernetes/kubernetes/pull/98728), [@stmcginnis](https://github.com/stmcginnis)) [SIG Cluster Lifecycle] +- Kubeadm: fix a bug where external credentials in an existing admin.conf prevented the CA certificate to be written in the cluster-info ConfigMap. ([#98882](https://github.com/kubernetes/kubernetes/pull/98882), [@kvaps](https://github.com/kvaps)) [SIG Cluster Lifecycle] +- Kubeadm: fix bad token placeholder text in "config print *-defaults --help" ([#98839](https://github.com/kubernetes/kubernetes/pull/98839), [@Mattias-](https://github.com/Mattias-)) [SIG Cluster Lifecycle] +- Kubeadm: get k8s CI version markers from k8s infra bucket ([#98836](https://github.com/kubernetes/kubernetes/pull/98836), [@hasheddan](https://github.com/hasheddan)) [SIG Cluster Lifecycle and Release] +- Mitigate CVE-2020-8555 for kube-up using GCE by preventing local loopback folume hosts. ([#97934](https://github.com/kubernetes/kubernetes/pull/97934), [@mattcary](https://github.com/mattcary)) [SIG Cloud Provider and Storage] +- Remove CSI topology from migrated in-tree gcepd volume. ([#97823](https://github.com/kubernetes/kubernetes/pull/97823), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Cloud Provider and Storage] +- Sync node status during kubelet node shutdown. + Adds an pod admission handler that rejects new pods when the node is in progress of shutting down. ([#98005](https://github.com/kubernetes/kubernetes/pull/98005), [@wzshiming](https://github.com/wzshiming)) [SIG Node] +- Truncates a message if it hits the NoteLengthLimit when the scheduler records an event for the pod that indicates the pod has failed to schedule. ([#98715](https://github.com/kubernetes/kubernetes/pull/98715), [@carlory](https://github.com/carlory)) [SIG Scheduling] +- We will no longer automatically delete all data when a failure is detected during creation of the volume data file on a CSI volume. Now we will only remove the data file and volume path. ([#96021](https://github.com/kubernetes/kubernetes/pull/96021), [@huffmanca](https://github.com/huffmanca)) [SIG Storage] + +### Other (Cleanup or Flake) + +- Fix the description of command line flags that can override --config ([#98254](https://github.com/kubernetes/kubernetes/pull/98254), [@changshuchao](https://github.com/changshuchao)) [SIG Scheduling] +- Migrate scheduler/taint_manager.go structured logging ([#98259](https://github.com/kubernetes/kubernetes/pull/98259), [@tanjing2020](https://github.com/tanjing2020)) [SIG Apps] +- Migrate staging/src/k8s.io/apiserver/pkg/admission logs to structured logging ([#98138](https://github.com/kubernetes/kubernetes/pull/98138), [@lala123912](https://github.com/lala123912)) [SIG API Machinery] +- Resolves flakes in the Ingress conformance tests due to conflicts with controllers updating the Ingress object ([#98430](https://github.com/kubernetes/kubernetes/pull/98430), [@liggitt](https://github.com/liggitt)) [SIG Network and Testing] +- The default delegating authorization options now allow unauthenticated access to healthz, readyz, and livez. A system:masters user connecting to an authz delegator will not perform an authz check. ([#98325](https://github.com/kubernetes/kubernetes/pull/98325), [@deads2k](https://github.com/deads2k)) [SIG API Machinery, Auth, Cloud Provider and Scheduling] +- The e2e suite can be instructed not to wait for pods in kube-system to be ready or for all nodes to be ready by passing `--allowed-not-ready-nodes=-1` when invoking the e2e.test program. This allows callers to run subsets of the e2e suite in scenarios other than perfectly healthy clusters. ([#98781](https://github.com/kubernetes/kubernetes/pull/98781), [@smarterclayton](https://github.com/smarterclayton)) [SIG Testing] +- The feature gates `WindowsGMSA` and `WindowsRunAsUserName` that are GA since v1.18 are now removed. ([#96531](https://github.com/kubernetes/kubernetes/pull/96531), [@ialidzhikov](https://github.com/ialidzhikov)) [SIG Node and Windows] +- The new `-gce-zones` flag on the `e2e.test` binary instructs tests that check for information about how the cluster interacts with the cloud to limit their queries to the provided zone list. If not specified, the current behavior of asking the cloud provider for all available zones in multi zone clusters is preserved. ([#98787](https://github.com/kubernetes/kubernetes/pull/98787), [@smarterclayton](https://github.com/smarterclayton)) [SIG API Machinery, Cluster Lifecycle and Testing] + +## Dependencies + +### Added +- github.com/moby/spdystream: [v0.2.0](https://github.com/moby/spdystream/tree/v0.2.0) + +### Changed +- github.com/NYTimes/gziphandler: [56545f4 → v1.1.1](https://github.com/NYTimes/gziphandler/compare/56545f4...v1.1.1) +- github.com/container-storage-interface/spec: [v1.2.0 → v1.3.0](https://github.com/container-storage-interface/spec/compare/v1.2.0...v1.3.0) +- github.com/go-logr/logr: [v0.2.0 → v0.4.0](https://github.com/go-logr/logr/compare/v0.2.0...v0.4.0) +- github.com/gogo/protobuf: [v1.3.1 → v1.3.2](https://github.com/gogo/protobuf/compare/v1.3.1...v1.3.2) +- github.com/kisielk/errcheck: [v1.2.0 → v1.5.0](https://github.com/kisielk/errcheck/compare/v1.2.0...v1.5.0) +- github.com/yuin/goldmark: [v1.1.27 → v1.2.1](https://github.com/yuin/goldmark/compare/v1.1.27...v1.2.1) +- golang.org/x/sync: cd5d95a → 67f06af +- golang.org/x/tools: c1934b7 → 113979e +- k8s.io/klog/v2: v2.4.0 → v2.5.0 +- sigs.k8s.io/apiserver-network-proxy/konnectivity-client: v0.0.14 → v0.0.15 + +### Removed +- github.com/docker/spdystream: [449fdfc](https://github.com/docker/spdystream/tree/449fdfc) + + + +# v1.21.0-alpha.2 + + +## Downloads for v1.21.0-alpha.2 + +### Source Code + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes.tar.gz) | 6836f6c8514253fe0831fd171fc4ed92eb6d9a773491c8dc82b90d171a1b10076bd6bfaea56ec1e199c5f46c273265bdb9f174f0b2d99c5af1de4c99b862329e +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-src.tar.gz) | d137694804741a05ab09e5f9a418448b66aba0146c028eafce61bcd9d7c276521e345ce9223ffbc703e8172041d58dfc56a3242a4df3686f24905a4541fcd306 + +### Client binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-client-darwin-amd64.tar.gz) | 9478b047a97717953f365c13a098feb7e3cb30a3df22e1b82aa945f2208dcc5cb90afc441ba059a3ae7aafb4ee000ec3a52dc65a8c043a5ac7255a391c875330 +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-client-linux-386.tar.gz) | 44c8dd4b1ddfc256d35786c8abf45b0eb5f0794f5e310d2efc865748adddc50e8bf38aa71295ae8a82884cb65f2e0b9b0737b000f96fd8f2d5c19971d7c4d8e8 +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-client-linux-amd64.tar.gz) | e1291989892769de6b978c17b8612b94da6f3b735a4d895100af622ca9ebb968c75548afea7ab00445869625dd0da3afec979e333afbb445805f5d31c1c13cc7 +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-client-linux-arm.tar.gz) | 3c4bcb8cbe73822d68a2f62553a364e20bec56b638c71d0f58679b4f4b277d809142346f18506914e694f6122a3e0f767eab20b7b1c4dbb79e4c5089981ae0f1 +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-client-linux-arm64.tar.gz) | 9389974a790268522e187f5ba5237f3ee4684118c7db76bc3d4164de71d8208702747ec333b204c7a78073ab42553cbbce13a1883fab4fec617e093b05fab332 +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-client-linux-ppc64le.tar.gz) | 63399e53a083b5af3816c28ff162c9de6b64c75da4647f0d6bbaf97afdf896823cb1e556f2abac75c6516072293026d3ff9f30676fd75143ac6ca3f4d21f4327 +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-client-linux-s390x.tar.gz) | 50898f197a9d923971ff9046c9f02779b57f7b3cea7da02f3ea9bab8c08d65a9c4a7531a2470fa14783460f52111a52b96ebf916c0a1d8215b4070e4e861c1b0 +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-client-windows-386.tar.gz) | a7743e839e1aa19f5ee20b6ee5000ac8ef9e624ac5be63bb574fad6992e4b9167193ed07e03c9bc524e88bfeed66c95341a38a03bff1b10bc9910345f33019f0 +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-client-windows-amd64.tar.gz) | 5f1d19c230bd3542866d16051808d184e9dd3e2f8c001ed4cee7b5df91f872380c2bf56a3add8c9413ead9d8c369efce2bcab4412174df9b823d3592677bf74e + +### Server binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-server-linux-amd64.tar.gz) | ef2cac10febde231aeb6f131e589450c560eeaab8046b49504127a091cddc17bc518c2ad56894a6a033033ab6fc6e121b1cc23691683bc36f45fe6b1dd8e0510 +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-server-linux-arm.tar.gz) | d11c9730307f08e80b2b8a7c64c3e9a9e43c622002e377dfe3a386f4541e24adc79a199a6f280f40298bb36793194fd44ed45defe8a3ee54a9cb1386bc26e905 +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-server-linux-arm64.tar.gz) | 28f8c32bf98ee1add7edf5d341c3bac1afc0085f90dcbbfb8b27a92087f13e2b53c327c8935ee29bf1dc3160655b32bbe3e29d5741a8124a3848a777e7d42933 +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-server-linux-ppc64le.tar.gz) | 99ae8d44b0de3518c27fa8bbddd2ecf053dfb789fb9d65f8a4ecf4c8331cf63d2f09a41c2bcd5573247d5f66a1b2e51944379df1715017d920d521b98589508a +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-server-linux-s390x.tar.gz) | f8c0e954a2dfc6845614488dadeed069cc7f3f08e33c351d7a77c6ef97867af590932e8576d12998a820a0e4d35d2eee797c764e2810f09ab1e90a5acaeaad33 + +### Node binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-node-linux-amd64.tar.gz) | c5456d50bfbe0d75fb150b3662ed7468a0abd3970792c447824f326894382c47bbd3a2cc5a290f691c8c09585ff6fe505ab86b4aff2b7e5ccee11b5e6354ae6c +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-node-linux-arm.tar.gz) | 335b5cd8672e053302fd94d932fb2fa2e48eeeb1799650b3f93acdfa635e03a8453637569ab710c46885c8317759f4c60aaaf24dca9817d9fa47500fe4a3ca53 +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-node-linux-arm64.tar.gz) | 3ee87dbeed8ace9351ac89bdaf7274dd10b4faec3ceba0825f690ec7a2bb7eb7c634274a1065a0939eec8ff3e43f72385f058f4ec141841550109e775bc5eff9 +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-node-linux-ppc64le.tar.gz) | 6956f965b8d719b164214ec9195fdb2c776b907fe6d2c524082f00c27872a73475927fd7d2a994045ce78f6ad2aa5aeaf1eb5514df1810d2cfe342fd4e5ce4a1 +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-node-linux-s390x.tar.gz) | 3b643aa905c709c57083c28dd9e8ffd88cb64466cda1499da7fc54176b775003e08b9c7a07b0964064df67c8142f6f1e6c13bfc261bd65fb064049920bfa57d0 +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-node-windows-amd64.tar.gz) | b2e6d6fb0091f2541f9925018c2bdbb0138a95bab06b4c6b38abf4b7144b2575422263b78fb3c6fd09e76d90a25a8d35a6d4720dc169794d42c95aa22ecc6d5f + +## Changelog since v1.21.0-alpha.1 + +## Urgent Upgrade Notes + +### (No, really, you MUST read this before you upgrade) + + - Remove storage metrics `storage_operation_errors_total`, since we already have `storage_operation_status_count`.And add new field `status` for `storage_operation_duration_seconds`, so that we can know about all status storage operation latency. ([#98332](https://github.com/kubernetes/kubernetes/pull/98332), [@JornShen](https://github.com/JornShen)) [SIG Instrumentation and Storage] + +## Changes by Kind ### Deprecation -- Remove all the generators from kubectl run. It will now only create pods. Additionally, deprecates all the flags that are not relevant anymore. ([#87077](https://github.com/kubernetes/kubernetes/pull/87077), [@soltysh](https://github.com/soltysh)) [SIG Architecture, SIG CLI, and SIG Testing] -- kubeadm: kube-dns is deprecated and will not be supported in a future version ([#86574](https://github.com/kubernetes/kubernetes/pull/86574), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] +- Remove the TokenRequest and TokenRequestProjection feature gates ([#97148](https://github.com/kubernetes/kubernetes/pull/97148), [@wawa0210](https://github.com/wawa0210)) [SIG Node] +- Removing experimental windows container hyper-v support with Docker ([#97141](https://github.com/kubernetes/kubernetes/pull/97141), [@wawa0210](https://github.com/wawa0210)) [SIG Node and Windows] +- The `export` query parameter (inconsistently supported by API resources and deprecated in v1.14) is fully removed. Requests setting this query parameter will now receive a 400 status response. ([#98312](https://github.com/kubernetes/kubernetes/pull/98312), [@deads2k](https://github.com/deads2k)) [SIG API Machinery, Auth and Testing] ### API Change -- Add kubescheduler.config.k8s.io/v1alpha2 ([#87628](https://github.com/kubernetes/kubernetes/pull/87628), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling] -- --enable-cadvisor-endpoints is now disabled by default. If you need access to the cAdvisor v1 Json API please enable it explicitly in the kubelet command line. Please note that this flag was deprecated in 1.15 and will be removed in 1.19. ([#87440](https://github.com/kubernetes/kubernetes/pull/87440), [@dims](https://github.com/dims)) [SIG Instrumentation, SIG Node, and SIG Testing] -- The following feature gates are removed, because the associated features were unconditionally enabled in previous releases: CustomResourceValidation, CustomResourceSubresources, CustomResourceWebhookConversion, CustomResourcePublishOpenAPI, CustomResourceDefaulting ([#87475](https://github.com/kubernetes/kubernetes/pull/87475), [@liggitt](https://github.com/liggitt)) [SIG API Machinery] +- Enable SPDY pings to keep connections alive, so that `kubectl exec` and `kubectl port-forward` won't be interrupted. ([#97083](https://github.com/kubernetes/kubernetes/pull/97083), [@knight42](https://github.com/knight42)) [SIG API Machinery and CLI] + +### Documentation + +- Official support to build kubernetes with docker-machine / remote docker is removed. This change does not affect building kubernetes with docker locally. ([#97935](https://github.com/kubernetes/kubernetes/pull/97935), [@adeniyistephen](https://github.com/adeniyistephen)) [SIG Release and Testing] +- Set kubelet option `--volume-stats-agg-period` to negative value to disable volume calculations. ([#96675](https://github.com/kubernetes/kubernetes/pull/96675), [@pacoxu](https://github.com/pacoxu)) [SIG Node] + +### Bug or Regression + +- Clean ReplicaSet by revision instead of creation timestamp in deployment controller ([#97407](https://github.com/kubernetes/kubernetes/pull/97407), [@waynepeking348](https://github.com/waynepeking348)) [SIG Apps] +- Ensure that client-go's EventBroadcaster is safe (non-racy) during shutdown. ([#95664](https://github.com/kubernetes/kubernetes/pull/95664), [@DirectXMan12](https://github.com/DirectXMan12)) [SIG API Machinery] +- Fix azure file migration issue ([#97877](https://github.com/kubernetes/kubernetes/pull/97877), [@andyzhangx](https://github.com/andyzhangx)) [SIG Auth, Cloud Provider and Storage] +- Fix kubelet from panic after getting the wrong signal ([#98200](https://github.com/kubernetes/kubernetes/pull/98200), [@wzshiming](https://github.com/wzshiming)) [SIG Node] +- Fix repeatedly acquire the inhibit lock ([#98088](https://github.com/kubernetes/kubernetes/pull/98088), [@wzshiming](https://github.com/wzshiming)) [SIG Node] +- Fixed a bug that the kubelet cannot start on BtrfS. ([#98042](https://github.com/kubernetes/kubernetes/pull/98042), [@gjkim42](https://github.com/gjkim42)) [SIG Node] +- Fixed an issue with garbage collection failing to clean up namespaced children of an object also referenced incorrectly by cluster-scoped children ([#98068](https://github.com/kubernetes/kubernetes/pull/98068), [@liggitt](https://github.com/liggitt)) [SIG API Machinery and Apps] +- Fixed no effect namespace when exposing deployment with --dry-run=client. ([#97492](https://github.com/kubernetes/kubernetes/pull/97492), [@masap](https://github.com/masap)) [SIG CLI] +- Fixing a bug where a failed node may not have the NoExecute taint set correctly ([#96876](https://github.com/kubernetes/kubernetes/pull/96876), [@howieyuen](https://github.com/howieyuen)) [SIG Apps and Node] +- Indentation of `Resource Quota` block in kubectl describe namespaces output gets correct. ([#97946](https://github.com/kubernetes/kubernetes/pull/97946), [@dty1er](https://github.com/dty1er)) [SIG CLI] +- KUBECTL_EXTERNAL_DIFF now accepts equal sign for additional parameters. ([#98158](https://github.com/kubernetes/kubernetes/pull/98158), [@dougsland](https://github.com/dougsland)) [SIG CLI] +- Kubeadm: fix a bug where "kubeadm join" would not properly handle missing names for existing etcd members. ([#97372](https://github.com/kubernetes/kubernetes/pull/97372), [@ihgann](https://github.com/ihgann)) [SIG Cluster Lifecycle] +- Kubelet should ignore cgroup driver check on Windows node. ([#97764](https://github.com/kubernetes/kubernetes/pull/97764), [@pacoxu](https://github.com/pacoxu)) [SIG Node and Windows] +- Make podTopologyHints protected by lock ([#95111](https://github.com/kubernetes/kubernetes/pull/95111), [@choury](https://github.com/choury)) [SIG Node] +- Readjust kubelet_containers_per_pod_count bucket ([#98169](https://github.com/kubernetes/kubernetes/pull/98169), [@wawa0210](https://github.com/wawa0210)) [SIG Instrumentation and Node] +- Scores from InterPodAffinity have stronger differentiation. ([#98096](https://github.com/kubernetes/kubernetes/pull/98096), [@leileiwan](https://github.com/leileiwan)) [SIG Scheduling] +- Specifying the KUBE_TEST_REPO environment variable when e2e tests are executed will instruct the test infrastructure to load that image from a location within the specified repo, using a predefined pattern. ([#93510](https://github.com/kubernetes/kubernetes/pull/93510), [@smarterclayton](https://github.com/smarterclayton)) [SIG Testing] +- Static pods will be deleted gracefully. ([#98103](https://github.com/kubernetes/kubernetes/pull/98103), [@gjkim42](https://github.com/gjkim42)) [SIG Node] +- Use network.Interface.VirtualMachine.ID to get the binded VM + Skip standalone VM when reconciling LoadBalancer ([#97635](https://github.com/kubernetes/kubernetes/pull/97635), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] + +### Other (Cleanup or Flake) + +- Kubeadm: change the default image repository for CI images from 'gcr.io/kubernetes-ci-images' to 'gcr.io/k8s-staging-ci-images' ([#97087](https://github.com/kubernetes/kubernetes/pull/97087), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] +- Migrate generic_scheduler.go and types.go to structured logging. ([#98134](https://github.com/kubernetes/kubernetes/pull/98134), [@tanjing2020](https://github.com/tanjing2020)) [SIG Scheduling] +- Migrate proxy/winuserspace/proxier.go logs to structured logging ([#97941](https://github.com/kubernetes/kubernetes/pull/97941), [@JornShen](https://github.com/JornShen)) [SIG Network] +- Migrate staging/src/k8s.io/apiserver/pkg/audit/policy/reader.go logs to structured logging. ([#98252](https://github.com/kubernetes/kubernetes/pull/98252), [@lala123912](https://github.com/lala123912)) [SIG API Machinery and Auth] +- Migrate staging\src\k8s.io\apiserver\pkg\endpoints logs to structured logging ([#98093](https://github.com/kubernetes/kubernetes/pull/98093), [@lala123912](https://github.com/lala123912)) [SIG API Machinery] +- Node ([#96552](https://github.com/kubernetes/kubernetes/pull/96552), [@pandaamanda](https://github.com/pandaamanda)) [SIG Apps, Cloud Provider, Node and Scheduling] +- The kubectl alpha debug command was scheduled to be removed in v1.21. ([#98111](https://github.com/kubernetes/kubernetes/pull/98111), [@pandaamanda](https://github.com/pandaamanda)) [SIG CLI] +- Update cri-tools to [v1.20.0](https://github.com/kubernetes-sigs/cri-tools/releases/tag/v1.20.0) ([#97967](https://github.com/kubernetes/kubernetes/pull/97967), [@rajibmitra](https://github.com/rajibmitra)) [SIG Cloud Provider] +- Windows nodes on GCE will take longer to start due to dependencies installed at node creation time. ([#98284](https://github.com/kubernetes/kubernetes/pull/98284), [@pjh](https://github.com/pjh)) [SIG Cloud Provider] + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- github.com/google/cadvisor: [v0.38.6 → v0.38.7](https://github.com/google/cadvisor/compare/v0.38.6...v0.38.7) +- k8s.io/gengo: 83324d8 → b6c5ce2 + +### Removed +_Nothing has changed._ + + + +# v1.21.0-alpha.1 + + +## Downloads for v1.21.0-alpha.1 + +### Source Code + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes.tar.gz) | b2bacd5c3fc9f829e6269b7d2006b0c6e464ff848bb0a2a8f2fe52ad2d7c4438f099bd8be847d8d49ac6e4087f4d74d5c3a967acd798e0b0cb4d7a2bdb122997 +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-src.tar.gz) | 518ac5acbcf23902fb1b902b69dbf3e86deca5d8a9b5f57488a15f185176d5a109558f3e4df062366af874eca1bcd61751ee8098b0beb9bcdc025d9a1c9be693 + +### Client binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-client-darwin-amd64.tar.gz) | eaa7aea84a5ed954df5ec710cbeb6ec88b46465f43cb3d09aabe2f714b84a050a50bf5736089f09dbf1090f2e19b44823d656c917e3c8c877630756c3026f2b6 +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-client-linux-386.tar.gz) | 47f74b8d46ad1779c5b0b5f15aa15d5513a504eeb6f53db4201fbe9ff8956cb986b7c1b0e9d50a99f78e9e2a7f304f3fc1cc2fa239296d9a0dd408eb6069e975 +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-client-linux-amd64.tar.gz) | 1a148e282628b008c8abd03dd12ec177ced17584b5115d92cd33dd251e607097d42e9da8c7089bd947134b900f85eb75a4740b6a5dd580c105455b843559df39 +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-client-linux-arm.tar.gz) | d13d2feb73bd032dc01f7e2955b98d8215a39fe1107d037a73fa1f7d06c3b93ebaa53ed4952d845c64454ef3cca533edb97132d234d50b6fb3bcbd8a8ad990eb +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-client-linux-arm64.tar.gz) | 8252105a17b09a78e9ad2c024e4e401a69764ac869708a071aaa06f81714c17b9e7c5b2eb8efde33f24d0b59f75c5da607d5e1e72bdf12adfbb8c829205cd1c1 +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-client-linux-ppc64le.tar.gz) | 297a9082df4988389dc4be30eb636dff49f36f5d87047bab44745884e610f46a17ae3a08401e2cab155b7c439f38057bfd8288418215f7dd3bf6a49dbe61ea0e +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-client-linux-s390x.tar.gz) | 04c06490dd17cd5dccfd92bafa14acf64280ceaea370d9635f23aeb6984d1beae6d0d1d1506edc6f30f927deeb149b989d3e482b47fbe74008b371f629656e79 +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-client-windows-386.tar.gz) | ec6e9e87a7d685f8751d7e58f24f417753cff5554a7229218cb3a08195d461b2e12409344950228e9fbbc92a8a06d35dd86242da6ff1e6652ec1fae0365a88c1 +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-client-windows-amd64.tar.gz) | 51039e6221d3126b5d15e797002ae01d4f0b10789c5d2056532f27ef13f35c5a2e51be27764fda68e8303219963126559023aed9421313bec275c0827fbcaf8a + +### Server binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-server-linux-amd64.tar.gz) | 4edf820930c88716263560275e3bd7fadb8dc3700b9f8e1d266562e356e0abeb1a913f536377dab91218e3940b447d6bf1da343b85da25c2256dc4dcde5798dd +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-server-linux-arm.tar.gz) | b15213e53a8ab4ba512ce6ef9ad42dd197d419c61615cd23de344227fd846c90448d8f3d98e555b63ba5b565afa627cca6b7e3990ebbbba359c96f2391302df1 +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-server-linux-arm64.tar.gz) | 5be29cca9a9358fc68351ee63e99d57dc2ffce6e42fc3345753dbbf7542ff2d770c4852424158540435fa6e097ce3afa9b13affc40c8b3b69fe8406798f8068f +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-server-linux-ppc64le.tar.gz) | 89fd99ab9ce85db0b94b86709932105efc883cc93959cf7ea9a39e79a4acea23064d7010eeb577450cccabe521c04b7ba47bbec212ed37edeed7cb04bad34518 +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-server-linux-s390x.tar.gz) | 2fbc30862c77d247aa8d96ab9d1a144599505287b0033a3a2d0988958e7bb2f2e8b67f52c1fec74b4ec47d74ba22cd0f6cb5c4228acbaa72b1678d5fece0254d + +### Node binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-node-linux-amd64.tar.gz) | 95658d321a0a371c0900b401d1469d96915310afbc4e4b9b11f031438bb188513b57d5a60b5316c3b0c18f541cda6f0ac42f59a76495f8abc743a067115da23a +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-node-linux-arm.tar.gz) | f375acfb42aad6c65b833c270e7e3acfe9cd1d6b2441c33874e77faae263957f7acfe86f1b71f14298118595e4cc6952c7dea0c832f7f2e72428336f13034362 +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-node-linux-arm64.tar.gz) | 43b4baccd58d74e7f48d096ab92f2bbbcdf47e30e7a3d2b56c6cc9f90002cfd4fefaac894f69bd5f9f4dbdb09a4749a77eb76b1b97d91746bd96fe94457879ab +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-node-linux-ppc64le.tar.gz) | e7962b522c6c7c14b9ee4c1d254d8bdd9846b2b33b0443fc9c4a41be6c40e5e6981798b720f0148f36263d5cc45d5a2bb1dd2f9ab2838e3d002e45b9bddeb7bf +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-node-linux-s390x.tar.gz) | 49ebc97f01829e65f7de15be00b882513c44782eaadd1b1825a227e3bd3c73cc6aca8345af05b303d8c43aa2cb944a069755b2709effb8cc22eae621d25d4ba5 +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.0-alpha.1/kubernetes-node-windows-amd64.tar.gz) | 6e0fd7724b09e6befbcb53b33574e97f2db089f2eee4bbf391abb7f043103a5e6e32e3014c0531b88f9a3ca88887bbc68625752c44326f98dd53adb3a6d1bed8 + +## Changelog since v1.20.0 + +## Urgent Upgrade Notes + +### (No, really, you MUST read this before you upgrade) + + - Kube-proxy's IPVS proxy mode no longer sets the net.ipv4.conf.all.route_localnet sysctl parameter. Nodes upgrading will have net.ipv4.conf.all.route_localnet set to 1 but new nodes will inherit the system default (usually 0). If you relied on any behavior requiring net.ipv4.conf.all.route_localnet, you must set ensure it is enabled as kube-proxy will no longer set it automatically. This change helps to further mitigate CVE-2020-8558. ([#92938](https://github.com/kubernetes/kubernetes/pull/92938), [@lbernail](https://github.com/lbernail)) [SIG Network and Release] + +## Changes by Kind + +### Deprecation + +- Deprecate the `topologyKeys` field in Service. This capability will be replaced with upcoming work around Topology Aware Subsetting and Service Internal Traffic Policy. ([#96736](https://github.com/kubernetes/kubernetes/pull/96736), [@andrewsykim](https://github.com/andrewsykim)) [SIG Apps] +- Kubeadm: deprecated command "alpha selfhosting pivot" is removed now. ([#97627](https://github.com/kubernetes/kubernetes/pull/97627), [@knight42](https://github.com/knight42)) [SIG Cluster Lifecycle] +- Kubeadm: graduate the command `kubeadm alpha kubeconfig user` to `kubeadm kubeconfig user`. The `kubeadm alpha kubeconfig user` command is deprecated now. ([#97583](https://github.com/kubernetes/kubernetes/pull/97583), [@knight42](https://github.com/knight42)) [SIG Cluster Lifecycle] +- Kubeadm: the "kubeadm alpha certs" command is removed now, please use "kubeadm certs" instead. ([#97706](https://github.com/kubernetes/kubernetes/pull/97706), [@knight42](https://github.com/knight42)) [SIG Cluster Lifecycle] +- Remove the deprecated metrics "scheduling_algorithm_preemption_evaluation_seconds" and "binding_duration_seconds", suggest to use "scheduler_framework_extension_point_duration_seconds" instead. ([#96447](https://github.com/kubernetes/kubernetes/pull/96447), [@chendave](https://github.com/chendave)) [SIG Cluster Lifecycle, Instrumentation, Scheduling and Testing] +- The PodSecurityPolicy API is deprecated in 1.21, and will no longer be served starting in 1.25. ([#97171](https://github.com/kubernetes/kubernetes/pull/97171), [@deads2k](https://github.com/deads2k)) [SIG Auth and CLI] + +### API Change + +- Change the APIVersion proto name of BoundObjectRef from aPIVersion to apiVersion. ([#97379](https://github.com/kubernetes/kubernetes/pull/97379), [@kebe7jun](https://github.com/kebe7jun)) [SIG Auth] +- Promote Immutable Secrets/ConfigMaps feature to Stable. + This allows to set `Immutable` field in Secrets or ConfigMap object to mark their contents as immutable. ([#97615](https://github.com/kubernetes/kubernetes/pull/97615), [@wojtek-t](https://github.com/wojtek-t)) [SIG Apps, Architecture, Node and Testing] ### Feature -- aggragation api will have alpha support for network proxy ([#87515](https://github.com/kubernetes/kubernetes/pull/87515), [@Sh4d1](https://github.com/Sh4d1)) [SIG API Machinery] -- API request throttling (due to a high rate of requests) is now reported in client-go logs at log level 2. The messages are of the form +- Add flag --lease-max-object-size and metric etcd_lease_object_counts for kube-apiserver to config and observe max objects attached to a single etcd lease. ([#97480](https://github.com/kubernetes/kubernetes/pull/97480), [@lingsamuel](https://github.com/lingsamuel)) [SIG API Machinery, Instrumentation and Scalability] +- Add flag --lease-reuse-duration-seconds for kube-apiserver to config etcd lease reuse duration. ([#97009](https://github.com/kubernetes/kubernetes/pull/97009), [@lingsamuel](https://github.com/lingsamuel)) [SIG API Machinery and Scalability] +- Adds the ability to pass --strict-transport-security-directives to the kube-apiserver to set the HSTS header appropriately. Be sure you understand the consequences to browsers before setting this field. ([#96502](https://github.com/kubernetes/kubernetes/pull/96502), [@249043822](https://github.com/249043822)) [SIG Auth] +- Kubeadm now includes CoreDNS v1.8.0. ([#96429](https://github.com/kubernetes/kubernetes/pull/96429), [@rajansandeep](https://github.com/rajansandeep)) [SIG Cluster Lifecycle] +- Kubeadm: add support for certificate chain validation. When using kubeadm in external CA mode, this allows an intermediate CA to be used to sign the certificates. The intermediate CA certificate must be appended to each signed certificate for this to work correctly. ([#97266](https://github.com/kubernetes/kubernetes/pull/97266), [@robbiemcmichael](https://github.com/robbiemcmichael)) [SIG Cluster Lifecycle] +- Kubeadm: amend the node kernel validation to treat CGROUP_PIDS, FAIR_GROUP_SCHED as required and CFS_BANDWIDTH, CGROUP_HUGETLB as optional ([#96378](https://github.com/kubernetes/kubernetes/pull/96378), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle and Node] +- The Kubernetes pause image manifest list now contains an image for Windows Server 20H2. ([#97322](https://github.com/kubernetes/kubernetes/pull/97322), [@claudiubelu](https://github.com/claudiubelu)) [SIG Windows] +- The apimachinery util/net function used to detect the bind address `ResolveBindAddress()` + takes into consideration global ip addresses on loopback interfaces when: + - the host has default routes + - there are no global IPs on those interfaces. + in order to support more complex network scenarios like BGP Unnumbered RFC 5549 ([#95790](https://github.com/kubernetes/kubernetes/pull/95790), [@aojea](https://github.com/aojea)) [SIG Network] + +### Bug or Regression + +- ## Changelog - Throttling request took 1.50705208s, request: GET: + ### General + - Fix priority expander falling back to a random choice even though there is a higher priority option to choose + - Clone `kubernetes/kubernetes` in `update-vendor.sh` shallowly, instead of fetching all revisions + - Speed up binpacking by reducing the number of PreFilter calls (call once per pod instead of #pods*#nodes times) + - Speed up finding unneeded nodes by 5x+ in very large clusters by reducing the number of PreFilter calls + - Expose `--max-nodes-total` as a metric + - Errors in `IncreaseSize` changed from type `apiError` to `cloudProviderError` + - Make `build-in-docker` and `test-in-docker` work on Linux systems with SELinux enabled + - Fix an error where existing nodes were not considered as destinations while finding place for pods in scale-down simulations + - Remove redundant log lines and reduce severity around parsing kubeEnv + - Don't treat nodes created by virtual kubelet as nodes from non-autoscaled node groups + - Remove redundant logging around calculating node utilization + - Add configurable `--network` and `--rm` flags for docker in `Makefile` + - Subtract DaemonSet pods' requests from node allocatable in the denominator while computing node utilization + - Include taints by condition when determining if a node is unready/still starting + - Fix `update-vendor.sh` to work on OSX and zsh + - Add best-effort eviction for DaemonSet pods while scaling down non-empty nodes + - Add build support for ARM64 - The presence of these messages, may indicate to the administrator the need to tune the cluster accordingly. ([#87740](https://github.com/kubernetes/kubernetes/pull/87740), [@jennybuckley](https://github.com/jennybuckley)) [SIG API Machinery] -- kubeadm: reject a node joining the cluster if a node with the same name already exists ([#81056](https://github.com/kubernetes/kubernetes/pull/81056), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- disableAvailabilitySetNodes is added to avoid VM list for VMSS clusters. It should only be used when vmType is "vmss" and all the nodes (including masters) are VMSS virtual machines. ([#87685](https://github.com/kubernetes/kubernetes/pull/87685), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- The kubectl --dry-run flag now accepts the values 'client', 'server', and 'none', to support client-side and server-side dry-run strategies. The boolean and unset values for the --dry-run flag are deprecated and a value will be required in a future version. ([#87580](https://github.com/kubernetes/kubernetes/pull/87580), [@julianvmodesto](https://github.com/julianvmodesto)) [SIG CLI] -- Add support for pre-allocated hugepages for more than one page size ([#82820](https://github.com/kubernetes/kubernetes/pull/82820), [@odinuge](https://github.com/odinuge)) [SIG Apps] -- Update CNI version to v0.8.5 ([#78819](https://github.com/kubernetes/kubernetes/pull/78819), [@justaugustus](https://github.com/justaugustus)) [SIG API Machinery, SIG Cluster Lifecycle, SIG Network, SIG Release, and SIG Testing] -- Skip default spreading scoring plugin for pods that define TopologySpreadConstraints ([#87566](https://github.com/kubernetes/kubernetes/pull/87566), [@skilxn-go](https://github.com/skilxn-go)) [SIG Scheduling] -- Added more details to taint toleration errors ([#87250](https://github.com/kubernetes/kubernetes/pull/87250), [@starizard](https://github.com/starizard)) [SIG Apps, and SIG Scheduling] -- Scheduler: Add DefaultBinder plugin ([#87430](https://github.com/kubernetes/kubernetes/pull/87430), [@alculquicondor](https://github.com/alculquicondor)) [SIG Scheduling, and SIG Testing] -- Kube-apiserver metrics will now include request counts, latencies, and response sizes for /healthz, /livez, and /readyz requests. ([#83598](https://github.com/kubernetes/kubernetes/pull/83598), [@jktomer](https://github.com/jktomer)) [SIG API Machinery] + ### AliCloud + - Add missing daemonsets and replicasets to ALI example cluster role + + ### Apache CloudStack + - Add support for Apache CloudStack + + ### AWS + - Regenerate list of EC2 instances + - Fix pricing endpoint in AWS China Region + + ### Azure + - Add optional jitter on initial VMSS VM cache refresh, keep the refreshes spread over time + - Serve from cache for the whole period of ongoing throttling + - Fix unwanted VMSS VMs cache invalidations + - Enforce setting the number of retries if cloud provider backoff is enabled + - Don't update capacity if VMSS provisioning state is updating + - Support allocatable resources overrides via VMSS tags + - Add missing stable labels in template nodes + - Proactively set instance status to deleting on node deletions + + ### Cluster API + - Migrate interaction with the API from using internal types to using Unstructured + - Improve tests to work better with constrained resources + - Add support for node autodiscovery + - Add support for `--cloud-config` + - Update group identifier to use for Cluster API annotations + + ### Exoscale + - Add support for Exoscale + + ### GCE + - Decrease the number of GCE Read Requests made while deleting nodes + - Base pricing of custom instances on their instance family type + - Add pricing information for missing machine types + - Add pricing information for different GPU types + - Ignore the new `topology.gke.io/zone` label when comparing groups + - Add missing stable labels to template nodes + + ### HuaweiCloud + - Add auto scaling group support + - Implement node group by AS + - Implement getting desired instance number of node group + - Implement increasing node group size + - Implement TemplateNodeInfo + - Implement caching instances + + ### IONOS + - Add support for IONOS + + ### Kubemark + - Skip non-kubemark nodes while computing node infos for node groups. + + ### Magnum + - Add Magnum support in the Cluster Autoscaler helm chart + + ### Packet + - Allow empty nodepools + - Add support for multiple nodepools + - Add pricing support + + ## Image + Image: `k8s.gcr.io/autoscaling/cluster-autoscaler:v1.20.0` ([#97011](https://github.com/kubernetes/kubernetes/pull/97011), [@towca](https://github.com/towca)) [SIG Cloud Provider] +- AcceleratorStats will be available in the Summary API of kubelet when cri_stats_provider is used. ([#96873](https://github.com/kubernetes/kubernetes/pull/96873), [@ruiwen-zhao](https://github.com/ruiwen-zhao)) [SIG Node] +- Add limited lines to log when having tail option ([#93920](https://github.com/kubernetes/kubernetes/pull/93920), [@zhouya0](https://github.com/zhouya0)) [SIG Node] +- Avoid systemd-logind loading configuration warning ([#97950](https://github.com/kubernetes/kubernetes/pull/97950), [@wzshiming](https://github.com/wzshiming)) [SIG Node] +- Cloud-controller-manager: routes controller should not depend on --allocate-node-cidrs ([#97029](https://github.com/kubernetes/kubernetes/pull/97029), [@andrewsykim](https://github.com/andrewsykim)) [SIG Cloud Provider and Testing] +- Copy annotations with empty value when deployment rolls back ([#94858](https://github.com/kubernetes/kubernetes/pull/94858), [@waynepeking348](https://github.com/waynepeking348)) [SIG Apps] +- Detach volumes from vSphere nodes not tracked by attach-detach controller ([#96689](https://github.com/kubernetes/kubernetes/pull/96689), [@gnufied](https://github.com/gnufied)) [SIG Cloud Provider and Storage] +- Fix kubectl label error when local=true is set. ([#97440](https://github.com/kubernetes/kubernetes/pull/97440), [@pandaamanda](https://github.com/pandaamanda)) [SIG CLI] +- Fix Azure file share not deleted issue when the namespace is deleted ([#97417](https://github.com/kubernetes/kubernetes/pull/97417), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] +- Fix CVE-2020-8555 for Gluster client connections. ([#97922](https://github.com/kubernetes/kubernetes/pull/97922), [@liggitt](https://github.com/liggitt)) [SIG Storage] +- Fix counting error in service/nodeport/loadbalancer quota check ([#97451](https://github.com/kubernetes/kubernetes/pull/97451), [@pacoxu](https://github.com/pacoxu)) [SIG API Machinery, Network and Testing] +- Fix kubectl-convert import known versions ([#97754](https://github.com/kubernetes/kubernetes/pull/97754), [@wzshiming](https://github.com/wzshiming)) [SIG CLI and Testing] +- Fix missing cadvisor machine metrics. ([#97006](https://github.com/kubernetes/kubernetes/pull/97006), [@lingsamuel](https://github.com/lingsamuel)) [SIG Node] +- Fix nil VMSS name when setting service to auto mode ([#97366](https://github.com/kubernetes/kubernetes/pull/97366), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] +- Fix the panic when kubelet registers if a node object already exists with no Status.Capacity or Status.Allocatable ([#95269](https://github.com/kubernetes/kubernetes/pull/95269), [@SataQiu](https://github.com/SataQiu)) [SIG Node] +- Fix the regression with the slow pods termination. Before this fix pods may take an additional time to terminate - up to one minute. Reversing the change that ensured that CNI resources cleaned up when the pod is removed on API server. ([#97980](https://github.com/kubernetes/kubernetes/pull/97980), [@SergeyKanzhelev](https://github.com/SergeyKanzhelev)) [SIG Node] +- Fix to recover CSI volumes from certain dangling attachments ([#96617](https://github.com/kubernetes/kubernetes/pull/96617), [@yuga711](https://github.com/yuga711)) [SIG Apps and Storage] +- Fix: azure file latency issue for metadata-heavy workloads ([#97082](https://github.com/kubernetes/kubernetes/pull/97082), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] +- Fixed Cinder volume IDs on OpenStack Train ([#96673](https://github.com/kubernetes/kubernetes/pull/96673), [@jsafrane](https://github.com/jsafrane)) [SIG Cloud Provider] +- Fixed FibreChannel volume plugin corrupting filesystems on detach of multipath volumes. ([#97013](https://github.com/kubernetes/kubernetes/pull/97013), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] +- Fixed a bug in kubelet that will saturate CPU utilization after containerd got restarted. ([#97174](https://github.com/kubernetes/kubernetes/pull/97174), [@hanlins](https://github.com/hanlins)) [SIG Node] +- Fixed bug in CPUManager with race on container map access ([#97427](https://github.com/kubernetes/kubernetes/pull/97427), [@klueska](https://github.com/klueska)) [SIG Node] +- Fixed cleanup of block devices when /var/lib/kubelet is a symlink. ([#96889](https://github.com/kubernetes/kubernetes/pull/96889), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] +- GCE Internal LoadBalancer sync loop will now release the ILB IP address upon sync failure. An error in ILB forwarding rule creation will no longer leak IP addresses. ([#97740](https://github.com/kubernetes/kubernetes/pull/97740), [@prameshj](https://github.com/prameshj)) [SIG Cloud Provider and Network] +- Ignore update pod with no new images in alwaysPullImages admission controller ([#96668](https://github.com/kubernetes/kubernetes/pull/96668), [@pacoxu](https://github.com/pacoxu)) [SIG Apps, Auth and Node] +- Kubeadm now installs version 3.4.13 of etcd when creating a cluster with v1.19 ([#97244](https://github.com/kubernetes/kubernetes/pull/97244), [@pacoxu](https://github.com/pacoxu)) [SIG Cluster Lifecycle] +- Kubeadm: avoid detection of the container runtime for commands that do not need it ([#97625](https://github.com/kubernetes/kubernetes/pull/97625), [@pacoxu](https://github.com/pacoxu)) [SIG Cluster Lifecycle] +- Kubeadm: fix a bug in the host memory detection code on 32bit Linux platforms ([#97403](https://github.com/kubernetes/kubernetes/pull/97403), [@abelbarrera15](https://github.com/abelbarrera15)) [SIG Cluster Lifecycle] +- Kubeadm: fix a bug where "kubeadm upgrade" commands can fail if CoreDNS v1.8.0 is installed. ([#97919](https://github.com/kubernetes/kubernetes/pull/97919), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] +- Performance regression [#97685](https://github.com/kubernetes/kubernetes/issues/97685) has been fixed. ([#97860](https://github.com/kubernetes/kubernetes/pull/97860), [@MikeSpreitzer](https://github.com/MikeSpreitzer)) [SIG API Machinery] +- Remove deprecated --cleanup-ipvs flag of kube-proxy, and make --cleanup flag always to flush IPVS ([#97336](https://github.com/kubernetes/kubernetes/pull/97336), [@maaoBit](https://github.com/maaoBit)) [SIG Network] +- The current version of the container image publicly exposed IP serving a /metrics endpoint to the Internet. The new version of the container image serves /metrics endpoint on a different port. ([#97621](https://github.com/kubernetes/kubernetes/pull/97621), [@vbannai](https://github.com/vbannai)) [SIG Cloud Provider] +- Use force unmount for NFS volumes if regular mount fails after 1 minute timeout ([#96844](https://github.com/kubernetes/kubernetes/pull/96844), [@gnufied](https://github.com/gnufied)) [SIG Storage] +- Users will see increase in time for deletion of pods and also guarantee that removal of pod from api server would mean deletion of all the resources from container runtime. ([#92817](https://github.com/kubernetes/kubernetes/pull/92817), [@kmala](https://github.com/kmala)) [SIG Node] +- Using exec auth plugins with kubectl no longer results in warnings about constructing many client instances from the same exec auth config. ([#97857](https://github.com/kubernetes/kubernetes/pull/97857), [@liggitt](https://github.com/liggitt)) [SIG API Machinery and Auth] +- Warning about using a deprecated volume plugin is logged only once. ([#96751](https://github.com/kubernetes/kubernetes/pull/96751), [@jsafrane](https://github.com/jsafrane)) [SIG Storage] -### Other (Bug, Cleanup or Flake) +### Other (Cleanup or Flake) -- Fix the masters rolling upgrade causing thundering herd of LISTs on etcd leading to control plane unavailability. ([#86430](https://github.com/kubernetes/kubernetes/pull/86430), [@wojtek-t](https://github.com/wojtek-t)) [SIG API Machinery, SIG Node, and SIG Testing] -- `kubectl diff` now returns 1 only on diff finding changes, and >1 on kubectl errors. The "exit status code 1" message as also been muted. ([#87437](https://github.com/kubernetes/kubernetes/pull/87437), [@apelisse](https://github.com/apelisse)) [SIG CLI, and SIG Testing] -- To reduce chances of throttling, VM cache is set to nil when Azure node provisioning state is deleting ([#87635](https://github.com/kubernetes/kubernetes/pull/87635), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] -- Fix regression in statefulset conversion which prevented applying a statefulset multiple times. ([#87706](https://github.com/kubernetes/kubernetes/pull/87706), [@liggitt](https://github.com/liggitt)) [SIG Apps, and SIG Testing] -- fixed two scheduler metrics (pending_pods and schedule_attempts_total) not being recorded ([#87692](https://github.com/kubernetes/kubernetes/pull/87692), [@everpeace](https://github.com/everpeace)) [SIG Scheduling] -- Resolved a performance issue in the node authorizer index maintenance. ([#87693](https://github.com/kubernetes/kubernetes/pull/87693), [@liggitt](https://github.com/liggitt)) [SIG Auth] -- Removed the 'client' label from apiserver_request_total. ([#87669](https://github.com/kubernetes/kubernetes/pull/87669), [@logicalhan](https://github.com/logicalhan)) [SIG API Machinery, and SIG Instrumentation] -- `(*"k8s.io/client-go/rest".Request).{Do,DoRaw,Stream,Watch}` now require callers to pass a `context.Context` as an argument. The context is used for timeout and cancellation signaling and to pass supplementary information to round trippers in the wrapped transport chain. If you don't need any of this functionality, it is sufficient to pass a context created with `context.Background()` to these functions. The `(*"k8s.io/client-go/rest".Request).Context` method is removed now that all methods that execute a request accept a context directly. ([#87597](https://github.com/kubernetes/kubernetes/pull/87597), [@mikedanese](https://github.com/mikedanese)) [SIG API Machinery, SIG Apps, SIG Auth, SIG Autoscaling, SIG CLI, SIG Cloud Provider, SIG Cluster Lifecycle, SIG Instrumentation, SIG Network, SIG Node, SIG Scheduling, SIG Storage, and SIG Testing] -- For volumes that allow attaches across multiple nodes, attach and detach operations across different nodes are now executed in parallel. ([#87258](https://github.com/kubernetes/kubernetes/pull/87258), [@verult](https://github.com/verult)) [SIG Apps, SIG Node, and SIG Storage] -- kubeadm: apply further improvements to the tentative support for concurrent etcd member join. Fixes a bug where multiple members can receive the same hostname. Increase the etcd client dial timeout and retry timeout for add/remove/... operations. ([#87505](https://github.com/kubernetes/kubernetes/pull/87505), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- Reverted a kubectl azure auth module change where oidc claim spn: prefix was omitted resulting a breaking behavior with existing Azure AD OIDC enabled api-server ([#87507](https://github.com/kubernetes/kubernetes/pull/87507), [@weinong](https://github.com/weinong)) [SIG API Machinery, SIG Auth, and SIG Cloud Provider] -- Update cri-tools to v1.17.0 ([#86305](https://github.com/kubernetes/kubernetes/pull/86305), [@saschagrunert](https://github.com/saschagrunert)) [SIG Cluster Lifecycle, and SIG Release] -- kubeadm: remove the deprecated CoreDNS feature-gate. It was set to "true" since v1.11 when the feature went GA. In v1.13 it was marked as deprecated and hidden from the CLI. ([#87400](https://github.com/kubernetes/kubernetes/pull/87400), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] -- Shared informers are now more reliable in the face of network disruption. ([#86015](https://github.com/kubernetes/kubernetes/pull/86015), [@squeed](https://github.com/squeed)) [SIG API Machinery] -- the CSR signing cert/key pairs will be reloaded from disk like the kube-apiserver cert/key pairs ([#86816](https://github.com/kubernetes/kubernetes/pull/86816), [@deads2k](https://github.com/deads2k)) [SIG API Machinery, SIG Apps, and SIG Auth] -- "kubectl describe statefulsets.apps" prints garbage for rolling update partition ([#85846](https://github.com/kubernetes/kubernetes/pull/85846), [@phil9909](https://github.com/phil9909)) [SIG CLI] +- Bump github.com/Azure/go-autorest/autorest to v0.11.12 ([#97033](https://github.com/kubernetes/kubernetes/pull/97033), [@patrickshan](https://github.com/patrickshan)) [SIG API Machinery, CLI, Cloud Provider and Cluster Lifecycle] +- Delete deprecated mixed protocol annotation ([#97096](https://github.com/kubernetes/kubernetes/pull/97096), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] +- Kube-proxy: Traffic from the cluster directed to ExternalIPs is always sent directly to the Service. ([#96296](https://github.com/kubernetes/kubernetes/pull/96296), [@aojea](https://github.com/aojea)) [SIG Network and Testing] +- Kubeadm: fix a whitespace issue in the output of the "kubeadm join" command shown as the output of "kubeadm init" and "kubeadm token create --print-join-command" ([#97413](https://github.com/kubernetes/kubernetes/pull/97413), [@SataQiu](https://github.com/SataQiu)) [SIG Cluster Lifecycle] +- Kubeadm: improve the error messaging when the user provides an invalid discovery token CA certificate hash. ([#97290](https://github.com/kubernetes/kubernetes/pull/97290), [@neolit123](https://github.com/neolit123)) [SIG Cluster Lifecycle] +- Migrate log messages in pkg/scheduler/{scheduler.go,factory.go} to structured logging ([#97509](https://github.com/kubernetes/kubernetes/pull/97509), [@aldudko](https://github.com/aldudko)) [SIG Scheduling] +- Migrate proxy/iptables/proxier.go logs to structured logging ([#97678](https://github.com/kubernetes/kubernetes/pull/97678), [@JornShen](https://github.com/JornShen)) [SIG Network] +- Migrate some scheduler log messages to structured logging ([#97349](https://github.com/kubernetes/kubernetes/pull/97349), [@aldudko](https://github.com/aldudko)) [SIG Scheduling] +- NONE ([#97167](https://github.com/kubernetes/kubernetes/pull/97167), [@geegeea](https://github.com/geegeea)) [SIG Node] +- NetworkPolicy validation framework optimizations for rapidly verifying CNI's work correctly across several pods and namespaces ([#91592](https://github.com/kubernetes/kubernetes/pull/91592), [@jayunit100](https://github.com/jayunit100)) [SIG Network, Storage and Testing] +- Official support to build kubernetes with docker-machine / remote docker is removed. This change does not affect building kubernetes with docker locally. ([#97618](https://github.com/kubernetes/kubernetes/pull/97618), [@jherrera123](https://github.com/jherrera123)) [SIG Release and Testing] +- Scheduler plugin validation now provides all errors detected instead of the first one. ([#96745](https://github.com/kubernetes/kubernetes/pull/96745), [@lingsamuel](https://github.com/lingsamuel)) [SIG Node, Scheduling and Testing] +- Storage related e2e testsuite redesign & cleanup ([#96573](https://github.com/kubernetes/kubernetes/pull/96573), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Storage and Testing] +- The OIDC authenticator no longer waits 10 seconds before attempting to fetch the metadata required to verify tokens. ([#97693](https://github.com/kubernetes/kubernetes/pull/97693), [@enj](https://github.com/enj)) [SIG API Machinery and Auth] +- The `AttachVolumeLimit` feature gate that is GA since v1.17 is now removed. ([#96539](https://github.com/kubernetes/kubernetes/pull/96539), [@ialidzhikov](https://github.com/ialidzhikov)) [SIG Storage] +- The `CSINodeInfo` feature gate that is GA since v1.17 is unconditionally enabled, and can no longer be specified via the `--feature-gates` argument. ([#96561](https://github.com/kubernetes/kubernetes/pull/96561), [@ialidzhikov](https://github.com/ialidzhikov)) [SIG Apps, Auth, Scheduling, Storage and Testing] +- The deprecated feature gates `RotateKubeletClientCertificate`, `AttachVolumeLimit`, `VolumePVCDataSource` and `EvenPodsSpread` are now unconditionally enabled and can no longer be specified in component invocations. ([#97306](https://github.com/kubernetes/kubernetes/pull/97306), [@gavinfish](https://github.com/gavinfish)) [SIG Node, Scheduling and Storage] +- `ServiceNodeExclusion`, `NodeDisruptionExclusion` and `LegacyNodeRoleBehavior`(locked to false) features have been promoted to GA. + To prevent control plane nodes being added to load balancers automatically, upgrade users need to add "node.kubernetes.io/exclude-from-external-load-balancers" label to control plane nodes. ([#97543](https://github.com/kubernetes/kubernetes/pull/97543), [@pacoxu](https://github.com/pacoxu)) [SIG API Machinery, Apps, Cloud Provider and Network] +### Uncategorized - +- Adding Brazilian Portuguese translation for kubectl ([#61595](https://github.com/kubernetes/kubernetes/pull/61595), [@cpanato](https://github.com/cpanato)) [SIG CLI] +## Dependencies -# v1.18.0-alpha.2 +### Added +_Nothing has changed._ -[Documentation](https://docs.k8s.io) +### Changed +- github.com/Azure/go-autorest/autorest: [v0.11.1 → v0.11.12](https://github.com/Azure/go-autorest/autorest/compare/v0.11.1...v0.11.12) +- github.com/coredns/corefile-migration: [v1.0.10 → v1.0.11](https://github.com/coredns/corefile-migration/compare/v1.0.10...v1.0.11) +- github.com/golang/mock: [v1.4.1 → v1.4.4](https://github.com/golang/mock/compare/v1.4.1...v1.4.4) +- github.com/google/cadvisor: [v0.38.5 → v0.38.6](https://github.com/google/cadvisor/compare/v0.38.5...v0.38.6) +- github.com/heketi/heketi: [c2e2a4a → v10.2.0+incompatible](https://github.com/heketi/heketi/compare/c2e2a4a...v10.2.0) +- github.com/miekg/dns: [v1.1.4 → v1.1.35](https://github.com/miekg/dns/compare/v1.1.4...v1.1.35) +- k8s.io/system-validators: v1.2.0 → v1.3.0 -## Downloads for v1.18.0-alpha.2 - - -filename | sha512 hash --------- | ----------- -[kubernetes.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes.tar.gz) | `7af83386b4b35353f0aa1bdaf73599eb08b1d1ca11ecc2c606854aff754db69f3cd3dc761b6d7fc86f01052f615ca53185f33dbf9e53b2f926b0f02fc103fbd3` -[kubernetes-src.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-src.tar.gz) | `a14b02a0a0bde97795a836a8f5897b0ee6b43e010e13e43dd4cca80a5b962a1ef3704eedc7916fed1c38ec663a71db48c228c91e5daacba7d9370df98c7ddfb6` - -### Client Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-client-darwin-386.tar.gz) | `427f214d47ded44519007de2ae87160c56c2920358130e474b768299751a9affcbc1b1f0f936c39c6138837bca2a97792a6700896976e98c4beee8a1944cfde1` -[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-client-darwin-amd64.tar.gz) | `861fd81ac3bd45765575bedf5e002a2294aba48ef9e15980fc7d6783985f7d7fcde990ea0aef34690977a88df758722ec0a2e170d5dcc3eb01372e64e5439192` -[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-client-linux-386.tar.gz) | `7d59b05d6247e2606a8321c72cd239713373d876dbb43b0fb7f1cb857fa6c998038b41eeed78d9eb67ce77b0b71776ceed428cce0f8d2203c5181b473e0bd86c` -[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-client-linux-amd64.tar.gz) | `7cdefb4e32bad9d2df5bb8e7e0a6f4dab2ae6b7afef5d801ac5c342d4effdeacd799081fa2dec699ecf549200786c7623c3176252010f12494a95240dd63311d` -[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-client-linux-arm.tar.gz) | `6212bbf0fa1d01ced77dcca2c4b76b73956cd3c6b70e0701c1fe0df5ff37160835f6b84fa2481e0e6979516551b14d8232d1c72764a559a3652bfe2a1e7488ff` -[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-client-linux-arm64.tar.gz) | `1f0d9990700510165ee471acb2f88222f1b80e8f6deb351ce14cf50a70a9840fb99606781e416a13231c74b2bd7576981b5348171aa33b628d2666e366cd4629` -[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-client-linux-ppc64le.tar.gz) | `77e00ba12a32db81e96f8de84609de93f32c61bb3f53875a57496d213aa6d1b92c09ad5a6de240a78e1a5bf77fac587ff92874f34a10f8909ae08ca32fda45d2` -[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-client-linux-s390x.tar.gz) | `a39ec2044bed5a4570e9c83068e0fc0ce923ccffa44380f8bbc3247426beaff79c8a84613bcb58b05f0eb3afbc34c79fe3309aa2e0b81abcfd0aa04770e62e05` -[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-client-windows-386.tar.gz) | `1a0ab88f9b7e34b60ab31d5538e97202a256ad8b7b7ed5070cae5f2f12d5d4edeae615db7a34ebbe254004b6393c6b2480100b09e30e59c9139492a3019a596a` -[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-client-windows-amd64.tar.gz) | `1966eb5dfb78c1bc33aaa6389f32512e3aa92584250a0164182f3566c81d901b59ec78ee4e25df658bc1dd221b5a9527d6ce3b6c487ca3e3c0b319a077caa735` - -### Server Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-server-linux-amd64.tar.gz) | `f814d6a3872e4572aa4da297c29def4c1fad8eba0903946780b6bf9788c72b99d71085c5aef9e12c01133b26fa4563c1766ba724ad2a8af2670a24397951a94d` -[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-server-linux-arm.tar.gz) | `56aa08225e546c92c2ff88ac57d3db7dd5e63640772ea72a429f080f7069827138cbc206f6f5fe3a0c01bfca043a9eda305ecdc1dcb864649114893e46b6dc84` -[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-server-linux-arm64.tar.gz) | `fb87128d905211ba097aa860244a376575ae2edbaca6e51402a24bc2964854b9b273e09df3d31a2bcffc91509f7eecb2118b183fb0e0eb544f33403fa235c274` -[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-server-linux-ppc64le.tar.gz) | `6d21fbf39b9d3a0df9642407d6f698fabdc809aca83af197bceb58a81b25846072f407f8fb7caae2e02dc90912e3e0f5894f062f91bcb69f8c2329625d3dfeb7` -[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-server-linux-s390x.tar.gz) | `ddcda4dc360ca97705f71bf2a18ddacd7b7ddf77535b62e699e97a1b2dd24843751313351d0112e238afe69558e8271eba4d27ab77bb67b4b9e3fbde6eec85c9` - -### Node Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-node-linux-amd64.tar.gz) | `78915a9bde35c70c67014f0cea8754849db4f6a84491a3ad9678fd3bc0203e43af5a63cfafe104ae1d56b05ce74893a87a6dcd008d7859e1af6b3bce65425b5d` -[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-node-linux-arm.tar.gz) | `3218e811abcb0cb09d80742def339be3916db5e9bbc62c0dc8e6d87085f7e3d9eeed79dea081906f1de78ddd07b7e3acdbd7765fdb838d262bb35602fd1df106` -[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-node-linux-arm64.tar.gz) | `fa22de9c4440b8fb27f4e77a5a63c5e1c8aa8aa30bb79eda843b0f40498c21b8c0ad79fff1d841bb9fef53fe20da272506de9a86f81a0b36d028dbeab2e482ce` -[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-node-linux-ppc64le.tar.gz) | `bbda9b5cc66e8f13d235703b2a85e2c4f02fa16af047be4d27a3e198e11eb11706e4a0fbb6c20978c770b069cd4cd9894b661f09937df9d507411548c36576e0` -[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-node-linux-s390x.tar.gz) | `b2ed1eda013069adce2aac00b86d75b84e006cfce9bafac0b5a2bafcb60f8f2cb346b5ea44eafa72d777871abef1ea890eb3a2a05de28968f9316fa88886a8ed` -[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.2/kubernetes-node-windows-amd64.tar.gz) | `bd8eb23dba711f31b5148257076b1bbe9629f2a75de213b2c779bd5b29279e9bf22f8bde32f4bc814f4c0cc49e19671eb8b24f4105f0fe2c1490c4b78ec3c704` - -## Changelog since v1.18.0-alpha.1 - -### Other notable changes - -* Bump golang/mock version to v1.3.1 ([#87326](https://github.com/kubernetes/kubernetes/pull/87326), [@wawa0210](https://github.com/wawa0210)) -* fix a bug that orphan revision cannot be adopted and statefulset cannot be synced ([#86801](https://github.com/kubernetes/kubernetes/pull/86801), [@likakuli](https://github.com/likakuli)) -* Azure storage clients now suppress requests on throttling ([#87306](https://github.com/kubernetes/kubernetes/pull/87306), [@feiskyer](https://github.com/feiskyer)) -* Introduce Alpha field `Immutable` in both Secret and ConfigMap objects to mark their contents as immutable. The implementation is hidden behind feature gate `ImmutableEphemeralVolumes` (currently in Alpha stage). ([#86377](https://github.com/kubernetes/kubernetes/pull/86377), [@wojtek-t](https://github.com/wojtek-t)) -* EndpointSlices will now be enabled by default. A new `EndpointSliceProxying` feature gate determines if kube-proxy will use EndpointSlices, this is disabled by default. ([#86137](https://github.com/kubernetes/kubernetes/pull/86137), [@robscott](https://github.com/robscott)) -* kubeadm upgrades always persist the etcd backup for stacked ([#86861](https://github.com/kubernetes/kubernetes/pull/86861), [@SataQiu](https://github.com/SataQiu)) -* Fix the bug PIP's DNS is deleted if no DNS label service annotation isn't set. ([#87246](https://github.com/kubernetes/kubernetes/pull/87246), [@nilo19](https://github.com/nilo19)) -* New flag `--show-hidden-metrics-for-version` in kube-controller-manager can be used to show all hidden metrics that deprecated in the previous minor release. ([#85281](https://github.com/kubernetes/kubernetes/pull/85281), [@RainbowMango](https://github.com/RainbowMango)) -* Azure network and VM clients now suppress requests on throttling ([#87122](https://github.com/kubernetes/kubernetes/pull/87122), [@feiskyer](https://github.com/feiskyer)) -* `kubectl apply -f --prune -n ` should prune all resources not defined in the file in the cli specified namespace. ([#85613](https://github.com/kubernetes/kubernetes/pull/85613), [@MartinKaburu](https://github.com/MartinKaburu)) -* Fixes service account token admission error in clusters that do not run the service account token controller ([#87029](https://github.com/kubernetes/kubernetes/pull/87029), [@liggitt](https://github.com/liggitt)) -* CustomResourceDefinition status fields are no longer required for client validation when submitting manifests. ([#87213](https://github.com/kubernetes/kubernetes/pull/87213), [@hasheddan](https://github.com/hasheddan)) -* All apiservers log request lines in a more greppable format. ([#87203](https://github.com/kubernetes/kubernetes/pull/87203), [@lavalamp](https://github.com/lavalamp)) -* provider/azure: Network security groups can now be in a separate resource group. ([#87035](https://github.com/kubernetes/kubernetes/pull/87035), [@CecileRobertMichon](https://github.com/CecileRobertMichon)) -* Cleaned up the output from `kubectl describe CSINode `. ([#85283](https://github.com/kubernetes/kubernetes/pull/85283), [@huffmanca](https://github.com/huffmanca)) -* Fixed the following ([#84265](https://github.com/kubernetes/kubernetes/pull/84265), [@bhagwat070919](https://github.com/bhagwat070919)) - * - AWS Cloud Provider attempts to delete LoadBalancer security group it didn’t provision - * - AWS Cloud Provider creates default LoadBalancer security group even if annotation [service.beta.kubernetes.io/aws-load-balancer-security-groups] is present -* kubelet: resource metrics endpoint `/metrics/resource/v1alpha1` as well as all metrics under this endpoint have been deprecated. ([#86282](https://github.com/kubernetes/kubernetes/pull/86282), [@RainbowMango](https://github.com/RainbowMango)) - * Please convert to the following metrics emitted by endpoint `/metrics/resource`: - * - scrape_error --> scrape_error - * - node_cpu_usage_seconds_total --> node_cpu_usage_seconds - * - node_memory_working_set_bytes --> node_memory_working_set_bytes - * - container_cpu_usage_seconds_total --> container_cpu_usage_seconds - * - container_memory_working_set_bytes --> container_memory_working_set_bytes - * - scrape_error --> scrape_error -* You can now pass "--node-ip ::" to kubelet to indicate that it should autodetect an IPv6 address to use as the node's primary address. ([#85850](https://github.com/kubernetes/kubernetes/pull/85850), [@danwinship](https://github.com/danwinship)) -* kubeadm: support automatic retry after failing to pull image ([#86899](https://github.com/kubernetes/kubernetes/pull/86899), [@SataQiu](https://github.com/SataQiu)) -* TODO ([#87044](https://github.com/kubernetes/kubernetes/pull/87044), [@jennybuckley](https://github.com/jennybuckley)) -* Improved yaml parsing performance ([#85458](https://github.com/kubernetes/kubernetes/pull/85458), [@cjcullen](https://github.com/cjcullen)) -* Fixed a bug which could prevent a provider ID from ever being set for node if an error occurred determining the provider ID when the node was added. ([#87043](https://github.com/kubernetes/kubernetes/pull/87043), [@zjs](https://github.com/zjs)) -* fix a regression in kubenet that prevent pods to obtain ip addresses ([#85993](https://github.com/kubernetes/kubernetes/pull/85993), [@chendotjs](https://github.com/chendotjs)) -* Bind kube-dns containers to linux nodes to avoid Windows scheduling ([#83358](https://github.com/kubernetes/kubernetes/pull/83358), [@wawa0210](https://github.com/wawa0210)) -* The following features are unconditionally enabled and the corresponding `--feature-gates` flags have been removed: `PodPriority`, `TaintNodesByCondition`, `ResourceQuotaScopeSelectors` and `ScheduleDaemonSetPods` ([#86210](https://github.com/kubernetes/kubernetes/pull/86210), [@draveness](https://github.com/draveness)) -* Bind dns-horizontal containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83364](https://github.com/kubernetes/kubernetes/pull/83364), [@wawa0210](https://github.com/wawa0210)) -* fix kubectl annotate error when local=true is set ([#86952](https://github.com/kubernetes/kubernetes/pull/86952), [@zhouya0](https://github.com/zhouya0)) -* Bug fixes: ([#84163](https://github.com/kubernetes/kubernetes/pull/84163), [@david-tigera](https://github.com/david-tigera)) - * Make sure we include latest packages node #351 ([@caseydavenport](https://github.com/caseydavenport)) -* fix kuebctl apply set-last-applied namespaces error ([#86474](https://github.com/kubernetes/kubernetes/pull/86474), [@zhouya0](https://github.com/zhouya0)) -* Add VolumeBinder method to FrameworkHandle interface, which allows user to get the volume binder when implementing scheduler framework plugins. ([#86940](https://github.com/kubernetes/kubernetes/pull/86940), [@skilxn-go](https://github.com/skilxn-go)) -* elasticsearch supports automatically setting the advertise address ([#85944](https://github.com/kubernetes/kubernetes/pull/85944), [@SataQiu](https://github.com/SataQiu)) -* If a serving certificates param specifies a name that is an IP for an SNI certificate, it will have priority for replying to server connections. ([#85308](https://github.com/kubernetes/kubernetes/pull/85308), [@deads2k](https://github.com/deads2k)) -* kube-proxy: Added dual-stack IPv4/IPv6 support to the iptables proxier. ([#82462](https://github.com/kubernetes/kubernetes/pull/82462), [@vllry](https://github.com/vllry)) -* Azure VMSS/VMSSVM clients now suppress requests on throttling ([#86740](https://github.com/kubernetes/kubernetes/pull/86740), [@feiskyer](https://github.com/feiskyer)) -* New metric kubelet_pleg_last_seen_seconds to aid diagnosis of PLEG not healthy issues. ([#86251](https://github.com/kubernetes/kubernetes/pull/86251), [@bboreham](https://github.com/bboreham)) -* For subprotocol negotiation, both client and server protocol is required now. ([#86646](https://github.com/kubernetes/kubernetes/pull/86646), [@tedyu](https://github.com/tedyu)) -* kubeadm: use bind-address option to configure the kube-controller-manager and kube-scheduler http probes ([#86493](https://github.com/kubernetes/kubernetes/pull/86493), [@aojea](https://github.com/aojea)) -* Marked scheduler's metrics scheduling_algorithm_predicate_evaluation_seconds and ([#86584](https://github.com/kubernetes/kubernetes/pull/86584), [@xiaoanyunfei](https://github.com/xiaoanyunfei)) - * scheduling_algorithm_priority_evaluation_seconds as deprecated. Those are replaced by framework_extension_point_duration_seconds[extenstion_point="Filter"] and framework_extension_point_duration_seconds[extenstion_point="Score"] respectively. -* Marked scheduler's scheduling_duration_seconds Summary metric as deprecated ([#86586](https://github.com/kubernetes/kubernetes/pull/86586), [@xiaoanyunfei](https://github.com/xiaoanyunfei)) -* Add instructions about how to bring up e2e test cluster ([#85836](https://github.com/kubernetes/kubernetes/pull/85836), [@YangLu1031](https://github.com/YangLu1031)) -* If a required flag is not provided to a command, the user will only see the required flag error message, instead of the entire usage menu. ([#86693](https://github.com/kubernetes/kubernetes/pull/86693), [@sallyom](https://github.com/sallyom)) -* kubeadm: tolerate whitespace when validating certificate authority PEM data in kubeconfig files ([#86705](https://github.com/kubernetes/kubernetes/pull/86705), [@neolit123](https://github.com/neolit123)) -* kubeadm: add support for the "ci/k8s-master" version label as a replacement for "ci-cross/*", which no longer exists. ([#86609](https://github.com/kubernetes/kubernetes/pull/86609), [@Pensu](https://github.com/Pensu)) -* Fix EndpointSlice controller race condition and ensure that it handles external changes to EndpointSlices. ([#85703](https://github.com/kubernetes/kubernetes/pull/85703), [@robscott](https://github.com/robscott)) -* Fix nil pointer dereference in azure cloud provider ([#85975](https://github.com/kubernetes/kubernetes/pull/85975), [@ldx](https://github.com/ldx)) -* fix: azure disk could not mounted on Standard_DC4s/DC2s instances ([#86612](https://github.com/kubernetes/kubernetes/pull/86612), [@andyzhangx](https://github.com/andyzhangx)) -* Fixes v1.17.0 regression in --service-cluster-ip-range handling with IPv4 ranges larger than 65536 IP addresses ([#86534](https://github.com/kubernetes/kubernetes/pull/86534), [@liggitt](https://github.com/liggitt)) -* Adds back support for AlwaysCheckAllPredicates flag. ([#86496](https://github.com/kubernetes/kubernetes/pull/86496), [@ahg-g](https://github.com/ahg-g)) -* Azure global rate limit is switched to per-client. A set of new rate limit configure options are introduced, including routeRateLimit, SubnetsRateLimit, InterfaceRateLimit, RouteTableRateLimit, LoadBalancerRateLimit, PublicIPAddressRateLimit, SecurityGroupRateLimit, VirtualMachineRateLimit, StorageAccountRateLimit, DiskRateLimit, SnapshotRateLimit, VirtualMachineScaleSetRateLimit and VirtualMachineSizeRateLimit. ([#86515](https://github.com/kubernetes/kubernetes/pull/86515), [@feiskyer](https://github.com/feiskyer)) - * The original rate limit options would be default values for those new client's rate limiter. -* Fix issue [#85805](https://github.com/kubernetes/kubernetes/pull/85805) about resource not found in azure cloud provider when lb specified in other resource group. ([#86502](https://github.com/kubernetes/kubernetes/pull/86502), [@levimm](https://github.com/levimm)) -* `AlwaysCheckAllPredicates` is deprecated in scheduler Policy API. ([#86369](https://github.com/kubernetes/kubernetes/pull/86369), [@Huang-Wei](https://github.com/Huang-Wei)) -* Kubernetes KMS provider for data encryption now supports disabling the in-memory data encryption key (DEK) cache by setting cachesize to a negative value. ([#86294](https://github.com/kubernetes/kubernetes/pull/86294), [@enj](https://github.com/enj)) -* option `preConfiguredBackendPoolLoadBalancerTypes` is added to azure cloud provider for the pre-configured load balancers, possible values: `""`, `"internal"`, "external"`, `"all"` ([#86338](https://github.com/kubernetes/kubernetes/pull/86338), [@gossion](https://github.com/gossion)) -* Promote StartupProbe to beta for 1.18 release ([#83437](https://github.com/kubernetes/kubernetes/pull/83437), [@matthyx](https://github.com/matthyx)) -* Fixes issue where AAD token obtained by kubectl is incompatible with on-behalf-of flow and oidc. ([#86412](https://github.com/kubernetes/kubernetes/pull/86412), [@weinong](https://github.com/weinong)) - * The audience claim before this fix has "spn:" prefix. After this fix, "spn:" prefix is omitted. -* change CounterVec to Counter about PLEGDiscardEvent ([#86167](https://github.com/kubernetes/kubernetes/pull/86167), [@yiyang5055](https://github.com/yiyang5055)) -* hollow-node do not use remote CRI anymore ([#86425](https://github.com/kubernetes/kubernetes/pull/86425), [@jkaniuk](https://github.com/jkaniuk)) -* hollow-node use fake CRI ([#85879](https://github.com/kubernetes/kubernetes/pull/85879), [@gongguan](https://github.com/gongguan)) - - - -# v1.18.0-alpha.1 - -[Documentation](https://docs.k8s.io) - -## Downloads for v1.18.0-alpha.1 - - -filename | sha512 hash --------- | ----------- -[kubernetes.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes.tar.gz) | `0c4904efc7f4f1436119c91dc1b6c93b3bd9c7490362a394bff10099c18e1e7600c4f6e2fcbaeb2d342a36c4b20692715cf7aa8ada6dfac369f44cc9292529d7` -[kubernetes-src.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-src.tar.gz) | `0a50fc6816c730ca5ae4c4f26d5ad7b049607d29f6a782a4e5b4b05ac50e016486e269dafcc6a163bd15e1a192780a9a987f1bb959696993641c603ed1e841c8` - -### Client Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-client-darwin-386.tar.gz) | `c6d75f7f3f20bef17fc7564a619b54e6f4a673d041b7c9ec93663763a1cc8dd16aecd7a2af70e8d54825a0eecb9762cf2edfdade840604c9a32ecd9cc2d5ac3c` -[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-client-darwin-amd64.tar.gz) | `ca1f19db289933beace6daee6fc30af19b0e260634ef6e89f773464a05e24551c791be58b67da7a7e2a863e28b7cbcc7b24b6b9bf467113c26da76ac8f54fdb6` -[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-client-linux-386.tar.gz) | `af2e673653eb39c3f24a54efc68e1055f9258bdf6cf8fea42faf42c05abefc2da853f42faac3b166c37e2a7533020b8993b98c0d6d80a5b66f39e91d8ae0a3fb` -[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-client-linux-amd64.tar.gz) | `9009032c3f94ac8a78c1322a28e16644ce3b20989eb762685a1819148aed6e883ca8e1200e5ec37ec0853f115c67e09b5d697d6cf5d4c45f653788a2d3a2f84f` -[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-client-linux-arm.tar.gz) | `afba9595b37a3f2eead6e3418573f7ce093b55467dce4da0b8de860028576b96b837a2fd942f9c276e965da694e31fbd523eeb39aefb902d7e7a2f169344d271` -[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-client-linux-arm64.tar.gz) | `04fc3b2fe3f271807f0bc6c61be52456f26a1af904964400be819b7914519edc72cbab9afab2bb2e2ba1a108963079367cedfb253c9364c0175d1fcc64d52f5c` -[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-client-linux-ppc64le.tar.gz) | `04c7edab874b33175ff7bebfff5b3a032bc6eb088fcd7387ffcd5b3fa71395ca8c5f9427b7ddb496e92087dfdb09eaf14a46e9513071d3bd73df76c182922d38` -[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-client-linux-s390x.tar.gz) | `499287dbbc33399a37b9f3b35e0124ff20b17b6619f25a207ee9c606ef261af61fa0c328dde18c7ce2d3dfb2eea2376623bc3425d16bc8515932a68b44f8bede` -[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-client-windows-386.tar.gz) | `cf84aeddf00f126fb13c0436b116dd0464a625659e44c84bf863517db0406afb4eefd86807e7543c4f96006d275772fbf66214ae7d582db5865c84ac3545b3e6` -[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-client-windows-amd64.tar.gz) | `69f20558ccd5cd6dbaccf29307210db4e687af21f6d71f68c69d3a39766862686ac1333ab8a5012010ca5c5e3c11676b45e498e3d4c38773da7d24bcefc46d95` - -### Server Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-server-linux-amd64.tar.gz) | `3f29df2ce904a0f10db4c1d7a425a36f420867b595da3fa158ae430bfead90def2f2139f51425b349faa8a9303dcf20ea01657cb6ea28eb6ad64f5bb32ce2ed1` -[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-server-linux-arm.tar.gz) | `4a21073b2273d721fbf062c254840be5c8471a010bcc0c731b101729e36e61f637cb7fcb521a22e8d24808510242f4fff8a6ca40f10e9acd849c2a47bf135f27` -[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-server-linux-arm64.tar.gz) | `7f1cb6d721bedc90e28b16f99bea7e59f5ad6267c31ef39c14d34db6ad6aad87ee51d2acdd01b6903307c1c00b58ff6b785a03d5a491cc3f8a4df9a1d76d406c` -[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-server-linux-ppc64le.tar.gz) | `8f2b552030b5274b1c2c7c166eacd5a14b0c6ca0f23042f4c52efe87e22a167ba4460dcd66615a5ecd26d9e88336be1fb555548392e70efe59070dd2c314da98` -[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-server-linux-s390x.tar.gz) | `8d9f2c96f66edafb7c8b3aa90960d29b41471743842aede6b47b3b2e61f4306fb6fc60b9ebc18820c547ee200bfedfe254c1cde962d447c791097dd30e79abdb` - -### Node Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-node-linux-amd64.tar.gz) | `84194cb081d1502f8ca68143569f9707d96f1a28fcf0c574ebd203321463a8b605f67bb2a365eaffb14fbeb8d55c8d3fa17431780b242fb9cba3a14426a0cd4a` -[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-node-linux-arm.tar.gz) | `0091e108ab94fd8683b89c597c4fdc2fbf4920b007cfcd5297072c44bc3a230dfe5ceed16473e15c3e6cf5edab866d7004b53edab95be0400cc60e009eee0d9d` -[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-node-linux-arm64.tar.gz) | `b7e85682cc2848a35d52fd6f01c247f039ee1b5dd03345713821ea10a7fa9939b944f91087baae95eaa0665d11857c1b81c454f720add077287b091f9f19e5d3` -[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-node-linux-ppc64le.tar.gz) | `cd1f0849e9c62b5d2c93ff0cebf58843e178d8a88317f45f76de0db5ae020b8027e9503a5fccc96445184e0d77ecdf6f57787176ac31dbcbd01323cd0a190cbb` -[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-node-linux-s390x.tar.gz) | `e1e697a34424c75d75415b613b81c8af5f64384226c5152d869f12fd7db1a3e25724975b73fa3d89e56e4bf78d5fd07e68a709ba8566f53691ba6a88addc79ea` -[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.18.0-alpha.1/kubernetes-node-windows-amd64.tar.gz) | `c725a19a4013c74e22383ad3fb4cb799b3e161c4318fdad066daf806730a89bc3be3ff0f75678d02b3cbe52b2ef0c411c0639968e200b9df470be40bb2c015cc` - -## Changelog since v1.17.0 - -### Action Required - -* action required ([#85363](https://github.com/kubernetes/kubernetes/pull/85363), [@immutableT](https://github.com/immutableT)) - * 1. Currently, if users were to explicitly specify CacheSize of 0 for KMS provider, they would end-up with a provider that caches up to 1000 keys. This PR changes this behavior. - * Post this PR, when users supply 0 for CacheSize this will result in a validation error. - * 2. CacheSize type was changed from int32 to *int32. This allows defaulting logic to differentiate between cases where users explicitly supplied 0 vs. not supplied any value. - * 3. KMS Provider's endpoint (path to Unix socket) is now validated when the EncryptionConfiguration files is loaded. This used to be handled by the GRPCService. - -### Other notable changes - -* fix: azure data disk should use same key as os disk by default ([#86351](https://github.com/kubernetes/kubernetes/pull/86351), [@andyzhangx](https://github.com/andyzhangx)) -* New flag `--show-hidden-metrics-for-version` in kube-proxy can be used to show all hidden metrics that deprecated in the previous minor release. ([#85279](https://github.com/kubernetes/kubernetes/pull/85279), [@RainbowMango](https://github.com/RainbowMango)) -* Remove cluster-monitoring addon ([#85512](https://github.com/kubernetes/kubernetes/pull/85512), [@serathius](https://github.com/serathius)) -* Changed core_pattern on COS nodes to be an absolute path. ([#86329](https://github.com/kubernetes/kubernetes/pull/86329), [@mml](https://github.com/mml)) -* Track mount operations as uncertain if operation fails with non-final error ([#82492](https://github.com/kubernetes/kubernetes/pull/82492), [@gnufied](https://github.com/gnufied)) -* add kube-proxy flags --ipvs-tcp-timeout, --ipvs-tcpfin-timeout, --ipvs-udp-timeout to configure IPVS connection timeouts. ([#85517](https://github.com/kubernetes/kubernetes/pull/85517), [@andrewsykim](https://github.com/andrewsykim)) -* The sample-apiserver aggregated conformance test has updated to use the Kubernetes v1.17.0 sample apiserver ([#84735](https://github.com/kubernetes/kubernetes/pull/84735), [@liggitt](https://github.com/liggitt)) -* The underlying format of the `CPUManager` state file has changed. Upgrades should be seamless, but any third-party tools that rely on reading the previous format need to be updated. ([#84462](https://github.com/kubernetes/kubernetes/pull/84462), [@klueska](https://github.com/klueska)) -* kubernetes will try to acquire the iptables lock every 100 msec during 5 seconds instead of every second. This specially useful for environments using kube-proxy in iptables mode with a high churn rate of services. ([#85771](https://github.com/kubernetes/kubernetes/pull/85771), [@aojea](https://github.com/aojea)) -* Fixed a panic in the kubelet cleaning up pod volumes ([#86277](https://github.com/kubernetes/kubernetes/pull/86277), [@tedyu](https://github.com/tedyu)) -* azure cloud provider cache TTL is configurable, list of the azure cloud provider is as following: ([#86266](https://github.com/kubernetes/kubernetes/pull/86266), [@zqingqing1](https://github.com/zqingqing1)) - * - "availabilitySetNodesCacheTTLInSeconds" - * - "vmssCacheTTLInSeconds" - * - "vmssVirtualMachinesCacheTTLInSeconds" - * - "vmCacheTTLInSeconds" - * - "loadBalancerCacheTTLInSeconds" - * - "nsgCacheTTLInSeconds" - * - "routeTableCacheTTLInSeconds" -* Fixes kube-proxy when EndpointSlice feature gate is enabled on Windows. ([#86016](https://github.com/kubernetes/kubernetes/pull/86016), [@robscott](https://github.com/robscott)) -* Fixes wrong validation result of NetworkPolicy PolicyTypes ([#85747](https://github.com/kubernetes/kubernetes/pull/85747), [@tnqn](https://github.com/tnqn)) -* Fixes an issue with kubelet-reported pod status on deleted/recreated pods. ([#86320](https://github.com/kubernetes/kubernetes/pull/86320), [@liggitt](https://github.com/liggitt)) -* kube-apiserver no longer serves the following deprecated APIs: ([#85903](https://github.com/kubernetes/kubernetes/pull/85903), [@liggitt](https://github.com/liggitt)) - * All resources under `apps/v1beta1` and `apps/v1beta2` - use `apps/v1` instead - * `daemonsets`, `deployments`, `replicasets` resources under `extensions/v1beta1` - use `apps/v1` instead - * `networkpolicies` resources under `extensions/v1beta1` - use `networking.k8s.io/v1` instead - * `podsecuritypolicies` resources under `extensions/v1beta1` - use `policy/v1beta1` instead -* kubeadm: fix potential panic when executing "kubeadm reset" with a corrupted kubelet.conf file ([#86216](https://github.com/kubernetes/kubernetes/pull/86216), [@neolit123](https://github.com/neolit123)) -* Fix a bug in port-forward: named port not working with service ([#85511](https://github.com/kubernetes/kubernetes/pull/85511), [@oke-py](https://github.com/oke-py)) -* kube-proxy no longer modifies shared EndpointSlices. ([#86092](https://github.com/kubernetes/kubernetes/pull/86092), [@robscott](https://github.com/robscott)) -* allow for configuration of CoreDNS replica count ([#85837](https://github.com/kubernetes/kubernetes/pull/85837), [@pickledrick](https://github.com/pickledrick)) -* Fixed a regression where the kubelet would fail to update the ready status of pods. ([#84951](https://github.com/kubernetes/kubernetes/pull/84951), [@tedyu](https://github.com/tedyu)) -* Resolves performance regression in client-go discovery clients constructed using `NewDiscoveryClientForConfig` or `NewDiscoveryClientForConfigOrDie`. ([#86168](https://github.com/kubernetes/kubernetes/pull/86168), [@liggitt](https://github.com/liggitt)) -* Make error message and service event message more clear ([#86078](https://github.com/kubernetes/kubernetes/pull/86078), [@feiskyer](https://github.com/feiskyer)) -* e2e-test-framework: add e2e test namespace dump if all tests succeed but the cleanup fails. ([#85542](https://github.com/kubernetes/kubernetes/pull/85542), [@schrodit](https://github.com/schrodit)) -* SafeSysctlWhitelist: add net.ipv4.ping_group_range ([#85463](https://github.com/kubernetes/kubernetes/pull/85463), [@AkihiroSuda](https://github.com/AkihiroSuda)) -* kubelet: the metric process_start_time_seconds be marked as with the ALPHA stability level. ([#85446](https://github.com/kubernetes/kubernetes/pull/85446), [@RainbowMango](https://github.com/RainbowMango)) -* API request throttling (due to a high rate of requests) is now reported in the kubelet (and other component) logs by default. The messages are of the form ([#80649](https://github.com/kubernetes/kubernetes/pull/80649), [@RobertKrawitz](https://github.com/RobertKrawitz)) - * Throttling request took 1.50705208s, request: GET: - * The presence of large numbers of these messages, particularly with long delay times, may indicate to the administrator the need to tune the cluster accordingly. -* Fix API Server potential memory leak issue in processing watch request. ([#85410](https://github.com/kubernetes/kubernetes/pull/85410), [@answer1991](https://github.com/answer1991)) -* Verify kubelet & kube-proxy can recover after being killed on Windows nodes ([#84886](https://github.com/kubernetes/kubernetes/pull/84886), [@YangLu1031](https://github.com/YangLu1031)) -* Fixed an issue that the scheduler only returns the first failure reason. ([#86022](https://github.com/kubernetes/kubernetes/pull/86022), [@Huang-Wei](https://github.com/Huang-Wei)) -* kubectl/drain: add skip-wait-for-delete-timeout option. ([#85577](https://github.com/kubernetes/kubernetes/pull/85577), [@michaelgugino](https://github.com/michaelgugino)) - * If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip. -* Following metrics have been turned off: ([#83841](https://github.com/kubernetes/kubernetes/pull/83841), [@RainbowMango](https://github.com/RainbowMango)) - * - kubelet_pod_worker_latency_microseconds - * - kubelet_pod_start_latency_microseconds - * - kubelet_cgroup_manager_latency_microseconds - * - kubelet_pod_worker_start_latency_microseconds - * - kubelet_pleg_relist_latency_microseconds - * - kubelet_pleg_relist_interval_microseconds - * - kubelet_eviction_stats_age_microseconds - * - kubelet_runtime_operations - * - kubelet_runtime_operations_latency_microseconds - * - kubelet_runtime_operations_errors - * - kubelet_device_plugin_registration_count - * - kubelet_device_plugin_alloc_latency_microseconds - * - kubelet_docker_operations - * - kubelet_docker_operations_latency_microseconds - * - kubelet_docker_operations_errors - * - kubelet_docker_operations_timeout - * - network_plugin_operations_latency_microseconds -* - Renamed Kubelet metric certificate_manager_server_expiration_seconds to certificate_manager_server_ttl_seconds and changed to report the second until expiration at read time rather than absolute time of expiry. ([#85874](https://github.com/kubernetes/kubernetes/pull/85874), [@sambdavidson](https://github.com/sambdavidson)) - * - Improved accuracy of Kubelet metric rest_client_exec_plugin_ttl_seconds. -* Bind metadata-agent containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83363](https://github.com/kubernetes/kubernetes/pull/83363), [@wawa0210](https://github.com/wawa0210)) -* Bind metrics-server containers to linux nodes to avoid Windows scheduling on kubernetes cluster includes linux nodes and windows nodes ([#83362](https://github.com/kubernetes/kubernetes/pull/83362), [@wawa0210](https://github.com/wawa0210)) -* During initialization phase (preflight), kubeadm now verifies the presence of the conntrack executable ([#85857](https://github.com/kubernetes/kubernetes/pull/85857), [@hnanni](https://github.com/hnanni)) -* VMSS cache is added so that less chances of VMSS GET throttling ([#85885](https://github.com/kubernetes/kubernetes/pull/85885), [@nilo19](https://github.com/nilo19)) -* Update go-winio module version from 0.4.11 to 0.4.14 ([#85739](https://github.com/kubernetes/kubernetes/pull/85739), [@wawa0210](https://github.com/wawa0210)) -* Fix LoadBalancer rule checking so that no unexpected LoadBalancer updates are made ([#85990](https://github.com/kubernetes/kubernetes/pull/85990), [@feiskyer](https://github.com/feiskyer)) -* kubectl drain node --dry-run will list pods that would be evicted or deleted ([#82660](https://github.com/kubernetes/kubernetes/pull/82660), [@sallyom](https://github.com/sallyom)) -* Windows nodes on GCE can use TPM-based authentication to the master. ([#85466](https://github.com/kubernetes/kubernetes/pull/85466), [@pjh](https://github.com/pjh)) -* kubectl/drain: add disable-eviction option. ([#85571](https://github.com/kubernetes/kubernetes/pull/85571), [@michaelgugino](https://github.com/michaelgugino)) - * Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, and should be used with caution. -* kubeadm now errors out whenever a not supported component config version is supplied for the kubelet and kube-proxy ([#85639](https://github.com/kubernetes/kubernetes/pull/85639), [@rosti](https://github.com/rosti)) -* Fixed issue with addon-resizer using deprecated extensions APIs ([#85793](https://github.com/kubernetes/kubernetes/pull/85793), [@bskiba](https://github.com/bskiba)) -* Includes FSType when describing CSI persistent volumes. ([#85293](https://github.com/kubernetes/kubernetes/pull/85293), [@huffmanca](https://github.com/huffmanca)) -* kubelet now exports a "server_expiration_renew_failure" and "client_expiration_renew_failure" metric counter if the certificate rotations cannot be performed. ([#84614](https://github.com/kubernetes/kubernetes/pull/84614), [@rphillips](https://github.com/rphillips)) -* kubeadm: don't write the kubelet environment file on "upgrade apply" ([#85412](https://github.com/kubernetes/kubernetes/pull/85412), [@boluisa](https://github.com/boluisa)) -* fix azure file AuthorizationFailure ([#85475](https://github.com/kubernetes/kubernetes/pull/85475), [@andyzhangx](https://github.com/andyzhangx)) -* Resolved regression in admission, authentication, and authorization webhook performance in v1.17.0-rc.1 ([#85810](https://github.com/kubernetes/kubernetes/pull/85810), [@liggitt](https://github.com/liggitt)) -* kubeadm: uses the apiserver AdvertiseAddress IP family to choose the etcd endpoint IP family for non external etcd clusters ([#85745](https://github.com/kubernetes/kubernetes/pull/85745), [@aojea](https://github.com/aojea)) -* kubeadm: Forward cluster name to the controller-manager arguments ([#85817](https://github.com/kubernetes/kubernetes/pull/85817), [@ereslibre](https://github.com/ereslibre)) -* Fixed "requested device X but found Y" attach error on AWS. ([#85675](https://github.com/kubernetes/kubernetes/pull/85675), [@jsafrane](https://github.com/jsafrane)) -* addons: elasticsearch discovery supports IPv6 ([#85543](https://github.com/kubernetes/kubernetes/pull/85543), [@SataQiu](https://github.com/SataQiu)) -* kubeadm: retry `kubeadm-config` ConfigMap creation or mutation if the apiserver is not responding. This will improve resiliency when joining new control plane nodes. ([#85763](https://github.com/kubernetes/kubernetes/pull/85763), [@ereslibre](https://github.com/ereslibre)) -* Update Cluster Autoscaler to 1.17.0; changelog: https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.17.0 ([#85610](https://github.com/kubernetes/kubernetes/pull/85610), [@losipiuk](https://github.com/losipiuk)) -* Filter published OpenAPI schema by making nullable, required fields non-required in order to avoid kubectl to wrongly reject null values. ([#85722](https://github.com/kubernetes/kubernetes/pull/85722), [@sttts](https://github.com/sttts)) -* kubectl set resources will no longer return an error if passed an empty change for a resource. ([#85490](https://github.com/kubernetes/kubernetes/pull/85490), [@sallyom](https://github.com/sallyom)) - * kubectl set subject will no longer return an error if passed an empty change for a resource. -* kube-apiserver: fixed a conflict error encountered attempting to delete a pod with gracePeriodSeconds=0 and a resourceVersion precondition ([#85516](https://github.com/kubernetes/kubernetes/pull/85516), [@michaelgugino](https://github.com/michaelgugino)) -* kubeadm: add a upgrade health check that deploys a Job ([#81319](https://github.com/kubernetes/kubernetes/pull/81319), [@neolit123](https://github.com/neolit123)) -* kubeadm: make sure images are pre-pulled even if a tag did not change but their contents changed ([#85603](https://github.com/kubernetes/kubernetes/pull/85603), [@bart0sh](https://github.com/bart0sh)) -* kube-apiserver: Fixes a bug that hidden metrics can not be enabled by the command-line option `--show-hidden-metrics-for-version`. ([#85444](https://github.com/kubernetes/kubernetes/pull/85444), [@RainbowMango](https://github.com/RainbowMango)) -* kubeadm now supports automatic calculations of dual-stack node cidr masks to kube-controller-manager. ([#85609](https://github.com/kubernetes/kubernetes/pull/85609), [@Arvinderpal](https://github.com/Arvinderpal)) -* Fix bug where EndpointSlice controller would attempt to modify shared objects. ([#85368](https://github.com/kubernetes/kubernetes/pull/85368), [@robscott](https://github.com/robscott)) -* Use context to check client closed instead of http.CloseNotifier in processing watch request which will reduce 1 goroutine for each request if proto is HTTP/2.x . ([#85408](https://github.com/kubernetes/kubernetes/pull/85408), [@answer1991](https://github.com/answer1991)) -* kubeadm: reset raises warnings if it cannot delete folders ([#85265](https://github.com/kubernetes/kubernetes/pull/85265), [@SataQiu](https://github.com/SataQiu)) -* Wait for kubelet & kube-proxy to be ready on Windows node within 10s ([#85228](https://github.com/kubernetes/kubernetes/pull/85228), [@YangLu1031](https://github.com/YangLu1031)) +### Removed +- rsc.io/quote/v3: v3.1.0 +- rsc.io/sampler: v1.3.0 From c860159ea83e814d906f4f600bec40aaa94a353d Mon Sep 17 00:00:00 2001 From: toshokan Date: Mon, 10 May 2021 22:07:40 -0400 Subject: [PATCH 002/175] [en] Fix secret -> secrets in titles --- content/en/docs/concepts/configuration/secret.md | 12 ++++++------ .../managing-secret-using-config-file.md | 6 +++--- .../managing-secret-using-kustomize.md | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/content/en/docs/concepts/configuration/secret.md b/content/en/docs/concepts/configuration/secret.md index 111a405a7a..73c9487eeb 100644 --- a/content/en/docs/concepts/configuration/secret.md +++ b/content/en/docs/concepts/configuration/secret.md @@ -407,9 +407,9 @@ stringData: There are several options to create a Secret: -- [create Secret using `kubectl` command](/docs/tasks/configmap-secret/managing-secret-using-kubectl/) -- [create Secret from config file](/docs/tasks/configmap-secret/managing-secret-using-config-file/) -- [create Secret using kustomize](/docs/tasks/configmap-secret/managing-secret-using-kustomize/) +- [create Secrets using `kubectl` command](/docs/tasks/configmap-secret/managing-secret-using-kubectl/) +- [create Secrets from config file](/docs/tasks/configmap-secret/managing-secret-using-config-file/) +- [create Secrets using kustomize](/docs/tasks/configmap-secret/managing-secret-using-kustomize/) ## Editing a Secret @@ -1243,7 +1243,7 @@ for secret data, so that the secrets are not stored in the clear into {{< glossa ## {{% heading "whatsnext" %}} -- Learn how to [manage Secret using `kubectl`](/docs/tasks/configmap-secret/managing-secret-using-kubectl/) -- Learn how to [manage Secret using config file](/docs/tasks/configmap-secret/managing-secret-using-config-file/) -- Learn how to [manage Secret using kustomize](/docs/tasks/configmap-secret/managing-secret-using-kustomize/) +- Learn how to [manage Secrets using `kubectl`](/docs/tasks/configmap-secret/managing-secret-using-kubectl/) +- Learn how to [manage Secrets using config file](/docs/tasks/configmap-secret/managing-secret-using-config-file/) +- Learn how to [manage Secrets using kustomize](/docs/tasks/configmap-secret/managing-secret-using-kustomize/) diff --git a/content/en/docs/tasks/configmap-secret/managing-secret-using-config-file.md b/content/en/docs/tasks/configmap-secret/managing-secret-using-config-file.md index b405d57baf..e72593af92 100644 --- a/content/en/docs/tasks/configmap-secret/managing-secret-using-config-file.md +++ b/content/en/docs/tasks/configmap-secret/managing-secret-using-config-file.md @@ -1,5 +1,5 @@ --- -title: Managing Secret using Configuration File +title: Managing Secrets using Configuration File content_type: task weight: 20 description: Creating Secret objects using resource configuration file. @@ -193,6 +193,6 @@ kubectl delete secret mysecret ## {{% heading "whatsnext" %}} - Read more about the [Secret concept](/docs/concepts/configuration/secret/) -- Learn how to [manage Secret with the `kubectl` command](/docs/tasks/configmap-secret/managing-secret-using-kubectl/) -- Learn how to [manage Secret using kustomize](/docs/tasks/configmap-secret/managing-secret-using-kustomize/) +- Learn how to [manage Secrets with the `kubectl` command](/docs/tasks/configmap-secret/managing-secret-using-kubectl/) +- Learn how to [manage Secrets using kustomize](/docs/tasks/configmap-secret/managing-secret-using-kustomize/) diff --git a/content/en/docs/tasks/configmap-secret/managing-secret-using-kustomize.md b/content/en/docs/tasks/configmap-secret/managing-secret-using-kustomize.md index fb257a6026..51ae9adac0 100644 --- a/content/en/docs/tasks/configmap-secret/managing-secret-using-kustomize.md +++ b/content/en/docs/tasks/configmap-secret/managing-secret-using-kustomize.md @@ -1,5 +1,5 @@ --- -title: Managing Secret using Kustomize +title: Managing Secrets using Kustomize content_type: task weight: 30 description: Creating Secret objects using kustomization.yaml file. @@ -123,6 +123,6 @@ kubectl delete secret db-user-pass-96mffmfh4k ## {{% heading "whatsnext" %}} - Read more about the [Secret concept](/docs/concepts/configuration/secret/) -- Learn how to [manage Secret with the `kubectl` command](/docs/tasks/configmap-secret/managing-secret-using-kubectl/) -- Learn how to [manage Secret using config file](/docs/tasks/configmap-secret/managing-secret-using-config-file/) +- Learn how to [manage Secrets with the `kubectl` command](/docs/tasks/configmap-secret/managing-secret-using-kubectl/) +- Learn how to [manage Secrets using config file](/docs/tasks/configmap-secret/managing-secret-using-config-file/) From bd7a705b3bda83e695cff29260bf953865109bc5 Mon Sep 17 00:00:00 2001 From: Qiming Teng Date: Fri, 4 Jun 2021 16:28:12 +0800 Subject: [PATCH 003/175] [zh] Fix and resync RBAC --- content/zh/docs/reference/access-authn-authz/rbac.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/content/zh/docs/reference/access-authn-authz/rbac.md b/content/zh/docs/reference/access-authn-authz/rbac.md index de429ce909..ee0a4f1007 100644 --- a/content/zh/docs/reference/access-authn-authz/rbac.md +++ b/content/zh/docs/reference/access-authn-authz/rbac.md @@ -273,13 +273,13 @@ RoleBinding 所在名字空间的资源。这种引用使得你可以跨整个 ```yaml apiVersion: rbac.authorization.k8s.io/v1 -# 此角色绑定使得用户 "dave" 能够读取 "default" 名字空间中的 Secrets +# 此角色绑定使得用户 "dave" 能够读取 "development" 名字空间中的 Secrets # 你需要一个名为 "secret-reader" 的 ClusterRole kind: RoleBinding metadata: name: read-secrets # RoleBinding 的名字空间决定了访问权限的授予范围。 - # 这里仅授权在 "development" 名字空间内的访问权限。 + # 这里隐含授权仅在 "development" 名字空间内的访问权限。 namespace: development subjects: - kind: User @@ -338,15 +338,15 @@ There are two reasons for this restriction: 1. A binding to a different role is a fundamentally different binding. Requiring a binding to be deleted/recreated in order to change the `roleRef` ensures the full list of subjects in the binding is intended to be granted -the new role (as opposed to enabling accidentally modifying just the roleRef +the new role (as opposed to enabling or accidentally modifying only the roleRef without verifying all of the existing subjects should be given the new role's permissions). 2. Making `roleRef` immutable allows giving `update` permission on an existing binding object to a user, which lets them manage the list of subjects, without being able to change the role that is granted to those subjects. --> 1. 针对不同角色的绑定是完全不一样的绑定。要求通过删除/重建绑定来更改 `roleRef`, - 这样可以确保要赋予绑定的所有主体会被授予新的角色（而不是在允许修改 - `roleRef` 的情况下导致所有现有主体未经验证即被授予新角色对应的权限）。 + 这样可以确保要赋予绑定的所有主体会被授予新的角色（而不是在允许或者不小心修改 + 了 `roleRef` 的情况下导致所有现有主体未经验证即被授予新角色对应的权限）。 1. 将 `roleRef` 设置为不可以改变，这使得可以为用户授予对现有绑定对象的 `update` 权限， 这样可以让他们管理主体列表，同时不能更改被授予这些主体的角色。 @@ -503,7 +503,7 @@ as a cluster administrator, include rules for custom resources, such as those se or aggregated API servers, to extend the default roles. For example: the following ClusterRoles let the "admin" and "edit" default roles manage the custom resource -named CronTab, whereas the "view" role can perform just read actions on CronTab resources. +named CronTab, whereas the "view" role can perform only read actions on CronTab resources. You can assume that CronTab objects are named `"crontabs"` in URLs as seen by the API server. --> 默认的[面向用户的角色](#default-roles-and-role-bindings) 使用 ClusterRole 聚合。 From 2193b58d8670e854767c2c4a21a3ef9fb01fde3f Mon Sep 17 00:00:00 2001 From: James Strong Date: Sun, 30 May 2021 15:32:26 -0400 Subject: [PATCH 004/175] create blog post for ingress-nginx stabilization discussion --- .../2021-05-30-update-with-ingress-nginx.md | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md diff --git a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md new file mode 100644 index 0000000000..c579e53c66 --- /dev/null +++ b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md @@ -0,0 +1,59 @@ +--- +layout: blog +title: 'Updates to Ingress Nginx with network v1' +date: 2021-05-30 +slug: update-with-ingress-nginx +--- + +**Authors:** James Strong, Richardo Katz + +With all Kubernetes APIs, there is a process to creating, maintaining, and +ultimately deprecating them once they become GA. The network API is no +different. With the release of Kubernetes 1.22, the network API goes from +v1beta1 to v1, with v1beta is deprecated. This move has been in discussion +for quite some time now in 2017 in +[issue #43214](https://github.com/kubernetes/kubernetes/issues/43214), +[2019 with 1.16 api deprecation](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) +, and more recently in +[KEP-1453:Graduate Ingress API to GA.](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122) + +During our last community meeting the group has decided to continue to +support Kubernetes versions older than 1.22 with ingress version 0.46.0 +for six months after the release of Kubernetes 1.22 with only CVE and +crashes only fixes, on a rolling basis. There will be a separate branch and +release of Ingress-NGINX to support this very similar to how Kubernetes +project proper does. As we advance the Ingress-NGINX project, the mainline +will support Kubernetes 1.22 and above, similar to the maintainers of the +Kubernetes project as a whole. + +{{< table caption="Ingress NGINX supported version with Kubernetes Versions" >}} +Kubernetes Version | Ingress-NGINX version | Support Notes +:-------------------|:----------------------|:------------ +`1.22` | TBD | On Going +`1.21` | `v0.46.0` | On Going but only CVE and crashes +`1.20` | `v0.46.0` | On Going but only CVE and crashes +`1.19` | `v0.46.0` | Support will drop 6 months after 1.22 release +{{< /table >}} + +Because of the networking v1 update going GA in Kubernetes 1.22, `v0.46.0` will +not work with Kubernetes 1.22. + +# What do I need to do + +The team is currently in the process of upgrading ingress-nginx to support +the v1 migration, you can track the progress +[here](https://github.com/kubernetes/ingress-nginx/pull/7156). This also +means the feature requests are being frozen until the v1 update is completed. + +In the meantime to ensure no compatibility issues: + +* Update to the latest version of Ingress-NGINX, currently [v0.46.0](https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v0.46.0) +* After Kubernetes 1.22 is released, ensure you are using the latest version of + Ingress-NGINX that supports v1 networking api. + +The community’s feedback and support in this effort is welcome. The +Ingress-Nginx Sub-project regularly hold community meetings where we discuss +this and other issues facing the project, meeting notes and details are +[here](https://github.com/kubernetes/community/tree/master/sig-network) + + From c787c02b81d9b405498829e3eb41eca37605afd3 Mon Sep 17 00:00:00 2001 From: timyinshi Date: Mon, 7 Jun 2021 22:31:24 +0800 Subject: [PATCH 005/175] delete extra chinese words Signed-off-by: timyinshi --- content/zh/docs/reference/access-authn-authz/rbac.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/zh/docs/reference/access-authn-authz/rbac.md b/content/zh/docs/reference/access-authn-authz/rbac.md index de429ce909..1ad4e3fe49 100644 --- a/content/zh/docs/reference/access-authn-authz/rbac.md +++ b/content/zh/docs/reference/access-authn-authz/rbac.md @@ -870,7 +870,7 @@ Auto-reconciliation is enabled by default if the RBAC authorizer is active. ### 自动协商 {#auto-reconciliation} 在每次启动时，API 服务器都会更新默认 ClusterRole 以添加缺失的各种权限，并更新 -默认的 ClusterRoleBinding 以增加缺失的的各类主体。 +默认的 ClusterRoleBinding 以增加缺失的各类主体。 这种自动协商机制允许集群去修复一些不小心发生的修改，并且有助于保证角色和角色绑定 在新的发行版本中有权限或主体变更时仍然保持最新。 From fa66b1c72474faf72974650ffd90aa745a17f22f Mon Sep 17 00:00:00 2001 From: James Strong Date: Tue, 8 Jun 2021 15:41:22 -0400 Subject: [PATCH 006/175] Update content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md Co-authored-by: Taylor Dolezal --- .../en/blog/_posts/2021-05-30-update-with-ingress-nginx.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md index c579e53c66..1b571182ed 100644 --- a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md @@ -11,8 +11,7 @@ With all Kubernetes APIs, there is a process to creating, maintaining, and ultimately deprecating them once they become GA. The network API is no different. With the release of Kubernetes 1.22, the network API goes from v1beta1 to v1, with v1beta is deprecated. This move has been in discussion -for quite some time now in 2017 in -[issue #43214](https://github.com/kubernetes/kubernetes/issues/43214), +since [2017](https://github.com/kubernetes/kubernetes/issues/43214), [2019 with 1.16 api deprecation](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) , and more recently in [KEP-1453:Graduate Ingress API to GA.](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122) @@ -56,4 +55,3 @@ Ingress-Nginx Sub-project regularly hold community meetings where we discuss this and other issues facing the project, meeting notes and details are [here](https://github.com/kubernetes/community/tree/master/sig-network) - From 6efa69b3fd4a52c63a4db6422732eb0e9b258242 Mon Sep 17 00:00:00 2001 From: James Strong Date: Tue, 8 Jun 2021 15:41:49 -0400 Subject: [PATCH 007/175] Update content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md Co-authored-by: Taylor Dolezal --- content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md index 1b571182ed..87477c4080 100644 --- a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md @@ -12,7 +12,7 @@ ultimately deprecating them once they become GA. The network API is no different. With the release of Kubernetes 1.22, the network API goes from v1beta1 to v1, with v1beta is deprecated. This move has been in discussion since [2017](https://github.com/kubernetes/kubernetes/issues/43214), -[2019 with 1.16 api deprecation](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) +[2019](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) with 1.16 Kubernetes API deprecations, , and more recently in [KEP-1453:Graduate Ingress API to GA.](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122) @@ -54,4 +54,3 @@ The community’s feedback and support in this effort is welcome. The Ingress-Nginx Sub-project regularly hold community meetings where we discuss this and other issues facing the project, meeting notes and details are [here](https://github.com/kubernetes/community/tree/master/sig-network) - From 4f62d704055f94a9aee2757e33c5c8a835b6221f Mon Sep 17 00:00:00 2001 From: James Strong Date: Tue, 8 Jun 2021 15:42:13 -0400 Subject: [PATCH 008/175] Update content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md Co-authored-by: Taylor Dolezal --- content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md index 87477c4080..cbd2301e02 100644 --- a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md @@ -13,8 +13,7 @@ different. With the release of Kubernetes 1.22, the network API goes from v1beta1 to v1, with v1beta is deprecated. This move has been in discussion since [2017](https://github.com/kubernetes/kubernetes/issues/43214), [2019](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) with 1.16 Kubernetes API deprecations, -, and more recently in -[KEP-1453:Graduate Ingress API to GA.](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122) +and most recently in KEP-1453: [Graduate Ingress API to GA](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122). During our last community meeting the group has decided to continue to support Kubernetes versions older than 1.22 with ingress version 0.46.0 From df7088934eb5fc0b0a460a54547119487f4f67dc Mon Sep 17 00:00:00 2001 From: James Strong Date: Tue, 8 Jun 2021 15:42:31 -0400 Subject: [PATCH 009/175] Update content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md Co-authored-by: Taylor Dolezal --- .../_posts/2021-05-30-update-with-ingress-nginx.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md index cbd2301e02..8c8b05eb71 100644 --- a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md @@ -15,14 +15,9 @@ since [2017](https://github.com/kubernetes/kubernetes/issues/43214), [2019](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) with 1.16 Kubernetes API deprecations, and most recently in KEP-1453: [Graduate Ingress API to GA](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122). -During our last community meeting the group has decided to continue to -support Kubernetes versions older than 1.22 with ingress version 0.46.0 -for six months after the release of Kubernetes 1.22 with only CVE and -crashes only fixes, on a rolling basis. There will be a separate branch and -release of Ingress-NGINX to support this very similar to how Kubernetes -project proper does. As we advance the Ingress-NGINX project, the mainline -will support Kubernetes 1.22 and above, similar to the maintainers of the -Kubernetes project as a whole. +During our last community meeting, the group has decided to continue supporting Kubernetes versions older than 1.22 with Ingress version 0.46.0. Support for Ingress-NGINX will continue for six months after Kubernetes 1.22 is released. Any additional bug fixes and CVEs for Ingress-NGINX will be addressed on a need-by-need basis. + +Ingress-NGINX will have separate branches and releases of Ingress-NGINX to support this model, mirroring the Kubernetes project process. Future releases of the Ingress-NGINX project will track and support the latest versions of Kubernetes. {{< table caption="Ingress NGINX supported version with Kubernetes Versions" >}} Kubernetes Version | Ingress-NGINX version | Support Notes From af063002a9352f13ff1c180375c50f4bac6e4fac Mon Sep 17 00:00:00 2001 From: James Strong Date: Tue, 8 Jun 2021 15:43:30 -0400 Subject: [PATCH 010/175] Update content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md Co-authored-by: Taylor Dolezal --- content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md index 8c8b05eb71..9c9cc17d42 100644 --- a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md @@ -35,8 +35,7 @@ not work with Kubernetes 1.22. The team is currently in the process of upgrading ingress-nginx to support the v1 migration, you can track the progress -[here](https://github.com/kubernetes/ingress-nginx/pull/7156). This also -means the feature requests are being frozen until the v1 update is completed. +[here](https://github.com/kubernetes/ingress-nginx/pull/7156). Feature requests are being frozen until the v1 update is completed. In the meantime to ensure no compatibility issues: From 6731e5cb152b40346eb5fa9047ddcaf387839382 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:06:47 -0400 Subject: [PATCH 011/175] updates post and date --- .../2021-05-30-update-with-ingress-nginx.md | 49 --------------- .../2021-06-21-update-with-ingress-nginx.md | 63 +++++++++++++++++++ 2 files changed, 63 insertions(+), 49 deletions(-) delete mode 100644 content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md create mode 100644 content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md diff --git a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md deleted file mode 100644 index 9c9cc17d42..0000000000 --- a/content/en/blog/_posts/2021-05-30-update-with-ingress-nginx.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -layout: blog -title: 'Updates to Ingress Nginx with network v1' -date: 2021-05-30 -slug: update-with-ingress-nginx ---- - -**Authors:** James Strong, Richardo Katz - -With all Kubernetes APIs, there is a process to creating, maintaining, and -ultimately deprecating them once they become GA. The network API is no -different. With the release of Kubernetes 1.22, the network API goes from -v1beta1 to v1, with v1beta is deprecated. This move has been in discussion -since [2017](https://github.com/kubernetes/kubernetes/issues/43214), -[2019](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) with 1.16 Kubernetes API deprecations, -and most recently in KEP-1453: [Graduate Ingress API to GA](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122). - -During our last community meeting, the group has decided to continue supporting Kubernetes versions older than 1.22 with Ingress version 0.46.0. Support for Ingress-NGINX will continue for six months after Kubernetes 1.22 is released. Any additional bug fixes and CVEs for Ingress-NGINX will be addressed on a need-by-need basis. - -Ingress-NGINX will have separate branches and releases of Ingress-NGINX to support this model, mirroring the Kubernetes project process. Future releases of the Ingress-NGINX project will track and support the latest versions of Kubernetes. - -{{< table caption="Ingress NGINX supported version with Kubernetes Versions" >}} -Kubernetes Version | Ingress-NGINX version | Support Notes -:-------------------|:----------------------|:------------ -`1.22` | TBD | On Going -`1.21` | `v0.46.0` | On Going but only CVE and crashes -`1.20` | `v0.46.0` | On Going but only CVE and crashes -`1.19` | `v0.46.0` | Support will drop 6 months after 1.22 release -{{< /table >}} - -Because of the networking v1 update going GA in Kubernetes 1.22, `v0.46.0` will -not work with Kubernetes 1.22. - -# What do I need to do - -The team is currently in the process of upgrading ingress-nginx to support -the v1 migration, you can track the progress -[here](https://github.com/kubernetes/ingress-nginx/pull/7156). Feature requests are being frozen until the v1 update is completed. - -In the meantime to ensure no compatibility issues: - -* Update to the latest version of Ingress-NGINX, currently [v0.46.0](https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v0.46.0) -* After Kubernetes 1.22 is released, ensure you are using the latest version of - Ingress-NGINX that supports v1 networking api. - -The community’s feedback and support in this effort is welcome. The -Ingress-Nginx Sub-project regularly hold community meetings where we discuss -this and other issues facing the project, meeting notes and details are -[here](https://github.com/kubernetes/community/tree/master/sig-network) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md new file mode 100644 index 0000000000..28bf1ccb1c --- /dev/null +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -0,0 +1,63 @@ +--- +layout: blog +title: 'Updates to Ingress Nginx with network v1' +date: 2021-06-21 +slug: update-with-ingress-nginx +--- + +**Authors:** James Strong, Richardo Katz + +With all Kubernetes APIs, there is a process to creating, maintaining, and +ultimately deprecating them once they become GA. The networking.k8s.io API is no +different. With the release of Kubernetes 1.22, The networking.k8s. +io/v1beta1 API version of IngressClass will no longer be served in +Kubernetes v1.22. The extensions/v1beta1 and networking.k8s.io/v1beta1 API +versions of Ingress also will no longer be served in v1.22. This move has been +in discussion since +[2017](https://github.com/kubernetes/kubernetes/issues/43214), +[2019](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) with +1.16 Kubernetes API deprecations,and most recently in +KEP-1453: +[Graduate Ingress API to GA](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122). + +During our community meetings, the group has decided to continue +supporting Kubernetes versions older than 1.22 with Ingress version 0.47.0. +Support for Ingress-NGINX will continue for six months after Kubernetes 1.22 +is released. Any additional bug fixes and CVEs for Ingress-NGINX will be +addressed on a need-by-need basis. + +Ingress-NGINX will have separate branches and releases of Ingress-NGINX to +support this model, mirroring the Kubernetes project process. Future +releases of the Ingress-NGINX project will track and support the latest +versions of Kubernetes. + +{{< table caption="Ingress NGINX supported version with Kubernetes Versions" >}} +Kubernetes Version | Ingress-NGINX version | Support Notes +:-------------------|:----------------------|:------------ +`1.22` | TBD | On Going +`1.21` | `v0.47.0` | On Going but only CVE and crashes +`1.20` | `v0.47.0` | On Going but only CVE and crashes +`1.19` | `v0.47.0` | Support will drop 6 months after 1.22 release +{{< /table >}} + +Because of the updates in Kubernetes 1.22, **v0.47.0** will not work with +Kubernetes 1.22. + +# What do I need to do + +The team is currently in the process of upgrading ingress-nginx to support +the v1 migration, you can track the progress +[here](https://github.com/kubernetes/ingress-nginx/pull/7156). +Feature requests are being frozen until the v1 update is completed. + +In the meantime to ensure no compatibility issues: + +* Update to the latest version of Ingress-NGINX, currently + [v0.47.0](https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v0.47.0) +* After Kubernetes 1.22 is released, ensure you are using the latest version of + Ingress-NGINX that supports v1 networking api. + +The community’s feedback and support in this effort is welcome. The +Ingress-Nginx Sub-project regularly hold community meetings where we discuss +this and other issues facing the project, meeting notes and details are +[here](https://github.com/kubernetes/community/tree/master/sig-network) From 2d46bfb913f80bae9e736bdcef502d885eb2cc59 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:19:36 -0400 Subject: [PATCH 012/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index 28bf1ccb1c..a358b6d03a 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -5,7 +5,7 @@ date: 2021-06-21 slug: update-with-ingress-nginx --- -**Authors:** James Strong, Richardo Katz +**Authors:** James Strong, Ricardo Katz With all Kubernetes APIs, there is a process to creating, maintaining, and ultimately deprecating them once they become GA. The networking.k8s.io API is no From ad177b2e71cf36361a1d0280310257631699b68d Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:48:26 -0400 Subject: [PATCH 013/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index a358b6d03a..b380b7a46b 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -8,7 +8,7 @@ slug: update-with-ingress-nginx **Authors:** James Strong, Ricardo Katz With all Kubernetes APIs, there is a process to creating, maintaining, and -ultimately deprecating them once they become GA. The networking.k8s.io API is no +ultimately deprecating them once they become GA. The networking.k8s.io API group is no different. With the release of Kubernetes 1.22, The networking.k8s. io/v1beta1 API version of IngressClass will no longer be served in Kubernetes v1.22. The extensions/v1beta1 and networking.k8s.io/v1beta1 API From 5aab352c816c9665100f9790c4b057218b86fe12 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:48:34 -0400 Subject: [PATCH 014/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index b380b7a46b..3dc840f4e6 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -20,7 +20,7 @@ in discussion since KEP-1453: [Graduate Ingress API to GA](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122). -During our community meetings, the group has decided to continue +During community meetings, the networking Special Interest Group has decided to continue supporting Kubernetes versions older than 1.22 with Ingress version 0.47.0. Support for Ingress-NGINX will continue for six months after Kubernetes 1.22 is released. Any additional bug fixes and CVEs for Ingress-NGINX will be From e60c68f756c87aca44561ac38014e096dec57f20 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:48:43 -0400 Subject: [PATCH 015/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index 3dc840f4e6..9e78868d41 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -52,7 +52,7 @@ Feature requests are being frozen until the v1 update is completed. In the meantime to ensure no compatibility issues: -* Update to the latest version of Ingress-NGINX, currently +* Update to the latest version of Ingress-NGINX; currently [v0.47.0](https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v0.47.0) * After Kubernetes 1.22 is released, ensure you are using the latest version of Ingress-NGINX that supports v1 networking api. From 3a74cb72752623521e24bcbc0a6a3c97f3891246 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:48:51 -0400 Subject: [PATCH 016/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index 9e78868d41..cff7b73dcd 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -58,6 +58,6 @@ In the meantime to ensure no compatibility issues: Ingress-NGINX that supports v1 networking api. The community’s feedback and support in this effort is welcome. The -Ingress-Nginx Sub-project regularly hold community meetings where we discuss +Ingress-NGINX Sub-project regularly holds community meetings where we discuss this and other issues facing the project, meeting notes and details are [here](https://github.com/kubernetes/community/tree/master/sig-network) From 9a611b2e2aec3f332a98d4573eb3248d2a66392e Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:49:05 -0400 Subject: [PATCH 017/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- .../en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index cff7b73dcd..5199b92c97 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -59,5 +59,5 @@ In the meantime to ensure no compatibility issues: The community’s feedback and support in this effort is welcome. The Ingress-NGINX Sub-project regularly holds community meetings where we discuss -this and other issues facing the project, meeting notes and details are -[here](https://github.com/kubernetes/community/tree/master/sig-network) +this and other issues facing the project. For more information on the sub-project, +please see [SIG Network](https://github.com/kubernetes/community/tree/master/sig-network). From ef178c5a3ee229176625bf27d3ea097ad9ed5b03 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:49:36 -0400 Subject: [PATCH 018/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index 5199b92c97..ae14e604b4 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -55,7 +55,7 @@ In the meantime to ensure no compatibility issues: * Update to the latest version of Ingress-NGINX; currently [v0.47.0](https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v0.47.0) * After Kubernetes 1.22 is released, ensure you are using the latest version of - Ingress-NGINX that supports v1 networking api. + Ingress-NGINX that supports the stable APIs for Ingress and IngressClass. The community’s feedback and support in this effort is welcome. The Ingress-NGINX Sub-project regularly holds community meetings where we discuss From ae18bd4806b7688d844554ddb44929a3f301d8bb Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:49:52 -0400 Subject: [PATCH 019/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index ae14e604b4..8278a009db 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -48,7 +48,8 @@ Kubernetes 1.22. The team is currently in the process of upgrading ingress-nginx to support the v1 migration, you can track the progress [here](https://github.com/kubernetes/ingress-nginx/pull/7156). -Feature requests are being frozen until the v1 update is completed. +We're not making feature improvements to `ingress-nginx` until after the support for +Ingress v1 is complete. In the meantime to ensure no compatibility issues: From cad573d5fa19a100847065eee30925051aecdd06 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:50:01 -0400 Subject: [PATCH 020/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index 8278a009db..140c4cf451 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -43,7 +43,7 @@ Kubernetes Version | Ingress-NGINX version | Support Notes Because of the updates in Kubernetes 1.22, **v0.47.0** will not work with Kubernetes 1.22. -# What do I need to do +# What you need to do The team is currently in the process of upgrading ingress-nginx to support the v1 migration, you can track the progress From 939f7d0c8099d1b710e0a018885be3a21a813ff5 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:51:11 -0400 Subject: [PATCH 021/175] clarifying ingress --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index 140c4cf451..0e78b1fcdf 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -21,7 +21,7 @@ KEP-1453: [Graduate Ingress API to GA](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122). During community meetings, the networking Special Interest Group has decided to continue -supporting Kubernetes versions older than 1.22 with Ingress version 0.47.0. +supporting Kubernetes versions older than 1.22 with Ingress-NGINX version 0.47.0. Support for Ingress-NGINX will continue for six months after Kubernetes 1.22 is released. Any additional bug fixes and CVEs for Ingress-NGINX will be addressed on a need-by-need basis. From efafee8ee0b9ddc76ca8815b731c8affdc7ee084 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 11:52:50 -0400 Subject: [PATCH 022/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- .../_posts/2021-06-21-update-with-ingress-nginx.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index 0e78b1fcdf..228dccda4d 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -9,10 +9,15 @@ slug: update-with-ingress-nginx With all Kubernetes APIs, there is a process to creating, maintaining, and ultimately deprecating them once they become GA. The networking.k8s.io API group is no -different. With the release of Kubernetes 1.22, The networking.k8s. -io/v1beta1 API version of IngressClass will no longer be served in -Kubernetes v1.22. The extensions/v1beta1 and networking.k8s.io/v1beta1 API -versions of Ingress also will no longer be served in v1.22. This move has been +different. The upcoming Kubernetes 1.22 release will remove several deprecated APIs +that are relevant to networking: + +- the `networking.k8s.io/v1beta1` API version of [IngressClass](/docs/concepts/services-networking/ingress/#ingress-class) +- all beta versions of [Ingress](/docs/concepts/services-networking/ingress/): `extensions/v1beta1` and `networking.k8s.io/v1beta1` + +On a v1.22 Kubernetes cluster, you'll be able to access Ingress and IngressClass +objects through the stable (v1) APIs, but access via their beta APIs won't be possible. +This change has been in in discussion since [2017](https://github.com/kubernetes/kubernetes/issues/43214), [2019](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) with From 1fabeed318c9816a93bd19d30d9226b802cf82a0 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 13:11:44 -0400 Subject: [PATCH 023/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index 228dccda4d..01528223d5 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -1,6 +1,6 @@ --- layout: blog -title: 'Updates to Ingress Nginx with network v1' +title: 'Updating NGINX-Ingress to use the stable Ingress API' date: 2021-06-21 slug: update-with-ingress-nginx --- From e31b30576b9f21cb3bed77cfcea2b75a39eba431 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 13:12:19 -0400 Subject: [PATCH 024/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index 01528223d5..e9132c74a3 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -37,7 +37,7 @@ releases of the Ingress-NGINX project will track and support the latest versions of Kubernetes. {{< table caption="Ingress NGINX supported version with Kubernetes Versions" >}} -Kubernetes Version | Ingress-NGINX version | Support Notes +Kubernetes version | Ingress-NGINX version | Notes :-------------------|:----------------------|:------------ `1.22` | TBD | On Going `1.21` | `v0.47.0` | On Going but only CVE and crashes From 701fe1d73366abb1a104b8449943707539121135 Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 14:18:14 -0400 Subject: [PATCH 025/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index e9132c74a3..de4696ac5d 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -21,7 +21,7 @@ This change has been in in discussion since [2017](https://github.com/kubernetes/kubernetes/issues/43214), [2019](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) with -1.16 Kubernetes API deprecations,and most recently in +1.16 Kubernetes API deprecations, and most recently in KEP-1453: [Graduate Ingress API to GA](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1453-ingress-api#122). From e7600828ebc3b4e2afc13f22edb6376ddf5fbdeb Mon Sep 17 00:00:00 2001 From: James Strong Date: Mon, 14 Jun 2021 14:18:28 -0400 Subject: [PATCH 026/175] Update content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md Co-authored-by: Tim Bannister --- .../blog/_posts/2021-06-21-update-with-ingress-nginx.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index de4696ac5d..a61bd9a844 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -39,10 +39,10 @@ versions of Kubernetes. {{< table caption="Ingress NGINX supported version with Kubernetes Versions" >}} Kubernetes version | Ingress-NGINX version | Notes :-------------------|:----------------------|:------------ -`1.22` | TBD | On Going -`1.21` | `v0.47.0` | On Going but only CVE and crashes -`1.20` | `v0.47.0` | On Going but only CVE and crashes -`1.19` | `v0.47.0` | Support will drop 6 months after 1.22 release +v1.22 | ≥ v.0.47.0 | New features, plus bug fixes. +v1.21 | v0.47.x | Bugfixes only, and just for security issues or crashes. No end-of-support date announced. +v1.20 | v0.47.x | Bugfixes only, and just for security issues or crashes. No end-of-support date announced. +v1.19 | v0.47.x | Bugfixes only, and just for security issues or crashes. Fixes only provided until 6 months after Kubernetes v1.22.0 is released. {{< /table >}} Because of the updates in Kubernetes 1.22, **v0.47.0** will not work with From 480a5f737f371f461bc3bb87fee0ba5a773f9dae Mon Sep 17 00:00:00 2001 From: yJunS Date: Thu, 17 Jun 2021 13:01:27 +0800 Subject: [PATCH 027/175] Add Dockershim Deprecation part --- .../2020-12-08-kubernetes-release-1.20.md | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md diff --git a/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md b/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md new file mode 100644 index 0000000000..18a694d63a --- /dev/null +++ b/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md @@ -0,0 +1,230 @@ +--- +layout: blog +title: 'Kubernetes 1.20: 最新版本' +date: 2020-12-08 +slug: kubernetes-1-20-release-announcement +--- + + + +**作者:** [Kubernetes 1.20 发布团队](https://github.com/kubernetes/sig-release/blob/master/releases/release-1.20/release_team.md) + + +我们很高兴地宣布 Kubernetes 1.20 的发布，这是我们 2020 年的第三个也是最后一个版本！此版本包含 42 项增强功能：11 项增强功能已升级到稳定版，15 项增强功能正在进入测试版，16 项增强功能正在进入 Alpha 版。 + + +1.20 发布周期在上一个延长的发布周期之后恢复到 11 周的正常节奏。这是一段时间以来功能最密集的版本之一：Kubernetes 创新周期仍呈上升趋势。此版本具有更多的 alpha 而非稳定的增强功能，表明云原生生态系统仍有许多需要探索的地方。 + + +## 主题 {#Major Themes} + + +### Volume 快照操作变得稳定 {$Volume Snapshot Operations Goes Stable} + + +此功能提供了触发卷快照操作的标准方法，并允许用户以可移植的方式在任何 Kubernetes 环境和支持的存储提供程序上合并快照操作。 + + +此外，这些 Kubernetes 快照原语充当基本构建块，解锁为 Kubernetes 开发高级企业级存储管理功能的能力，包括应用程序或集群级备份解决方案。 + + +请注意，快照支持要求 Kubernetes 分销商捆绑 Snapshot 控制器、Snapshot CRD 和验证 webhook。还必须在集群上部署支持快照功能的 CSI 驱动程序。 + + + + +### Kubectl Debug 功能升级到 Beta ${#Kubectl Debug Graduates to Beta} + + +`kubectl alpha debug` 功能在 1.20 中升级到测试版，成为 `kubectl debug`. 该功能直接从 kubectl 提供对常见调试工作流的支持。此版本的 kubectl 支持的故障排除场景包括： + + +* 通过创建使用不同容器映像或命令的 pod 副本，对在启动时崩溃的工作负载进行故障排除。 +* 通过在 pod 的新副本或使用临时容器中添加带有调试工具的新容器来对 distroless 容器进行故障排除。（临时容器是默认未启用的 alpha 功能。） +* 通过创建在主机命名空间中运行并可以访问主机文件系统的容器来对节点进行故障排除。 + + +请注意，作为新的内置命令，`kubectl debug` 优先于任何名为 “debug” 的 kubectl 插件。您必须重命名受影响的插件。 + + +`kubectl alpha debug` 现在不推荐使用，并将在后续版本中删除。更新您的脚本以使用 `kubectl debug`. 有关更多信息 `kubectl debug`，请参阅[调试正在运行的 Pod]((https://kubernetes.io/docs/tasks/debug-application-cluster/debug-running-pod/)。 + + +### 测试版：API 优先级和公平性 ${#Beta: API Priority and Fairness) + + +Kubernetes 1.20 由 1.18 引入，现在默认启用 API 优先级和公平性 (APF)。这允许 `kube-apiserver` 按优先级对传入请求进行分类。 + + +### Alpha 更新：IPV4/IPV6 ${#Alpha with updates: IPV4/IPV6} + + +重新实现了 IPv4/IPv6 双栈，以支持基于用户和社区反馈的双栈服务。这允许将 IPv4 和 IPv6 服务集群 IP 地址分配给单个服务，还允许服务从单 IP 堆栈转换为双 IP 堆栈，反之亦然。 + + +### GA：进程 PID 稳定性限制 ${#GA: Process PID Limiting for Stability} + + +进程 ID (pid) 是 Linux 主机上的基本资源。达到任务限制而不达到任何其他资源限制并导致主机不稳定是微不足道的。 + + + +管理员需要机制来确保用户 pod 不会导致 pid 耗尽，从而阻止主机守护程序（运行时、kubelet 等）运行。此外，重要的是要确保 pod 之间的 pid 受到限制，以确保它们对节点上的其他工作负载的影响有限。 +默认启用一年后，SIG 节点在 `SupportNodePidsLimit`（节点到 Pod PID 隔离）和 `SupportPodPidsLimit`（限制每个 Pod 的 PID 的能力）上都将 PID 限制升级为 GA 。 + + +### Alpha：节点优雅的关闭 ${#Alpha: Graceful node shutdown} + + +用户和集群管理员希望 Pod 遵守预期的 Pod 生命周期，包括 Pod 终止。目前，当一个节点关闭时，Pod 不会遵循预期的 Pod 终止生命周期，也不会正常终止，这可能会导致某些工作负载出现问题。 +该 `GracefulNodeShutdown` 功能现在处于 Alpha 阶段。`GracefulNodeShutdown` 使 kubelet 知道节点系统关闭，从而在系统关闭期间正常终止 pod。 + + +## 主要变化 ${#Major Changes} + + +### Dockershim 弃用 ${#Dockershim Deprecation} + + +Dockershim，Docker 的容器运行时接口 (CRI) shim 已被弃用。不推荐使用对 Docker 的支持，并将在未来版本中删除。由于 Docker 映像遵循开放容器计划 (OCI) 映像规范，因此 Docker 生成的映像将继续在具有所有 CRI 兼容运行时的集群中工作。Kubernetes 社区写了一篇关于弃用的详细[博客文章](https://blog.k8s.io/2020/12/02/dont-panic-kubernetes-and-docker/)，并为其提供了一个专门的常见问题[解答页面](https://blog.k8s.io/2020/12/02/dockershim-faq/)。 + + +### Exec 探测超时处理 ${#Exec Probe Timeout Handling} + + +一个关于 exec 探测超时的长期错误可能会影响现有的 pod 定义，已得到修复。在此修复之前 `timeoutSeconds`，exec 探测器不考虑该字段。相反，探测将无限期运行，甚至超过其配置的截止日期，直到返回结果。通过此更改，`1 second` 如果未指定值，则将应用默认值，并且如果探测时间超过一秒，现有 pod 定义可能不再足够。`ExecProbeTimeout` 此修复程序添加了一个名为 的功能门，使集群操作员能够恢复到以前的行为，但这将在后续版本中锁定并删除。为了恢复到以前的行为，集群运营商应该将此功能门设置为 `false`。 + + +有关更多详细信息，请查看有关配置探针的[更新文档](/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes)。 + + +## 其他更新 ${#Other Updates} + + +### 稳定版 ${#Graduated to Stable} + + +* [运行时类](https://github.com/kubernetes/enhancements/issues/585) +* [内置 API 类型默认值](https://github.com/kubernetes/enhancements/issues/1929) +* [添加 Pod-Startup Liveness-Probe Holdoff](https://github.com/kubernetes/enhancements/issues/950) +* [在 Windows 上支持 CRI-ContainerD](https://github.com/kubernetes/enhancements/issues/1001) +* [SCTP 对服务的支持](https://github.com/kubernetes/enhancements/issues/614) +* [将 AppProtocol 添加到服务和终端上](https://github.com/kubernetes/enhancements/issues/1507) + + +### 值得注意的功能更新 ${#Notable Feature Updates} + + +* [定时任务](https://github.com/kubernetes/enhancements/issues/19) + + +# 发行说明 ${#Release notes} + + +您可以在[发行说明](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.20.md)中查看 1.20 发行版的完整详细信息。 + + +# 可用的发布 ${#Availability of release} + + +Kubernetes 1.20 可在 [GitHub](https://github.com/kubernetes/kubernetes/releases/tag/v1.20.0) 上下载。有一些很棒的资源可以帮助您开始使用 Kubernetes。您可以在 Kubernetes 主站点上查看一些[交互式教程](https://kubernetes.io/docs/tutorials/)，或者使用 [kind](https://kind.sigs.k8s.io) 的 Docker 容器在您的机器上运行本地集群。如果您想尝试从头开始构建集群，请查看 Kelsey Hightower 的 [Kubernetes the Hard Way](https://github.com/kelseyhightower/kubernetes-the-hard-way) 教程。 + + +# 发布团队 ${#Release Team} + + +这个版本是由一群非常敬业的人促成的，他们在世界上发生的许多事情中作为一个团队走到了一起。非常感谢发布负责人 Jeremy Rickard 以及发布团队中的其他所有人，感谢他们相互支持，并努力为社区发布 1.20 版本。 + + +# 发布 Logo ${#Release Logo} + +![Kubernetes 1.20 Release Logo](/images/blog/2020-12-08-kubernetes-1.20-release-announcement/laser.png) + +[raddest](https://www.dictionary.com/browse/rad): *adjective*, Slang. excellent; wonderful; cool: + + +> Kubernetes 1.20 版本是迄今为止最激动人心的版本。 + + +2020 年对我们中的许多人来说都是充满挑战的一年，但 Kubernetes 贡献者在此版本中提供了创纪录的增强功能。这是一项了不起的成就，因此发布负责人希望以一点轻松的方式结束这一年，并向 [Kubernetes 1.14 - Caturnetes](https://github.com/kubernetes/sig-release/tree/master/releases/release-1.14) 和一只名叫 Humphrey 的 “rad” 猫致敬。 + + +Humphrey是发布负责人的猫，有一个永久的 `blep`. 在 1990 年代，*Rad* 是美国非常普遍的俚语，激光背景也是如此。Humphrey 在 1990 年代风格的学校照片中感觉像是结束这一年的有趣方式。希望 Humphrey 和它的 *blep* 在 2020 年底给你带来一点快乐！ + + +发布标志由 [Henry Hsu - @robotdancebattle](https://www.instagram.com/robotdancebattle/) 创建。 + + +# 用户亮点 ${#User Highlights} + + +- Apple 正在世界各地的数据中心运行数千个节点的 Kubernetes 集群。观看 [Alena Prokarchyk](https://youtu.be/Tx8qXC-U3KM) 的 KubeCon NA 主题演讲，了解有关他们的云原生之旅的更多信息。 + + +# 项目速度 ${#Project Velocity} + + +[CNCF K8S DevStats 项目](https://k8s.devstats.cncf.io/)聚集了许多有关Kubernetes和各分项目的速度有趣的数据点。这包括从个人贡献到做出贡献的公司数量的所有内容，并且清楚地说明了为发展这个生态系统所做的努力的深度和广度。 + + +在持续 11 周（9 月 25 日至 12 月 9 日）的 v1.20 发布周期中，我们看到了来自 [26 个国家/地区](https://k8s.devstats.cncf.io/d/50/countries-stats?orgId=1&from=1601006400000&to=1607576399000&var-period_name=Quarter&var-countries=All&var-repogroup_name=Kubernetes&var-metric=rcommitters&var-cum=countries) 的 [967 家公司](https://k8s.devstats.cncf.io/d/9/companies-table?orgId=1&var-period_name=v1.19.0%20-%20now&var-metric=contributions) 和 [1335 名个人](https://k8s.devstats.cncf.io/d/66/developer-activity-counts-by-companies?orgId=1&var-period_name=v1.19.0%20-%20now&var-metric=contributions&var-repogroup_name=Kubernetes&var-country_name=All&var-companies=All)（其中 [44 人](https://k8s.devstats.cncf.io/d/52/new-contributors?orgId=1&from=1601006400000&to=1607576399000&var-repogroup_name=Kubernetes)首次为 Kubernetes 做出贡献）的贡献。 + + +# 生态系统更新 ${#Ecosystem Updates} + + +- KubeCon North America 三周前刚刚结束，这是第二个虚拟的此类活动！现在所有演讲都可以[点播](https://www.youtube.com/playlist?list=PLj6h78yzYM2Pn8RxfLh2qrXBDftr6Qjut)，供任何需要赶上的人使用！ +- 6 月，Kubernetes 社区成立了一个新的工作组，作为对美国各地发生的 Black Lives Matter 抗议活动的直接回应。WG Naming 的目标是尽可能彻底地删除 Kubernetes 项目中有害和不清楚的语言，并以可移植到其他 CNCF 项目的方式进行。在 [KubeCon 2020 North America](https://sched.co/eukp) 上就这项重要工作及其如何进行进行了精彩的介绍性演讲，这项工作的初步影响[实际上可以在 v1.20 版本中看到](https://github.com/kubernetes/enhancements/issues/2067)。 +- 此前于今年夏天宣布，在 Kubecon NA 期间发布了经认证的 [Kubernetes 安全专家 (CKS) 认证](https://www.cncf.io/announcements/2020/11/17/kubernetes-security-specialist-certification-now-available/) ，以便立即安排！遵循 CKA 和 CKAD 的模型，CKS 是一项基于性能的考试，侧重于以安全为主题的能力和领域。该考试面向当前的 CKA 持有者，尤其是那些想要完善其在保护云工作负载方面的基础知识的人（这是我们所有人，对吧？）。 + + + + +# 活动更新 ${#Event Updates} + + +KubeCon + CloudNativeCon Europe 2021 将于 2021 年 5 月 4 日至 7 日举行！注册将于 1 月 11 日开放。您可以在[此处](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/)找到有关会议的更多信息。请记住，[CFP](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/program/cfp/) 将于太平洋标准时间 12 月 13 日星期日晚上 11:59 关闭！ + + +# 即将发布的网络研讨会 ${#Upcoming release webinar} + + +请继续关注今年 1 月即将举行的发布网络研讨会。 + + +# 参与其中 ${#Get Involved} + + +如果您有兴趣为 Kubernetes 社区做出贡献，那么特别兴趣小组 (SIG) 是一个很好的起点。其中许多可能符合您的兴趣！如果您有什么想与社区分享的内容，您可以参加每周的社区会议，或使用以下任一渠道： + + + +* 在新的 [Kubernetes Contributor 网站](https://www.kubernetes.dev/)上了解更多关于为Kubernetes 做出贡献的信息 +* 在 Twitter [@Kubernetesio](https://twitter.com/kubernetesio) 上关注我们以获取最新更新 +* 加入关于讨论的[社区](https://discuss.kubernetes.io/)讨论 +* 加入 [Slack 社区](http://slack.k8s.io/) +* 分享您的 [Kubernetes 故事](https://docs.google.com/a/linuxfoundation.org/forms/d/e/1FAIpQLScuI7Ye3VQHQTwBASrgkjQDSS5TP0g3AXfFhwSM9YpHgxRKFA/viewform) +* 在[博客](https://kubernetes.io/blog/)上阅读更多关于 Kubernetes 发生的事情 +* 了解有关 [Kubernetes 发布团队](https://github.com/kubernetes/sig-release/tree/master/release-team)的更多信息 From ddd130c37504c598ad67d26a33b22795caeb4631 Mon Sep 17 00:00:00 2001 From: Qiming Teng Date: Fri, 4 Jun 2021 15:56:57 +0800 Subject: [PATCH 028/175] [zh] Fix ccm glossary --- .../glossary/cloud-controller-manager.md | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/content/zh/docs/reference/glossary/cloud-controller-manager.md b/content/zh/docs/reference/glossary/cloud-controller-manager.md index d7ea6223c6..3d33b8c7fe 100644 --- a/content/zh/docs/reference/glossary/cloud-controller-manager.md +++ b/content/zh/docs/reference/glossary/cloud-controller-manager.md @@ -2,9 +2,9 @@ title: 云控制器管理器（Cloud Controller Manager） id: cloud-controller-manager date: 2018-04-12 -full_link: /zh/docs/tasks/administer-cluster/running-cloud-controller/ +full_link: /zh/docs/concepts/architecture/cloud-controller/ short_description: > - 云控制器管理器是 1.8 的 alpha 特性。在未来发布的版本中，这是将 Kubernetes 与任何其他云集成的最佳方式。 + 将 Kubernetes 与第三方云提供商进行集成的控制面组件。 aka: tags: @@ -12,36 +12,31 @@ tags: - architecture - operation --- - - - 云控制器管理器是指嵌入特定云的控制逻辑的 {{< glossary_tooltip text="控制平面" term_id="control-plane" >}}组件。 -云控制器管理器允许您链接集群到云提供商的应用编程接口中， -并把和该云平台交互的组件与只和您的集群交互的组件分离开。 +云控制器管理器使得你可以将你的集群连接到云提供商的 API 之上， +并将与该云平台交互的组件同与你的集群交互的组件分离开来。 @@ -51,4 +46,5 @@ infrastructure, the cloud-controller-manager component enables cloud providers t features at a different pace compared to the main Kubernetes project. --> 通过分离 Kubernetes 和底层云基础设置之间的互操作性逻辑， -云控制器管理器组件使云提供商能够以不同于 Kubernetes 主项目的速度进行发布新特征。 \ No newline at end of file +云控制器管理器组件使云提供商能够以不同于 Kubernetes 主项目的 +步调发布新特征。 From 0aef916e813af03814ef10dc2d1bb6f8c9b6ee94 Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Sat, 19 Jun 2021 00:57:55 +0100 Subject: [PATCH 029/175] Remove orphaned page from French localization --- content/fr/docs/setup/pick-right-solution.md | 303 ------------------- 1 file changed, 303 deletions(-) delete mode 100644 content/fr/docs/setup/pick-right-solution.md diff --git a/content/fr/docs/setup/pick-right-solution.md b/content/fr/docs/setup/pick-right-solution.md deleted file mode 100644 index a730ce8eb1..0000000000 --- a/content/fr/docs/setup/pick-right-solution.md +++ /dev/null @@ -1,303 +0,0 @@ ---- -reviewers: -- yastij -title: Choisir la bonne solution -description: Panorama de solutions Kubernetes -weight: 10 -content_type: concept ---- - - - -Kubernetes peut fonctionner sur des plateformes variées: sur votre PC portable, sur des VMs d'un fournisseur de cloud, ou un rack -de serveurs bare-metal. L'effort demandé pour configurer un cluster varie de l'éxécution d'une simple commande à la création -de votre propre cluster personnalisé. Utilisez ce guide pour choisir la solution qui correspond le mieux à vos besoins. - -Si vous voulez simplement jeter un coup d'oeil rapide, utilisez alors de préférence les [solutions locales basées sur Docker](#solutions-locales). - -Lorsque vous êtes prêts à augmenter le nombre de machines et souhaitez bénéficier de la haute disponibilité, une -[solution hébergée](#solutions-hebergées) est la plus simple à déployer et à maintenir. - -[Les solutions cloud clés en main](#solutions-clés-en-main) ne demandent que peu de commande pour déployer et couvrent un large panel de - fournisseurs de cloud. [Les solutions clés en main pour cloud privé](#solutions-on-premises-clés-en-main) possèdent la simplicité des solutions cloud clés en main combinées avec la sécurité de votre propre réseau privé. - -Si vous avez déjà un moyen de configurer vos resources, utilisez [kubeadm](/fr/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/) pour facilement -déployer un cluster grâce à une seule ligne de commande par machine. - -[Les solutions personnalisées](#solutions-personnalisées) varient d'instructions pas à pas, à des conseils relativement généraux pour déployer un - -cluster Kubernetes en partant du début. - - - - - -## Solutions locales - -* [Minikube](/fr/docs/setup/learning-environment/minikube/) est une méthode pour créer un cluster Kubernetes local à noeud unique pour le développement et le test. L'installation est entièrement automatisée et ne nécessite pas de compte de fournisseur de cloud. - -* [Docker Desktop](https://www.docker.com/products/docker-desktop) est une -application facile à installer pour votre environnement Mac ou Windows qui vous permet de -commencer à coder et déployer votre code dans des conteneurs en quelques minutes sur un nœud unique Kubernetes. - -* [Minishift](https://docs.okd.io/latest/minishift/) installe la version communautaire de la plate-forme d'entreprise OpenShift -de Kubernetes pour le développement local et les tests. Il offre une VM tout-en-un (`minishift start`) pour Windows, macOS et Linux, - le `oc cluster up` containerisé (Linux uniquement) et [est livré avec quelques Add Ons faciles à installer](https://github.com/minishift/minishift-addons/tree/master/add-ons). - -* [MicroK8s](https://microk8s.io/) fournit une commande unique d'installation de la dernière version de Kubernetes sur une machine locale -pour le développement et les tests. L'installation est rapide (~30 sec) et supporte de nombreux plugins dont Istio avec une seule commande. - -* [IBM Cloud Private-CE (Community Edition)](https://github.com/IBM/deploy-ibm-cloud-private) peut utiliser VirtualBox sur votre machine -pour déployer Kubernetes sur une ou plusieurs machines virtuelles afin de développer et réaliser des scénarios de test. Cette solution -peut créer un cluster multi-nœuds complet. - -* [IBM Cloud Private-CE (Community Edition) sur Linux Containers](https://github.com/HSBawa/icp-ce-on-linux-containers) est un script IaC (Infrastructure as Code) basé sur Terraform/Packer/BASH pour créer un cluster LXD à sept nœuds (1 Boot, 1 Master, 1 Management, 1 Proxy et 3 Workers) sur une machine Linux. - -* [Kubeadm-dind](https://github.com/kubernetes-sigs/kubeadm-dind-cluster) est un cluster Kubernetes multi-nœuds (tandis que minikube est -un nœud unique) qui ne nécessite qu'un docker-engine. Il utilise la technique du docker-in-docker pour déployer le cluster Kubernetes. - -* [Ubuntu sur LXD](/docs/getting-start-guides/ubuntu/local/) supporte un déploiement de 9 instances sur votre machine locale. - -## Solutions hebergées - -* [AppsCode.com](https://appscode.com/products/cloud-deployment/) fournit des clusters Kubernetes managés pour divers clouds publics, dont AWS et Google Cloud Platform. - -* [APPUiO](https://appuio.ch) propose une plate-forme de cloud public OpenShift, supportant n'importe quel workload Kubernetes. De plus, APPUiO propose des Clusters OpenShift privés et managés, fonctionnant sur n'importe quel cloud public ou privé. - -* [Amazon Elastic Container Service for Kubernetes](https://aws.amazon.com/eks/) offre un service managé de Kubernetes. - -* [Azure Kubernetes Service](https://azure.microsoft.com/services/container-service/) offre des clusters Kubernetes managés. - -* [Containership Kubernetes Engine (CKE)](https://containership.io/containership-platform) Approvisionnement et gestion intuitive de clusters - Kubernetes sur GCP, Azure, AWS, Packet, et DigitalOcean. Mises à niveau transparentes, auto-scaling, métriques, création de -workloads, et plus encore. - -* [DigitalOcean Kubernetes](https://www.digitalocean.com/products/kubernetes/) offre un service managé de Kubernetes. - -* [Giant Swarm](https://giantswarm.io/product/) offre des clusters Kubernetes managés dans leur propre centre de données, on-premises ou sur des clouds public. - -* [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/) offre des clusters Kubernetes managés. - -* [IBM Cloud Kubernetes Service](https://cloud.ibm.com/docs/containers?topic=containers-getting-started) offre des clusters Kubernetes managés avec choix d'isolation, des outils opérationnels, une vision intégrée de la sécurité des images et des conteneurs et une intégration avec Watson, IoT et les données. - -* [Kubermatic](https://www.loodse.com) fournit des clusters Kubernetes managés pour divers clouds publics, y compris AWS et Digital Ocean, ainsi que sur site avec intégration OpenStack. - -* [Kublr](https://kublr.com) offre des clusters Kubernetes sécurisés, évolutifs et hautement fiables sur AWS, Azure, GCP et on-premises, - de qualité professionnelle. Il inclut la sauvegarde et la reprise après sinistre prêtes à l'emploi, la journalisation et la surveillance centralisées multi-clusters, ainsi qu'une fonction d'alerte intégrée. - -* [Madcore.Ai](https://madcore.ai) est un outil CLI orienté développement pour déployer l'infrastructure Kubernetes dans AWS. Les masters, un groupe d'autoscaling pour les workers sur des spot instances, les ingress-ssl-lego, Heapster, et Grafana. - -* [Nutanix Karbon](https://www.nutanix.com/products/karbon/) est une plateforme de gestion et d'exploitation Kubernetes multi-clusters hautement disponibles qui simplifie l'approvisionnement, les opérations et la gestion du cycle de vie de Kubernetes. - -* [OpenShift Dedicated](https://www.openshift.com/dedicated/) offre des clusters Kubernetes gérés et optimisés par OpenShift. - -* [OpenShift Online](https://www.openshift.com/features/) fournit un accès hébergé gratuit aux applications Kubernetes. - -* [Oracle Container Engine for Kubernetes](https://docs.us-phoenix-1.oraclecloud.com/Content/ContEng/Concepts/contengoverview.htm) est un service entièrement géré, évolutif et hautement disponible que vous pouvez utiliser pour déployer vos applications conteneurisées dans le cloud. - -* [Platform9](https://platform9.com/products/kubernetes/) offre des Kubernetes gérés on-premises ou sur n'importe quel cloud public, et fournit une surveillance et des alertes de santé 24h/24 et 7j/7. (Kube2go, une plate-forme de service de déploiement de cluster Kubernetes pour le déploiement de l'interface utilisateur Web9, a été intégrée à Platform9 Sandbox.) - -* [Stackpoint.io](https://stackpoint.io) fournit l'automatisation et la gestion de l'infrastructure Kubernetes pour plusieurs clouds publics. - -* [SysEleven MetaKube](https://www.syseleven.io/products-services/managed-kubernetes/) offre un Kubernetes-as-a-Service sur un cloud public OpenStack. Il inclut la gestion du cycle de vie, les tableaux de bord d'administration, la surveillance, la mise à l'échelle automatique et bien plus encore. - -* [VMware Cloud PKS](https://cloud.vmware.com/vmware-cloud-pks) est une offre d'entreprise Kubernetes-as-a-Service faisant partie du catalogue de services Cloud VMware qui fournit des clusters Kubernetes faciles à utiliser, sécurisés par défaut, rentables et basés sur du SaaS. - -## Solutions clés en main - -Ces solutions vous permettent de créer des clusters Kubernetes sur une gamme de fournisseurs de Cloud IaaaS avec seulement -quelques commandes. Ces solutions sont activement développées et bénéficient du soutien actif de la communauté. - -* [Agile Stacks](https://www.agilestacks.com/products/kubernetes) -* [Alibaba Cloud](/docs/setup/turnkey/alibaba-cloud/) -* [APPUiO](https://appuio.ch) -* [AWS](/docs/setup/turnkey/aws/) -* [Azure](/docs/setup/turnkey/azure/) -* [CenturyLink Cloud](/docs/setup/turnkey/clc/) -* [Conjure-up Kubernetes with Ubuntu on AWS, Azure, Google Cloud, Oracle Cloud](/docs/getting-started-guides/ubuntu/) -* [Containership](https://containership.io/containership-platform) -* [Docker Enterprise](https://www.docker.com/products/docker-enterprise) -* [Gardener](https://gardener.cloud/) -* [Giant Swarm](https://giantswarm.io) -* [Google Compute Engine (GCE)](/docs/setup/turnkey/gce/) -* [IBM Cloud](https://github.com/patrocinio/kubernetes-softlayer) -* [Kontena Pharos](https://kontena.io/pharos/) -* [Kubermatic](https://cloud.kubermatic.io) -* [Kublr](https://kublr.com/) -* [Madcore.Ai](https://madcore.ai/) -* [Nirmata](https://nirmata.com/) -* [Nutanix Karbon](https://www.nutanix.com/products/karbon/) -* [Oracle Container Engine for K8s](https://docs.us-phoenix-1.oraclecloud.com/Content/ContEng/Concepts/contengprerequisites.htm) -* [Pivotal Container Service](https://pivotal.io/platform/pivotal-container-service) -* [Rancher 2.0](https://rancher.com/docs/rancher/v2.x/en/) -* [Stackpoint.io](/docs/setup/turnkey/stackpoint/) -* [Tectonic by CoreOS](https://coreos.com/tectonic) -* [VMware Cloud PKS](https://cloud.vmware.com/vmware-cloud-pks) - -## Solutions On-Premises clés en main - -Ces solutions vous permettent de créer des clusters Kubernetes sur votre cloud privé sécurisé avec seulement quelques commandes. - -* [Agile Stacks](https://www.agilestacks.com/products/kubernetes) -* [APPUiO](https://appuio.ch) -* [Docker Enterprise](https://www.docker.com/products/docker-enterprise) -* [Giant Swarm](https://giantswarm.io) -* [GKE On-Prem | Google Cloud](https://cloud.google.com/gke-on-prem/) -* [IBM Cloud Private](https://www.ibm.com/cloud-computing/products/ibm-cloud-private/) -* [Kontena Pharos](https://kontena.io/pharos/) -* [Kubermatic](https://www.loodse.com) -* [Kublr](https://kublr.com/) -* [Mirantis Cloud Platform](https://www.mirantis.com/software/kubernetes/) -* [Nirmata](https://nirmata.com/) -* [OpenShift Container Platform](https://www.openshift.com/products/container-platform/) (OCP) by [Red Hat](https://www.redhat.com) -* [Pivotal Container Service](https://pivotal.io/platform/pivotal-container-service) -* [Rancher 2.0](https://rancher.com/docs/rancher/v2.x/en/) -* [SUSE CaaS Platform](https://www.suse.com/products/caas-platform) -* [SUSE Cloud Application Platform](https://www.suse.com/products/cloud-application-platform/) - -## Solutions personnalisées - -Kubernetes peut fonctionner sur une large gamme de fournisseurs de Cloud et d'environnements bare-metal, ainsi qu'avec de nombreux -systèmes d'exploitation. - -Si vous pouvez trouver un guide ci-dessous qui correspond à vos besoins, utilisez-le. C'est peut-être un peu dépassé, mais... -ce sera plus facile que de partir de zéro. Si vous voulez repartir de zéro, soit parce que vous avez des exigences particulières, -ou simplement parce que vous voulez comprendre ce qu'il y a à l'interieur de Kubernetes -essayez le guide [Getting Started from Scratch](/docs/setup/release/building-from-source/). - -### Universel - -Si vous avez déjà un moyen de configurer les ressources d'hébergement, utilisez -[kubeadm](/fr/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/) pour déployer facilement un cluster -avec une seule commande par machine. - -### Cloud - -Ces solutions sont des combinaisons de fournisseurs de cloud computing et de systèmes d'exploitation qui ne sont pas couverts par les solutions ci-dessus. - -* [Cloud Foundry Container Runtime (CFCR)](https://docs-cfcr.cfapps.io/) -* [CoreOS on AWS or GCE](/docs/setup/custom-cloud/coreos/) -* [Gardener](https://gardener.cloud/) -* [Kublr](https://kublr.com/) -* [Kubernetes on Ubuntu](/docs/getting-started-guides/ubuntu/) -* [Kubespray](/docs/setup/custom-cloud/kubespray/) -* [Rancher Kubernetes Engine (RKE)](https://github.com/rancher/rke) - -### VMs On-Premises - -* [Cloud Foundry Container Runtime (CFCR)](https://docs-cfcr.cfapps.io/) -* [CloudStack](/docs/setup/on-premises-vm/cloudstack/) (uses Ansible, CoreOS and flannel) -* [Fedora (Multi Node)](/docs/getting-started-guides/fedora/flannel_multi_node_cluster/) (uses Fedora and flannel) -* [Nutanix AHV](https://www.nutanix.com/products/acropolis/virtualization/) -* [OpenShift Container Platform](https://www.openshift.com/products/container-platform/) (OCP) Kubernetes platform by [Red Hat](https://www.redhat.com) -* [oVirt](/docs/setup/on-premises-vm/ovirt/) -* [Vagrant](/docs/setup/custom-cloud/coreos/) (uses CoreOS and flannel) -* [VMware](/docs/setup/custom-cloud/coreos/) (uses CoreOS and flannel) -* [VMware vSphere](https://vmware.github.io/vsphere-storage-for-kubernetes/documentation/) -* [VMware vSphere, OpenStack, or Bare Metal](/docs/getting-started-guides/ubuntu/) (uses Juju, Ubuntu and flannel) - -### Bare Metal - -* [CoreOS](/docs/setup/custom-cloud/coreos/) -* [Digital Rebar](/docs/setup/on-premises-metal/krib/) -* [Docker Enterprise](https://www.docker.com/products/docker-enterprise) -* [Fedora (Single Node)](/docs/getting-started-guides/fedora/fedora_manual_config/) -* [Fedora (Multi Node)](/docs/getting-started-guides/fedora/flannel_multi_node_cluster/) -* [Kubernetes on Ubuntu](/docs/getting-started-guides/ubuntu/) -* [OpenShift Container Platform](https://www.openshift.com/products/container-platform/) (OCP) Kubernetes platform by [Red Hat](https://www.redhat.com) - -### Integrations - -Ces solutions fournissent une intégration avec des orchestrateurs, des resources managers ou des plateformes tierces. - -* [DCOS](/docs/setup/on-premises-vm/dcos/) - * Community Edition DCOS utilise AWS - * Enterprise Edition DCOS supporte l'hébergement cloud, les VMs on-premises, et le bare-metal - -## Tableau des Solutions - -Ci-dessous vous trouverez un tableau récapitulatif de toutes les solutions listées précédemment. - -| Fournisseur de IaaS | Config. Mgmt. | OS | Réseau | Docs | Niveau de support | -|------------------------------------------------|------------------------------------------------------------------------------|--------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| tous | tous | multi-support | tout les CNI | [docs](/fr/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/) | Project ([SIG-cluster-lifecycle](https://git.k8s.io/community/sig-cluster-lifecycle)) | -| Google Kubernetes Engine | | | GCE | [docs](https://cloud.google.com/kubernetes-engine/docs/) | Commercial | -| Docker Enterprise | personnalisé | [multi-support](https://success.docker.com/article/compatibility-matrix) | [multi-support](https://docs.docker.com/ee/ucp/kubernetes/install-cni-plugin/) | [docs](https://docs.docker.com/ee/) | Commercial | -| IBM Cloud Private | Ansible | multi-support | multi-support | [docs](https://www.ibm.com/support/knowledgecenter/SSBS6K/product_welcome_cloud_private.html) | [Commercial](https://www.ibm.com/mysupport/s/topic/0TO500000001o0fGAA/ibm-cloud-private?language=en_US&productId=01t50000004X1PWAA0) and [Community](https://www.ibm.com/support/knowledgecenter/SSBS6K_3.1.2/troubleshoot/support_types.html) | -| Red Hat OpenShift | Ansible & CoreOS | RHEL & CoreOS | [multi-support](https://docs.openshift.com/container-platform/3.11/architecture/networking/network_plugins.html) | [docs](https://docs.openshift.com/container-platform/3.11/welcome/index.html) | Commercial | -| Stackpoint.io | | multi-support | multi-support | [docs](https://stackpoint.io/) | Commercial | -| AppsCode.com | Saltstack | Debian | multi-support | [docs](https://appscode.com/products/cloud-deployment/) | Commercial | -| Madcore.Ai | Jenkins DSL | Ubuntu | flannel | [docs](https://madcore.ai) | Community ([@madcore-ai](https://github.com/madcore-ai)) | -| Platform9 | | multi-support | multi-support | [docs](https://platform9.com/managed-kubernetes/) | Commercial | -| Kublr | personnalisé | multi-support | multi-support | [docs](http://docs.kublr.com/) | Commercial | -| Kubermatic | | multi-support | multi-support | [docs](http://docs.kubermatic.io/) | Commercial | -| IBM Cloud Kubernetes Service | | Ubuntu | IBM Cloud Networking + Calico | [docs](https://cloud.ibm.com/docs/containers?topic=containers-getting-started) | Commercial | -| Giant Swarm | | CoreOS | flannel and/or Calico | [docs](https://docs.giantswarm.io/) | Commercial | -| GCE | Saltstack | Debian | GCE | [docs](/docs/setup/turnkey/gce/) | Project | -| Azure Kubernetes Service | | Ubuntu | Azure | [docs](https://docs.microsoft.com/en-us/azure/aks/) | Commercial | -| Azure (IaaS) | | Ubuntu | Azure | [docs](/docs/setup/turnkey/azure/) | [Community (Microsoft)](https://github.com/Azure/acs-engine) | -| Bare-metal | personnalisé | Fedora | _none_ | [docs](/docs/getting-started-guides/fedora/fedora_manual_config/) | Project | -| Bare-metal | personnalisé | Fedora | flannel | [docs](/docs/getting-started-guides/fedora/flannel_multi_node_cluster/) | Community ([@aveshagarwal](https://github.com/aveshagarwal)) | -| libvirt | personnalisé | Fedora | flannel | [docs](/docs/getting-started-guides/fedora/flannel_multi_node_cluster/) | Community ([@aveshagarwal](https://github.com/aveshagarwal)) | -| KVM | personnalisé | Fedora | flannel | [docs](/docs/getting-started-guides/fedora/flannel_multi_node_cluster/) | Community ([@aveshagarwal](https://github.com/aveshagarwal)) | -| DCOS | Marathon | CoreOS/Alpine | personnalisé | [docs](/docs/getting-started-guides/dcos/) | Community ([Kubernetes-Mesos Authors](https://github.com/mesosphere/kubernetes-mesos/blob/master/AUTHORS.md)) | -| AWS | CoreOS | CoreOS | flannel | [docs](/docs/setup/turnkey/aws/) | Community | -| GCE | CoreOS | CoreOS | flannel | [docs](/docs/getting-started-guides/coreos/) | Community ([@pires](https://github.com/pires)) | -| Vagrant | CoreOS | CoreOS | flannel | [docs](/docs/getting-started-guides/coreos/) | Community ([@pires](https://github.com/pires), [@AntonioMeireles](https://github.com/AntonioMeireles)) | -| CloudStack | Ansible | CoreOS | flannel | [docs](/docs/getting-started-guides/cloudstack/) | Community ([@sebgoa](https://github.com/sebgoa)) | -| VMware vSphere | tous | multi-support | multi-support | [docs](https://vmware.github.io/vsphere-storage-for-kubernetes/documentation/) | [Community](https://vmware.github.io/vsphere-storage-for-kubernetes/documentation/contactus.html) | -| Bare-metal | personnalisé | CentOS | flannel | [docs](/docs/getting-started-guides/centos/centos_manual_config/) | Community ([@coolsvap](https://github.com/coolsvap)) | -| lxd | Juju | Ubuntu | flannel/canal | [docs](/docs/getting-started-guides/ubuntu/local/) | [Commercial](https://www.ubuntu.com/kubernetes) and [Community](https://jujucharms.com/kubernetes) | -| AWS | Juju | Ubuntu | flannel/calico/canal | [docs](/docs/getting-started-guides/ubuntu/) | [Commercial](https://www.ubuntu.com/kubernetes) and [Community](https://jujucharms.com/kubernetes) | -| Azure | Juju | Ubuntu | flannel/calico/canal | [docs](/docs/getting-started-guides/ubuntu/) | [Commercial](https://www.ubuntu.com/kubernetes) and [Community](https://jujucharms.com/kubernetes) | -| GCE | Juju | Ubuntu | flannel/calico/canal | [docs](/docs/getting-started-guides/ubuntu/) | [Commercial](https://www.ubuntu.com/kubernetes) and [Community](https://jujucharms.com/kubernetes) | -| Oracle Cloud | Juju | Ubuntu | flannel/calico/canal | [docs](/docs/getting-started-guides/ubuntu/) | [Commercial](https://www.ubuntu.com/kubernetes) and [Community](https://jujucharms.com/kubernetes) | -| Rackspace | personnalisé | CoreOS | flannel/calico/canal | [docs](https://developer.rackspace.com/docs/rkaas/latest/) | [Commercial](https://www.rackspace.com/managed-kubernetes) | -| VMware vSphere | Juju | Ubuntu | flannel/calico/canal | [docs](/docs/getting-started-guides/ubuntu/) | [Commercial](https://www.ubuntu.com/kubernetes) and [Community](https://jujucharms.com/kubernetes) | -| Bare Metal | Juju | Ubuntu | flannel/calico/canal | [docs](/docs/getting-started-guides/ubuntu/) | [Commercial](https://www.ubuntu.com/kubernetes) and [Community](https://jujucharms.com/kubernetes) | -| AWS | Saltstack | Debian | AWS | [docs](/docs/setup/turnkey/aws/) | Community ([@justinsb](https://github.com/justinsb)) | -| AWS | kops | Debian | AWS | [docs](https://github.com/kubernetes/kops/) | Community ([@justinsb](https://github.com/justinsb)) | -| Bare-metal | personnalisé | Ubuntu | flannel | [docs](/docs/getting-started-guides/ubuntu/) | Community ([@resouer](https://github.com/resouer), [@WIZARD-CXY](https://github.com/WIZARD-CXY)) | -| oVirt | | | | [docs](/docs/setup/on-premises-vm/ovirt/) | Community ([@simon3z](https://github.com/simon3z)) | -| tous | tous | tous | tous | [docs](/docs/setup/release/building-from-source/) | Community ([@erictune](https://github.com/erictune)) | -| tous | tous | tous | tous | [docs](http://docs.projectcalico.org/v2.2/getting-started/kubernetes/installation/) | Commercial and Community | -| tous | RKE | multi-support | flannel or canal | [docs](https://rancher.com/docs/rancher/v2.x/en/quick-start-guide/) | [Commercial](https://rancher.com/what-is-rancher/overview/) and [Community](https://github.com/rancher/rancher) | -| tous | [Gardener Cluster-Operator](https://kubernetes.io/blog/2018/05/17/gardener/) | multi-support | multi-support | [docs](https://gardener.cloud) | [Project/Community](https://github.com/gardener) and [Commercial]( https://cloudplatform.sap.com/) | -| Alibaba Cloud Container Service For Kubernetes | ROS | CentOS | flannel/Terway | [docs](https://www.aliyun.com/product/containerservice) | Commercial | -| Agile Stacks | Terraform | CoreOS | multi-support | [docs](https://www.agilestacks.com/products/kubernetes) | Commercial | -| IBM Cloud Kubernetes Service | | Ubuntu | calico | [docs](https://cloud.ibm.com/docs/containers?topic=containers-container_index#container_index) | Commercial | -| Digital Rebar | kubeadm | tous | metal | [docs](/docs/setup/on-premises-metal/krib/) | Community ([@digitalrebar](https://github.com/digitalrebar)) | -| VMware Cloud PKS | | Photon OS | Canal | [docs](https://docs.vmware.com/en/VMware-Kubernetes-Engine/index.html) | Commercial | -| Mirantis Cloud Platform | Salt | Ubuntu | multi-support | [docs](https://docs.mirantis.com/mcp/) | Commercial | - -{{< note >}} -Le tableau ci-dessus est ordonné par versions testées et utilisées dans les noeuds, suivis par leur niveau de support. -{{< /note >}} - -### Définition des colonnes - -* **IaaS Provider** est le produit ou l'organisation qui fournit les machines virtuelles ou physiques (nœuds) sur lesquelles Kubernetes fonctionne. -* **OS** est le système d'exploitation de base des nœuds. -* **Config. Mgmt.** est le système de gestion de configuration qui permet d'installer et de maintenir Kubernetes sur les - nœuds. -* **Le réseau** est ce qui implémente le [modèle de réseau](/docs/concepts/cluster-administration/networking/). Ceux qui ont le type de réseautage - Aucun_ ne peut pas prendre en charge plus d'un nœud unique, ou peut prendre en charge plusieurs nœuds VM dans un nœud physique unique. -* **Conformité** indique si un cluster créé avec cette configuration a passé la conformité du projet. - pour le support de l'API et des fonctionnalités de base de Kubernetes v1.0.0. -* **Niveaux de soutien** - * **Projet** : Les contributeurs de Kubernetes utilisent régulièrement cette configuration, donc elle fonctionne généralement avec la dernière version. - de Kubernetes. - * **Commercial** : Une offre commerciale avec son propre dispositif d'accompagnement. - * **Communauté** : Soutenu activement par les contributions de la communauté. Peut ne pas fonctionner avec les versions récentes de Kubernetes. - * **Inactif** : Pas de maintenance active. Déconseillé aux nouveaux utilisateurs de Kubernetes et peut être retiré. -* **Note** contient d'autres informations pertinentes, telles que la version de Kubernetes utilisée. - - - -[1]: https://gist.github.com/erictune/4cabc010906afbcc5061 - -[2]: https://gist.github.com/derekwaynecarr/505e56036cdf010bf6b6 - -[3]: https://gist.github.com/erictune/2f39b22f72565365e59b - - From 93a1a281b9cc56874520a71afb839b02182dc33e Mon Sep 17 00:00:00 2001 From: yJunS Date: Sat, 19 Jun 2021 20:18:40 +0800 Subject: [PATCH 030/175] Fix some format questions --- .../2020-12-08-kubernetes-release-1.20.md | 79 ++++++++++--------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md b/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md index 18a694d63a..efdbcad601 100644 --- a/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md +++ b/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md @@ -21,10 +21,10 @@ slug: kubernetes-1-20-release-announcement 1.20 发布周期在上一个延长的发布周期之后恢复到 11 周的正常节奏。这是一段时间以来功能最密集的版本之一：Kubernetes 创新周期仍呈上升趋势。此版本具有更多的 alpha 而非稳定的增强功能，表明云原生生态系统仍有许多需要探索的地方。 -## 主题 {#Major Themes} +## 主题 {#major-themes} -### Volume 快照操作变得稳定 {$Volume Snapshot Operations Goes Stable} +### Volume 快照操作变得稳定 {$volume-snapshot-operations-goes-stable} 此功能提供了触发卷快照操作的标准方法，并允许用户以可移植的方式在任何 Kubernetes 环境和支持的存储提供程序上合并快照操作。 @@ -38,7 +38,7 @@ slug: kubernetes-1-20-release-announcement -### Kubectl Debug 功能升级到 Beta ${#Kubectl Debug Graduates to Beta} +### Kubectl Debug 功能升级到 Beta ${#kubectl-debug-graduates-to-beta} `kubectl alpha debug` 功能在 1.20 中升级到测试版，成为 `kubectl debug`. 该功能直接从 kubectl 提供对常见调试工作流的支持。此版本的 kubectl 支持的故障排除场景包括： @@ -51,36 +51,36 @@ slug: kubernetes-1-20-release-announcement * 通过创建在主机命名空间中运行并可以访问主机文件系统的容器来对节点进行故障排除。 -请注意，作为新的内置命令，`kubectl debug` 优先于任何名为 “debug” 的 kubectl 插件。您必须重命名受影响的插件。 +请注意，作为新的内置命令，`kubectl debug` 优先于任何名为 “debug” 的 kubectl 插件。你必须重命名受影响的插件。 -`kubectl alpha debug` 现在不推荐使用，并将在后续版本中删除。更新您的脚本以使用 `kubectl debug`. 有关更多信息 `kubectl debug`，请参阅[调试正在运行的 Pod]((https://kubernetes.io/docs/tasks/debug-application-cluster/debug-running-pod/)。 +`kubectl alpha debug` 现在不推荐使用，并将在后续版本中删除。更新你的脚本以使用 `kubectl debug`. 有关更多信息 `kubectl debug`，请参阅[调试正在运行的 Pod]((https://kubernetes.io/docs/tasks/debug-application-cluster/debug-running-pod/)。 -### 测试版：API 优先级和公平性 ${#Beta: API Priority and Fairness) +### 测试版：API 优先级和公平性 ${#beta-api-priority-and-fairness) Kubernetes 1.20 由 1.18 引入，现在默认启用 API 优先级和公平性 (APF)。这允许 `kube-apiserver` 按优先级对传入请求进行分类。 -### Alpha 更新：IPV4/IPV6 ${#Alpha with updates: IPV4/IPV6} +### Alpha 更新：IPV4/IPV6 ${#alpha-with-updates-ipv4-ipv6} 重新实现了 IPv4/IPv6 双栈，以支持基于用户和社区反馈的双栈服务。这允许将 IPv4 和 IPv6 服务集群 IP 地址分配给单个服务，还允许服务从单 IP 堆栈转换为双 IP 堆栈，反之亦然。 -### GA：进程 PID 稳定性限制 ${#GA: Process PID Limiting for Stability} +### GA：进程 PID 稳定性限制 ${#ga-process-pid-limiting-for-stability} -进程 ID (pid) 是 Linux 主机上的基本资源。达到任务限制而不达到任何其他资源限制并导致主机不稳定是微不足道的。 +进程 ID (pid) 是 Linux 主机上的基本资源。达到任务限制而不达到任何其他资源限制并导致主机不稳定是很可能发生的。 管理员需要机制来确保用户 pod 不会导致 pid 耗尽，从而阻止主机守护程序（运行时、kubelet 等）运行。此外，重要的是要确保 pod 之间的 pid 受到限制，以确保它们对节点上的其他工作负载的影响有限。 -默认启用一年后，SIG 节点在 `SupportNodePidsLimit`（节点到 Pod PID 隔离）和 `SupportPodPidsLimit`（限制每个 Pod 的 PID 的能力）上都将 PID 限制升级为 GA 。 +默认启用一年后，SIG Node 在 `SupportNodePidsLimit`（节点到 Pod PID 隔离）和 `SupportPodPidsLimit`（限制每个 Pod 的 PID 的能力）上都将 PID 限制升级为 GA 。 -### Alpha：节点优雅的关闭 ${#Alpha: Graceful node shutdown} +### Alpha：节点体面地关闭 ${#alpha-graceful-node-shutdown} @@ -88,29 +88,31 @@ The `GracefulNodeShutdown` feature is now in Alpha. `GracefulNodeShutdown` makes 该 `GracefulNodeShutdown` 功能现在处于 Alpha 阶段。`GracefulNodeShutdown` 使 kubelet 知道节点系统关闭，从而在系统关闭期间正常终止 pod。 -## 主要变化 ${#Major Changes} +## 主要变化 ${#major-changes} -### Dockershim 弃用 ${#Dockershim Deprecation} +### Dockershim 弃用 ${#dockershim-deprecation} -Dockershim，Docker 的容器运行时接口 (CRI) shim 已被弃用。不推荐使用对 Docker 的支持，并将在未来版本中删除。由于 Docker 映像遵循开放容器计划 (OCI) 映像规范，因此 Docker 生成的映像将继续在具有所有 CRI 兼容运行时的集群中工作。Kubernetes 社区写了一篇关于弃用的详细[博客文章](https://blog.k8s.io/2020/12/02/dont-panic-kubernetes-and-docker/)，并为其提供了一个专门的常见问题[解答页面](https://blog.k8s.io/2020/12/02/dockershim-faq/)。 +Dockershim，Docker 的容器运行时接口 (CRI) shim 已被弃用。不推荐使用对 Docker 的支持，并将在未来版本中删除。由于 Docker 映像遵循开放容器计划 (OCI) 映像规范，因此 Docker 生成的映像将继续在具有所有 CRI 兼容运行时的集群中工作。 +Kubernetes 社区写了一篇关于弃用的详细[博客文章](https://blog.k8s.io/2020/12/02/dont-panic-kubernetes-and-docker/)，并为其提供了一个专门的常见问题[解答页面](https://blog.k8s.io/2020/12/02/dockershim-faq/)。 -### Exec 探测超时处理 ${#Exec Probe Timeout Handling} +### Exec 探测超时处理 ${#exec-probe-timeout-handling} -一个关于 exec 探测超时的长期错误可能会影响现有的 pod 定义，已得到修复。在此修复之前 `timeoutSeconds`，exec 探测器不考虑该字段。相反，探测将无限期运行，甚至超过其配置的截止日期，直到返回结果。通过此更改，`1 second` 如果未指定值，则将应用默认值，并且如果探测时间超过一秒，现有 pod 定义可能不再足够。`ExecProbeTimeout` 此修复程序添加了一个名为 的功能门，使集群操作员能够恢复到以前的行为，但这将在后续版本中锁定并删除。为了恢复到以前的行为，集群运营商应该将此功能门设置为 `false`。 +一个关于 exec 探测超时的长期错误可能会影响现有的 pod 定义，已得到修复。在此修复之前 `timeoutSeconds`，exec 探测器不考虑该字段。相反，探测将无限期运行，甚至超过其配置的截止日期，直到返回结果。通过此更改，`1 second` 如果未指定值，则将应用默认值，并且如果探测时间超过一秒，现有 pod 定义可能不再足够。 +`ExecProbeTimeout` 此修复程序添加了一个名为 的功能门，使集群操作员能够恢复到以前的行为，但这将在后续版本中锁定并删除。为了恢复到以前的行为，集群运营商应该将此功能门设置为 `false`。 有关更多详细信息，请查看有关配置探针的[更新文档](/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes)。 -## 其他更新 ${#Other Updates} +## 其他更新 ${#other-updates} -### 稳定版 ${#Graduated to Stable} +### 稳定版 ${#graduated-to-stable} -* [运行时类](https://github.com/kubernetes/enhancements/issues/585) +* [RuntimeClass](https://github.com/kubernetes/enhancements/issues/585) * [内置 API 类型默认值](https://github.com/kubernetes/enhancements/issues/1929) * [添加 Pod-Startup Liveness-Probe Holdoff](https://github.com/kubernetes/enhancements/issues/950) * [在 Windows 上支持 CRI-ContainerD](https://github.com/kubernetes/enhancements/issues/1001) -* [SCTP 对服务的支持](https://github.com/kubernetes/enhancements/issues/614) -* [将 AppProtocol 添加到服务和终端上](https://github.com/kubernetes/enhancements/issues/1507) +* [SCTP 对 Services 的支持](https://github.com/kubernetes/enhancements/issues/614) +* [将 AppProtocol 添加到 Services 和 Endpoints 上](https://github.com/kubernetes/enhancements/issues/1507) -### 值得注意的功能更新 ${#Notable Feature Updates} +### 值得注意的功能更新 ${#notable-feature-updates} -* [定时任务](https://github.com/kubernetes/enhancements/issues/19) +* [CronJobs](https://github.com/kubernetes/enhancements/issues/19) -# 发行说明 ${#Release notes} +# 发行说明 ${#release-notes} -您可以在[发行说明](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.20.md)中查看 1.20 发行版的完整详细信息。 +你可以在[发行说明](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.20.md)中查看 1.20 发行版的完整详细信息。 -# 可用的发布 ${#Availability of release} +# 可用的发布 ${#availability-of-release} -Kubernetes 1.20 可在 [GitHub](https://github.com/kubernetes/kubernetes/releases/tag/v1.20.0) 上下载。有一些很棒的资源可以帮助您开始使用 Kubernetes。您可以在 Kubernetes 主站点上查看一些[交互式教程](https://kubernetes.io/docs/tutorials/)，或者使用 [kind](https://kind.sigs.k8s.io) 的 Docker 容器在您的机器上运行本地集群。如果您想尝试从头开始构建集群，请查看 Kelsey Hightower 的 [Kubernetes the Hard Way](https://github.com/kelseyhightower/kubernetes-the-hard-way) 教程。 +Kubernetes 1.20 可在 [GitHub](https://github.com/kubernetes/kubernetes/releases/tag/v1.20.0) 上下载。有一些很棒的资源可以帮助你开始使用 Kubernetes。你可以在 Kubernetes 主站点上查看一些[交互式教程](https://kubernetes.io/docs/tutorials/)，或者使用 [kind](https://kind.sigs.k8s.io) 的 Docker 容器在你的机器上运行本地集群。如果你想尝试从头开始构建集群，请查看 Kelsey Hightower 的 [Kubernetes the Hard Way](https://github.com/kelseyhightower/kubernetes-the-hard-way) 教程。 -# 发布团队 ${#Release Team} +# 发布团队 ${#release-team} 这个版本是由一群非常敬业的人促成的，他们在世界上发生的许多事情中作为一个团队走到了一起。非常感谢发布负责人 Jeremy Rickard 以及发布团队中的其他所有人，感谢他们相互支持，并努力为社区发布 1.20 版本。 -# 发布 Logo ${#Release Logo} +# 发布 Logo ${#release-logo} ![Kubernetes 1.20 Release Logo](/images/blog/2020-12-08-kubernetes-1.20-release-announcement/laser.png) @@ -169,13 +171,13 @@ Humphrey是发布负责人的猫，有一个永久的 `blep`. 在 1990 年代， 发布标志由 [Henry Hsu - @robotdancebattle](https://www.instagram.com/robotdancebattle/) 创建。 -# 用户亮点 ${#User Highlights} +# 用户亮点 ${#user-highlights} - Apple 正在世界各地的数据中心运行数千个节点的 Kubernetes 集群。观看 [Alena Prokarchyk](https://youtu.be/Tx8qXC-U3KM) 的 KubeCon NA 主题演讲，了解有关他们的云原生之旅的更多信息。 -# 项目速度 ${#Project Velocity} +# 项目速度 ${#project-velocity} [CNCF K8S DevStats 项目](https://k8s.devstats.cncf.io/)聚集了许多有关Kubernetes和各分项目的速度有趣的数据点。这包括从个人贡献到做出贡献的公司数量的所有内容，并且清楚地说明了为发展这个生态系统所做的努力的深度和广度。 @@ -184,7 +186,7 @@ Humphrey是发布负责人的猫，有一个永久的 `blep`. 在 1990 年代， 在持续 11 周（9 月 25 日至 12 月 9 日）的 v1.20 发布周期中，我们看到了来自 [26 个国家/地区](https://k8s.devstats.cncf.io/d/50/countries-stats?orgId=1&from=1601006400000&to=1607576399000&var-period_name=Quarter&var-countries=All&var-repogroup_name=Kubernetes&var-metric=rcommitters&var-cum=countries) 的 [967 家公司](https://k8s.devstats.cncf.io/d/9/companies-table?orgId=1&var-period_name=v1.19.0%20-%20now&var-metric=contributions) 和 [1335 名个人](https://k8s.devstats.cncf.io/d/66/developer-activity-counts-by-companies?orgId=1&var-period_name=v1.19.0%20-%20now&var-metric=contributions&var-repogroup_name=Kubernetes&var-country_name=All&var-companies=All)（其中 [44 人](https://k8s.devstats.cncf.io/d/52/new-contributors?orgId=1&from=1601006400000&to=1607576399000&var-repogroup_name=Kubernetes)首次为 Kubernetes 做出贡献）的贡献。 -# 生态系统更新 ${#Ecosystem Updates} +# 生态系统更新 ${#ecosystem-updates} -# 活动更新 ${#Event Updates} +# 活动更新 ${#event-updates} -KubeCon + CloudNativeCon Europe 2021 将于 2021 年 5 月 4 日至 7 日举行！注册将于 1 月 11 日开放。您可以在[此处](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/)找到有关会议的更多信息。请记住，[CFP](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/program/cfp/) 将于太平洋标准时间 12 月 13 日星期日晚上 11:59 关闭！ +KubeCon + CloudNativeCon Europe 2021 将于 2021 年 5 月 4 日至 7 日举行！注册将于 1 月 11 日开放。你可以在[此处](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/)找到有关会议的更多信息。 +请记住，[CFP](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/program/cfp/) 将于太平洋标准时间 12 月 13 日星期日晚上 11:59 关闭！ -# 即将发布的网络研讨会 ${#Upcoming release webinar} +# 即将发布的网络研讨会 ${#upcoming-release-webinar} 请继续关注今年 1 月即将举行的发布网络研讨会。 -# 参与其中 ${#Get Involved} +# 参与其中 ${#get-involved} -如果您有兴趣为 Kubernetes 社区做出贡献，那么特别兴趣小组 (SIG) 是一个很好的起点。其中许多可能符合您的兴趣！如果您有什么想与社区分享的内容，您可以参加每周的社区会议，或使用以下任一渠道： +如果你有兴趣为 Kubernetes 社区做出贡献，那么特别兴趣小组 (SIG) 是一个很好的起点。其中许多可能符合你的兴趣！如果你有什么想与社区分享的内容，你可以参加每周的社区会议，或使用以下任一渠道： -### Volume 快照操作变得稳定 {$volume-snapshot-operations-goes-stable} +### Volume 快照操作变得稳定 {#volume-snapshot-operations-goes-stable} 此功能提供了触发卷快照操作的标准方法，并允许用户以可移植的方式在任何 Kubernetes 环境和支持的存储提供程序上合并快照操作。 @@ -38,7 +38,7 @@ slug: kubernetes-1-20-release-announcement -### Kubectl Debug 功能升级到 Beta ${#kubectl-debug-graduates-to-beta} +### Kubectl Debug 功能升级到 Beta {#kubectl-debug-graduates-to-beta} `kubectl alpha debug` 功能在 1.20 中升级到测试版，成为 `kubectl debug`. 该功能直接从 kubectl 提供对常见调试工作流的支持。此版本的 kubectl 支持的故障排除场景包括： @@ -54,22 +54,23 @@ slug: kubernetes-1-20-release-announcement 请注意，作为新的内置命令，`kubectl debug` 优先于任何名为 “debug” 的 kubectl 插件。你必须重命名受影响的插件。 -`kubectl alpha debug` 现在不推荐使用，并将在后续版本中删除。更新你的脚本以使用 `kubectl debug`. 有关更多信息 `kubectl debug`，请参阅[调试正在运行的 Pod]((https://kubernetes.io/docs/tasks/debug-application-cluster/debug-running-pod/)。 +`kubectl alpha debug` 现在不推荐使用，并将在后续版本中删除。更新你的脚本以使用 `kubectl debug`. 有关更多信息 `kubectl debug`，请参阅[调试正在运行的 Pod]((https://kubernetes.io/zh/docs/tasks/debug-application-cluster/debug-running-pod/)。 -### 测试版：API 优先级和公平性 ${#beta-api-priority-and-fairness) +### 测试版：API 优先级和公平性 {#beta-api-priority-and-fairness) Kubernetes 1.20 由 1.18 引入，现在默认启用 API 优先级和公平性 (APF)。这允许 `kube-apiserver` 按优先级对传入请求进行分类。 -### Alpha 更新：IPV4/IPV6 ${#alpha-with-updates-ipv4-ipv6} +### Alpha 更新：IPV4/IPV6 {#alpha-with-updates-ipv4-ipv6} -重新实现了 IPv4/IPv6 双栈，以支持基于用户和社区反馈的双栈服务。这允许将 IPv4 和 IPv6 服务集群 IP 地址分配给单个服务，还允许服务从单 IP 堆栈转换为双 IP 堆栈，反之亦然。 +基于用户和社区反馈，重新实现了 IPv4/IPv6 双栈以支持双栈服务。 +这允许将 IPv4 和 IPv6 服务集群 IP 地址分配给单个服务，还允许服务从单 IP 堆栈转换为双 IP 堆栈，反之亦然。 -### GA：进程 PID 稳定性限制 ${#ga-process-pid-limiting-for-stability} +### GA：进程 PID 稳定性限制 {#ga-process-pid-limiting-for-stability} 进程 ID (pid) 是 Linux 主机上的基本资源。达到任务限制而不达到任何其他资源限制并导致主机不稳定是很可能发生的。 @@ -80,7 +81,7 @@ Kubernetes 1.20 由 1.18 引入，现在默认启用 API 优先级和公平性 ( 默认启用一年后，SIG Node 在 `SupportNodePidsLimit`（节点到 Pod PID 隔离）和 `SupportPodPidsLimit`（限制每个 Pod 的 PID 的能力）上都将 PID 限制升级为 GA 。 -### Alpha：节点体面地关闭 ${#alpha-graceful-node-shutdown} +### Alpha：节点体面地关闭 {#alpha-graceful-node-shutdown} @@ -91,7 +92,7 @@ The `GracefulNodeShutdown` feature is now in Alpha. `GracefulNodeShutdown` makes ## 主要变化 ${#major-changes} -### Dockershim 弃用 ${#dockershim-deprecation} +### Dockershim 弃用 {#dockershim-deprecation} @@ -99,20 +100,21 @@ Dockershim，Docker 的容器运行时接口 (CRI) shim 已被弃用。不推荐 Kubernetes 社区写了一篇关于弃用的详细[博客文章](https://blog.k8s.io/2020/12/02/dont-panic-kubernetes-and-docker/)，并为其提供了一个专门的常见问题[解答页面](https://blog.k8s.io/2020/12/02/dockershim-faq/)。 -### Exec 探测超时处理 ${#exec-probe-timeout-handling} +### Exec 探测超时处理 {#exec-probe-timeout-handling} -一个关于 exec 探测超时的长期错误可能会影响现有的 pod 定义，已得到修复。在此修复之前 `timeoutSeconds`，exec 探测器不考虑该字段。相反，探测将无限期运行，甚至超过其配置的截止日期，直到返回结果。通过此更改，`1 second` 如果未指定值，则将应用默认值，并且如果探测时间超过一秒，现有 pod 定义可能不再足够。 -`ExecProbeTimeout` 此修复程序添加了一个名为 的功能门，使集群操作员能够恢复到以前的行为，但这将在后续版本中锁定并删除。为了恢复到以前的行为，集群运营商应该将此功能门设置为 `false`。 +一个关于 exec 探测超时的长期错误可能会影响现有的 pod 定义，已得到修复。在此修复之前，exec 探测器不考虑 `timeoutSeconds` 字段。相反，探测将无限期运行，甚至超过其配置的截止日期，直到返回结果。 +通过此更改，如果未指定值，将应用默认值 `1 second`，并且如果探测时间超过一秒，现有 pod 定义可能不再足够。 +新引入的 `ExecProbeTimeout` 特性门控所提供的修复使集群操作员能够恢复到以前的行为，但这种行为将在后续版本中锁定并删除。为了恢复到以前的行为，集群运营商应该将此特性门控设置为 `false`。 -有关更多详细信息，请查看有关配置探针的[更新文档](/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes)。 +有关更多详细信息，请查看有关配置探针的[更新文档](/zh/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes)。 -## 其他更新 ${#other-updates} +## 其他更新 {#other-updates} -### 稳定版 ${#graduated-to-stable} +### 稳定版 {#graduated-to-stable} * [RuntimeClass](https://github.com/kubernetes/enhancements/issues/585) * [内置 API 类型默认值](https://github.com/kubernetes/enhancements/issues/1929) -* [添加 Pod-Startup Liveness-Probe Holdoff](https://github.com/kubernetes/enhancements/issues/950) +* [添加了对 Pod 层面启动探针和活跃性探针的扼制](https://github.com/kubernetes/enhancements/issues/950) * [在 Windows 上支持 CRI-ContainerD](https://github.com/kubernetes/enhancements/issues/1001) * [SCTP 对 Services 的支持](https://github.com/kubernetes/enhancements/issues/614) * [将 AppProtocol 添加到 Services 和 Endpoints 上](https://github.com/kubernetes/enhancements/issues/1507) -### 值得注意的功能更新 ${#notable-feature-updates} +### 值得注意的功能更新 {#notable-feature-updates} * [CronJobs](https://github.com/kubernetes/enhancements/issues/19) -# 发行说明 ${#release-notes} +# 发行说明 {#release-notes} 你可以在[发行说明](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.20.md)中查看 1.20 发行版的完整详细信息。 -# 可用的发布 ${#availability-of-release} +# 可用的发布 {#availability-of-release} Kubernetes 1.20 可在 [GitHub](https://github.com/kubernetes/kubernetes/releases/tag/v1.20.0) 上下载。有一些很棒的资源可以帮助你开始使用 Kubernetes。你可以在 Kubernetes 主站点上查看一些[交互式教程](https://kubernetes.io/docs/tutorials/)，或者使用 [kind](https://kind.sigs.k8s.io) 的 Docker 容器在你的机器上运行本地集群。如果你想尝试从头开始构建集群，请查看 Kelsey Hightower 的 [Kubernetes the Hard Way](https://github.com/kelseyhightower/kubernetes-the-hard-way) 教程。 -# 发布团队 ${#release-team} +# 发布团队 {#release-team} -这个版本是由一群非常敬业的人促成的，他们在世界上发生的许多事情中作为一个团队走到了一起。非常感谢发布负责人 Jeremy Rickard 以及发布团队中的其他所有人，感谢他们相互支持，并努力为社区发布 1.20 版本。 +这个版本是由一群非常敬业的人促成的，他们在世界上发生的许多事情的时段作为一个团队走到了一起。 +非常感谢发布负责人 Jeremy Rickard 以及发布团队中的其他所有人，感谢他们相互支持，并努力为社区发布 1.20 版本。 -# 发布 Logo ${#release-logo} +# 发布 Logo {#release-logo} ![Kubernetes 1.20 Release Logo](/images/blog/2020-12-08-kubernetes-1.20-release-announcement/laser.png) @@ -171,13 +174,13 @@ Humphrey是发布负责人的猫，有一个永久的 `blep`. 在 1990 年代， 发布标志由 [Henry Hsu - @robotdancebattle](https://www.instagram.com/robotdancebattle/) 创建。 -# 用户亮点 ${#user-highlights} +# 用户亮点 {#user-highlights} - Apple 正在世界各地的数据中心运行数千个节点的 Kubernetes 集群。观看 [Alena Prokarchyk](https://youtu.be/Tx8qXC-U3KM) 的 KubeCon NA 主题演讲，了解有关他们的云原生之旅的更多信息。 -# 项目速度 ${#project-velocity} +# 项目速度 {#project-velocity} [CNCF K8S DevStats 项目](https://k8s.devstats.cncf.io/)聚集了许多有关Kubernetes和各分项目的速度有趣的数据点。这包括从个人贡献到做出贡献的公司数量的所有内容，并且清楚地说明了为发展这个生态系统所做的努力的深度和广度。 @@ -186,7 +189,7 @@ Humphrey是发布负责人的猫，有一个永久的 `blep`. 在 1990 年代， 在持续 11 周（9 月 25 日至 12 月 9 日）的 v1.20 发布周期中，我们看到了来自 [26 个国家/地区](https://k8s.devstats.cncf.io/d/50/countries-stats?orgId=1&from=1601006400000&to=1607576399000&var-period_name=Quarter&var-countries=All&var-repogroup_name=Kubernetes&var-metric=rcommitters&var-cum=countries) 的 [967 家公司](https://k8s.devstats.cncf.io/d/9/companies-table?orgId=1&var-period_name=v1.19.0%20-%20now&var-metric=contributions) 和 [1335 名个人](https://k8s.devstats.cncf.io/d/66/developer-activity-counts-by-companies?orgId=1&var-period_name=v1.19.0%20-%20now&var-metric=contributions&var-repogroup_name=Kubernetes&var-country_name=All&var-companies=All)（其中 [44 人](https://k8s.devstats.cncf.io/d/52/new-contributors?orgId=1&from=1601006400000&to=1607576399000&var-repogroup_name=Kubernetes)首次为 Kubernetes 做出贡献）的贡献。 -# 生态系统更新 ${#ecosystem-updates} +# 生态系统更新 {#ecosystem-updates} -# 活动更新 ${#event-updates} +# 活动更新 {#event-updates} KubeCon + CloudNativeCon Europe 2021 将于 2021 年 5 月 4 日至 7 日举行！注册将于 1 月 11 日开放。你可以在[此处](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/)找到有关会议的更多信息。 请记住，[CFP](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/program/cfp/) 将于太平洋标准时间 12 月 13 日星期日晚上 11:59 关闭！ -# 即将发布的网络研讨会 ${#upcoming-release-webinar} +# 即将发布的网络研讨会 {#upcoming-release-webinar} 请继续关注今年 1 月即将举行的发布网络研讨会。 -# 参与其中 ${#get-involved} +# 参与其中 {#get-involved} 如果你有兴趣为 Kubernetes 社区做出贡献，那么特别兴趣小组 (SIG) 是一个很好的起点。其中许多可能符合你的兴趣！如果你有什么想与社区分享的内容，你可以参加每周的社区会议，或使用以下任一渠道： From 08a8887464d66b3e84820147c692aa94497f1e78 Mon Sep 17 00:00:00 2001 From: yJunS Date: Sun, 20 Jun 2021 23:35:32 +0800 Subject: [PATCH 032/175] Remove $ --- content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md b/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md index 2d7c223e5d..edd57b1a98 100644 --- a/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md +++ b/content/zh/blog/_posts/2020-12-08-kubernetes-release-1.20.md @@ -89,7 +89,7 @@ The `GracefulNodeShutdown` feature is now in Alpha. `GracefulNodeShutdown` makes 该 `GracefulNodeShutdown` 功能现在处于 Alpha 阶段。`GracefulNodeShutdown` 使 kubelet 知道节点系统关闭，从而在系统关闭期间正常终止 pod。 -## 主要变化 ${#major-changes} +## 主要变化 {#major-changes} ### Dockershim 弃用 {#dockershim-deprecation} From e33dea35cebd2162b0a7c686a5de72304a3e7aaa Mon Sep 17 00:00:00 2001 From: Jim Angel Date: Mon, 21 Jun 2021 03:54:19 +0000 Subject: [PATCH 033/175] cleaning up OWNERS --- OWNERS_ALIASES | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 2feb3cdb2d..227f4e9e05 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -1,11 +1,9 @@ aliases: sig-docs-blog-owners: # Approvers for blog content - - castrojo - kbarnard10 - onlydole - mrbobbytables sig-docs-blog-reviewers: # Reviewers for blog content - - castrojo - kbarnard10 - mrbobbytables - onlydole @@ -31,9 +29,7 @@ aliases: - reylejano - savitharaghunathan - sftim - - steveperry-53 - tengqm - - zparnold sig-docs-en-reviews: # PR reviews for English content - bradtopol - celestehorgan @@ -44,9 +40,7 @@ aliases: - onlydole - rajeshdeshpande02 - sftim - - steveperry-53 - tengqm - - zparnold sig-docs-es-owners: # Admins for Spanish content - raelga - electrocucaracha @@ -235,10 +229,12 @@ aliases: - parispittman # authoritative source: https://git.k8s.io/sig-release/OWNERS_ALIASES sig-release-leads: + - cpanato # SIG Technical Lead - hasheddan # SIG Technical Lead - jeremyrickard # SIG Technical Lead - justaugustus # SIG Chair - LappleApple # SIG Program Manager + - puerco # SIG Technical Lead - saschagrunert # SIG Chair release-engineering-approvers: - cpanato # Release Manager @@ -250,10 +246,11 @@ aliases: release-engineering-reviewers: - ameukam # Release Manager Associate - jimangel # Release Manager Associate + - markyjackson-taulia # Release Manager Associate - mkorbi # Release Manager Associate - palnabarun # Release Manager Associate - onlydole # Release Manager Associate - sethmccombs # Release Manager Associate - thejoycekung # Release Manager Associate - verolop # Release Manager Associate - - wilsonehusin # Release Manager Associate + - wilsonehusin # Release Manager Associate \ No newline at end of file From 21ca9a224fb45b5ce0845fff58de38091388f950 Mon Sep 17 00:00:00 2001 From: Shubham Kuchhal Date: Mon, 21 Jun 2021 18:27:52 +0530 Subject: [PATCH 034/175] Add MongoDB deployment and service. --- ...port-forward-access-application-cluster.md | 4 +-- .../application/mongodb/mongo-deployment.yaml | 31 +++++++++++++++++++ .../application/mongodb/mongo-service.yaml | 14 +++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 content/en/examples/application/mongodb/mongo-deployment.yaml create mode 100644 content/en/examples/application/mongodb/mongo-service.yaml diff --git a/content/en/docs/tasks/access-application-cluster/port-forward-access-application-cluster.md b/content/en/docs/tasks/access-application-cluster/port-forward-access-application-cluster.md index a0b4d78dab..ba8f7b1244 100644 --- a/content/en/docs/tasks/access-application-cluster/port-forward-access-application-cluster.md +++ b/content/en/docs/tasks/access-application-cluster/port-forward-access-application-cluster.md @@ -31,7 +31,7 @@ for database debugging. 1. Create a Deployment that runs MongoDB: ```shell - kubectl apply -f https://k8s.io/examples/application/guestbook/mongo-deployment.yaml + kubectl apply -f https://k8s.io/examples/application/mongodb/mongo-deployment.yaml ``` The output of a successful command verifies that the deployment was created: @@ -84,7 +84,7 @@ for database debugging. 2. Create a Service to expose MongoDB on the network: ```shell - kubectl apply -f https://k8s.io/examples/application/guestbook/mongo-service.yaml + kubectl apply -f https://k8s.io/examples/application/mongodb/mongo-service.yaml ``` The output of a successful command verifies that the Service was created: diff --git a/content/en/examples/application/mongodb/mongo-deployment.yaml b/content/en/examples/application/mongodb/mongo-deployment.yaml new file mode 100644 index 0000000000..04908ce25b --- /dev/null +++ b/content/en/examples/application/mongodb/mongo-deployment.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongo + labels: + app.kubernetes.io/name: mongo + app.kubernetes.io/component: backend +spec: + selector: + matchLabels: + app.kubernetes.io/name: mongo + app.kubernetes.io/component: backend + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: mongo + app.kubernetes.io/component: backend + spec: + containers: + - name: mongo + image: mongo:4.2 + args: + - --bind_ip + - 0.0.0.0 + resources: + requests: + cpu: 100m + memory: 100Mi + ports: + - containerPort: 27017 diff --git a/content/en/examples/application/mongodb/mongo-service.yaml b/content/en/examples/application/mongodb/mongo-service.yaml new file mode 100644 index 0000000000..b9cef607bc --- /dev/null +++ b/content/en/examples/application/mongodb/mongo-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongo + labels: + app.kubernetes.io/name: mongo + app.kubernetes.io/component: backend +spec: + ports: + - port: 27017 + targetPort: 27017 + selector: + app.kubernetes.io/name: mongo + app.kubernetes.io/component: backend From 045ed2d8c544997ed6b2b7a3d67bb963e2351c94 Mon Sep 17 00:00:00 2001 From: Shubham Kuchhal Date: Tue, 22 Jun 2021 17:17:07 +0530 Subject: [PATCH 035/175] Improvement: Runtime Class --- content/en/docs/concepts/containers/runtime-class.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/concepts/containers/runtime-class.md b/content/en/docs/concepts/containers/runtime-class.md index ed15910100..96858d32af 100644 --- a/content/en/docs/concepts/containers/runtime-class.md +++ b/content/en/docs/concepts/containers/runtime-class.md @@ -118,7 +118,7 @@ Runtime handlers are configured through containerd's configuration at `/etc/containerd/config.toml`. Valid handlers are configured under the runtimes section: ``` -[plugins.cri.containerd.runtimes.${HANDLER_NAME}] +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.${HANDLER_NAME}] ``` See containerd's config documentation for more details: From 8b3528af6113652cc85f4a8324cd06ba6482a1d3 Mon Sep 17 00:00:00 2001 From: James Strong Date: Wed, 23 Jun 2021 17:32:17 -0400 Subject: [PATCH 036/175] update the version for testing: --- .../en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index a61bd9a844..be79f3bdee 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -39,7 +39,7 @@ versions of Kubernetes. {{< table caption="Ingress NGINX supported version with Kubernetes Versions" >}} Kubernetes version | Ingress-NGINX version | Notes :-------------------|:----------------------|:------------ -v1.22 | ≥ v.0.47.0 | New features, plus bug fixes. +v1.22 | v1.0.0-alpha.1 | New features, plus bug fixes. v1.21 | v0.47.x | Bugfixes only, and just for security issues or crashes. No end-of-support date announced. v1.20 | v0.47.x | Bugfixes only, and just for security issues or crashes. No end-of-support date announced. v1.19 | v0.47.x | Bugfixes only, and just for security issues or crashes. Fixes only provided until 6 months after Kubernetes v1.22.0 is released. @@ -62,6 +62,8 @@ In the meantime to ensure no compatibility issues: [v0.47.0](https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v0.47.0) * After Kubernetes 1.22 is released, ensure you are using the latest version of Ingress-NGINX that supports the stable APIs for Ingress and IngressClass. +* Test Ingress-NGINX version v1.0.0-alpha.1 with Cluster versions >= 1.19 + and report any issues to the projects Github page. The community’s feedback and support in this effort is welcome. The Ingress-NGINX Sub-project regularly holds community meetings where we discuss From 626aaffb2cb07d883673ecfbe82f68650a8d8a71 Mon Sep 17 00:00:00 2001 From: Arhell Date: Sat, 26 Jun 2021 12:25:12 +0300 Subject: [PATCH 037/175] [ja] Modified the output when retrieving Secrets --- .../configmap-secret/managing-secret-using-config-file.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/ja/docs/tasks/configmap-secret/managing-secret-using-config-file.md b/content/ja/docs/tasks/configmap-secret/managing-secret-using-config-file.md index f9572ca1f4..e4c19098fe 100644 --- a/content/ja/docs/tasks/configmap-secret/managing-secret-using-config-file.md +++ b/content/ja/docs/tasks/configmap-secret/managing-secret-using-config-file.md @@ -119,6 +119,8 @@ kubectl get secret mysecret -o yaml ```yaml apiVersion: v1 +data: + config.yaml: YXBpVXJsOiAiaHR0cHM6Ly9teS5hcGkuY29tL2FwaS92MSIKdXNlcm5hbWU6IHt7dXNlcm5hbWV9fQpwYXNzd29yZDoge3twYXNzd29yZH19 kind: Secret metadata: creationTimestamp: 2018-11-15T20:40:59Z @@ -127,8 +129,6 @@ metadata: resourceVersion: "7225" uid: c280ad2e-e916-11e8-98f2-025000000001 type: Opaque -data: - config.yaml: YXBpVXJsOiAiaHR0cHM6Ly9teS5hcGkuY29tL2FwaS92MSIKdXNlcm5hbWU6IHt7dXNlcm5hbWV9fQpwYXNzd29yZDoge3twYXNzd29yZH19 ``` `kubectl get`と`kubectl describe`コマンドはデフォルトではSecretの内容を表示しません。 @@ -154,6 +154,8 @@ stringData: ```yaml apiVersion: v1 +data: + username: YWRtaW5pc3RyYXRvcg== kind: Secret metadata: creationTimestamp: 2018-11-15T20:46:46Z @@ -162,8 +164,6 @@ metadata: resourceVersion: "7579" uid: 91460ecb-e917-11e8-98f2-025000000001 type: Opaque -data: - username: YWRtaW5pc3RyYXRvcg== ``` `YWRtaW5pc3RyYXRvcg==`をデコードすると`administrator`となります。 From 9f3e905ea63f0ce299ed3db7786c809106988842 Mon Sep 17 00:00:00 2001 From: Arhell Date: Sun, 27 Jun 2021 11:37:58 +0300 Subject: [PATCH 038/175] [ja] Adds COF to the list of operators frameworks --- content/ja/docs/concepts/extend-kubernetes/operator.md | 1 + 1 file changed, 1 insertion(+) diff --git a/content/ja/docs/concepts/extend-kubernetes/operator.md b/content/ja/docs/concepts/extend-kubernetes/operator.md index c3857598f2..7383339160 100644 --- a/content/ja/docs/concepts/extend-kubernetes/operator.md +++ b/content/ja/docs/concepts/extend-kubernetes/operator.md @@ -87,6 +87,7 @@ kubectl edit SampleDB/example-database # 手動でいくつかの設定を変更 * [Custom Resources](/ja/docs/concepts/extend-kubernetes/api-extension/custom-resources/)をより深く学びます * ユースケースに合わせた、既製のオペレーターを[OperatorHub.io](https://operatorhub.io/)から見つけます * 自前のオペレーターを書くために既存のツールを使います、例: + * [Charmed Operator Framework](https://juju.is/) * [KUDO](https://kudo.dev/)（Kubernetes Universal Declarative Operator）を使います * [kubebuilder](https://book.kubebuilder.io/)を使います * [Metacontroller](https://metacontroller.app/)を自分で実装したWebHooksと一緒に使います From 3ccaf3a0b7fead25952f877d495672429c9d6ce9 Mon Sep 17 00:00:00 2001 From: liuwei10 Date: Mon, 28 Jun 2021 17:10:37 +0800 Subject: [PATCH 039/175] modify spelling mistakes in user guide --- .../windows/user-guide-windows-containers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/ja/docs/setup/production-environment/windows/user-guide-windows-containers.md b/content/ja/docs/setup/production-environment/windows/user-guide-windows-containers.md index 6f1ed4558e..2868afb182 100644 --- a/content/ja/docs/setup/production-environment/windows/user-guide-windows-containers.md +++ b/content/ja/docs/setup/production-environment/windows/user-guide-windows-containers.md @@ -134,7 +134,7 @@ Kubernetes v1.14以降、Windowsコンテナワークロードは、Group Manage Podの仕様で`"kubernetes.io/os": windows`のようなnodeSelectorが指定されていない場合、PodをWindowsまたはLinuxの任意のホストでスケジュールすることができます。WindowsコンテナはWindowsでのみ実行でき、LinuxコンテナはLinuxでのみ実行できるため、これは問題になる可能性があります。ベストプラクティスは、nodeSelectorを使用することです。 -ただし、多くの場合、ユーザーには既存の多数のLinuxコンテナのdepolyment、およびコミュニティHelmチャートのような既成構成のエコシステムやOperatorのようなプログラム的にPodを生成するケースがあることを理解しています。このような状況では、nodeSelectorsを追加するための構成変更をためらう可能性があります。代替策は、Taintsを使用することです。kubeletは登録中にTaintsを設定できるため、Windowsだけで実行する時に自動的にTaintを追加するように簡単に変更できます。 +ただし、多くの場合、ユーザーには既存の多数のLinuxコンテナのdeployment、およびコミュニティHelmチャートのような既成構成のエコシステムやOperatorのようなプログラム的にPodを生成するケースがあることを理解しています。このような状況では、nodeSelectorsを追加するための構成変更をためらう可能性があります。代替策は、Taintsを使用することです。kubeletは登録中にTaintsを設定できるため、Windowsだけで実行する時に自動的にTaintを追加するように簡単に変更できます。 例:`--register-with-taints='os=windows:NoSchedule'` From 7045fc791db079f1defedf1a00801c597f0de23b Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Mon, 14 Jun 2021 13:37:42 +0100 Subject: [PATCH 040/175] Revise explanation for when to use readiness probes Some cases in the existing text were better served by startup probes. Reword to make the distinction more clear. --- .../docs/concepts/workloads/pods/pod-lifecycle.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/content/en/docs/concepts/workloads/pods/pod-lifecycle.md b/content/en/docs/concepts/workloads/pods/pod-lifecycle.md index 778bee6c02..166d81e424 100644 --- a/content/en/docs/concepts/workloads/pods/pod-lifecycle.md +++ b/content/en/docs/concepts/workloads/pods/pod-lifecycle.md @@ -304,13 +304,23 @@ specify a readiness probe. In this case, the readiness probe might be the same as the liveness probe, but the existence of the readiness probe in the spec means that the Pod will start without receiving any traffic and only start receiving traffic after the probe starts succeeding. -If your container needs to work on loading large data, configuration files, or -migrations during startup, specify a readiness probe. If you want your container to be able to take itself down for maintenance, you can specify a readiness probe that checks an endpoint specific to readiness that is different from the liveness probe. +If your app has a strict dependency on back-end services, you can implement both +a liveness and a readiness probe. The liveness probe passes when the app itself +is healthy, but the readiness problem additionally checks that each required +back-end service is available. This helps you avoid directing traffic to Pods +that can only respond with error messages. + +If your container needs to work on loading large data, configuration files, or +migrations during startup, you can use a +[startup probe](#when-should-you-use-a-startup-probe). However, if you want to +detect the difference between an app that has failed and an app that is still +processing its startup data, you might prefer a readiness probe. + {{< note >}} If you want to be able to drain requests when the Pod is deleted, you do not necessarily need a readiness probe; on deletion, the Pod automatically puts itself From 538bf40eb3cf3502918c37910be904c807734039 Mon Sep 17 00:00:00 2001 From: yaohaoyun Date: Wed, 30 Jun 2021 11:38:15 +0800 Subject: [PATCH 041/175] modify kubernetes spelling error --- content/zh/docs/tasks/job/parallel-processing-expansion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/zh/docs/tasks/job/parallel-processing-expansion.md b/content/zh/docs/tasks/job/parallel-processing-expansion.md index 7dd67cb239..9e2c0a2c86 100644 --- a/content/zh/docs/tasks/job/parallel-processing-expansion.md +++ b/content/zh/docs/tasks/job/parallel-processing-expansion.md @@ -367,7 +367,7 @@ cat job.yaml.jinja2 | render_template | kubectl apply -f - -Kubernets 接收清单文件并执行你所创建的 Job。 +Kubernetes 接收清单文件并执行你所创建的 Job。 + ## デフォルトのhostsファイルの内容 From 1f42a309673888696e57d9e5d19df8af8044bcbd Mon Sep 17 00:00:00 2001 From: vaibhav Date: Thu, 1 Jul 2021 19:08:36 +0530 Subject: [PATCH 043/175] Add the links in docs/concepts/overview/working-with-objects/labels/ --- .../en/docs/concepts/overview/working-with-objects/labels.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/docs/concepts/overview/working-with-objects/labels.md b/content/en/docs/concepts/overview/working-with-objects/labels.md index 25eb5da66e..fe590402ae 100644 --- a/content/en/docs/concepts/overview/working-with-objects/labels.md +++ b/content/en/docs/concepts/overview/working-with-objects/labels.md @@ -42,7 +42,7 @@ Example labels: * `"partition" : "customerA"`, `"partition" : "customerB"` * `"track" : "daily"`, `"track" : "weekly"` -These are examples of commonly used labels; you are free to develop your own conventions. Keep in mind that label Key must be unique for a given object. +These are examples of [commonly used labels](/docs/concepts/overview/working-with-objects/common-labels/); you are free to develop your own conventions. Keep in mind that label Key must be unique for a given object. ## Syntax and character set @@ -50,7 +50,7 @@ _Labels_ are key/value pairs. Valid label keys have two segments: an optional pr If the prefix is omitted, the label Key is presumed to be private to the user. Automated system components (e.g. `kube-scheduler`, `kube-controller-manager`, `kube-apiserver`, `kubectl`, or other third-party automation) which add labels to end-user objects must specify a prefix. -The `kubernetes.io/` and `k8s.io/` prefixes are reserved for Kubernetes core components. +The `kubernetes.io/` and `k8s.io/` prefixes are [reserved](/docs/reference/labels-annotations-taints/) for Kubernetes core components. Valid label value: * must be 63 characters or less (can be empty), From 729414f65068d5e331f99d3853e0d68d13fd3f1f Mon Sep 17 00:00:00 2001 From: bhumijgupta Date: Thu, 1 Jul 2021 23:51:37 +0530 Subject: [PATCH 044/175] Add kubectl-convert install info Signed-off-by: bhumijgupta --- .../docs/tasks/tools/install-kubectl-linux.md | 45 +++++++++++++- .../docs/tasks/tools/install-kubectl-macos.md | 61 ++++++++++++++++++- .../tasks/tools/install-kubectl-windows.md | 31 +++++++++- 3 files changed, 134 insertions(+), 3 deletions(-) diff --git a/content/en/docs/tasks/tools/install-kubectl-linux.md b/content/en/docs/tasks/tools/install-kubectl-linux.md index cd04442614..ff8be80f19 100644 --- a/content/en/docs/tasks/tools/install-kubectl-linux.md +++ b/content/en/docs/tasks/tools/install-kubectl-linux.md @@ -41,7 +41,13 @@ For example, to download version {{< param "fullversion" >}} on Linux, type: ``` {{< /note >}} -1. Validate the binary (optional) + Download kubectl-convert (optional) + + ```bash + curl -LO https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl-convert + ``` + +1. Validate the kubectl binary (optional) Download the kubectl checksum file: @@ -72,12 +78,49 @@ For example, to download version {{< param "fullversion" >}} on Linux, type: Download the same version of the binary and checksum. {{< /note >}} +1. Validate the kubectl-convert binary (optional) + + Download the kubectl-convert checksum file: + + ```bash + curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl-convert.sha256" + ``` + + Validate the kubectl-convert binary against the checksum file: + + ```bash + echo "$(}} + Download the same version of the binary and checksum. + {{< /note >}} + 1. Install kubectl ```bash sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl ``` + Install kubectl-convert (optional) + + ```bash + sudo install -o root -g root -m 0755 kubectl-convert /usr/local/bin/kubectl-convert + ``` + {{< note >}} If you do not have root access on the target system, you can still install kubectl to the `~/.local/bin` directory: diff --git a/content/en/docs/tasks/tools/install-kubectl-macos.md b/content/en/docs/tasks/tools/install-kubectl-macos.md index 5bb3d918bb..b3231dc36f 100644 --- a/content/en/docs/tasks/tools/install-kubectl-macos.md +++ b/content/en/docs/tasks/tools/install-kubectl-macos.md @@ -36,6 +36,16 @@ The following methods exist for installing kubectl on macOS: {{< /tab >}} {{< /tabs >}} + Download kubectl-convert (optional) + {{< tabs name="download_binary_macos" >}} + {{< tab name="Intel" codelang="bash" >}} + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl-convert" + {{< /tab >}} + {{< tab name="Apple Silicon" codelang="bash" >}} + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/arm64/kubectl-convert" + {{< /tab >}} + {{< /tabs >}} + {{< note >}} To download a specific version, replace the `$(curl -L -s https://dl.k8s.io/release/stable.txt)` portion of the command with the specific version. @@ -53,7 +63,7 @@ The following methods exist for installing kubectl on macOS: {{< /note >}} -1. Validate the binary (optional) +1. Validate the kubectl binary (optional) Download the kubectl checksum file: @@ -89,12 +99,54 @@ The following methods exist for installing kubectl on macOS: Download the same version of the binary and checksum. {{< /note >}} +1. Validate the kubectl-convert binary (optional) + + Download the kubectl checksum file: + + {{< tabs name="download_checksum_macos" >}} + {{< tab name="Intel" codelang="bash" >}} + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl-convert.sha256" + {{< /tab >}} + {{< tab name="Apple Silicon" codelang="bash" >}} + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/arm64/kubectl-convert.sha256" + {{< /tab >}} + {{< /tabs >}} + + Validate the kubectl binary against the checksum file: + + ```bash + echo "$(}} + Download the same version of the binary and checksum. + {{< /note >}} + 1. Make the kubectl binary executable. ```bash chmod +x ./kubectl ``` + Make kubectl-convert binary executable (optional). + + ```bash + chmod +x ./kubectl-convert + ``` + 1. Move the kubectl binary to a file location on your system `PATH`. ```bash @@ -102,6 +154,13 @@ The following methods exist for installing kubectl on macOS: sudo chown root: /usr/local/bin/kubectl ``` + Move the kubectl-convert binary to your system `PATH` (optional) + + ```bash + sudo mv ./kubectl /usr/local/bin/kubectl-convert + sudo chown root: /usr/local/bin/kubectl-convert + ``` + {{< note >}} Make sure `/usr/local/bin` is in your PATH environment variable. {{< /note >}} diff --git a/content/en/docs/tasks/tools/install-kubectl-windows.md b/content/en/docs/tasks/tools/install-kubectl-windows.md index 45f7759df9..a02c861bfd 100644 --- a/content/en/docs/tasks/tools/install-kubectl-windows.md +++ b/content/en/docs/tasks/tools/install-kubectl-windows.md @@ -33,11 +33,17 @@ The following methods exist for installing kubectl on Windows: curl -LO https://dl.k8s.io/release/{{< param "fullversion" >}}/bin/windows/amd64/kubectl.exe ``` + Download kubectl-convert binary (optional) + + ```powershell + curl -LO https://dl.k8s.io/release/{{< param "fullversion" >}}/bin/windows/amd64/kubectl-convert.exe + ``` + {{< note >}} To find out the latest stable version (for example, for scripting), take a look at [https://dl.k8s.io/release/stable.txt](https://dl.k8s.io/release/stable.txt). {{< /note >}} -1. Validate the binary (optional) +1. Validate the kubectl binary (optional) Download the kubectl checksum file: @@ -60,6 +66,29 @@ The following methods exist for installing kubectl on Windows: $($(CertUtil -hashfile .\kubectl.exe SHA256)[1] -replace " ", "") -eq $(type .\kubectl.exe.sha256) ``` +1. Validate the kubectl-convert binary (optional) + + Download the kubectl-convert checksum file: + + ```powershell + curl -LO https://dl.k8s.io/{{< param "fullversion" >}}/bin/windows/amd64/kubectl-convert.exe.sha256 + ``` + + Validate the kubectl-convert binary against the checksum file: + + - Using Command Prompt to manually compare `CertUtil`'s output to the checksum file downloaded: + + ```cmd + CertUtil -hashfile kubectl-convert.exe SHA256 + type kubectl-convert.exe.sha256 + ``` + + - Using PowerShell to automate the verification using the `-eq` operator to get a `True` or `False` result: + + ```powershell + $($(CertUtil -hashfile .\kubectl-convert.exe SHA256)[1] -replace " ", "") -eq $(type .\kubectl-convert.exe.sha256) + ``` + 1. Add the binary in to your `PATH`. 1. Test to ensure the version of `kubectl` is the same as downloaded: From 34b701c10f3f60d6d3e6ae6fde50f8089512823f Mon Sep 17 00:00:00 2001 From: bhumijgupta Date: Thu, 1 Jul 2021 23:58:59 +0530 Subject: [PATCH 045/175] Add link to kubectl-convert in install tools footer Signed-off-by: bhumijgupta --- content/en/docs/tasks/tools/included/kubectl-whats-next.md | 1 + 1 file changed, 1 insertion(+) diff --git a/content/en/docs/tasks/tools/included/kubectl-whats-next.md b/content/en/docs/tasks/tools/included/kubectl-whats-next.md index 4b0da49bbc..c5d6839617 100644 --- a/content/en/docs/tasks/tools/included/kubectl-whats-next.md +++ b/content/en/docs/tasks/tools/included/kubectl-whats-next.md @@ -4,6 +4,7 @@ description: "What's next after installing kubectl." headless: true --- +* [What is kubectl-convert](/docs/reference/using-api/deprecation-guide/#migrate-to-non-deprecated-apis) * [Install Minikube](https://minikube.sigs.k8s.io/docs/start/) * See the [getting started guides](/docs/setup/) for more about creating clusters. * [Learn how to launch and expose your application.](/docs/tasks/access-application-cluster/service-access-application-cluster/) From 5f6f123db7adca12df802636cad07d103d8b9aa7 Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Fri, 2 Jul 2021 11:24:48 +0900 Subject: [PATCH 046/175] [ko] Fix nits --- content/ko/docs/concepts/workloads/pods/_index.md | 8 ++++---- content/ko/docs/setup/_index.md | 2 +- content/ko/docs/setup/best-practices/multiple-zones.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/content/ko/docs/concepts/workloads/pods/_index.md b/content/ko/docs/concepts/workloads/pods/_index.md index 8645053bef..22b98b705c 100644 --- a/content/ko/docs/concepts/workloads/pods/_index.md +++ b/content/ko/docs/concepts/workloads/pods/_index.md @@ -32,10 +32,10 @@ _파드_ (고래 떼(pod of whales)나 콩꼬투리(pea pod)와 마찬가지로) ## 파드란 무엇인가? {{< note >}} -[도커](https://www.docker.com/)가 가장 일반적으로 -잘 알려진 런타임이지만, 쿠버네티스는 도커보다 -{{< glossary_tooltip text="컨테이너 런타임" term_id="container-runtime" >}}을 -더 많이 지원하며, 도커의 일부 용어를 사용하면 파드를 설명하는 데 도움이 된다. +[도커](https://www.docker.com/)가 가장 일반적으로 잘 알려진 +{{< glossary_tooltip text="컨테이너 런타임" term_id="container-runtime" >}}이지만, +쿠버네티스는 도커 외에도 다양한 컨테이너 런타임을 지원하며, +파드를 설명할 때 도커 관련 용어를 사용하면 더 쉽게 설명할 수 있다. {{< /note >}} 파드의 공유 콘텍스트는 리눅스 네임스페이스, 컨트롤 그룹(cgroup) 및 diff --git a/content/ko/docs/setup/_index.md b/content/ko/docs/setup/_index.md index d6e1ea5a21..3c6013c590 100644 --- a/content/ko/docs/setup/_index.md +++ b/content/ko/docs/setup/_index.md @@ -58,6 +58,6 @@ card: - 클러스터 구성의 [모범 사례](/ko/docs/setup/best-practices/)를 확인한다. 쿠버네티스의 {{< glossary_tooltip term_id="control-plane" text="컨트롤 플레인" >}}은 -리눅스에서 실행되어야 한다. 클러스터 내에서는 리눅스 또는 +리눅스에서 실행되도록 설계되었다. 클러스터 내에서는 리눅스 또는 다른 운영 체제(예: 윈도우)에서 애플리케이션을 실행할 수 있다. - [윈도우 노드를 포함하는 클러스터 구성하기](/ko/docs/setup/production-environment/windows/)를 살펴본다. diff --git a/content/ko/docs/setup/best-practices/multiple-zones.md b/content/ko/docs/setup/best-practices/multiple-zones.md index 3d825ebd08..93ab353d37 100644 --- a/content/ko/docs/setup/best-practices/multiple-zones.md +++ b/content/ko/docs/setup/best-practices/multiple-zones.md @@ -55,7 +55,7 @@ content_type: concept 특정 kubelet을 나타내는 노드 오브젝트에 {{< glossary_tooltip text="레이블" term_id="label" >}}을 자동으로 추가한다. 이러한 레이블에는 -[영역 정보](/docs/reference/labels-annotations-taints/#topologykubernetesiozone)가 포함될 수 있다. +[영역 정보](/ko/docs/reference/labels-annotations-taints/#topologykubernetesiozone)가 포함될 수 있다. 클러스터가 여러 영역 또는 지역에 걸쳐있는 경우, [파드 토폴로지 분배 제약 조건](/ko/docs/concepts/workloads/pods/pod-topology-spread-constraints/)과 From d9f11f3b19af9f02208b5d688554cca5d1c031f8 Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Fri, 2 Jul 2021 11:48:07 +0900 Subject: [PATCH 047/175] [ko] Update links (http -> https) --- content/ko/_index.html | 2 +- content/ko/case-studies/box/index.html | 4 ++-- content/ko/docs/concepts/policy/pod-security-policy.md | 2 +- content/ko/docs/contribute/participate/_index.md | 2 +- content/ko/docs/tutorials/stateless-application/guestbook.md | 2 +- content/ko/includes/task-tutorial-prereqs.md | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/content/ko/_index.html b/content/ko/_index.html index 9db1ac0982..2b3d62cb44 100644 --- a/content/ko/_index.html +++ b/content/ko/_index.html @@ -10,7 +10,7 @@ sitemap: {{% blocks/feature image="flower" %}} K8s라고도 알려진 [쿠버네티스]({{< relref "/docs/concepts/overview/what-is-kubernetes" >}})는 컨테이너화된 애플리케이션을 자동으로 배포, 스케일링 및 관리해주는 오픈소스 시스템입니다. -애플리케이션을 구성하는 컨테이너들의 쉬운 관리 및 발견을 위해서 컨테이너들을 논리적인 단위로 그룹화합니다. 쿠버네티스는 [Google에서 15년간 프로덕션 워크로드 운영한 경험](http://queue.acm.org/detail.cfm?id=2898444)을 토대로 구축되었으며, 커뮤니티에서 제공한 최상의 아이디어와 방법들이 결합되어 있습니다. +애플리케이션을 구성하는 컨테이너들의 쉬운 관리 및 발견을 위해서 컨테이너들을 논리적인 단위로 그룹화합니다. 쿠버네티스는 [Google에서 15년간 프로덕션 워크로드 운영한 경험](https://queue.acm.org/detail.cfm?id=2898444)을 토대로 구축되었으며, 커뮤니티에서 제공한 최상의 아이디어와 방법들이 결합되어 있습니다. {{% /blocks/feature %}} {{% blocks/feature image="scalable" %}} diff --git a/content/ko/case-studies/box/index.html b/content/ko/case-studies/box/index.html index 058ff7f9a2..392e3d66bb 100644 --- a/content/ko/case-studies/box/index.html +++ b/content/ko/case-studies/box/index.html @@ -23,7 +23,7 @@ case_study_details:

Solution

-

Over the past couple of years, Box has been decomposing its infrastructure into microservices, and became an early adopter of, as well as contributor to, Kubernetes container orchestration. Kubernetes, Ghods says, has allowed Box's developers to "target a universal set of concepts that are portable across all clouds."

+

Over the past couple of years, Box has been decomposing its infrastructure into microservices, and became an early adopter of, as well as contributor to, Kubernetes container orchestration. Kubernetes, Ghods says, has allowed Box's developers to "target a universal set of concepts that are portable across all clouds."

Impact

@@ -37,7 +37,7 @@ case_study_details: In the summer of 2014, Box was feeling the pain of a decade's worth of hardware and software infrastructure that wasn't keeping up with the company's needs. {{< /case-studies/lead >}} -

A platform that allows its more than 50 million users (including governments and big businesses like General Electric) to manage and share content in the cloud, Box was originally a PHP monolith of millions of lines of code built exclusively with bare metal inside of its own data centers. It had already begun to slowly chip away at the monolith, decomposing it into microservices. And "as we've been expanding into regions around the globe, and as the public cloud wars have been heating up, we've been focusing a lot more on figuring out how we run our workload across many different environments and many different cloud infrastructure providers," says Box Cofounder and Services Architect Sam Ghods. "It's been a huge challenge thus far because of all these different providers, especially bare metal, have very different interfaces and ways in which you work with them."

+

A platform that allows its more than 50 million users (including governments and big businesses like General Electric) to manage and share content in the cloud, Box was originally a PHP monolith of millions of lines of code built exclusively with bare metal inside of its own data centers. It had already begun to slowly chip away at the monolith, decomposing it into microservices. And "as we've been expanding into regions around the globe, and as the public cloud wars have been heating up, we've been focusing a lot more on figuring out how we run our workload across many different environments and many different cloud infrastructure providers," says Box Cofounder and Services Architect Sam Ghods. "It's been a huge challenge thus far because of all these different providers, especially bare metal, have very different interfaces and ways in which you work with them."

Box's cloud native journey accelerated that June, when Ghods attended DockerCon. The company had come to the realization that it could no longer run its applications only off bare metal, and was researching containerizing with Docker, virtualizing with OpenStack, and supporting public cloud.

diff --git a/content/ko/docs/concepts/policy/pod-security-policy.md b/content/ko/docs/concepts/policy/pod-security-policy.md index eae69022e6..5941a0e13a 100644 --- a/content/ko/docs/concepts/policy/pod-security-policy.md +++ b/content/ko/docs/concepts/policy/pod-security-policy.md @@ -567,7 +567,7 @@ spec: 리눅스 기능은 전통적으로 슈퍼유저와 관련된 권한을 보다 세밀하게 분류한다. 이러한 기능 중 일부는 권한 에스컬레이션 또는 컨테이너 분류에 사용될 수 있으며 파드시큐리티폴리시에 의해 제한될 수 있다. 리눅스 기능에 대한 자세한 내용은 -[기능(7)](http://man7.org/linux/man-pages/man7/capabilities.7.html)을 +[기능(7)](https://man7.org/linux/man-pages/man7/capabilities.7.html)을 참고하길 바란다. 다음 필드는 대문자로 표기된 기능 이름 목록을 diff --git a/content/ko/docs/contribute/participate/_index.md b/content/ko/docs/contribute/participate/_index.md index ef271ca31c..46ae4e0d9c 100644 --- a/content/ko/docs/contribute/participate/_index.md +++ b/content/ko/docs/contribute/participate/_index.md @@ -94,7 +94,7 @@ PR 소유자에게 조언하는데 활용된다. ## 병합 작업 방식 풀 리퀘스트 요청이 콘텐츠를 발행하는데 사용하는 -브랜치에 병합되면, 해당 콘텐츠는 http://kubernetes.io 에 공개된다. 게시된 콘텐츠의 +브랜치에 병합되면, 해당 콘텐츠는 https://kubernetes.io 에 공개된다. 게시된 콘텐츠의 품질을 높히기 위해 SIG Docs 승인자가 풀 리퀘스트를 병합하는 것을 제한한다. 작동 방식은 다음과 같다. diff --git a/content/ko/docs/tutorials/stateless-application/guestbook.md b/content/ko/docs/tutorials/stateless-application/guestbook.md index 4a475563ba..0aea000cd7 100644 --- a/content/ko/docs/tutorials/stateless-application/guestbook.md +++ b/content/ko/docs/tutorials/stateless-application/guestbook.md @@ -269,7 +269,7 @@ Google Compute Engine 또는 Google Kubernetes Engine과 같은 일부 클라우 Forwarding from [::1]:8080 -> 80 ``` -1. 방명록을 보기위해 브라우저에서 [http://localhost:8080](http://localhost:8080) 페이지를 로드한다. +1. 방명록을 보기 위해 브라우저에서 [http://localhost:8080](http://localhost:8080) 페이지를 로드한다. ### `LoadBalancer`를 통해 프론트엔드 서비스 확인하기 diff --git a/content/ko/includes/task-tutorial-prereqs.md b/content/ko/includes/task-tutorial-prereqs.md index 65651286bd..e27f4b99e4 100644 --- a/content/ko/includes/task-tutorial-prereqs.md +++ b/content/ko/includes/task-tutorial-prereqs.md @@ -5,4 +5,4 @@ 다음의 쿠버네티스 플레이그라운드 중 하나를 사용할 수 있다. * [Katacoda](https://www.katacoda.com/courses/kubernetes/playground) -* [Play with Kubernetes](http://labs.play-with-k8s.com/) +* [Play with Kubernetes](https://labs.play-with-k8s.com/) From a7de73262a6b2fff90e5feebbfb636c3b5075a99 Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Fri, 2 Jul 2021 14:50:36 +0900 Subject: [PATCH 048/175] [ja] Update Netlify link address --- README-ja.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-ja.md b/README-ja.md index 49d0dd1bad..91e624c610 100644 --- a/README-ja.md +++ b/README-ja.md @@ -1,6 +1,6 @@ # Kubernetesのドキュメント -[![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-master-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest) +[![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-main-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest) このリポジトリには、[KubernetesのWebサイトとドキュメント](https://kubernetes.io/)をビルドするために必要な全アセットが格納されています。貢献に興味を持っていただきありがとうございます！ From 812e3fbb03f7764c5c3921cbc19cfb1a73bebb7d Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Fri, 2 Jul 2021 14:52:37 +0900 Subject: [PATCH 049/175] [ko] Update Netlify link address --- README-ko.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-ko.md b/README-ko.md index c4038212c6..c3e1068b2e 100644 --- a/README-ko.md +++ b/README-ko.md @@ -1,6 +1,6 @@ # 쿠버네티스 문서화 -[![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-master-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest) +[![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-main-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest) 이 저장소에는 [쿠버네티스 웹사이트 및 문서](https://kubernetes.io/)를 빌드하는 데 필요한 자산이 포함되어 있습니다. 기여해주셔서 감사합니다! From 4efba9b18a80bc32e5ee02b5e239ac72b8032987 Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Fri, 2 Jul 2021 14:54:34 +0900 Subject: [PATCH 050/175] [pl] Update Netlify link address --- README-pl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-pl.md b/README-pl.md index 5426aef445..ae25b89286 100644 --- a/README-pl.md +++ b/README-pl.md @@ -1,6 +1,6 @@ # Dokumentacja projektu Kubernetes -[![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-master-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest) +[![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-main-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest) W tym repozytorium znajdziesz wszystko, czego potrzebujesz do zbudowania [strony internetowej Kubernetesa wraz z dokumentacją](https://kubernetes.io/). Bardzo nam miło, że chcesz wziąć udział w jej współtworzeniu! From 0d6ae289a98d7337dd2ca7f54c437eb5243bc1b6 Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Fri, 2 Jul 2021 14:55:54 +0900 Subject: [PATCH 051/175] [pt] Update Netlify link address --- README-pt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-pt.md b/README-pt.md index e27bf544d1..d856bf7b42 100644 --- a/README-pt.md +++ b/README-pt.md @@ -1,6 +1,6 @@ # A documentação do Kubernetes -[![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-master-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest) +[![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-main-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest) Bem-vindos! Este repositório contém todos os recursos necessários para criar o [website e documentação do Kubernetes](https://kubernetes.io/). Estamos muito satisfeitos por você querer contribuir! From 956eb48e72d76d0dfa5290e95037d059e386562f Mon Sep 17 00:00:00 2001 From: bhumijgupta Date: Sat, 3 Jul 2021 09:36:30 +0530 Subject: [PATCH 052/175] Move kubectl-convert installation info under Optional kubectl configurations and plugins Signed-off-by: bhumijgupta --- .../included/kubectl-convert-overview.md | 9 ++ .../tools/included/kubectl-whats-next.md | 1 - .../docs/tasks/tools/install-kubectl-linux.md | 95 +++++++------ .../docs/tasks/tools/install-kubectl-macos.md | 131 ++++++++++-------- .../tasks/tools/install-kubectl-windows.md | 68 ++++----- 5 files changed, 166 insertions(+), 138 deletions(-) create mode 100644 content/en/docs/tasks/tools/included/kubectl-convert-overview.md diff --git a/content/en/docs/tasks/tools/included/kubectl-convert-overview.md b/content/en/docs/tasks/tools/included/kubectl-convert-overview.md new file mode 100644 index 0000000000..7490fc9832 --- /dev/null +++ b/content/en/docs/tasks/tools/included/kubectl-convert-overview.md @@ -0,0 +1,9 @@ +--- +title: "kubectl-convert overview" +description: "What is kubectl-convert plugin?" +headless: true +--- + +kubectl-convert is a kubectl plugin which allows to convert manifests between different api +versions. This can be particularly helpful to migrate manifests to a non-deprecated api version. +For more info, visit [migrate to non deprecated apis](/docs/reference/using-api/deprecation-guide/#migrate-to-non-deprecated-apis) \ No newline at end of file diff --git a/content/en/docs/tasks/tools/included/kubectl-whats-next.md b/content/en/docs/tasks/tools/included/kubectl-whats-next.md index c5d6839617..4b0da49bbc 100644 --- a/content/en/docs/tasks/tools/included/kubectl-whats-next.md +++ b/content/en/docs/tasks/tools/included/kubectl-whats-next.md @@ -4,7 +4,6 @@ description: "What's next after installing kubectl." headless: true --- -* [What is kubectl-convert](/docs/reference/using-api/deprecation-guide/#migrate-to-non-deprecated-apis) * [Install Minikube](https://minikube.sigs.k8s.io/docs/start/) * See the [getting started guides](/docs/setup/) for more about creating clusters. * [Learn how to launch and expose your application.](/docs/tasks/access-application-cluster/service-access-application-cluster/) diff --git a/content/en/docs/tasks/tools/install-kubectl-linux.md b/content/en/docs/tasks/tools/install-kubectl-linux.md index ff8be80f19..f771b9cb0c 100644 --- a/content/en/docs/tasks/tools/install-kubectl-linux.md +++ b/content/en/docs/tasks/tools/install-kubectl-linux.md @@ -41,13 +41,7 @@ For example, to download version {{< param "fullversion" >}} on Linux, type: ``` {{< /note >}} - Download kubectl-convert (optional) - - ```bash - curl -LO https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl-convert - ``` - -1. Validate the kubectl binary (optional) +1. Validate the binary (optional) Download the kubectl checksum file: @@ -78,49 +72,12 @@ For example, to download version {{< param "fullversion" >}} on Linux, type: Download the same version of the binary and checksum. {{< /note >}} -1. Validate the kubectl-convert binary (optional) - - Download the kubectl-convert checksum file: - - ```bash - curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl-convert.sha256" - ``` - - Validate the kubectl-convert binary against the checksum file: - - ```bash - echo "$(}} - Download the same version of the binary and checksum. - {{< /note >}} - 1. Install kubectl ```bash sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl ``` - Install kubectl-convert (optional) - - ```bash - sudo install -o root -g root -m 0755 kubectl-convert /usr/local/bin/kubectl-convert - ``` - {{< note >}} If you do not have root access on the target system, you can still install kubectl to the `~/.local/bin` directory: @@ -214,7 +171,7 @@ kubectl version --client {{< include "included/verify-kubectl.md" >}} -## Optional kubectl configurations +## Optional kubectl configurations and plugins ### Enable shell autocompletion @@ -227,6 +184,54 @@ Below are the procedures to set up autocompletion for Bash and Zsh. {{< tab name="Zsh" include="included/optional-kubectl-configs-zsh.md" />}} {{< /tabs >}} +### Install kubectl convert plugin + +{{< include "included/kubectl-convert-overview.md" >}} + +1. Download the latest release with the command: + + ```bash + curl -LO https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl-convert + ``` + +1. Validate the binary (optional) + + Download the kubectl-convert checksum file: + + ```bash + curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl-convert.sha256" + ``` + + Validate the kubectl-convert binary against the checksum file: + + ```bash + echo "$(}} + Download the same version of the binary and checksum. + {{< /note >}} + +1. Install kubectl-convert + + ```bash + sudo install -o root -g root -m 0755 kubectl-convert /usr/local/bin/kubectl-convert + ``` + + ## {{% heading "whatsnext" %}} {{< include "included/kubectl-whats-next.md" >}} diff --git a/content/en/docs/tasks/tools/install-kubectl-macos.md b/content/en/docs/tasks/tools/install-kubectl-macos.md index b3231dc36f..fed72a6dbd 100644 --- a/content/en/docs/tasks/tools/install-kubectl-macos.md +++ b/content/en/docs/tasks/tools/install-kubectl-macos.md @@ -36,16 +36,6 @@ The following methods exist for installing kubectl on macOS: {{< /tab >}} {{< /tabs >}} - Download kubectl-convert (optional) - {{< tabs name="download_binary_macos" >}} - {{< tab name="Intel" codelang="bash" >}} - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl-convert" - {{< /tab >}} - {{< tab name="Apple Silicon" codelang="bash" >}} - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/arm64/kubectl-convert" - {{< /tab >}} - {{< /tabs >}} - {{< note >}} To download a specific version, replace the `$(curl -L -s https://dl.k8s.io/release/stable.txt)` portion of the command with the specific version. @@ -63,7 +53,7 @@ The following methods exist for installing kubectl on macOS: {{< /note >}} -1. Validate the kubectl binary (optional) +1. Validate the binary (optional) Download the kubectl checksum file: @@ -99,54 +89,12 @@ The following methods exist for installing kubectl on macOS: Download the same version of the binary and checksum. {{< /note >}} -1. Validate the kubectl-convert binary (optional) - - Download the kubectl checksum file: - - {{< tabs name="download_checksum_macos" >}} - {{< tab name="Intel" codelang="bash" >}} - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl-convert.sha256" - {{< /tab >}} - {{< tab name="Apple Silicon" codelang="bash" >}} - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/arm64/kubectl-convert.sha256" - {{< /tab >}} - {{< /tabs >}} - - Validate the kubectl binary against the checksum file: - - ```bash - echo "$(}} - Download the same version of the binary and checksum. - {{< /note >}} - 1. Make the kubectl binary executable. ```bash chmod +x ./kubectl ``` - Make kubectl-convert binary executable (optional). - - ```bash - chmod +x ./kubectl-convert - ``` - 1. Move the kubectl binary to a file location on your system `PATH`. ```bash @@ -154,13 +102,6 @@ The following methods exist for installing kubectl on macOS: sudo chown root: /usr/local/bin/kubectl ``` - Move the kubectl-convert binary to your system `PATH` (optional) - - ```bash - sudo mv ./kubectl /usr/local/bin/kubectl-convert - sudo chown root: /usr/local/bin/kubectl-convert - ``` - {{< note >}} Make sure `/usr/local/bin` is in your PATH environment variable. {{< /note >}} @@ -214,7 +155,7 @@ If you are on macOS and using [Macports](https://macports.org/) package manager, {{< include "included/verify-kubectl.md" >}} -## Optional kubectl configurations +## Optional kubectl configurations and plugins ### Enable shell autocompletion @@ -227,6 +168,74 @@ Below are the procedures to set up autocompletion for Bash and Zsh. {{< tab name="Zsh" include="included/optional-kubectl-configs-zsh.md" />}} {{< /tabs >}} +### Install kubectl convert plugin + +{{< include "included/kubectl-convert-overview.md" >}} + +1. Download the latest release with the command: + + {{< tabs name="download_convert_binary_macos" >}} + {{< tab name="Intel" codelang="bash" >}} + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl-convert" + {{< /tab >}} + {{< tab name="Apple Silicon" codelang="bash" >}} + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/arm64/kubectl-convert" + {{< /tab >}} + {{< /tabs >}} + +1. Validate the binary (optional) + + Download the kubectl checksum file: + + {{< tabs name="download_convert_checksum_macos" >}} + {{< tab name="Intel" codelang="bash" >}} + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl-convert.sha256" + {{< /tab >}} + {{< tab name="Apple Silicon" codelang="bash" >}} + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/arm64/kubectl-convert.sha256" + {{< /tab >}} + {{< /tabs >}} + + Validate the kubectl-convert binary against the checksum file: + + ```bash + echo "$(}} + Download the same version of the binary and checksum. + {{< /note >}} + +1. Make kubectl-convert binary executable + + ```bash + chmod +x ./kubectl-convert + ``` + +1. Move the kubectl-convert binary to a file location on your system `PATH`. + + ```bash + sudo mv ./kubectl /usr/local/bin/kubectl-convert + sudo chown root: /usr/local/bin/kubectl-convert + ``` + + {{< note >}} + Make sure `/usr/local/bin` is in your PATH environment variable. + {{< /note >}} + ## {{% heading "whatsnext" %}} {{< include "included/kubectl-whats-next.md" >}} diff --git a/content/en/docs/tasks/tools/install-kubectl-windows.md b/content/en/docs/tasks/tools/install-kubectl-windows.md index a02c861bfd..956eb50a61 100644 --- a/content/en/docs/tasks/tools/install-kubectl-windows.md +++ b/content/en/docs/tasks/tools/install-kubectl-windows.md @@ -33,17 +33,11 @@ The following methods exist for installing kubectl on Windows: curl -LO https://dl.k8s.io/release/{{< param "fullversion" >}}/bin/windows/amd64/kubectl.exe ``` - Download kubectl-convert binary (optional) - - ```powershell - curl -LO https://dl.k8s.io/release/{{< param "fullversion" >}}/bin/windows/amd64/kubectl-convert.exe - ``` - {{< note >}} To find out the latest stable version (for example, for scripting), take a look at [https://dl.k8s.io/release/stable.txt](https://dl.k8s.io/release/stable.txt). {{< /note >}} -1. Validate the kubectl binary (optional) +1. Validate the binary (optional) Download the kubectl checksum file: @@ -66,29 +60,6 @@ The following methods exist for installing kubectl on Windows: $($(CertUtil -hashfile .\kubectl.exe SHA256)[1] -replace " ", "") -eq $(type .\kubectl.exe.sha256) ``` -1. Validate the kubectl-convert binary (optional) - - Download the kubectl-convert checksum file: - - ```powershell - curl -LO https://dl.k8s.io/{{< param "fullversion" >}}/bin/windows/amd64/kubectl-convert.exe.sha256 - ``` - - Validate the kubectl-convert binary against the checksum file: - - - Using Command Prompt to manually compare `CertUtil`'s output to the checksum file downloaded: - - ```cmd - CertUtil -hashfile kubectl-convert.exe SHA256 - type kubectl-convert.exe.sha256 - ``` - - - Using PowerShell to automate the verification using the `-eq` operator to get a `True` or `False` result: - - ```powershell - $($(CertUtil -hashfile .\kubectl-convert.exe SHA256)[1] -replace " ", "") -eq $(type .\kubectl-convert.exe.sha256) - ``` - 1. Add the binary in to your `PATH`. 1. Test to ensure the version of `kubectl` is the same as downloaded: @@ -159,7 +130,7 @@ Edit the config file with a text editor of your choice, such as Notepad. {{< include "included/verify-kubectl.md" >}} -## Optional kubectl configurations +## Optional kubectl configurations and plugins ### Enable shell autocompletion @@ -169,6 +140,41 @@ Below are the procedures to set up autocompletion for Zsh, if you are running th {{< include "included/optional-kubectl-configs-zsh.md" >}} +### Install kubectl-convert plugin + +{{< include "included/kubectl-convert-overview.md" >}} + +1. Download the latest release with the command: + + ```powershell + curl -LO https://dl.k8s.io/release/{{< param "fullversion" >}}/bin/windows/amd64/kubectl-convert.exe + ``` + +1. Validate the binary (optional) + + Download the kubectl-convert checksum file: + + ```powershell + curl -LO https://dl.k8s.io/{{< param "fullversion" >}}/bin/windows/amd64/kubectl-convert.exe.sha256 + ``` + + Validate the kubectl-convert binary against the checksum file: + + - Using Command Prompt to manually compare `CertUtil`'s output to the checksum file downloaded: + + ```cmd + CertUtil -hashfile kubectl-convert.exe SHA256 + type kubectl-convert.exe.sha256 + ``` + + - Using PowerShell to automate the verification using the `-eq` operator to get a `True` or `False` result: + + ```powershell + $($(CertUtil -hashfile .\kubectl-convert.exe SHA256)[1] -replace " ", "") -eq $(type .\kubectl-convert.exe.sha256) + ``` + +1. Add the binary in to your `PATH`. + ## {{% heading "whatsnext" %}} {{< include "included/kubectl-whats-next.md" >}} From 85adb64d7288cd90026245f64260619404fbda3a Mon Sep 17 00:00:00 2001 From: Nitesh Seram Date: Mon, 5 Jul 2021 08:56:43 +0530 Subject: [PATCH 053/175] fix link issue --- content/ko/_index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/ko/_index.html b/content/ko/_index.html index 2b3d62cb44..c6350f1559 100644 --- a/content/ko/_index.html +++ b/content/ko/_index.html @@ -48,7 +48,7 @@ Google이 일주일에 수십억 개의 컨테이너들을 운영하게 해준


- Revisit KubeCon EU 2021 + Attend KubeCon Europe on May 17-20, 2022
From 15b64da3ddc9880d446d9c62bb1ddfd6728e340d Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Mon, 5 Jul 2021 16:27:39 +0900 Subject: [PATCH 054/175] [ko] Update outdated files in dev-1.21-ko.6 (p2) --- .../ko/docs/concepts/architecture/nodes.md | 36 ++++++++++++++----- content/ko/docs/concepts/storage/volumes.md | 31 ++++++++++++---- .../pods/pod-topology-spread-constraints.md | 14 ++++---- .../generate-ref-docs/quickstart.md | 4 +-- .../docs/contribute/new-content/open-a-pr.md | 18 +++++----- .../docs/contribute/new-content/overview.md | 4 +-- .../ko/docs/contribute/participate/_index.md | 4 +-- .../contribute/participate/pr-wranglers.md | 4 +-- .../participate/roles-and-responsibilities.md | 4 +-- .../feature-gates.md | 3 +- 10 files changed, 81 insertions(+), 41 deletions(-) diff --git a/content/ko/docs/concepts/architecture/nodes.md b/content/ko/docs/concepts/architecture/nodes.md index 5bba08100e..3ab89472d8 100644 --- a/content/ko/docs/concepts/architecture/nodes.md +++ b/content/ko/docs/concepts/architecture/nodes.md @@ -1,4 +1,7 @@ --- + + + title: 노드 content_type: concept weight: 10 @@ -8,7 +11,8 @@ weight: 10 쿠버네티스는 컨테이너를 파드내에 배치하고 _노드_ 에서 실행함으로 워크로드를 구동한다. 노드는 클러스터에 따라 가상 또는 물리적 머신일 수 있다. 각 노드는 -{{< glossary_tooltip text="컨트롤 플레인" term_id="control-plane" >}}에 의해 관리되며 +{{< glossary_tooltip text="컨트롤 플레인" term_id="control-plane" >}}에 +의해 관리되며 {{< glossary_tooltip text="파드" term_id="pod" >}}를 실행하는 데 필요한 서비스를 포함한다. @@ -272,17 +276,18 @@ kubelet은 `NodeStatus` 와 리스 오브젝트를 생성하고 업데이트 할 #### 안정성 대부분의 경우, 노드 컨트롤러는 초당 `--node-eviction-rate`(기본값 0.1)로 -축출 비율을 제한한다. 이 말은 10초당 1개의 노드를 초과하여 +축출 속도를 제한한다. 이 말은 10초당 1개의 노드를 초과하여 파드 축출을 하지 않는다는 의미가 된다. 노드 축출 행위는 주어진 가용성 영역 내 하나의 노드가 상태가 불량할 경우 변화한다. 노드 컨트롤러는 영역 내 동시에 상태가 불량한 노드의 퍼센티지가 얼마나 되는지 체크한다(NodeReady 컨디션은 ConditionUnknown 또는 -ConditionFalse 다.). -- 상태가 불량한 노드의 일부가 최소 `--unhealthy-zone-threshold` - (기본값 0.55)가 되면 축출 비율은 감소한다. +ConditionFalse 다). +- 상태가 불량한 노드의 비율이 최소 `--unhealthy-zone-threshold` + (기본값 0.55)가 되면 축출 속도가 감소한다. - 클러스터가 작으면 (즉 `--large-cluster-size-threshold` - 노드 이하면 - 기본값 50) 축출은 중지되고, 그렇지 않으면 축출 비율은 초당 + 노드 이하면 - 기본값 50) 축출이 중지된다. +- 이외의 경우, 축출 속도는 초당 `--secondary-node-eviction-rate`(기본값 0.01)로 감소된다. 이 정책들이 가용성 영역 단위로 실행되어지는 이유는 나머지가 연결되어 있는 동안 @@ -293,7 +298,7 @@ ConditionFalse 다.). 노드가 가용성 영역들에 걸쳐 퍼져 있는 주된 이유는 하나의 전체 영역이 장애가 발생할 경우 워크로드가 상태 양호한 영역으로 이전되어질 수 있도록 하기 위해서이다. 그러므로, 하나의 영역 내 모든 노드들이 상태가 불량하면 노드 컨트롤러는 -`--node-eviction-rate` 의 정상 비율로 축출한다. 코너 케이스란 모든 영역이 +`--node-eviction-rate` 의 정상 속도로 축출한다. 코너 케이스란 모든 영역이 완전히 상태불량 (즉 클러스터 내 양호한 노드가 없는 경우) 한 경우이다. 이러한 경우, 노드 컨트롤러는 마스터 연결에 문제가 있어 일부 연결이 복원될 때까지 모든 축출을 중지하는 것으로 여긴다. @@ -347,7 +352,8 @@ Kubelet은 노드가 종료되는 동안 파드가 일반 [파드 종료 프로 사용하여 주어진 기간 동안 노드 종료를 지연시키므로 systemd에 의존한다. 그레이스풀 노드 셧다운은 1.21에서 기본적으로 활성화된 `GracefulNodeShutdown` -[기능 게이트](/ko/docs/reference/command-line-tools-reference/feature-gates/)로 제어된다. +[기능 게이트](/ko/docs/reference/command-line-tools-reference/feature-gates/)로 +제어된다. 기본적으로, 아래 설명된 두 구성 옵션, `ShutdownGracePeriod` 및 `ShutdownGracePeriodCriticalPods` 는 모두 0으로 설정되어 있으므로, @@ -371,6 +377,20 @@ Kubelet은 노드가 종료되는 동안 파드가 일반 [파드 종료 프로 유예 종료에 할당되고, 마지막 10초는 [중요 파드](/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/#marking-pod-as-critical)의 종료에 할당된다. +{{< note >}} +그레이스풀 노드 셧다운 과정에서 축출된 파드는 `Failed` 라고 표시된다. +`kubectl get pods` 명령을 실행하면 축출된 파드의 상태가 `Shutdown`으로 표시된다. +그리고 `kubectl describe pod` 명령을 실행하면 노드 셧다운으로 인해 파드가 축출되었음을 알 수 있다. + +``` +Status: Failed +Reason: Shutdown +Message: Node is shutting, evicting pods +``` + +실패한 파드 오브젝트는 명시적으로 삭제하거나 [가비지 콜렉션에 의해 정리](/ko/docs/concepts/workloads/pods/pod-lifecycle/#pod-garbage-collection)되기 전까지는 보존된다. +이는 갑작스러운 노드 종료의 경우와 비교했을 때 동작에 차이가 있다. +{{< /note >}} ## {{% heading "whatsnext" %}} diff --git a/content/ko/docs/concepts/storage/volumes.md b/content/ko/docs/concepts/storage/volumes.md index 3bdfead48d..29f4755172 100644 --- a/content/ko/docs/concepts/storage/volumes.md +++ b/content/ko/docs/concepts/storage/volumes.md @@ -1,4 +1,9 @@ --- + + + + + title: 볼륨 content_type: concept weight: 10 @@ -13,7 +18,6 @@ weight: 10 파일을 공유할 때 발생한다. 쿠버네티스 {{< glossary_tooltip text="볼륨" term_id="volume" >}} 추상화는 이러한 문제를 모두 해결한다. - [파드](/ko/docs/concepts/workloads/pods/)에 대해 익숙해지는 것을 추천한다. @@ -40,7 +44,6 @@ weight: 10 볼륨을 사용하려면, `.spec.volumes` 에서 파드에 제공할 볼륨을 지정하고 `.spec.containers[*].volumeMounts` 의 컨테이너에 해당 볼륨을 마운트할 위치를 선언한다. - 컨테이너의 프로세스는 도커 이미지와 볼륨으로 구성된 파일시스템 뷰를 본다. [도커 이미지](https://docs.docker.com/userguide/dockerimages/)는 파일시스템 계층의 루트에 있다. 볼륨은 이미지 내에 지정된 경로에 @@ -117,6 +120,7 @@ EBS 볼륨이 파티션된 경우, 선택적 필드인 `partition: "}} 컨트롤러 관리자와 kubelet에 의해 로드되지 않도록 `awsElasticBlockStore` 스토리지 @@ -257,6 +261,9 @@ spec: `path` 에서 파생된다. {{< note >}} +* [컨피그맵](/docs/tasks/configure-pod-container/configure-pod-configmap/)을 사용하기 위해서는 + 먼저 컨피그맵을 생성해야 한다. + * 컨피그맵을 [`subPath`](#subpath-사용하기) 볼륨 마운트로 사용하는 컨테이너는 컨피그맵 업데이트를 수신하지 않는다. @@ -522,6 +529,15 @@ glusterfs 볼륨에 데이터를 미리 채울 수 있으며, 파드 간에 데 ### hostPath {#hostpath} +{{< warning >}} +HostPath 볼륨에는 많은 보안 위험이 있으며, 가능하면 HostPath를 사용하지 않는 +것이 좋다. HostPath 볼륨을 사용해야 하는 경우, 필요한 파일 또는 디렉터리로만 +범위를 지정하고 ReadOnly로 마운트해야 한다. + +AdmissionPolicy를 사용하여 특정 디렉터리로의 HostPath 액세스를 제한하는 경우, +`readOnly` 마운트를 사용하는 정책이 유효하려면 `volumeMounts` 가 반드시 지정되어야 한다. +{{< /warning >}} + `hostPath` 볼륨은 호스트 노드의 파일시스템에 있는 파일이나 디렉터리를 파드에 마운트 한다. 이것은 대부분의 파드들이 필요한 것은 아니지만, 일부 애플리케이션에 강력한 탈출구를 제공한다. @@ -538,7 +554,6 @@ glusterfs 볼륨에 데이터를 미리 채울 수 있으며, 파드 간에 데 필드가 `type` 에 지원되는 값은 다음과 같다. - | 값 | 행동 | |:------|:---------| | | 빈 문자열 (기본값)은 이전 버전과의 호환성을 위한 것으로, hostPath 볼륨은 마운트 하기 전에 아무런 검사도 수행되지 않는다. | @@ -552,6 +567,9 @@ glusterfs 볼륨에 데이터를 미리 채울 수 있으며, 파드 간에 데 다음과 같은 이유로 이 유형의 볼륨 사용시 주의해야 한다. +* HostPath는 권한있는 시스템 자격 증명 (예 : Kubelet 용) 또는 권한있는 API + (예 : 컨테이너 런타임 소켓)를 노출 할 수 있으며, 이는 컨테이너 이스케이프 또는 + 클러스터의 다른 부분을 공격하는 데 사용될 수 있다. * 동일한 구성(파드템플릿으로 생성한 것과 같은)을 가진 파드는 노드에 있는 파일이 다르기 때문에 노드마다 다르게 동작할 수 있다. * 기본 호스트에 생성된 파일 또는 디렉터리는 root만 쓸 수 있다. @@ -909,7 +927,8 @@ API 서버에 대해 `--service-account-max-token-expiration` 옵션을 지정 상대 경로를 지정한다. {{< note >}} -projected 볼륨 소스를 [`subPath`](#subpath-사용하기) 볼륨으로 마운트해서 사용하는 컨테이너는 해당 볼륨 소스의 업데이트를 수신하지 않는다. +projected 볼륨 소스를 [`subPath`](#subpath-사용하기) 볼륨으로 마운트해서 사용하는 컨테이너는 +해당 볼륨 소스의 업데이트를 수신하지 않는다. {{< /note >}} ### quobyte @@ -1103,7 +1122,6 @@ vmware-vdiskmanager -c -t 0 -s 40GB -a lsilogic myDisk.vmdk {{< /tabs >}} - #### vSphere VMDK 구성 예시 {#vsphere-vmdk-configuration} ```yaml @@ -1133,8 +1151,7 @@ spec: {{< feature-state for_k8s_version="v1.19" state="beta" >}} `vsphereVolume` 용 `CSIMigration` 기능이 활성화되면, 기존 인-트리 플러그인에서 -`csi.vsphere.vmware.com` {{< glossary_tooltip text="CSI" term_id="csi" >}} 드라이버로 모든 플러그인 작업을 리디렉션한다. -이 기능을 사용하려면, +`csi.vsphere.vmware.com` {{< glossary_tooltip text="CSI" term_id="csi" >}} 드라이버로 모든 플러그인 작업을 리디렉션한다. 이 기능을 사용하려면, [vSphere CSI 드라이버](https://github.com/kubernetes-sigs/vsphere-csi-driver)가 클러스터에 설치되어야 하며 `CSIMigration` 및 `CSIMigrationvSphere` [기능 게이트](/ko/docs/reference/command-line-tools-reference/feature-gates/)가 활성화되어 있어야 한다. diff --git a/content/ko/docs/concepts/workloads/pods/pod-topology-spread-constraints.md b/content/ko/docs/concepts/workloads/pods/pod-topology-spread-constraints.md index 3c30e895b6..304471bb28 100644 --- a/content/ko/docs/concepts/workloads/pods/pod-topology-spread-constraints.md +++ b/content/ko/docs/concepts/workloads/pods/pod-topology-spread-constraints.md @@ -82,12 +82,11 @@ spec: 사용자는 하나 또는 다중 `topologySpreadConstraint` 를 정의해서 kube-scheduler 에게 클러스터에 걸쳐 있는 기존 파드와 시작하는 각각의 파드와 연관하여 배치하는 방법을 명령할 수 있다. 필드는 다음과 같다. - **maxSkew** 는 파드가 균등하지 않게 분산될 수 있는 정도를 나타낸다. - 이것은 주어진 토폴로지 유형의 임의의 두 토폴로지 도메인에 일치하는 - 파드의 수 사이에서 허용되는 차이의 최댓값이다. 이것은 0보다는 커야 - 한다. 그 의미는 `whenUnsatisfiable` 의 값에 따라 다르다. + 이것은 0보다는 커야 한다. 그 의미는 `whenUnsatisfiable` 의 값에 따라 다르다. - `whenUnsatisfiable` 이 "DoNotSchedule"과 같을 때, `maxSkew` 는 - 대상 토폴로지에서 일치하는 파드 수와 전역 최솟값 사이에 - 허용되는 최대 차이이다. + 대상 토폴로지에서 일치하는 파드 수와 전역 최솟값 + (토폴로지 도메인에서 레이블 셀렉터와 일치하는 최소 파드 수. 예를 들어 3개의 영역에 각각 0, 2, 3개의 일치하는 파드가 있으면, 전역 최솟값은 0) + 사이에 허용되는 최대 차이이다. - `whenUnsatisfiable` 이 "ScheduleAnyway"와 같으면, 스케줄러는 왜곡을 줄이는데 도움이 되는 토폴로지에 더 높은 우선 순위를 부여한다. - **topologyKey** 는 노드 레이블의 키다. 만약 두 노드가 이 키로 레이블이 지정되고, 레이블이 동일한 값을 가진다면 스케줄러는 두 노드를 같은 토폴로지에 있는것으로 여기게 된다. 스케줄러는 각 토폴로지 도메인에 균형잡힌 수의 파드를 배치하려고 시도한다. @@ -96,6 +95,8 @@ spec: - `ScheduleAnyway` 는 스케줄러에게 차이(skew)를 최소화하는 노드에 높은 우선 순위를 부여하면서, 스케줄링을 계속하도록 지시한다. - **labelSelector** 는 일치하는 파드를 찾는데 사용된다. 이 레이블 셀렉터와 일치하는 파드의 수를 계산하여 해당 토폴로지 도메인에 속할 파드의 수를 결정한다. 자세한 내용은 [레이블 셀렉터](/ko/docs/concepts/overview/working-with-objects/labels/#레이블-셀렉터)를 참조한다. +파드에 2개 이상의 `topologySpreadConstraint`가 정의되어 있으면, 각 제약 조건은 AND로 연결된다 - kube-scheduler는 새로운 파드의 모든 제약 조건을 만족하는 노드를 찾는다. + 사용자는 `kubectl explain Pod.spec.topologySpreadConstraints` 를 실행해서 이 필드에 대한 자세한 내용을 알 수 있다. ### 예시: 단수 토폴로지 분배 제약 조건 @@ -387,7 +388,8 @@ profiles: ## 알려진 제한사항 -- 디플로이먼트를 스케일링 다운하면 그 결과로 파드의 분포가 불균형이 될 수 있다. +- 파드가 제거된 이후에도 제약 조건이 계속 충족된다는 보장은 없다. 예를 들어 디플로이먼트를 스케일링 다운하면 그 결과로 파드의 분포가 불균형해질 수 있다. +[Descheduler](https://github.com/kubernetes-sigs/descheduler)를 사용하여 파드 분포를 다시 균형있게 만들 수 있다. - 파드와 일치하는 테인트(taint)가 된 노드가 존중된다. [이슈 80921](https://github.com/kubernetes/kubernetes/issues/80921)을 본다. ## {{% heading "whatsnext" %}} diff --git a/content/ko/docs/contribute/generate-ref-docs/quickstart.md b/content/ko/docs/contribute/generate-ref-docs/quickstart.md index 6e3fbb5263..6855696b9d 100644 --- a/content/ko/docs/contribute/generate-ref-docs/quickstart.md +++ b/content/ko/docs/contribute/generate-ref-docs/quickstart.md @@ -18,7 +18,7 @@ weight: 40 ## `website` 저장소 클론하기 {#Getting-the-docs-repository} -개인 계정에 있는 포크 버전의 `website` 저장소가 `kubernetes/website` 저장소의 master 브랜치만큼 최신인지 확인한 뒤, +개인 계정에 있는 포크 버전의 `website` 저장소가 GitHub에 있는 `kubernetes/website` 저장소(`main` 브랜치)의 최신 상태와 일치하는지 확인한 뒤, 개인 계정에 있는 포크 버전의 `website` 저장소를 로컬 개발 환경으로 클론한다. ```shell @@ -171,7 +171,7 @@ cd /update-imported-docs `release.yml` 환경설정 파일은 상대경로 링크를 수정하는 방법을 포함하고 있다. 임포트하는 파일 안에 있는 상대경로 링크를 수정하려면, `gen-absolute-links` 필드를 `true` 로 명시한다. 이에 대한 예시는 -[`release.yml`](https://github.com/kubernetes/website/blob/master/update-imported-docs/release.yml) 에서 볼 수 있다. +[`release.yml`](https://github.com/kubernetes/website/blob/main/update-imported-docs/release.yml) 에서 볼 수 있다. ## `kubernetes/website` 의 변경사항을 커밋하기 {#Adding-and-committing-changes-in-kubernetes-website} diff --git a/content/ko/docs/contribute/new-content/open-a-pr.md b/content/ko/docs/contribute/new-content/open-a-pr.md index 552a6e1a0c..a1f0178b3c 100644 --- a/content/ko/docs/contribute/new-content/open-a-pr.md +++ b/content/ko/docs/contribute/new-content/open-a-pr.md @@ -127,7 +127,7 @@ git에 익숙하거나, 변경 사항이 몇 줄보다 클 경우, upstream https://github.com/kubernetes/website.git (push) ``` -6. 포크의 `origin/master` 와 `kubernetes/website` 의 `upstream/master` 에서 커밋을 가져온다. +6. 포크의 `origin/main` 와 `kubernetes/website` 의 `upstream/main` 에서 커밋을 가져온다. ```bash git fetch origin @@ -137,15 +137,15 @@ git에 익숙하거나, 변경 사항이 몇 줄보다 클 경우, 이를 통해 변경을 시작하기 전에 로컬 리포지터리가 최신 상태인지 확인한다. {{< note >}} - 이 워크플로는 [쿠버네티스 커뮤니티 GitHub 워크플로](https://github.com/kubernetes/community/blob/master/contributors/guide/github-workflow.md)와 다르다. 포크에 업데이트를 푸시하기 전에 로컬의 `master` 복사본을 `upstream/master` 와 병합할 필요가 없다. + 이 워크플로는 [쿠버네티스 커뮤니티 GitHub 워크플로](https://github.com/kubernetes/community/blob/master/contributors/guide/github-workflow.md)와 다르다. 포크에 업데이트를 푸시하기 전에 로컬의 `main` 복사본을 `upstream/main` 와 병합할 필요가 없다. {{< /note >}} ### 브랜치 만들기 1. 작업할 브랜치 기반을 결정한다. - - 기존 콘텐츠를 개선하려면, `upstream/master` 를 사용한다. - - 기존 기능에 대한 새로운 콘텐츠를 작성하려면, `upstream/master` 를 사용한다. + - 기존 콘텐츠를 개선하려면, `upstream/main` 를 사용한다. + - 기존 기능에 대한 새로운 콘텐츠를 작성하려면, `upstream/main` 를 사용한다. - 현지화된 콘텐츠의 경우, 현지화 규칙을 사용한다. 자세한 내용은 [쿠버네티스 문서 현지화](/ko/docs/contribute/localization_ko/)를 참고한다. - 다가오는 쿠버네티스 릴리스의 새로운 기능에 대해서는 기능 브랜치(feature branch)를 사용한다. 자세한 정보는 [릴리스 문서화](/docs/contribute/new-content/new-features/)를 참고한다. - 콘텐츠 재구성과 같이 여러 SIG Docs 기여자들이 협업하는 장기적인 작업에는, @@ -154,10 +154,10 @@ git에 익숙하거나, 변경 사항이 몇 줄보다 클 경우, 브랜치 선택에 도움이 필요하면, 슬랙 채널 `#sig-docs` 에 문의한다. -2. 1단계에서 식별된 브랜치를 기반으로 새 브랜치를 작성한다. 이 예에서는 기본 브랜치가 `upstream/master` 라고 가정한다. +2. 1단계에서 식별된 브랜치를 기반으로 새 브랜치를 작성한다. 이 예에서는 기본 브랜치가 `upstream/main` 라고 가정한다. ```bash - git checkout -b upstream/master + git checkout -b upstream/main ``` 3. 텍스트 편집기를 사용하여 변경한다. @@ -264,7 +264,7 @@ website의 컨테이너 이미지를 만들거나 Hugo를 로컬에서 실행할 또는, 컴퓨터에 `hugo` 명령을 설치하여 사용한다. -1. [`website/netlify.toml`](https://raw.githubusercontent.com/kubernetes/website/master/netlify.toml)에 지정된 [Hugo](https://gohugo.io/getting-started/installing/) 버전을 설치한다. +1. [`website/netlify.toml`](https://raw.githubusercontent.com/kubernetes/website/main/netlify.toml)에 지정된 [Hugo](https://gohugo.io/getting-started/installing/) 버전을 설치한다. 2. website 리포지터리를 업데이트하지 않았다면, `website/themes/docsy` 디렉터리가 비어 있다. 테마의 로컬 복제본이 없으면 사이트를 빌드할 수 없다. website 테마를 업데이트하려면, 다음을 실행한다. @@ -372,11 +372,11 @@ PR을 연 후, GitHub는 자동 테스트를 실행하고 [Netlify](https://www. git push --force-with-lease origin ``` -2. `kubernetes/website` 의 `upstream/master` 에 대한 변경 사항을 가져오고 브랜치를 리베이스한다. +2. `kubernetes/website` 의 `upstream/main` 에 대한 변경 사항을 가져오고 브랜치를 리베이스한다. ```bash git fetch upstream - git rebase upstream/master + git rebase upstream/main ``` 3. 리베이스의 결과를 검사한다. diff --git a/content/ko/docs/contribute/new-content/overview.md b/content/ko/docs/contribute/new-content/overview.md index 5c8f6569da..540f1e6c12 100644 --- a/content/ko/docs/contribute/new-content/overview.md +++ b/content/ko/docs/contribute/new-content/overview.md @@ -42,7 +42,7 @@ CLA에 서명하지 않은 기여자의 풀 리퀘스트(pull request)는 자동 시나리오 | 브랜치 :---------|:------------ -현재 릴리스의 기존 또는 새로운 영어 콘텐츠 | `master` +현재 릴리스의 기존 또는 새로운 영어 콘텐츠 | `main` 기능 변경 릴리스의 콘텐츠 | `dev-` 패턴을 사용하여 기능 변경이 있는 주 버전과 부 버전에 해당하는 브랜치. 예를 들어, `v{{< skew nextMinorVersion >}}` 에서 기능이 변경된 경우, ``dev-{{< skew nextMinorVersion >}}`` 에 문서 변경을 추가한다. 다른 언어로된 콘텐츠(현지화) | 현지화 규칙을 사용. 자세한 내용은 [현지화 브랜치 전략](/docs/contribute/localization/#branching-strategy)을 참고한다. @@ -60,6 +60,6 @@ PR 당 하나의 언어로 풀 리퀘스트를 제한한다. 여러 언어로 ## 기여자를 위한 도구들 -`kubernetes/website` 리포지터리의 [문서 기여자를 위한 도구](https://github.com/kubernetes/website/tree/master/content/en/docs/doc-contributor-tools) 디렉터리에는 기여 여정이 좀 더 순조롭게 진행되도록 도와주는 도구들이 포함되어 있다. +`kubernetes/website` 리포지터리의 [문서 기여자를 위한 도구](https://github.com/kubernetes/website/tree/main/content/en/docs/doc-contributor-tools) 디렉터리에는 기여 여정이 좀 더 순조롭게 진행되도록 도와주는 도구들이 포함되어 있다. diff --git a/content/ko/docs/contribute/participate/_index.md b/content/ko/docs/contribute/participate/_index.md index 46ae4e0d9c..c2d9aed771 100644 --- a/content/ko/docs/contribute/participate/_index.md +++ b/content/ko/docs/contribute/participate/_index.md @@ -73,8 +73,8 @@ GitHub의 SIG Docs [팀]에는 두 분류가 있다. - approve 이 두 플러그인은 `kubernetes/website` GitHub 리포지터리 최상위 수준에 있는 -[OWNERS](https://github.com/kubernetes/website/blob/master/OWNERS)와 -[OWNERS_ALIASES](https://github.com/kubernetes/website/blob/master/OWNERS_ALIASES) +[OWNERS](https://github.com/kubernetes/website/blob/main/OWNERS)와 +[OWNERS_ALIASES](https://github.com/kubernetes/website/blob/main/OWNERS_ALIASES) 파일을 사용해서 해당 리포지터리에 대해 prow가 작동하는 방식을 제어한다. diff --git a/content/ko/docs/contribute/participate/pr-wranglers.md b/content/ko/docs/contribute/participate/pr-wranglers.md index f3333890d2..674696ee90 100644 --- a/content/ko/docs/contribute/participate/pr-wranglers.md +++ b/content/ko/docs/contribute/participate/pr-wranglers.md @@ -45,8 +45,8 @@ PR 랭글러는 일주일 간 매일 다음의 일을 해야 한다. 지정한다. 콘텐츠에 대한 작업이 필요하다면, 제안하거나 인라인 피드백을 추가한다. - [LGTM 보유, 문서 승인 필요](https://github.com/kubernetes/website/pulls?q=is%3Aopen+is%3Apr+-label%3Ado-not-merge%2Fwork-in-progress+-label%3Ado-not-merge%2Fhold+label%3Alanguage%2Fen+label%3Algtm+): 병합을 위해 `/approve` 코멘트가 필요한 PR을 나열한다. -- [퀵윈(Quick Wins)](https://github.com/kubernetes/website/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+base%3Amaster+-label%3A%22do-not-merge%2Fwork-in-progress%22+-label%3A%22do-not-merge%2Fhold%22+label%3A%22cncf-cla%3A+yes%22+label%3A%22size%2FXS%22+label%3A%22language%2Fen%22): 명확한 결격 사유가 없는 메인 브랜치에 대한 PR을 나열한다. ([XS, S, M, L, XL, XXL] 크기의 PR을 작업할 때 크기 레이블에서 "XS"를 변경한다) -- [메인 브랜치이외의 브랜치에 대한 PR](https://github.com/kubernetes/website/pulls?q=is%3Aopen+is%3Apr+label%3Alanguage%2Fen+-base%3Amaster): `dev-` 브랜치에 대한 것일 경우, 곧 출시될 예정인 릴리스이다. `/assign @` 을 사용하여 [문서 릴리스 관리자](https://github.com/kubernetes/sig-release/tree/master/release-team#kubernetes-release-team-roles)를 할당한다. 오래된 브랜치에 대한 PR인 경우, PR 작성자가 가장 적합한 브랜치를 대상으로 하고 있는지 여부를 파악할 수 있도록 도와준다. +- [퀵윈(Quick Wins)](https://github.com/kubernetes/website/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+base%3Amain+-label%3A%22do-not-merge%2Fwork-in-progress%22+-label%3A%22do-not-merge%2Fhold%22+label%3A%22cncf-cla%3A+yes%22+label%3A%22size%2FXS%22+label%3A%22language%2Fen%22): 명확한 결격 사유가 없는 메인 브랜치에 대한 PR을 나열한다. ([XS, S, M, L, XL, XXL] 크기의 PR을 작업할 때 크기 레이블에서 "XS"를 변경한다) +- [메인 브랜치이외의 브랜치에 대한 PR](https://github.com/kubernetes/website/pulls?q=is%3Aopen+is%3Apr+label%3Alanguage%2Fen+-base%3Amain): `dev-` 브랜치에 대한 것일 경우, 곧 출시될 예정인 릴리스이다. `/assign @` 을 사용하여 [문서 릴리스 관리자](https://github.com/kubernetes/sig-release/tree/master/release-team#kubernetes-release-team-roles)를 할당한다. 오래된 브랜치에 대한 PR인 경우, PR 작성자가 가장 적합한 브랜치를 대상으로 하고 있는지 여부를 파악할 수 있도록 도와준다. ### 랭글러를 위한 유용한 Prow 명령어 diff --git a/content/ko/docs/contribute/participate/roles-and-responsibilities.md b/content/ko/docs/contribute/participate/roles-and-responsibilities.md index 897d638435..c8e32e2a77 100644 --- a/content/ko/docs/contribute/participate/roles-and-responsibilities.md +++ b/content/ko/docs/contribute/participate/roles-and-responsibilities.md @@ -144,7 +144,7 @@ LGTM은 "Looks good to me"의 약자이며 풀 리퀘스트가 기술적으로 지원하려면, 다음을 수행한다. 1. `kubernetes/website` 리포지터리 내 - [OWNERS_ALIASES](https://github.com/kubernetes/website/blob/master/OWNERS) 파일의 섹션에 + [OWNERS_ALIASES](https://github.com/kubernetes/website/blob/main/OWNERS) 파일의 섹션에 여러분의 GitHub 사용자 이름을 추가하는 풀 리퀘스트를 연다. {{< note >}} @@ -216,7 +216,7 @@ PR은 자동으로 병합된다. SIG Docs 승인자는 추가적인 기술 리 지원하려면 다음을 수행한다. 1. `kubernetes/website` 리포지터리 내 - [OWNERS_ALIASES](https://github.com/kubernetes/website/blob/master/OWNERS) + [OWNERS_ALIASES](https://github.com/kubernetes/website/blob/main/OWNERS) 파일의 섹션에 자신을 추가하는 풀 리퀘스트를 연다. {{< note >}} diff --git a/content/ko/docs/reference/command-line-tools-reference/feature-gates.md b/content/ko/docs/reference/command-line-tools-reference/feature-gates.md index aef90d8db5..e5e6f1570d 100644 --- a/content/ko/docs/reference/command-line-tools-reference/feature-gates.md +++ b/content/ko/docs/reference/command-line-tools-reference/feature-gates.md @@ -152,7 +152,8 @@ kubelet과 같은 컴포넌트의 기능 게이트를 설정하려면, 기능 | `ProbeTerminationGracePeriod` | `false` | 알파 | 1.21 | | | `ProcMountType` | `false` | 알파 | 1.12 | | | `QOSReserved` | `false` | 알파 | 1.11 | | -| `RemainingItemCount` | `false` | 알파 | 1.15 | | +| `RemainingItemCount` | `false` | 알파 | 1.15 | 1.15 | +| `RemainingItemCount` | `true` | 베타 | 1.16 | | | `RemoveSelfLink` | `false` | 알파 | 1.16 | 1.19 | | `RemoveSelfLink` | `true` | 베타 | 1.20 | | | `RotateKubeletServerCertificate` | `false` | 알파 | 1.7 | 1.11 | From 20eea3b75b149c4af6f8971bb04bda9d7e5d56e8 Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Mon, 5 Jul 2021 18:31:09 +0900 Subject: [PATCH 055/175] Update links --- .../docs/concepts/scheduling-eviction/taint-and-toleration.md | 2 +- content/en/docs/reference/glossary/api-eviction.md | 2 +- content/en/docs/reference/glossary/kube-scheduler.md | 2 +- static/_redirects | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/content/en/docs/concepts/scheduling-eviction/taint-and-toleration.md b/content/en/docs/concepts/scheduling-eviction/taint-and-toleration.md index 710be78c88..2f9b4d3f06 100644 --- a/content/en/docs/concepts/scheduling-eviction/taint-and-toleration.md +++ b/content/en/docs/concepts/scheduling-eviction/taint-and-toleration.md @@ -267,7 +267,7 @@ This ensures that DaemonSet pods are never evicted due to these problems. ## Taint Nodes by Condition The control plane, using the node {{}}, -automatically creates taints with a `NoSchedule` effect for [node conditions](/docs/concepts/scheduling-eviction/pod-eviction#node-conditions). +automatically creates taints with a `NoSchedule` effect for [node conditions](/docs/concepts/scheduling-eviction/node-pressure-eviction/). The scheduler checks taints, not node conditions, when it makes scheduling decisions. This ensures that node conditions don't directly affect scheduling. diff --git a/content/en/docs/reference/glossary/api-eviction.md b/content/en/docs/reference/glossary/api-eviction.md index b13238c955..69fc9d9b0c 100644 --- a/content/en/docs/reference/glossary/api-eviction.md +++ b/content/en/docs/reference/glossary/api-eviction.md @@ -2,7 +2,7 @@ title: API-initiated eviction id: api-eviction date: 2021-04-27 -full_link: /docs/concepts/scheduling-eviction/pod-eviction/#api-eviction +full_link: /docs/concepts/scheduling-eviction/api-eviction/ short_description: > API-initiated eviction is the process by which you use the Eviction API to create an Eviction object that triggers graceful pod termination. diff --git a/content/en/docs/reference/glossary/kube-scheduler.md b/content/en/docs/reference/glossary/kube-scheduler.md index a1a91a1527..96fc11a71d 100644 --- a/content/en/docs/reference/glossary/kube-scheduler.md +++ b/content/en/docs/reference/glossary/kube-scheduler.md @@ -2,7 +2,7 @@ title: kube-scheduler id: kube-scheduler date: 2018-04-12 -full_link: /docs/reference/generated/kube-scheduler/ +full_link: /docs/reference/command-line-tools-reference/kube-scheduler/ short_description: > Control plane component that watches for newly created pods with no assigned node, and selects a node for them to run on. diff --git a/static/_redirects b/static/_redirects index daaa22d497..ab803fc378 100644 --- a/static/_redirects +++ b/static/_redirects @@ -128,6 +128,7 @@ /docs/concepts/scheduling/scheduling-framework/ /docs/concepts/scheduling-eviction/scheduling-framework/ 301 /id/docs/concepts/scheduling/scheduling-framework/ /id/docs/concepts/scheduling-eviction/scheduling-framework/ 301 /docs/concepts/scheduling-eviction/eviction-policy/ /docs/concepts/scheduling-eviction/node-pressure-eviction/ 301 +/docs/concepts/scheduling-eviction/pod-eviction/ /docs/concepts/scheduling-eviction/ 301 /docs/concepts/service-catalog/ /docs/concepts/extend-kubernetes/service-catalog/ 301 /docs/concepts/services-networking/networkpolicies/ /docs/concepts/services-networking/network-policies/ 301 /docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ /docs/tasks/network/customize-hosts-file-for-pods/ 301 From 437e2a74f8516d40e564ffb4437b8bcdd47d17ff Mon Sep 17 00:00:00 2001 From: bhumijgupta Date: Mon, 5 Jul 2021 17:15:04 +0530 Subject: [PATCH 056/175] Add info to verify successfull installation and address other suggestions Signed-off-by: bhumijgupta --- .../tasks/tools/included/kubectl-convert-overview.md | 8 +++++--- content/en/docs/tasks/tools/install-kubectl-linux.md | 9 ++++++++- content/en/docs/tasks/tools/install-kubectl-macos.md | 10 +++++++++- content/en/docs/tasks/tools/install-kubectl-windows.md | 10 +++++++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/content/en/docs/tasks/tools/included/kubectl-convert-overview.md b/content/en/docs/tasks/tools/included/kubectl-convert-overview.md index 7490fc9832..b1799d52ea 100644 --- a/content/en/docs/tasks/tools/included/kubectl-convert-overview.md +++ b/content/en/docs/tasks/tools/included/kubectl-convert-overview.md @@ -1,9 +1,11 @@ --- title: "kubectl-convert overview" -description: "What is kubectl-convert plugin?" +description: >- + A kubectl plugin that allows you to convert manifests from one version + of a Kubernetes API to a different version. headless: true --- -kubectl-convert is a kubectl plugin which allows to convert manifests between different api -versions. This can be particularly helpful to migrate manifests to a non-deprecated api version. +A plugin for Kubernetes command-line tool `kubectl`, which allows you to convert manifests between different API +versions. This can be particularly helpful to migrate manifests to a non-deprecated api version with newer Kubernetes release. For more info, visit [migrate to non deprecated apis](/docs/reference/using-api/deprecation-guide/#migrate-to-non-deprecated-apis) \ No newline at end of file diff --git a/content/en/docs/tasks/tools/install-kubectl-linux.md b/content/en/docs/tasks/tools/install-kubectl-linux.md index f771b9cb0c..099e678a56 100644 --- a/content/en/docs/tasks/tools/install-kubectl-linux.md +++ b/content/en/docs/tasks/tools/install-kubectl-linux.md @@ -184,7 +184,7 @@ Below are the procedures to set up autocompletion for Bash and Zsh. {{< tab name="Zsh" include="included/optional-kubectl-configs-zsh.md" />}} {{< /tabs >}} -### Install kubectl convert plugin +### Install `kubectl convert` plugin {{< include "included/kubectl-convert-overview.md" >}} @@ -231,6 +231,13 @@ Below are the procedures to set up autocompletion for Bash and Zsh. sudo install -o root -g root -m 0755 kubectl-convert /usr/local/bin/kubectl-convert ``` +1. Verify plugin is successfully installed + + ```shell + kubectl convert --help + ``` + + If you do not see an error, it means the plugin is successfully installed. ## {{% heading "whatsnext" %}} diff --git a/content/en/docs/tasks/tools/install-kubectl-macos.md b/content/en/docs/tasks/tools/install-kubectl-macos.md index fed72a6dbd..2b214f6396 100644 --- a/content/en/docs/tasks/tools/install-kubectl-macos.md +++ b/content/en/docs/tasks/tools/install-kubectl-macos.md @@ -168,7 +168,7 @@ Below are the procedures to set up autocompletion for Bash and Zsh. {{< tab name="Zsh" include="included/optional-kubectl-configs-zsh.md" />}} {{< /tabs >}} -### Install kubectl convert plugin +### Install `kubectl convert` plugin {{< include "included/kubectl-convert-overview.md" >}} @@ -236,6 +236,14 @@ Below are the procedures to set up autocompletion for Bash and Zsh. Make sure `/usr/local/bin` is in your PATH environment variable. {{< /note >}} +1. Verify plugin is successfully installed + + ```shell + kubectl convert --help + ``` + + If you do not see an error, it means the plugin is successfully installed. + ## {{% heading "whatsnext" %}} {{< include "included/kubectl-whats-next.md" >}} diff --git a/content/en/docs/tasks/tools/install-kubectl-windows.md b/content/en/docs/tasks/tools/install-kubectl-windows.md index 956eb50a61..ef9f4e5815 100644 --- a/content/en/docs/tasks/tools/install-kubectl-windows.md +++ b/content/en/docs/tasks/tools/install-kubectl-windows.md @@ -140,7 +140,7 @@ Below are the procedures to set up autocompletion for Zsh, if you are running th {{< include "included/optional-kubectl-configs-zsh.md" >}} -### Install kubectl-convert plugin +### Install `kubectl convert` plugin {{< include "included/kubectl-convert-overview.md" >}} @@ -175,6 +175,14 @@ Below are the procedures to set up autocompletion for Zsh, if you are running th 1. Add the binary in to your `PATH`. +1. Verify plugin is successfully installed + + ```shell + kubectl convert --help + ``` + + If you do not see an error, it means the plugin is successfully installed. + ## {{% heading "whatsnext" %}} {{< include "included/kubectl-whats-next.md" >}} From 1fac17f5320ac0420ca12385cc87a8abc6b4a88c Mon Sep 17 00:00:00 2001 From: Qiming Teng Date: Fri, 25 Jun 2021 12:23:06 +0800 Subject: [PATCH 057/175] Remove extra horizontal line and spacing --- layouts/docs/baseof.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/layouts/docs/baseof.html b/layouts/docs/baseof.html index ec0b4d575a..cc6237938b 100644 --- a/layouts/docs/baseof.html +++ b/layouts/docs/baseof.html @@ -25,7 +25,7 @@ {{ if (and (not .Params.hide_feedback) (.Site.Params.ui.feedback.enable) (.Site.GoogleAnalytics)) }} {{ partial "feedback.html" .Site.Params.ui.feedback }} {{ end }} -
{{ partial "page-meta-lastmod.html" . }}
+ {{ partial "page-meta-lastmod.html" . }} {{ if (.Site.DisqusShortname) }}
{{ partial "disqus-comment.html" . }} @@ -40,4 +40,4 @@
{{ partial "scripts.html" . }} - \ No newline at end of file + From f7c96a4c00f2da51cb071da2250fa11a8b05cbb8 Mon Sep 17 00:00:00 2001 From: Shubham Kuchhal Date: Tue, 6 Jul 2021 12:54:30 +0530 Subject: [PATCH 058/175] Mention Security Embargo Policy. --- content/en/releases/security-embargo-policy.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 content/en/releases/security-embargo-policy.md diff --git a/content/en/releases/security-embargo-policy.md b/content/en/releases/security-embargo-policy.md new file mode 100644 index 0000000000..917ddb9700 --- /dev/null +++ b/content/en/releases/security-embargo-policy.md @@ -0,0 +1,6 @@ +--- +title: Security Embargo Policy +type: docs +--- + +Some information about releases is subject to embargo and we have defined policy about how those embargos are set. Please refer [Security Embargo Policy](https://github.com/kubernetes/security/blob/master/private-distributors-list.md#embargo-policy) here for more information. \ No newline at end of file From 3de9fc63e46b070babeef6d598f434d77fbcffe6 Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Tue, 6 Jul 2021 19:14:51 +0100 Subject: [PATCH 059/175] Fix outdated / incorrect links --- content/id/docs/contribute/localization_id.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/id/docs/contribute/localization_id.md b/content/id/docs/contribute/localization_id.md index 5a9c491297..a9be38700e 100644 --- a/content/id/docs/contribute/localization_id.md +++ b/content/id/docs/contribute/localization_id.md @@ -107,7 +107,7 @@ dapat menemukan kata-kata tersebut dalam bahasa Indonesia. ### Panduan untuk kata-kata API Objek Kubernetes Gunakan gaya "CamelCase" untuk menulis objek API Kubernetes, lihat daftar -lengkapnya [di sini](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/). +lengkapnya [di sini](/docs/reference/kubernetes-api/). Sebagai contoh: * *Benar*: PersistentVolume. *Salah*: volume persisten, `PersistentVolume`, @@ -130,7 +130,7 @@ ditulis dalam huruf kapital pada halaman asli bahasa Inggris. ### Panduan untuk "Feature Gate" Kubernetes -Istilah [_functional gate_](https://kubernetes.io/ko/docs/reference/command-line-tools-reference/feature-gates/) +Istilah [_functional gate_](/docs/reference/command-line-tools-reference/feature-gates/) Kubernetes tidak perlu diterjemahkan ke dalam bahasa Indonesia dan tetap dipertahankan dalam bentuk aslinya. From 081c5fa7aa3bef0b9e4a3f5519f05594de932ba3 Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Wed, 7 Jul 2021 11:43:10 +0100 Subject: [PATCH 060/175] =?UTF-8?q?Use=20correct=20original=20term=20for?= =?UTF-8?q?=20=E2=80=9Cfeature=20gate=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jihoon Seo <46767780+jihoon-seo@users.noreply.github.com> --- content/id/docs/contribute/localization_id.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/id/docs/contribute/localization_id.md b/content/id/docs/contribute/localization_id.md index a9be38700e..beffa5be6c 100644 --- a/content/id/docs/contribute/localization_id.md +++ b/content/id/docs/contribute/localization_id.md @@ -130,7 +130,7 @@ ditulis dalam huruf kapital pada halaman asli bahasa Inggris. ### Panduan untuk "Feature Gate" Kubernetes -Istilah [_functional gate_](/docs/reference/command-line-tools-reference/feature-gates/) +Istilah [_feature gate_](/docs/reference/command-line-tools-reference/feature-gates/) Kubernetes tidak perlu diterjemahkan ke dalam bahasa Indonesia dan tetap dipertahankan dalam bentuk aslinya. @@ -175,4 +175,4 @@ scale | | skala | | process | kata kerja | memproses | https://kbbi.web.id/proses | replica | kata benda | replika | https://kbbi.web.id/replika | flag | | tanda, parameter, argumen | | -event | | _event_ | | \ No newline at end of file +event | | _event_ | | From 74ac7a96f661669c60bcd55fe9b50ab1d31f7b79 Mon Sep 17 00:00:00 2001 From: howieyuen Date: Thu, 8 Jul 2021 16:09:08 +0800 Subject: [PATCH 061/175] [zh] translate analytics.md in contribute section --- content/zh/docs/contribute/analytics.md | 51 +++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 content/zh/docs/contribute/analytics.md diff --git a/content/zh/docs/contribute/analytics.md b/content/zh/docs/contribute/analytics.md new file mode 100644 index 0000000000..b6175a2670 --- /dev/null +++ b/content/zh/docs/contribute/analytics.md @@ -0,0 +1,51 @@ +--- +title: 查看站点分析 +content_type: concept +weight: 100 +card: + name: contribute + weight: 100 +--- + + + + + + +此页面包含有关 kubernetes.io 分析仪表板的信息。 + + + + +[查看仪表板](https://datastudio.google.com/reporting/fede2672-b2fd-402a-91d2-7473bdb10f04)。 + +此仪表板使用 Google Data Studio 构建，显示使用 Google Analytics 在 kubernetes.io 上收集的信息。 + + +### 使用仪表板 + +默认情况下，仪表板显示过去 30 天收集的所有分析。 +使用日期选择器查看来自不同日期范围的数据。 +其他过滤选项允许你根据用户位置、用于访问站点的设备、所用文档的翻译等查看数据。 + +如果你发现此仪表板存在问题，或者想要请求任何改进， +请[开启一个问题](https://github.com/kubernetes/website/issues/new/choose)。 From 374bb0547acbf4da5336de916c49657a6a94f3ef Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Fri, 9 Jul 2021 11:02:29 +0900 Subject: [PATCH 062/175] Update links (2) --- ...2018-08-03-make-kubernetes-production-grade-anywhere.md | 2 +- .../scheduling-eviction/pod-priority-preemption.md | 6 +++--- .../concepts/scheduling-eviction/taint-and-toleration.md | 6 +++--- content/en/docs/concepts/workloads/pods/disruptions.md | 2 +- static/_redirects | 7 ++++--- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/content/en/blog/_posts/2018-08-03-make-kubernetes-production-grade-anywhere.md b/content/en/blog/_posts/2018-08-03-make-kubernetes-production-grade-anywhere.md index a786475a67..329b2c4de7 100644 --- a/content/en/blog/_posts/2018-08-03-make-kubernetes-production-grade-anywhere.md +++ b/content/en/blog/_posts/2018-08-03-make-kubernetes-production-grade-anywhere.md @@ -104,7 +104,7 @@ Master and Worker nodes should be protected from overload and resource exhaustio Resource consumption by the control plane will correlate with the number of pods and the pod churn rate. Very large and very small clusters will benefit from non-default [settings](/docs/reference/command-line-tools-reference/kube-apiserver/) of kube-apiserver request throttling and memory. Having these too high can lead to request limit exceeded and out of memory errors. -On worker nodes, [Node Allocatable](/docs/tasks/administer-cluster/reserve-compute-resources/) should be configured based on a reasonable supportable workload density at each node. Namespaces can be created to subdivide the worker node cluster into multiple virtual clusters with resource CPU and memory [quotas](/docs/tasks/administer-cluster/manage-resources/memory-default-namespace/). Kubelet handling of [out of resource](/docs/tasks/administer-cluster/out-of-resource/) conditions can be configured. +On worker nodes, [Node Allocatable](/docs/tasks/administer-cluster/reserve-compute-resources/) should be configured based on a reasonable supportable workload density at each node. Namespaces can be created to subdivide the worker node cluster into multiple virtual clusters with resource CPU and memory [quotas](/docs/tasks/administer-cluster/manage-resources/memory-default-namespace/). Kubelet handling of [out of resource](/docs/concepts/scheduling-eviction/node-pressure-eviction/) conditions can be configured. ## Security diff --git a/content/en/docs/concepts/scheduling-eviction/pod-priority-preemption.md b/content/en/docs/concepts/scheduling-eviction/pod-priority-preemption.md index fe49f2acd4..fff925b6c5 100644 --- a/content/en/docs/concepts/scheduling-eviction/pod-priority-preemption.md +++ b/content/en/docs/concepts/scheduling-eviction/pod-priority-preemption.md @@ -353,7 +353,7 @@ the removal of the lowest priority Pods is not sufficient to allow the scheduler to schedule the preemptor Pod, or if the lowest priority Pods are protected by `PodDisruptionBudget`. -The kubelet uses Priority to determine pod order for [out-of-resource eviction](/docs/tasks/administer-cluster/out-of-resource/). +The kubelet uses Priority to determine pod order for [node-pressure eviction](/docs/concepts/scheduling-eviction/node-pressure-eviction/). You can use the QoS class to estimate the order in which pods are most likely to get evicted. The kubelet ranks pods for eviction based on the following factors: @@ -361,10 +361,10 @@ to get evicted. The kubelet ranks pods for eviction based on the following facto 1. Pod Priority 1. Amount of resource usage relative to requests -See [evicting end-user pods](/docs/tasks/administer-cluster/out-of-resource/#evicting-end-user-pods) +See [Pod selection for kubelet eviction](/docs/concepts/scheduling-eviction/node-pressure-eviction/#pod-selection-for-kubelet-eviction) for more details. -kubelet out-of-resource eviction does not evict Pods when their +kubelet node-pressure eviction does not evict Pods when their usage does not exceed their requests. If a Pod with lower priority is not exceeding its requests, it won't be evicted. Another Pod with higher priority that exceeds its requests may be evicted. diff --git a/content/en/docs/concepts/scheduling-eviction/taint-and-toleration.md b/content/en/docs/concepts/scheduling-eviction/taint-and-toleration.md index 2f9b4d3f06..030f28e7d1 100644 --- a/content/en/docs/concepts/scheduling-eviction/taint-and-toleration.md +++ b/content/en/docs/concepts/scheduling-eviction/taint-and-toleration.md @@ -267,7 +267,7 @@ This ensures that DaemonSet pods are never evicted due to these problems. ## Taint Nodes by Condition The control plane, using the node {{}}, -automatically creates taints with a `NoSchedule` effect for [node conditions](/docs/concepts/scheduling-eviction/node-pressure-eviction/). +automatically creates taints with a `NoSchedule` effect for [node conditions](/docs/concepts/scheduling-eviction/node-pressure-eviction/#node-conditions). The scheduler checks taints, not node conditions, when it makes scheduling decisions. This ensures that node conditions don't directly affect scheduling. @@ -298,7 +298,7 @@ arbitrary tolerations to DaemonSets. ## {{% heading "whatsnext" %}} -* Read about [out of resource handling](/docs/concepts/scheduling-eviction/out-of-resource/) and how you can configure it -* Read about [pod priority](/docs/concepts/scheduling-eviction/pod-priority-preemption/) +* Read about [Node-pressure Eviction](/docs/concepts/scheduling-eviction/node-pressure-eviction/) and how you can configure it +* Read about [Pod Priority](/docs/concepts/scheduling-eviction/pod-priority-preemption/) diff --git a/content/en/docs/concepts/workloads/pods/disruptions.md b/content/en/docs/concepts/workloads/pods/disruptions.md index 5836a11fb4..430d4b7155 100644 --- a/content/en/docs/concepts/workloads/pods/disruptions.md +++ b/content/en/docs/concepts/workloads/pods/disruptions.md @@ -31,7 +31,7 @@ an application. Examples are: - cloud provider or hypervisor failure makes VM disappear - a kernel panic - the node disappears from the cluster due to cluster network partition -- eviction of a pod due to the node being [out-of-resources](/docs/tasks/administer-cluster/out-of-resource/). +- eviction of a pod due to the node being [out-of-resources](/docs/concepts/scheduling-eviction/node-pressure-eviction/). Except for the out-of-resources condition, all these conditions should be familiar to most users; they are not specific diff --git a/static/_redirects b/static/_redirects index ab803fc378..9e1012a932 100644 --- a/static/_redirects +++ b/static/_redirects @@ -57,7 +57,7 @@ /docs/admin/node-conformance.md /docs/admin/node-conformance/ 301 /docs/admin/node-conformance/ /docs/setup/best-practices/node-conformance/ 301 /docs/admin/node-problem/ /docs/tasks/debug-application-cluster/monitor-node-health/ 301 -/docs/admin/out-of-resource/ /docs/tasks/administer-cluster/out-of-resource/ 301 +/docs/admin/out-of-resource/ /docs/concepts/scheduling-eviction/node-pressure-eviction/ 301 /docs/admin/rescheduler/ /docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ 301 /docs/admin/resourcequota/* /docs/concepts/policy/resource-quotas/ 301 /docs/admin/resourcequota/limitstorageconsumption/ /docs/tasks/administer-cluster/limit-storage-consumption/ 301 @@ -128,7 +128,8 @@ /docs/concepts/scheduling/scheduling-framework/ /docs/concepts/scheduling-eviction/scheduling-framework/ 301 /id/docs/concepts/scheduling/scheduling-framework/ /id/docs/concepts/scheduling-eviction/scheduling-framework/ 301 /docs/concepts/scheduling-eviction/eviction-policy/ /docs/concepts/scheduling-eviction/node-pressure-eviction/ 301 -/docs/concepts/scheduling-eviction/pod-eviction/ /docs/concepts/scheduling-eviction/ 301 +/docs/concepts/scheduling-eviction/out-of-resource/ /docs/concepts/scheduling-eviction/node-pressure-eviction/ 301 +/docs/concepts/scheduling-eviction/pod-eviction/ /docs/concepts/scheduling-eviction/#pod-disruption 301 /docs/concepts/service-catalog/ /docs/concepts/extend-kubernetes/service-catalog/ 301 /docs/concepts/services-networking/networkpolicies/ /docs/concepts/services-networking/network-policies/ 301 /docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ /docs/tasks/network/customize-hosts-file-for-pods/ 301 @@ -265,7 +266,7 @@ /docs/tasks/administer-cluster/overview/ /docs/concepts/cluster-administration/ 301 /docs/tasks/administer-cluster/quota-memory-cpu-namespace/ /docs/tasks/administer-cluster/manage-resources/quota-memory-cpu-namespace/ 301 /docs/tasks/administer-cluster/quota-pod-namespace/ /docs/tasks/administer-cluster/manage-resources/quota-pod-namespace/ 301 -/docs/tasks/administer-cluster/reserve-compute-resources/out-of-resource.md /docs/tasks/administer-cluster/out-of-resource/ 301 +/docs/tasks/administer-cluster/reserve-compute-resources/out-of-resource.md /docs/concepts/scheduling-eviction/node-pressure-eviction/ 301 /docs/tasks/administer-cluster/out-of-resource/ /docs/concepts/scheduling-eviction/node-pressure-eviction/ 301 /docs/tasks/administer-cluster/romana-network-policy/ /docs/tasks/administer-cluster/network-policy-provider/romana-network-policy/ 301 /docs/tasks/administer-cluster/running-cloud-controller.md /docs/tasks/administer-cluster/running-cloud-controller/ 301 From 6e6c1f89208f62e68485d7614a6891f74bb1d8b8 Mon Sep 17 00:00:00 2001 From: Qiming Teng Date: Fri, 9 Jul 2021 12:54:18 +0800 Subject: [PATCH 063/175] Redirect kubeadm config reference Instead of redirecting users to the godoc site for kubeadm configuration options, we'd better point them to the generated reference. --- .../setup-tools/kubeadm/kubeadm-init-phase.md | 2 +- .../setup-tools/kubeadm/kubeadm-init.md | 4 ++-- .../setup-tools/kubeadm/kubeadm-join.md | 4 ++-- .../tools/kubeadm/control-plane-flags.md | 2 +- .../tools/kubeadm/dual-stack-support.md | 17 +++++++++++++---- .../tools/kubeadm/high-availability.md | 2 +- .../tools/kubeadm/troubleshooting-kubeadm.md | 11 ++++++++--- .../administer-cluster/kubeadm/kubeadm-certs.md | 2 +- 8 files changed, 29 insertions(+), 15 deletions(-) diff --git a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init-phase.md b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init-phase.md index 2b6939bac6..4cb7ff351f 100644 --- a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init-phase.md +++ b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init-phase.md @@ -144,7 +144,7 @@ install them selectively. {{< /tabs >}} For more details on each field in the `v1beta2` configuration you can navigate to our -[API reference pages.] (https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2) +[API reference pages.] (/docs/reference/config-api/kubeadm-config.v1beta2/) ## {{% heading "whatsnext" %}} diff --git a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md index 2260c2ca22..9c0eabedd5 100644 --- a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md +++ b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md @@ -122,8 +122,8 @@ The default configuration can be printed out using the If your configuration is not using the latest version it is **recommended** that you migrate using the [kubeadm config migrate](/docs/reference/setup-tools/kubeadm/kubeadm-config/) command. -For more information on the fields and usage of the configuration you can navigate to our API reference -page and pick a version from [the list](https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm#section-directories). +For more information on the fields and usage of the configuration you can navigate to our +[API reference page](/docs/reference/config-api/kubeadm-config.v1beta2/). ### Adding kube-proxy parameters {#kube-proxy} diff --git a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-join.md b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-join.md index 53ca4a789b..ead51c1405 100644 --- a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-join.md +++ b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-join.md @@ -282,8 +282,8 @@ The default configuration can be printed out using the If your configuration is not using the latest version it is **recommended** that you migrate using the [kubeadm config migrate](/docs/reference/setup-tools/kubeadm/kubeadm-config/) command. -For more information on the fields and usage of the configuration you can navigate to our API reference -page and pick a version from [the list](https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm#pkg-subdirectories). +For more information on the fields and usage of the configuration you can navigate to our +[API reference](/docs/reference/config-api/kubeadm-config.v1beta2/). ## {{% heading "whatsnext" %}} diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md b/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md index 8a9828a7ad..df1ebcd91c 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md @@ -24,7 +24,7 @@ The `extraArgs` field consist of `key: value` pairs. To override a flag for a co 3. Run `kubeadm init` with `--config `. For more details on each field in the configuration you can navigate to our -[API reference pages](https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2#ClusterConfiguration). +[API reference pages](/docs/reference/config-api/kubeadm-config.v1beta2/). {{< note >}} You can generate a `ClusterConfiguration` object with default values by running `kubeadm config print init-defaults` and saving the output to a file of your choice. diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md b/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md index 767787179c..b26158eb5a 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md @@ -45,7 +45,9 @@ similar to the following example: kubeadm init --pod-network-cidr=10.244.0.0/16,2001:db8:42:0::/56 --service-cidr=10.96.0.0/16,2001:db8:42:1::/112 ``` -To make things clearer, here is an example kubeadm [configuration file](https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2) `kubeadm-config.yaml` for the primary dual-stack control plane node. +To make things clearer, here is an example kubeadm +[configuration file](/docs/reference/config-api/kubeadm-config.v1beta2/) `kubeadm-config.yaml` +for the primary dual-stack control plane node. ```yaml --- @@ -85,7 +87,8 @@ The `--apiserver-advertise-address` flag does not support dual-stack. Before joining a node, make sure that the node has IPv6 routable network interface and allows IPv6 forwarding. -Here is an example kubeadm [configuration file](https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2) `kubeadm-config.yaml` for joining a worker node to the cluster. +Here is an example kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta2/) +`kubeadm-config.yaml` for joining a worker node to the cluster. ```yaml apiVersion: kubeadm.k8s.io/v1beta2 @@ -98,7 +101,9 @@ nodeRegistration: node-ip: 10.100.0.3,fd00:1:2:3::3 ``` -Also, here is an example kubeadm [configuration file](https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2) `kubeadm-config.yaml` for joining another control plane node to the cluster. +Also, here is an example kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta2/) +`kubeadm-config.yaml` for joining another control plane node to the cluster. + ```yaml apiVersion: kubeadm.k8s.io/v1beta2 kind: JoinConfiguration @@ -134,7 +139,9 @@ In 1.21 the `IPv6DualStack` feature is Beta and the feature gate is defaulted to kubeadm init --feature-gates IPv6DualStack=false ``` -To make things more clear, here is an example kubeadm [configuration file](https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2) `kubeadm-config.yaml` for the single-stack control plane node. +To make things more clear, here is an example kubeadm +[configuration file](/docs/reference/config-api/kubeadm-config.v1beta2/) +`kubeadm-config.yaml` for the single-stack control plane node. ```yaml apiVersion: kubeadm.k8s.io/v1beta2 @@ -150,3 +157,5 @@ networking: * [Validate IPv4/IPv6 dual-stack](/docs/tasks/network/validate-dual-stack) networking * Read about [Dual-stack](/docs/concepts/services-networking/dual-stack/) cluster networking +* Learn more about the kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta2/) + diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/high-availability.md b/content/en/docs/setup/production-environment/tools/kubeadm/high-availability.md index e387e2d41c..83438bf788 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/high-availability.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/high-availability.md @@ -115,7 +115,7 @@ option. Your cluster requirements may need a different configuration. {{< note >}} The `kubeadm init` flags `--config` and `--certificate-key` cannot be mixed, therefore if you want - to use the [kubeadm configuration](https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2) + to use the [kubeadm configuration](/docs/reference/config-api/kubeadm-config.v1beta2/) you must add the `certificateKey` field in the appropriate config locations (under `InitConfiguration` and `JoinConfiguration: controlPlane`). {{< /note >}} diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md b/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md index 9f868f0637..42a2f4c59e 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md @@ -258,7 +258,12 @@ Error from server: Get https://10.19.0.41:10250/containerLogs/default/mysql-ddc6 curl http://169.254.169.254/metadata/v1/interfaces/public/0/anchor_ipv4/address ``` - The workaround is to tell `kubelet` which IP to use using `--node-ip`. When using DigitalOcean, it can be the public one (assigned to `eth0`) or the private one (assigned to `eth1`) should you want to use the optional private network. The [`KubeletExtraArgs` section of the kubeadm `NodeRegistrationOptions` structure](https://github.com/kubernetes/kubernetes/blob/release-1.13/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go) can be used for this. + The workaround is to tell `kubelet` which IP to use using `--node-ip`. + When using DigitalOcean, it can be the public one (assigned to `eth0`) or + the private one (assigned to `eth1`) should you want to use the optional + private network. The `kubeletExtraArgs` section of the kubeadm + [`NodeRegistrationOptions` structure](/docs/reference/config-api/kubeadm-config.v1beta2/#kubeadm-k8s-io-v1beta2-NodeRegistrationOptions) + can be used for this. Then restart `kubelet`: @@ -331,7 +336,7 @@ Alternatively, you can try separating the `key=value` pairs like so: `--apiserver-extra-args "enable-admission-plugins=LimitRanger,enable-admission-plugins=NamespaceExists"` but this will result in the key `enable-admission-plugins` only having the value of `NamespaceExists`. -A known workaround is to use the kubeadm [configuration file](/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#apiserver-flags). +A known workaround is to use the kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta2/). ## kube-proxy scheduled before node is initialized by cloud-controller-manager @@ -381,7 +386,7 @@ Kubernetes components like the kubelet and kube-controller-manager use the defau for the feature to work. To workaround this issue you can configure the flex-volume directory using the kubeadm -[configuration file](https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2). +[configuration file](/docs/reference/config-api/kubeadm-config.v1beta2/). On the primary control-plane Node (created using `kubeadm init`) pass the following file using `--config`: diff --git a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-certs.md b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-certs.md index be98558659..6d61184b52 100644 --- a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-certs.md +++ b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-certs.md @@ -161,7 +161,7 @@ The built-in signer is part of [`kube-controller-manager`](/docs/reference/comma To activate the built-in signer, you must pass the `--cluster-signing-cert-file` and `--cluster-signing-key-file` flags. -If you're creating a new cluster, you can use a kubeadm [configuration file](https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2): +If you're creating a new cluster, you can use a kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta2/): ```yaml apiVersion: kubeadm.k8s.io/v1beta2 From 6c542e88c2f5e7478942fd25cd55704245749ea8 Mon Sep 17 00:00:00 2001 From: Qiming Teng Date: Fri, 9 Jul 2021 19:55:59 +0800 Subject: [PATCH 064/175] Update content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md Co-authored-by: Tim Bannister --- .../production-environment/tools/kubeadm/dual-stack-support.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md b/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md index b26158eb5a..8772db093b 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md @@ -157,5 +157,4 @@ networking: * [Validate IPv4/IPv6 dual-stack](/docs/tasks/network/validate-dual-stack) networking * Read about [Dual-stack](/docs/concepts/services-networking/dual-stack/) cluster networking -* Learn more about the kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta2/) - +* Learn more about the kubeadm [configuration format](/docs/reference/config-api/kubeadm-config.v1beta2/) From 07627bd7d391d313fdebed2770245d740676ea24 Mon Sep 17 00:00:00 2001 From: Arhell Date: Sat, 10 Jul 2021 12:16:47 +0300 Subject: [PATCH 065/175] [zh] Write Vixie in title case --- content/zh/docs/tasks/job/automated-tasks-with-cron-jobs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/zh/docs/tasks/job/automated-tasks-with-cron-jobs.md b/content/zh/docs/tasks/job/automated-tasks-with-cron-jobs.md index b4039ee281..0f2f5e90b6 100644 --- a/content/zh/docs/tasks/job/automated-tasks-with-cron-jobs.md +++ b/content/zh/docs/tasks/job/automated-tasks-with-cron-jobs.md @@ -230,9 +230,9 @@ It takes a [Cron](https://en.wikipedia.org/wiki/Cron) format string, such as `0 格式串，例如 `0 * * * *` or `@hourly` ，作为它的任务被创建和执行的调度时间。 -该格式也包含了扩展的 `vixie cron` 步长值。 +该格式也包含了扩展的 "Vixie cron" 步长值。 [FreeBSD 手册](https://www.freebsd.org/cgi/man.cgi?crontab%285%29)中解释如下: -1.20 发布周期在上一个延长的发布周期之后恢复到 11 周的正常节奏。这是一段时间以来功能最密集的版本之一：Kubernetes 创新周期仍呈上升趋势。此版本具有更多的 alpha 而非稳定的增强功能，表明云原生生态系统仍有许多需要探索的地方。 +1.20 发布周期在上一个延长的发布周期之后恢复到 11 周的正常节奏。这是一段时间以来功能最密集的版本之一：Kubernetes 创新周期仍呈上升趋势。此版本具有更多的 Alpha 而非稳定的增强功能，表明云原生生态系统仍有许多需要探索的地方。 ## 主题 {#major-themes} @@ -54,7 +54,7 @@ slug: kubernetes-1-20-release-announcement 请注意，作为新的内置命令，`kubectl debug` 优先于任何名为 “debug” 的 kubectl 插件。你必须重命名受影响的插件。 -`kubectl alpha debug` 现在不推荐使用，并将在后续版本中删除。更新你的脚本以使用 `kubectl debug`. 有关更多信息 `kubectl debug`，请参阅[调试正在运行的 Pod]((https://kubernetes.io/zh/docs/tasks/debug-application-cluster/debug-running-pod/)。 +`kubectl alpha debug` 现在不推荐使用，并将在后续版本中删除。更新你的脚本以使用 `kubectl debug`。 有关更多信息 `kubectl debug`，请参阅[调试正在运行的 Pod]((https://kubernetes.io/zh/docs/tasks/debug-application-cluster/debug-running-pod/)。 ### 测试版：API 优先级和公平性 {#beta-api-priority-and-fairness) @@ -78,7 +78,7 @@ Kubernetes 1.20 由 1.18 引入，现在默认启用 API 优先级和公平性 ( 管理员需要机制来确保用户 pod 不会导致 pid 耗尽，从而阻止主机守护程序（运行时、kubelet 等）运行。此外，重要的是要确保 pod 之间的 pid 受到限制，以确保它们对节点上的其他工作负载的影响有限。 -默认启用一年后，SIG Node 在 `SupportNodePidsLimit`（节点到 Pod PID 隔离）和 `SupportPodPidsLimit`（限制每个 Pod 的 PID 的能力）上都将 PID 限制升级为 GA 。 +默认启用一年后，SIG Node 在 `SupportNodePidsLimit`（节点到 Pod PID 隔离）和 `SupportPodPidsLimit`（限制每个 Pod 的 PID 的能力）上都将 PID 限制升级为 GA。 ### Alpha：节点体面地关闭 {#alpha-graceful-node-shutdown} From 477383508cc87bef752d4d740a6e031bef48ca5b Mon Sep 17 00:00:00 2001 From: Meysam Date: Sun, 11 Jul 2021 22:45:06 +0430 Subject: [PATCH 072/175] =?UTF-8?q?Update=20the=20link=20to=20cloud=20inte?= =?UTF-8?q?rface=20=F0=9F=93=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- content/en/docs/concepts/architecture/cloud-controller.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/concepts/architecture/cloud-controller.md b/content/en/docs/concepts/architecture/cloud-controller.md index 9b64289e82..229cc489f9 100644 --- a/content/en/docs/concepts/architecture/cloud-controller.md +++ b/content/en/docs/concepts/architecture/cloud-controller.md @@ -210,7 +210,7 @@ To upgrade a HA control plane to use the cloud controller manager, see [Migrate Want to know how to implement your own cloud controller manager, or extend an existing project? -The cloud controller manager uses Go interfaces to allow implementations from any cloud to be plugged in. Specifically, it uses the `CloudProvider` interface defined in [`cloud.go`](https://github.com/kubernetes/cloud-provider/blob/release-1.17/cloud.go#L42-L62) from [kubernetes/cloud-provider](https://github.com/kubernetes/cloud-provider). +The cloud controller manager uses Go interfaces to allow implementations from any cloud to be plugged in. Specifically, it uses the `CloudProvider` interface defined in [`cloud.go`](https://github.com/kubernetes/cloud-provider/blob/release-1.21/cloud.go#L42-L69) from [kubernetes/cloud-provider](https://github.com/kubernetes/cloud-provider). The implementation of the shared controllers highlighted in this document (Node, Route, and Service), and some scaffolding along with the shared cloudprovider interface, is part of the Kubernetes core. Implementations specific to cloud providers are outside the core of Kubernetes and implement the `CloudProvider` interface. From 2c9ebb38f2ec78ccde60f6a42394fda5ad7f58c3 Mon Sep 17 00:00:00 2001 From: Arhell Date: Mon, 12 Jul 2021 00:47:38 +0300 Subject: [PATCH 073/175] [ru] Fixed variable substitution typo --- .../ru/docs/contribute/generate-ref-docs/kubernetes-api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/ru/docs/contribute/generate-ref-docs/kubernetes-api.md b/content/ru/docs/contribute/generate-ref-docs/kubernetes-api.md index 883fc8d16c..e96bc29ea7 100644 --- a/content/ru/docs/contribute/generate-ref-docs/kubernetes-api.md +++ b/content/ru/docs/contribute/generate-ref-docs/kubernetes-api.md @@ -82,8 +82,8 @@ git clone https://github.com/kubernetes/kubernetes $GOPATH/src/k8s.io/kubernetes Примеры: ```shell -export K8S_WEBROOT=$(GOPATH)/src/github.com//website -export K8S_ROOT=$(GOPATH)/src/k8s.io/kubernetes +export K8S_WEBROOT=${GOPATH}/src/github.com//website +export K8S_ROOT=${GOPATH}/src/k8s.io/kubernetes export K8S_RELEASE=1.17.0 ``` From 9fedbf68ea4bc52cdcfb39fd3f81d03b6651f658 Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Mon, 5 Jul 2021 18:21:22 +0900 Subject: [PATCH 074/175] [ko] Update outdated files in dev-1.21-ko.6 (p3) --- .../manage-deployment.md | 2 +- .../organize-cluster-access-kubeconfig.md | 6 +- content/ko/docs/concepts/containers/images.md | 14 ++++ .../concepts/extend-kubernetes/operator.md | 3 +- .../concepts/policy/pod-security-policy.md | 65 +++++++++++++--- .../taint-and-toleration.md | 29 ++++++-- .../services-networking/dns-pod-service.md | 2 +- content/ko/examples/policy/baseline-psp.yaml | 74 +++++++++++++++++++ 8 files changed, 174 insertions(+), 21 deletions(-) create mode 100644 content/ko/examples/policy/baseline-psp.yaml diff --git a/content/ko/docs/concepts/cluster-administration/manage-deployment.md b/content/ko/docs/concepts/cluster-administration/manage-deployment.md index abcc4c2cd5..7e3093d51e 100644 --- a/content/ko/docs/concepts/cluster-administration/manage-deployment.md +++ b/content/ko/docs/concepts/cluster-administration/manage-deployment.md @@ -50,7 +50,7 @@ kubectl apply -f https://k8s.io/examples/application/nginx/ URL을 구성 소스로 지정할 수도 있다. 이는 GitHub에 체크인된 구성 파일에서 직접 배포하는 데 편리하다. ```shell -kubectl apply -f https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/application/nginx/nginx-deployment.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/application/nginx/nginx-deployment.yaml ``` ```shell diff --git a/content/ko/docs/concepts/configuration/organize-cluster-access-kubeconfig.md b/content/ko/docs/concepts/configuration/organize-cluster-access-kubeconfig.md index a002414b67..aa739b381c 100644 --- a/content/ko/docs/concepts/configuration/organize-cluster-access-kubeconfig.md +++ b/content/ko/docs/concepts/configuration/organize-cluster-access-kubeconfig.md @@ -17,6 +17,11 @@ kubeconfig 파일들을 사용하여 클러스터, 사용자, 네임스페이스 `kubeconfig`라는 이름의 파일이 있다는 의미는 아니다. {{< /note >}} +{{< warning >}} +신뢰할 수 있는 소스의 kubeconfig 파일만 사용한다. 특수 제작된 kubeconfig 파일을 사용하면 악성 코드가 실행되거나 파일이 노출될 수 있다. +신뢰할 수 없는 kubeconfig 파일을 사용해야 하는 경우 셸 스크립트를 사용하는 경우처럼 먼저 신중하게 검사한다. +{{< /warning>}} + 기본적으로 `kubectl`은 `$HOME/.kube` 디렉터리에서 `config`라는 이름의 파일을 찾는다. `KUBECONFIG` 환경 변수를 설정하거나 [`--kubeconfig`](/docs/reference/generated/kubectl/kubectl/) 플래그를 지정해서 @@ -154,4 +159,3 @@ kubeconfig 파일에서 파일과 경로 참조는 kubeconfig 파일의 위치 - diff --git a/content/ko/docs/concepts/containers/images.md b/content/ko/docs/concepts/containers/images.md index 9dd72e335f..886f8247a3 100644 --- a/content/ko/docs/concepts/containers/images.md +++ b/content/ko/docs/concepts/containers/images.md @@ -77,6 +77,20 @@ weight: 10 `imagePullPolicy` 가 특정값 없이 정의되면, `Always` 로 설정된다. +### 이미지풀백오프(ImagePullBackOff) + +kubelet이 컨테이너 런타임을 사용하여 파드의 컨테이너 생성을 시작할 때, +`ImagePullBackOff`로 인해 컨테이너가 +[Waiting](/ko/docs/concepts/workloads/pods/pod-lifecycle/#container-state-waiting) 상태에 있을 수 있다. + +`ImagePullBackOff`라는 상태는 (이미지 이름이 잘못됨, 또는 `imagePullSecret` 없이 +비공개 레지스트리에서 풀링 시도 등의 이유로) 쿠버네티스가 컨테이너 이미지를 +가져올 수 없기 때문에 컨테이너를 실행할 수 없음을 의미한다. `BackOff`라는 단어는 +쿠버네티스가 백오프 딜레이를 증가시키면서 이미지 풀링을 계속 시도할 것임을 나타낸다. + +쿠버네티스는 시간 간격을 늘려가면서 시도를 계속하며, 시간 간격의 상한은 쿠버네티스 코드에 +300초(5분)로 정해져 있다. + ## 이미지 인덱스가 있는 다중 아키텍처 이미지 바이너리 이미지를 제공할 뿐만 아니라, 컨테이너 레지스트리는 [컨테이너 이미지 인덱스](https://github.com/opencontainers/image-spec/blob/master/image-index.md)를 제공할 수도 있다. 이미지 인덱스는 컨테이너의 아키텍처별 버전에 대한 여러 [이미지 매니페스트](https://github.com/opencontainers/image-spec/blob/master/manifest.md)를 가리킬 수 있다. 아이디어는 이미지의 이름(예를 들어, `pause`, `example/mycontainer`, `kube-apiserver`)을 가질 수 있다는 것이다. 그래서 다른 시스템들이 사용하고 있는 컴퓨터 아키텍처에 적합한 바이너리 이미지를 가져올 수 있다. diff --git a/content/ko/docs/concepts/extend-kubernetes/operator.md b/content/ko/docs/concepts/extend-kubernetes/operator.md index a0959f83dc..aba13a59c2 100644 --- a/content/ko/docs/concepts/extend-kubernetes/operator.md +++ b/content/ko/docs/concepts/extend-kubernetes/operator.md @@ -116,7 +116,7 @@ kubectl edit SampleDB/example-database # 일부 설정을 수동으로 변경하 * [Charmed Operator Framework](https://juju.is/) * [kubebuilder](https://book.kubebuilder.io/) 사용하기 * [KUDO](https://kudo.dev/) (Kubernetes Universal Declarative Operator) -* 웹훅(WebHook)과 함께 [Metacontroller](https://metacontroller.app/)를 +* 웹훅(WebHook)과 함께 [Metacontroller](https://metacontroller.github.io/metacontroller/intro.html)를 사용하여 직접 구현하기 * [오퍼레이터 프레임워크](https://operatorframework.io) * [shell-operator](https://github.com/flant/shell-operator) @@ -124,6 +124,7 @@ kubectl edit SampleDB/example-database # 일부 설정을 수동으로 변경하 ## {{% heading "whatsnext" %}} +* {{< glossary_tooltip text="CNCF" term_id="cncf" >}} [오퍼레이터 백서](https://github.com/cncf/tag-app-delivery/blob/eece8f7307f2970f46f100f51932db106db46968/operator-wg/whitepaper/Operator-WhitePaper_v1-0.md) 읽어보기 * [사용자 정의 리소스](/ko/docs/concepts/extend-kubernetes/api-extension/custom-resources/)에 대해 더 알아보기 * [OperatorHub.io](https://operatorhub.io/)에서 유스케이스에 맞는 이미 만들어진 오퍼레이터 찾기 * 다른 사람들이 사용할 수 있도록 자신의 오퍼레이터를 [게시](https://operatorhub.io/)하기 diff --git a/content/ko/docs/concepts/policy/pod-security-policy.md b/content/ko/docs/concepts/policy/pod-security-policy.md index 5941a0e13a..ff98e134eb 100644 --- a/content/ko/docs/concepts/policy/pod-security-policy.md +++ b/content/ko/docs/concepts/policy/pod-security-policy.md @@ -11,7 +11,8 @@ weight: 30 {{< feature-state for_k8s_version="v1.21" state="deprecated" >}} -파드시큐리티폴리시(PodSecurityPolicy)는 쿠버네티스 v1.21부터 더이상 사용되지 않으며, v1.25에서 제거된다. +파드시큐리티폴리시(PodSecurityPolicy)는 쿠버네티스 v1.21부터 더이상 사용되지 않으며, v1.25에서 제거된다. 사용 중단에 대한 상세 사항은 +[파드시큐리티폴리시 사용 중단: 과거, 현재, 그리고 미래](/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/)를 참조한다. 파드 시큐리티 폴리시를 사용하면 파드 생성 및 업데이트에 대한 세분화된 권한을 부여할 수 있다. @@ -48,10 +49,9 @@ _Pod Security Policy_ 는 파드 명세의 보안 관련 측면을 제어하는 ## 파드 시큐리티 폴리시 활성화 -파드 시큐리티 폴리시 제어는 선택 사항(하지만 권장함)인 -[어드미션 -컨트롤러](/docs/reference/access-authn-authz/admission-controllers/#podsecuritypolicy)로 -구현된다. [어드미션 컨트롤러 활성화](/docs/reference/access-authn-authz/admission-controllers/#how-do-i-turn-on-an-admission-control-plug-in)하면 +파드 시큐리티 폴리시 제어는 선택 사항인 [어드미션 +컨트롤러](/docs/reference/access-authn-authz/admission-controllers/#podsecuritypolicy)로 구현된다. +[어드미션 컨트롤러를 활성화](/docs/reference/access-authn-authz/admission-controllers/#how-do-i-turn-on-an-admission-control-plug-in)하면 파드시큐리티폴리시가 적용되지만, 정책을 승인하지 않고 활성화하면 클러스터에 **파드가 생성되지 않는다.** @@ -110,11 +110,15 @@ roleRef: name: apiGroup: rbac.authorization.k8s.io subjects: -# Authorize specific service accounts: +# 네임스페이스의 모든 서비스 어카운트 승인(권장): +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:serviceaccounts: +# 특정 서비스 어카운트 승인(권장하지 않음): - kind: ServiceAccount name: namespace: -# Authorize specific users (not recommended): +# 특정 사용자 승인(권장하지 않음): - kind: User apiGroup: rbac.authorization.k8s.io name: @@ -124,21 +128,55 @@ subjects: 실행되는 파드에 대해서만 사용 권한을 부여한다. 네임스페이스에서 실행되는 모든 파드에 접근 권한을 부여하기 위해 시스템 그룹과 쌍을 이룰 수 있다. ```yaml -# Authorize all service accounts in a namespace: +# 네임스페이스의 모든 서비스 어카운트 승인: - kind: Group apiGroup: rbac.authorization.k8s.io name: system:serviceaccounts -# Or equivalently, all authenticated users in a namespace: +# 또는 동일하게, 네임스페이스의 모든 승인된 사용자에게 사용 권한 부여 - kind: Group apiGroup: rbac.authorization.k8s.io name: system:authenticated ``` RBAC 바인딩에 대한 자세한 예는, -[역할 바인딩 예제](/docs/reference/access-authn-authz/rbac#role-binding-examples)를 참고하길 바란다. +[역할 바인딩 예제](/docs/reference/access-authn-authz/rbac#role-binding-examples)를 참고한다. 파드시큐리티폴리시 인증에 대한 전체 예제는 -[아래](#예제)를 참고하길 바란다. +[아래](#예제)를 참고한다. +### 추천 예제 + +파드시큐리티폴리시는 새롭고 간결해진 `PodSecurity` {{< glossary_tooltip +text="어드미션 컨트롤러" term_id="admission-controller" >}}로 대체되고 있다. +이 변경에 대한 상세사항은 +[파드시큐리티폴리시 사용 중단: 과거, 현재, 그리고 미래](/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/)를 참조한다. +다음 가이드라인을 참조하여 파드시큐리티폴리시를 새로운 어드미션 컨트롤러로 쉽게 전환할 수 있다. + +1. 파드시큐리티폴리시를 [파드 보안 표준](/docs/concepts/security/pod-security-standards/)에 의해 정의된 폴리시로 한정한다. + - {{< example file="policy/privileged-psp.yaml" >}}Privileged{{< /example >}} + - {{< example file="policy/baseline-psp.yaml" >}}Baseline{{< /example >}} + - {{< example file="policy/restricted-psp.yaml" >}}Restricted{{< /example >}} + +2. `system:serviceaccounts:` (여기서 ``는 타겟 네임스페이스) 그룹을 사용하여 + 파드시큐리티폴리시를 전체 네임스페이스에만 바인드한다. 예시는 다음과 같다. + + ```yaml + apiVersion: rbac.authorization.k8s.io/v1 + # 이 클러스터롤바인딩(ClusterRoleBinding)을 통해 "development" 네임스페이스의 모든 파드가 기준 파드시큐리티폴리시(PSP)를 사용할 수 있다. + kind: ClusterRoleBinding + metadata: + name: psp-baseline-namespaces + roleRef: + kind: ClusterRole + name: psp-baseline + apiGroup: rbac.authorization.k8s.io + subjects: + - kind: Group + name: system:serviceaccounts:development + apiGroup: rbac.authorization.k8s.io + - kind: Group + name: system:serviceaccounts:canary + apiGroup: rbac.authorization.k8s.io + ``` ### 문제 해결 @@ -661,5 +699,10 @@ spec: ## {{% heading "whatsnext" %}} +- [파드시큐리티폴리시 사용 중단: 과거, 현재, 그리고 +미래](/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/)에서 +파드시큐리티폴리시의 미래에 대해 알아본다. + - 폴리시 권장 사항에 대해서는 [파드 보안 표준](/docs/concepts/security/pod-security-standards/)을 참조한다. + - API 세부 정보는 [파드 시큐리티 폴리시 레퍼런스](/docs/reference/generated/kubernetes-api/{{< param "version" >}}/#podsecuritypolicy-v1beta1-policy) 참조한다. diff --git a/content/ko/docs/concepts/scheduling-eviction/taint-and-toleration.md b/content/ko/docs/concepts/scheduling-eviction/taint-and-toleration.md index 4465b8a149..c47f5f995b 100644 --- a/content/ko/docs/concepts/scheduling-eviction/taint-and-toleration.md +++ b/content/ko/docs/concepts/scheduling-eviction/taint-and-toleration.md @@ -1,4 +1,8 @@ --- + + + + title: 테인트(Taints)와 톨러레이션(Tolerations) content_type: concept weight: 40 @@ -260,13 +264,27 @@ tolerations: 이렇게 하면 이러한 문제로 인해 데몬셋 파드가 축출되지 않는다. -## 컨디션별 노드 테인트하기 +## 컨디션을 기준으로 노드 테인트하기 -노드 라이프사이클 컨트롤러는 `NoSchedule` 이펙트가 있는 노드 컨디션에 해당하는 -테인트를 자동으로 생성한다. -마찬가지로 스케줄러는 노드 컨디션을 확인하지 않는다. 대신 스케줄러는 테인트를 확인한다. 이렇게 하면 노드 컨디션이 노드에 스케줄된 내용에 영향을 미치지 않는다. 사용자는 적절한 파드 톨러레이션을 추가하여 노드의 일부 문제(노드 컨디션으로 표시)를 무시하도록 선택할 수 있다. +컨트롤 플레인은 노드 {{}}를 이용하여 +[노드 조건](/docs/concepts/scheduling-eviction/node-pressure-eviction/)에 대한 `NoSchedule` 효과를 사용하여 자동으로 테인트를 생성한다. -쿠버네티스 1.8 버전부터 데몬셋 컨트롤러는 다음의 `NoSchedule` 톨러레이션을 +스케줄러는 스케줄링 결정을 내릴 때 노드 조건을 확인하는 것이 아니라 테인트를 확인한다. +이렇게 하면 노드 조건이 스케줄링에 직접적인 영향을 주지 않는다. +예를 들어 `DiskPressure` 노드 조건이 활성화된 경우 +컨트롤 플레인은 `node.kubernetes.io/disk-pressure` 테인트를 추가하고 영향을 받는 노드에 새 파드를 할당하지 않는다. +`MemoryPressure` 노드 조건이 활성화되면 +컨트롤 플레인이 `node.kubernetes.io/memory-pressure` 테인트를 추가한다. + +새로 생성된 파드에 파드 톨러레이션을 추가하여 노드 조건을 무시하도록 할 수 있다. +또한 컨트롤 플레인은 `BestEffort` 이외의 +{{< glossary_tooltip text="QoS 클래스" term_id="qos-class" >}}를 가지는 파드에 +`node.kubernetes.io/memory-pressure` 톨러레이션을 추가한다. +이는 쿠버네티스가 `Guaranteed` 또는 `Burstable` QoS 클래스를 갖는 파드(메모리 요청이 설정되지 않은 파드 포함)를 +마치 그 파드들이 메모리 압박에 대처 가능한 것처럼 다루는 반면, +새로운 `BestEffort` 파드는 영향을 받는 노드에 할당하지 않기 때문이다. + +데몬셋 컨트롤러는 다음의 `NoSchedule` 톨러레이션을 모든 데몬에 자동으로 추가하여, 데몬셋이 중단되는 것을 방지한다. * `node.kubernetes.io/memory-pressure` @@ -278,7 +296,6 @@ tolerations: 이러한 톨러레이션을 추가하면 이전 버전과의 호환성이 보장된다. 데몬셋에 임의의 톨러레이션을 추가할 수도 있다. - ## {{% heading "whatsnext" %}} * [리소스 부족 다루기](/docs/concepts/scheduling-eviction/node-pressure-eviction/)와 어떻게 구성하는지에 대해 알아보기 diff --git a/content/ko/docs/concepts/services-networking/dns-pod-service.md b/content/ko/docs/concepts/services-networking/dns-pod-service.md index 8a35c2c6a3..b405617118 100644 --- a/content/ko/docs/concepts/services-networking/dns-pod-service.md +++ b/content/ko/docs/concepts/services-networking/dns-pod-service.md @@ -50,7 +50,7 @@ options ndots:5 ``` 요약하면, _test_ 네임스페이스에 있는 파드는 `data.prod` 또는 -`data.prod.cluster.local` 중 하나를 통해 성공적으로 해석될 수 있다. +`data.prod.svc.cluster.local` 중 하나를 통해 성공적으로 해석될 수 있다. ### DNS 레코드 diff --git a/content/ko/examples/policy/baseline-psp.yaml b/content/ko/examples/policy/baseline-psp.yaml new file mode 100644 index 0000000000..679b780096 --- /dev/null +++ b/content/ko/examples/policy/baseline-psp.yaml @@ -0,0 +1,74 @@ +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: baseline + annotations: + # 선택 사항: 기본 AppArmor 프로파일을 활성화한다. 이 경우 기본값을 설정해야 한다. + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + privileged: false + # Moby의 기본 캐퍼빌리티 집합(NET_RAW는 제외되었음) + allowedCapabilities: + - 'CHOWN' + - 'DAC_OVERRIDE' + - 'FSETID' + - 'FOWNER' + - 'MKNOD' + - 'SETGID' + - 'SETUID' + - 'SETFCAP' + - 'SETPCAP' + - 'NET_BIND_SERVICE' + - 'SYS_CHROOT' + - 'KILL' + - 'AUDIT_WRITE' + # hostpath를 제외한 모든 볼륨 타입을 허용 + volumes: + # '코어' 볼륨 타입 + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + # 클러스터 관리자에 의해 구성된 휘발성 CSI 드라이버와 퍼시스턴트볼륨(PersistentVolume)은 사용하기에 안전하다고 가정한다. + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + # hostpath 타입이 아닌 다른 모든 볼륨 타입을 허용 + - 'awsElasticBlockStore' + - 'azureDisk' + - 'azureFile' + - 'cephFS' + - 'cinder' + - 'fc' + - 'flexVolume' + - 'flocker' + - 'gcePersistentDisk' + - 'gitRepo' + - 'glusterfs' + - 'iscsi' + - 'nfs' + - 'photonPersistentDisk' + - 'portworxVolume' + - 'quobyte' + - 'rbd' + - 'scaleIO' + - 'storageos' + - 'vsphereVolume' + hostNetwork: false + hostIPC: false + hostPID: false + readOnlyRootFilesystem: false + runAsUser: + rule: 'RunAsAny' + seLinux: + # 이 파드시큐리티폴리시는 노드가 SELinux가 아닌 AppArmor를 사용하고 있다고 가정한다. + # 파드시큐리티폴리시 SELinux API는 SELinux 파드 보안 표준을 표현할 수 없으므로, + # SELinux를 사용하는 경우 더 제한적인 기본값을 선택해야 한다. + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' From fdcc9de6fe680e526c8a718b6c7c688fc7c0bd9a Mon Sep 17 00:00:00 2001 From: llhuii Date: Mon, 12 Jul 2021 17:16:47 +0800 Subject: [PATCH 075/175] [zh] fix the beta summary indentation in using-api --- content/zh/docs/reference/using-api/_index.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/zh/docs/reference/using-api/_index.md b/content/zh/docs/reference/using-api/_index.md index 303f2152ac..837adf9dc6 100644 --- a/content/zh/docs/reference/using-api/_index.md +++ b/content/zh/docs/reference/using-api/_index.md @@ -111,7 +111,7 @@ Here's a summary of each level: 特性默认开启。 - 尽管一些特性会发生细节上的变化，但它们将会被长期支持。 - + --> - 在随后的 Beta 版或稳定版中，对象的模式和（或）语义可能以不兼容的方式改变。 当这种情况发生时，将提供迁移说明。 模式更改可能需要删除、编辑和重建 API 对象。 @@ -130,10 +130,10 @@ Here's a summary of each level: 后续发布版本可能会有不兼容的变动。 如果你有多个集群可以独立升级，可以放宽这一限制。 - + --> {{< note >}} 请试用测试版特性时并提供反馈。特性完成 Beta 阶段测试后， 就可能不会有太多的变更了。 From 9f501d0f73f5da5489991c2c182b07fc4762e760 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 12 Jul 2021 08:56:43 -0600 Subject: [PATCH 076/175] Adding a blog to discuss release cadence change --- .../index.md | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md diff --git a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md new file mode 100644 index 0000000000..7c86f985c5 --- /dev/null +++ b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md @@ -0,0 +1,84 @@ +--- +layout: blog +title: "Kuberentes Release Cadence Change: Here’s What You Need To Know" +date: 2021-07-01 +slug: new-kubernetes-release-cadence +--- + +REF: https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/ + + +**Authors**: Celeste Horgan, Adolfo García Veytia, James Laverack, Jeremy Rickard + +On April 23, 2021, the Release Team merged a Kubernetes Enhancement Proposal (KEP) changing the Kubernetes release cycle from four releases a year (once a quarter) to three releases a year. + +This blog post is a high level overview post for what this means for the Kubernetes community's contributors and maintainers. + +## What's changing and when + +Starting with the [Kubernetes 1.22 release](https://www.kubernetes.dev/resources/release/), a lightweight policy will drive the creation of each release schedule. This policy states: + +* The first Kubernetes release of a calendar year should start at the second or third + week of January to provide people more room after coming back from the + winter holidays. +* The last Kubernetes release of a calendar year should be finished by the middle of + December. +* A Kubernetes release cycle has a length of approximately 15 weeks. +* The week of KubeCon + CloudNativeCon is not considered a 'working week' for SIG Release. The Release Team will not hold meetings or make decisions in this period. +* An explicit SIG Release break of at least two weeks between each cycle will + be enforced. + +As a result, Kubernetes will follow a three release pear year cadence. Kubernetes 1.23 will be the final release of the 2021 calendar year. This new schedule creation policy results in a very predictible release schedule, allowing us to forecast upcoming release dates: + + +*Proposed Kubernetes Release Schedule for the remainder of 2021* + +| Week Number in Year | Release Number | Release Week | Note | +| -------- | -------- | -------- | -------- | +| 35 | 3 | 1 (August 23) | | +| 50 | 3 | 16 (December 07) | KubeCon + CloudNativeCon NA Break (Oct 11-15) | + +*Proposed Kubernetes Release Schedule for 2022* + +| Week Number in Year | Release Number | Release Week | Note | +| -------- | -------- | -------- | -------- | +| 1 | 1 | 1 (January 03) | | +| 15 | 1 | 15 (April 12) | | +| 17 | 2 | 1 (April 26) | KubeCon + CloudNativeCon EU likely to occur | +| 32 | 2 | 15 (August 09) | | +| 34 | 3 | 1 (August 22 | KubeCon + CloudNativeCon NA likely to occur | +| 49 | 3 | 14 (December 06) | + +These proposed dates reflect only the start and end dates, and they are subject to change. The Release Team will select dates for enhancement freeze, code freeze, and other milestones at the start of each release. Feedback from prior releases will feed into this process. + +## What this means for end users + +The only major change end users will experience is a slower release cadence. Kubernetes release artifacts, release notes, and all other aspects of any given release will stay the same. + +With less releases to consume per year, end user organizations will spend less time on upgrades and more time on supporting their Kubernetes clusters. It also means that Kubernetes releases are in support for a slightly longer period of time. + + +## What this means for Kubernetes contributors + +With a lower release cadence, contributors have more time for project enhancements, feature development, planning, and testing. A slower release cadence also provides more room for maintaining their mental health, preparing for events like KubeCon + CloudNativeCon or work on downstream integrations. + + +## Why we decided to change the release cadence + +The Kubernetes 1.19 cycle was far longer than usual. SIG Release extended it to lessen the burden on both Kubernetes contributors and Kubernetes end users due the COVID-19 pandemic. Following this extended release, the Kubernetes 1.20 release became the third, and final, release for 2020. + +As the Kubernetes project matures, the number of enhancements per cycle grows, along with the burden on contributors, the Release Engineering team. Downstream consumers and integrators also face increased challenges keeping up with [ever more feature-packed releases](https://kubernetes.io/blog/2021/04/08/kubernetes-1-21-release-announcement/). A wider project adoption means the complexity of supporting a rapidly evolving platform affects a bigger downstream chain of consumers. + +Changing the release cadence from four to three releases per year balances a variety of factors for stakeholders: while it's not strictly an LTS policy, consumers and integrators will get longer support terms for each minor version as the extended release cycles lead to the [last three releases being supported](https://kubernetes.io/blog/2020/08/31/kubernetes-1-19-feature-one-year-support/) for a longer period. Contributors get more time to [mature enhancements](https://www.cncf.io/blog/2021/04/12/enhancing-the-kubernetes-enhancements-process/) and [get them ready for production](https://github.com/kubernetes/community/blob/master/sig-architecture/production-readiness.md). + +Finally, the management overhead for SIG Release and the Release Engineering team diminishes allowing the team to spend more time on improving the quality of the software releases and the tooling that drives them. + +## How you can help + +Join the [discussion](https://github.com/kubernetes/sig-release/discussions/1566) about communicating future release dates and be sure to be on the lookout for post release surveys. + +## Where you can find out more + +- Read the KEP [here](https://github.com/kubernetes/enhancements/tree/master/keps/sig-release/2572-release-cadence) +- Join the [kubernetes-dev](https://groups.google.com/g/kubernetes-dev) mailing list +- Join [Kubernetes Slack](https://slack.k8s.io) and follow the #announcements channel \ No newline at end of file From 2616547b98f2898d562cdddd688a15bbbc75d8f7 Mon Sep 17 00:00:00 2001 From: Arhell Date: Tue, 13 Jul 2021 01:13:31 +0300 Subject: [PATCH 077/175] [zh] Fixed variable substitution --- .../zh/docs/contribute/generate-ref-docs/kubernetes-api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/zh/docs/contribute/generate-ref-docs/kubernetes-api.md b/content/zh/docs/contribute/generate-ref-docs/kubernetes-api.md index 9059cdef1e..5563e174a0 100644 --- a/content/zh/docs/contribute/generate-ref-docs/kubernetes-api.md +++ b/content/zh/docs/contribute/generate-ref-docs/kubernetes-api.md @@ -143,8 +143,8 @@ For example: 例如： ```shell -export K8S_WEBROOT=$(GOPATH)/src/github.com//website -export K8S_ROOT=$(GOPATH)/src/k8s.io/kubernetes +export K8S_WEBROOT=${GOPATH}/src/github.com//website +export K8S_ROOT=${GOPATH}/src/k8s.io/kubernetes export K8S_RELEASE=1.17.0 ``` From 875b67c219b046cf23464d148de0201eea8ecf64 Mon Sep 17 00:00:00 2001 From: chenxuc Date: Tue, 13 Jul 2021 11:34:15 +0800 Subject: [PATCH 078/175] [zh]Add a new rule and fix format --- content/zh/docs/contribute/localization_zh.md | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/content/zh/docs/contribute/localization_zh.md b/content/zh/docs/contribute/localization_zh.md index d00456b598..8b157d4a18 100644 --- a/content/zh/docs/contribute/localization_zh.md +++ b/content/zh/docs/contribute/localization_zh.md @@ -6,8 +6,7 @@ content_type: concept 本节详述文档中文本地化过程中须注意的事项。 -这里列举的内容包含了*中文本地化小组*早期给出的指导性建议和后续实践过程中 -积累的经验。 +这里列举的内容包含了*中文本地化小组*早期给出的指导性建议和后续实践过程中积累的经验。 在阅读、贡献、评阅中文本地化文档的过程中，如果对本文的指南有任何改进建议， 都请直接提出 PR。我们欢迎任何形式的补充和更正！ @@ -167,8 +166,8 @@ English text {{}} ``` -评阅人应该不难理解中英文段落的对应关系。但是如果采用下面的方式，则会出现 -两个 `note`，因此需要避免。这是因为被注释起来的短代码仍会起作用！ +评阅人应该不难理解中英文段落的对应关系。但是如果采用下面的方式， +则会出现两个 `note`，因此需要避免。这是因为被注释起来的短代码仍会起作用！ ``` + +* Beta versions of the `ValidatingWebhookConfiguration` and `MutatingWebhookConfiguration` API (the **admissionregistration.k8s.io/v1beta1** API versions) +* The beta `CustomResourceDefinition` API (**apiextensions.k8s.io/v1beta1**) +* The beta `APIService` API (**apiregistration.k8s.io/v1beta1**) +* The beta `TokenReview` API (**authentication.k8s.io/v1beta1**) +* Beta API versions of `SubjectAccessReview`, `LocalSubjectAccessReview`, `SelfSubjectAccessReview` (API versions from **authorization.k8s.io/v1beta1**) +* The beta `CertificateSigningRequest` API (**certificates.k8s.io/v1beta1**) +* The beta `Lease` API (**coordination.k8s.io/v1beta1**) +* All beta `Ingress` APIs (the **extensions/v1beta1** and **networking.k8s.io/v1beta1** API versions) + + +## What to do + +We're going to run through each of the resources that are affected by these removals +and explain the steps you'll need to take. + +`Ingress` +: Migrate to use the **networking.k8s.io/v1** + [Ingress](/docs/reference/kubernetes-api/service-resources/ingress-v1/) API, + [available since v1.19](/blog/2020/08/26/kubernetes-release-1.19-accentuate-the-paw-sitive/#ingress-graduates-to-general-availability). + The related API [IngressClass](/docs/reference/kubernetes-api/service-resources/ingress-class-v1/) + is designed to complement the [Ingress](/docs/concepts/services-networking/ingress/) + concept, allowing you to configure multiple kinds of Ingress within one cluster. + If you're currently using the deprecated + [`kubernetes.io/ingress.class`](https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-ingress-class-deprecated) + annotation, plan to switch to using the `.spec.ingressClassName` field instead. + On any cluster running Kubernetes v1.19 or later, you can use the v1 API to + retrieve or update existing Ingress objects, even if they were created using an + older API version. + + When you convert an Ingress to the v1 API, you should review each rule in that Ingress. + Older Ingresses use the legacy `ImplementationSpecific` path type. Instead of `ImplementationSpecific`, switch [path matching](/docs/concepts/services-networking/ingress/#path-types) to either `Prefix` or `Exact`. One of the benefits of moving to these alternative path types is that it becomes easier to migrate between different Ingress classes. + + **ⓘ** As well as upgrading _your_ own use of the Ingress API as a client, make sure that + every ingress controller that you use is compatible with the v1 Ingress API. + Read [Ingress Prerequisites](/docs/concepts/services-networking/ingress/#prerequisites) + for more context about Ingress and ingress controllers. + +`ValidatingWebhookConfiguration` and `MutatingWebhookConfiguration` +: Migrate to use the **admissionregistration.k8s.io/v1** API versions of + [ValidatingWebhookConfiguration](/docs/reference/kubernetes-api/extend-resources/validating-webhook-configuration-v1/) + and [MutatingWebhookConfiguration](/docs/reference/kubernetes-api/extend-resources/mutating-webhook-configuration-v1/), + available since v1.16. + You can use the v1 API to retrieve or update existing objects, even if they were created using an older API version. + +`CustomResourceDefinition` +: Migrate to use the [CustomResourceDefinition](/docs/reference/kubernetes-api/extend-resources/custom-resource-definition-v1/) + **apiextensions.k8s.io/v1** API, available since v1.16. + You can use the v1 API to retrieve or update existing objects, even if they were created + using an older API version. If you defined any custom resources in your cluster, those + are still served after you upgrade. + + If you're using external CustomResourceDefinitions, you can use + [`kubectl convert`](#kubectl-convert) to translate existing manifests to use the newer API. + Because there are some functional differences between beta and stable CustomResourceDefinitions, + our advice is to test out each one to make sure it works how you expect after the upgrade. + +`APIService` +: Migrate to use the **apiregistration.k8s.io/v1** [APIService](/docs/reference/kubernetes-api/cluster-resources/api-service-v1/) + API, available since v1.10. + You can use the v1 API to retrieve or update existing objects, even if they were created using an older API version. + If you already have API aggregation using an APIService object, this aggregation continues + to work after you upgrade. + +`TokenReview` +: Migrate to use the **authentication.k8s.io/v1** [TokenReview](/docs/reference/kubernetes-api/authentication-resources/token-review-v1/) + API, available since v1.10. + + In Kubernetes version v1.21 and earlier, the Kubernetes API server + [defaults](/docs/reference/access-authn-authz/authentication/#webhook-token-authentication) + to sending _beta_ TokenReviews to webhooks. See [Rehearse for the upgrade](#rehearse-for-the-upgrade) + for some specific tips about switching to the stable API. + +`SubjectAccessReview`, `SelfSubjectAccessReview` and `LocalSubjectAccessReview` +: Migrate to use the **authorization.k8s.io/v1** versions of those + [authorization APIs](/docs/reference/kubernetes-api/authorization-resources/), available since v1.6. + +`CertificateSigningRequest` +: Migrate to use the **certificates.k8s.io/v1** + [CertificateSigningRequest](/docs/reference/kubernetes-api/authentication-resources/certificate-signing-request-v1/) + API, available since v1.19. + You can use the v1 API to retrieve or update existing objects, even if they were created + using an older API version. Existing issued certificates retain their validity when you upgrade. + +`Lease` +: Migrate to use the **coordination.k8s.io/v1** [Lease](/docs/reference/kubernetes-api/cluster-resources/lease-v1/) + API, available since v1.14. + You can use the v1 API to retrieve or update existing objects, even if they were created + using an older API version. + +### `kubectl convert` + +There is a plugin to `kubectl` that provides the `kubectl convert` subcommand. +It's an official plugin that you can download as part of Kubernetes. +See [Download Kubernetes](/releases/download/) for more details. + +You can use `kubectl convert` to update manifest files to use a different API +version. For example, if you have a manifest in source control that uses the beta +Ingress API, you can check that definition out, +and run +`kubectl convert -f --output-version /`. +You can use the `kubectl convert` command to automatically convert an +existing manifest. + +For example, to convert an older Ingress definition to +`networking.k8s.io/v1`, you can run: +```bash +kubectl convert -f ./legacy-ingress.yaml --output-version networking.k8s.io/v1 +``` + +The automatic conversion uses a similar technique to how the Kubernetes control plane +updates objects that were originally created using an older API version. Because it's +a mechanical conversion, you might need to go in and change the manifest to adjust +defaults etc. + +### Rehearse for the upgrade + +If you manage your cluster's API server component, you can try out these API +removals before you upgrade to Kubernetes v1.22. + +To do that, add the following to the kube-apiserver command line arguments: + +`--runtime-config=admissionregistration.k8s.io/v1beta1=false,apiextensions.k8s.io/v1beta1=false,apiregistration.k8s.io/v1beta1=false,authentication.k8s.io/v1beta1=false,authorization.k9s.io/v1=false,certificates.k8s.io/v1beta=false,coordination.k8s.io/v1beta1=false,extensions/v1beta1/ingresses=false,networking.k8s.io/v1beta1/ingresses=false,networking.k8s.io/v1beta1/ingressclasses=false` + +Once you've switched all the kube-apiservers in your cluster to use that setting, +those beta APIs are removed. You can test that API clients (`kubectl`, deployment +tools, custom controllers etc) still work how you expect, and you can revert if +you need to without having to plan a more disruptive downgrade. + + +There's another setting that's relevant if you use webhook authentication checks. +Kubernetes v1.22 is the first version of Kubernetes that sends TokenReview objects +to webhooks using the `authentication.k8s.io/v1` API by default. However, you can +switch over earlier to try it out. +Add `--authentication-token-webhook-version=v1` to the command line options for +the kube-apiserver, and check that webhooks for authentication still work how you +expected. + +Once you're happy it works, you can leave the `--authentication-token-webhook-version=v1` +option set through and after the upgrade to v1.22. + +### Advice for software authors + +Maybe you're reading this because you're a developer of an addon or other +component that integrates with Kubernetes? + +If you develop an Ingress controller, webhook authenticator, an API aggregation, or +any other tool that relies on these deprecated APIs, you should already have started +to switch your software over. + +You can use the tips in +[Rehearse for the upgrade](#rehearse-for-the-upgrade) to run your own Kubernetes +cluster that only uses the new APIs, and make sure that your code works OK. +For your documentation, make sure readers are aware of any steps they should take +for the Kubernetes v1.22 upgrade. + +Where possible, give your users a hand to adopt the new APIs early - perhaps in a +test environment - so they can give you feedback about any problems. + +There are some [more deprecations](#looking-ahead) coming in Kubernetes v1.25, +so plan to have those covered too. + +## Kubernetes API removals + +Here's some background about why Kubernetes removes some APIs, and also a promise +about _stable_ APIs in Kubernetes. + +Kubernetes follows a defined +[deprecation policy](/docs/reference/using-api/deprecation-policy/) for its +features, including the Kubernetes API. That policy allows for replacing stable +(“GA”) APIs from Kubernetes. Importantly, this policy means that a stable API only +be deprecated when a newer stable version of that same API is available. + +That stability guarantee matters: if you're using a stable Kubernetes API, there +won't ever be a new version released that forces you to switch to an alpha or beta +feature. + +Earlier stages are different. Alpha features are under test and potentially +incomplete. Almost always, alpha features are disabled by default. +Kubernetes releases can and do remove alpha features that haven't worked out. + +After alpha, comes beta. These features are typically enabled by default; if the +testing works out, the feature can graduate to stable. If not, it might need +a redesign. + +Last year, Kubernetes officially +[adopted](/blog/2020/08/21/moving-forward-from-beta/#avoiding-permanent-beta) +a policy for APIs that have reached their beta phase: + +> For Kubernetes REST APIs, when a new feature's API reaches beta, that starts +> a countdown. The beta-quality API now has three releases … +> to either: +> +> * reach GA, and deprecate the beta, or +> * have a new beta version (and deprecate the previous beta). + +_At the time of that article, three Kubernetes releases equated to roughly nine +calendar months. Later that same month, Kubernetes +adopted a new +release cadence of three releases per calendar year, so the countdown period is +now roughly twelve calendar months._ + +Whether an API removal is because of a beta feature graduating to stable, or +because that API hasn't proved successful, Kubernetes will continue to remove +APIs by following its deprecation policy and making sure that migration options +are documented. + +### Looking ahead + +The upcoming **v1.25** release will stop serving beta versions of several Kubernetes APIs +that have been stable for some time. The same release **removes** PodSecurityPolicy, +which is deprecated and won't graduate to stable. See +[PodSecurityPolicy Deprecation: Past, Present, and Future](/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/) +for more information. + +The official [list of API removals](/docs/reference/using-api/deprecation-guide/#v1-25) +planned for Kubernetes 1.25 is: + +* The beta `CronJob` API (**batch/v1beta1**) +* The beta `EndpointSlice` API (**networking.k8s.io/v1beta1**) +* The beta `PodDisruptionBudget` API (**policy/v1beta1**) +* The beta `PodSecurityPolicy` API (**policy/v1beta1**) + +## Want to know more? + +Deprecations are announced in the Kubernetes release notes. You can see the announcements +of pending deprecations in the release notes for +[1.19](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.19.md#deprecations), +[1.20](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.20.md#deprecation), +and [1.21](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.21.md#deprecation). + +For information on the process of deprecation and removal, check out the official Kubernetes +[deprecation policy](/docs/reference/using-api/deprecation-policy/#deprecating-parts-of-the-api) +document. From 96d89ba3f4ec651fecd7a9fb28e187c2c14cf839 Mon Sep 17 00:00:00 2001 From: Chris Negus Date: Tue, 13 Jul 2021 05:04:31 -0400 Subject: [PATCH 080/175] Added examples of Lists to api-concepts (#28608) * Added List descriptions to api-concepts * Corrected note and example per comments --- .../docs/reference/using-api/api-concepts.md | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/content/en/docs/reference/using-api/api-concepts.md b/content/en/docs/reference/using-api/api-concepts.md index d87e4c25de..7ff6028eb3 100644 --- a/content/en/docs/reference/using-api/api-concepts.md +++ b/content/en/docs/reference/using-api/api-concepts.md @@ -208,6 +208,77 @@ more remaining items and the API server does not include a `remainingItemCount` field in its response. The intended use of the `remainingItemCount` is estimating the size of a collection. +## Lists + +There are dozens of list types (such as `PodList`, `ServiceList`, and `NodeList`) defined in the Kubernetes API. +You can get more information about each list type from the [Kubernetes API](https://kubernetes.io/docs/reference/kubernetes-api/) documentation. + +When you query the API for a particular type, all items returned by that query are of that type. For example, when you +ask for a list of services, the list type is shown as `kind: ServiceList` and each item in that list represents a single Service. For example: + +```console + +GET /api/v1/services +--- +{ + "kind": "ServiceList", + "apiVersion": "v1", + "metadata": { + "resourceVersion": "2947301" + }, + "items": [ + { + "metadata": { + "name": "kubernetes", + "namespace": "default", +... + "metadata": { + "name": "kube-dns", + "namespace": "kube-system", +... +``` + +Some tools, such as `kubectl` provide another way to query the Kubernetes API. Because the output of `kubectl` might include multiple list types, the list of items is represented as `kind: List`. For example: + +```console + +$ kubectl get services -A -o yaml + +apiVersion: v1 +kind: List +metadata: + resourceVersion: "" + selfLink: "" +items: +- apiVersion: v1 + kind: Service + metadata: + creationTimestamp: "2021-06-03T14:54:12Z" + labels: + component: apiserver + provider: kubernetes + name: kubernetes + namespace: default +... +- apiVersion: v1 + kind: Service + metadata: + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + creationTimestamp: "2021-06-03T14:54:14Z" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: kube-dns + namespace: kube-system +``` + +{{< note >}} +Keep in mind that the Kubernetes API does not have a `kind: List` type. `kind: List` is an internal mechanism type for lists of mixed resources and should not be depended upon. +{{< /note >}} + ## Receiving resources as Tables From 63d1cec41e0cc312c04bbbb971af711f747b30d1 Mon Sep 17 00:00:00 2001 From: Anubhav Vardhan Date: Tue, 13 Jul 2021 16:09:52 +0530 Subject: [PATCH 081/175] Update statefulset.md --- content/fr/docs/concepts/workloads/controllers/statefulset.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/fr/docs/concepts/workloads/controllers/statefulset.md b/content/fr/docs/concepts/workloads/controllers/statefulset.md index 87286aeaa4..f223a8432f 100644 --- a/content/fr/docs/concepts/workloads/controllers/statefulset.md +++ b/content/fr/docs/concepts/workloads/controllers/statefulset.md @@ -178,7 +178,7 @@ Lorsque le StatefulSet {{< glossary_tooltip term_id="controller" >}} crée un Po il ajoute une étiquette, `statefulset.kubernetes.io/pod-name`, renseignée avec le nom du Pod. Cette étiquette vous permet d'attacher un Service à un Pod spécifique du StatefulSet. -## Garanties de déploiment et de mise à l'échelle +## Garanties de déploiement et de mise à l'échelle * Pour un StatefulSet avec N réplicas, lorsque les Pods sont déployés, ils sont créés de manière séquentielle, dans l'ordre {0..N-1}. * Lorsque les Pods sont supprimés, ils sont terminés dans l'ordre inverse, {N-1..0}. From 7790ca7dc368853aa8851d5e7a1894ab58a4ee0f Mon Sep 17 00:00:00 2001 From: chenxuc Date: Tue, 13 Jul 2021 21:06:22 +0800 Subject: [PATCH 082/175] [zh]sync with en to fix broken links --- .../zh/docs/tasks/administer-cluster/cluster-upgrade.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/zh/docs/tasks/administer-cluster/cluster-upgrade.md b/content/zh/docs/tasks/administer-cluster/cluster-upgrade.md index e9b9d0dec5..72535b27e8 100644 --- a/content/zh/docs/tasks/administer-cluster/cluster-upgrade.md +++ b/content/zh/docs/tasks/administer-cluster/cluster-upgrade.md @@ -62,14 +62,14 @@ If your cluster was deployed using the `kubeadm` tool, refer to for detailed information on how to upgrade the cluster. Once you have upgraded the cluster, remember to -[install the latest version of `kubectl`](/docs/tasks/tools/install-kubectl/). +[install the latest version of `kubectl`](/docs/tasks/tools/). --> 如果你的集群是使用 `kubeadm` 安装工具部署而来， 那么升级群集的详细信息，请参阅 [升级 kubeadm 集群](/zh/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/)。 升级集群之后，要记得 -[安装最新版本的 `kubectl`](/zh/docs/tasks/tools/install-kubectl/). +[安装最新版本的 `kubectl`](/zh/docs/tasks/tools/). ### 手动部署 {#manual-deployments} @@ -101,7 +101,7 @@ You should manually update the control plane following this sequence: 现在，你应该 -[安装最新版本的 `kubectl`](/zh/docs/tasks/tools/install-kubectl/). +[安装最新版本的 `kubectl`](/zh/docs/tasks/tools/). 对于群集中的每个节点， [排空](/zh/docs/tasks/administer-cluster/safely-drain-node/) From f522a71e5e9adfb112ed73af83a92e92924c30ae Mon Sep 17 00:00:00 2001 From: kaushambisharma Date: Tue, 13 Jul 2021 18:53:02 +0530 Subject: [PATCH 083/175] removing dup images Signed-off-by: kaushambisharma --- content/en/blog/_posts/image01.png | Bin 13525 -> 0 bytes content/en/blog/_posts/image02.png | Bin 30770 -> 0 bytes content/en/blog/_posts/image03.png | Bin 28564 -> 0 bytes content/en/blog/_posts/image04.png | Bin 19665 -> 0 bytes content/en/blog/_posts/image05.png | Bin 18311 -> 0 bytes content/en/blog/_posts/image06.png | Bin 19689 -> 0 bytes content/en/blog/_posts/image07.png | Bin 20712 -> 0 bytes 7 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 content/en/blog/_posts/image01.png delete mode 100644 content/en/blog/_posts/image02.png delete mode 100644 content/en/blog/_posts/image03.png delete mode 100644 content/en/blog/_posts/image04.png delete mode 100644 content/en/blog/_posts/image05.png delete mode 100644 content/en/blog/_posts/image06.png delete mode 100644 content/en/blog/_posts/image07.png diff --git a/content/en/blog/_posts/image01.png b/content/en/blog/_posts/image01.png deleted file mode 100644 index 91e885613945a9e2624a4984c0bebdcf3eca6c33..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13525 zcmaibbyQT{_ct+gNJ*z42#PfG0K$L>g2aH7hyp4d<0zfdse~xfLpP%2AUQPBT_X(L zGBgbJUY_{Y`mOc;!#bRM&)$3P*=NUR?-Q=0{g8^Bl^hQbk4jzb!DBo;0vIrEASDL= zz9$R3j)y0iqW(Zh-+gK;)yLUjH1V=F{_A&*j>6GFL;H@Ysj0$S=BAx&9@lty_~xW& z!f427h-gt*Ru6S<0@n9_=^z>%#q?Y0q&FCa7=@?}O)4uPa(a1e&1zq0AHXIZlS2ziv^1pPfZyXSpaj;QZ9`bcb<#`)>y)lXI`j zxctdy&WkL>t@W2K6Z7F|xV84=hMKrp9+h(~TbYwmpPRM$`_U`!E)fx#`4m5vlkATU zB?kte3@+bg9llrQ`TjT=n~uWDW#>1lyM-{@+={@c>Vm)#H;}pvs1KQ>u>Tx0sKB#s z1+e`&|Mysh7EKEK_xSJS0c^nhze)Y)`2Qx$`wzQ+llot>f3p5BvV+inzx_8eV%C34 z9$sPzeXT#t`5aTI*LY?$+i%@{@yg#v8ubA)#+K3R=|QGIF*Uy7#*T5hfee0R>YG%% z*rnKKeH1%vH$U=F|6<^+RM`*1O2_GrjR){OGl>-uJti zP1YHLss}pP1W0%0ydcN1+baZ5&T4Nr6#8$48*E55q4bWs>{Z|t`m4I>V#I@4GVbe4 zvYq%-c+qv7pz3e--doD+xOYhPdXGbiSrkk_2DD^ zz7{P1`q{2$$I`*MHz{mHg(Wb#@xbv{k(k*#8(7xwrF4IS{2m@Aher64MCx{)#p3D$ zrrycyOhqg41WBy4`tY1M{|hx@|IFKhaHx!8WDRk8YY+wae8-kIxC5{g(O})UW99*6Z3%OpZLLS}_REPxTs7 z_VffHXZG-r0v?j?e+h<^8z(>P!7o?CDrG$+XZsd%WF=x}SUuzFWnG0|f761?T)Miy zaV@!ewb+lDs70<$8dNPx&cOfzRe0>VID6CF!-%=_-+mny@5gz@=TF8Rddb_=#D%4k z1s`*q&PZ@~IVBKd__4$<uwI)Do!Sa%K4#JhUT zYnXAqa5>+lRZG3*GpH2sjZ0tGV0cX>k7hiDgC_#Rh=jii6-$w)9_hks>KQj@u$Tj( zd)^a-gz&uzZI9YAeYb6t+HB3uEdwzt!zcJ4%Av{lAeG}vW2d6dBLdZH*~ z9}rmk2_5(1Q%q6JlQ<`|Y7Egv$MF3*O(CBmnrvSK=oQ>5C*ya1_`xtSf9lv5H&V|wcj-{xnKKEefs0XcH6{SM23X# zcFZSiVXySVf?EN}uj6RzV9T9J86myiRBINAsf=W3a{?9|L(O&DmKZ;2khFxH0I-$X zz)2ugmg=mkv5cU-lfr^`jL+HJuZAIhogOvfeGSztEQcY2eSBlS?1y2@?II0>hUvlH zD9g$(Z!p)ONVWOk^s5p+A`BtZJeJM+Wc*bi9P=X|XJw#LdGgDT?qtEt?5stQQ#-`v-a!X-RF$%k&q?{sMH! z7iK-Ju|9)NAE*AXHiYs^B6ITV$yEgdk{#`}Jv`J2NYlxD@!&5q3HVq7)i73L{x8kk ziB%^090FO=Q&#+%7JAb3Kwy*LCv<`eJipV=Bcm}C?i#ZE&;8jBgF`LM+;0!v7@gou zy(74Wf*ubDv#HUd=lS>VpgqN{+Ix!y{CH9%?&qw$I@C;Q0q5iUh1YrB{3FEZ1K#0| z7u6l$yp-gzIEaWOIx;(Q5RLY+Dby{o7ryaqL4$|*2-!QjyBSczl+E>!$KK-8kTX29 zgi_S)z6pG@1kImiH7SFgwS`M5>v9t@Brr+f@F~pFZZ0jQHRRN(O-!fWsjAL=Ctj78L zxyU4Se!_ZB0n|W6<#W}J3 zpv;Upi2QH3E`GQ&w{QV-5u>p0$MMvNqNhoS(U!dLUj;tMAm@xN)acV7r$i#`*WK#R z=9NCDq;T$;zoJ08;MOS59|Vs>rTy7{QpMjfZrRW}Sr#0LYp0ueNe=r%OjGfXH~Z`& z%u7XLySt(7k~bX*ueLDrxQh`!U9l0vYA78sp1Aus$)8u4>e>UvmGe2KxeUfT)^v}ksqDXR$0^-pCa zOohq zYm|E4i%_P5O+gd)=`F(%FTXud8^9bcC-&>NVZYbq>SG9QAB^dV;e(4A?!7aW5Fx*g zWaTu;9IMXUSE0ze2t~+1^}u1SEPy--#`9&gNjQm3d0EH@Ys6p=*h4qJ6q(FZVLIC zDbSHBVXqr~6#Om5VsteLwjJ7db1>4G40cFjNEbQ z?yW9NK@t7hWjJ)4MJ5AwXQRfof~!gE;jfESIp;vhS>>$FH7z+A!IF;C=4_*9Z-Q~{ zTHuGjnrLqo71R3xjE!k^MdW!iEyL{i6h(&pnn5 zbW6TPr+Z9Ve%Ux*6$mh)xj(Hu?RMdV>^NFBogMc1-n7$nA-Ep3{i|Sst&Xg;MX%hd zC%TXq1ik5L9fnz^N6ry%_mCp}r@N~AHgecgScuk$y1ZZ*;w)~jfl(RZ=XFnijlWJW zEcMbqvdZjIGxuV?GNTl%=z%%KjuwXv7$zi&?ri=uPx}S-6C0Ap&wTc488M7gRhr#j zV92-g?K8Kn@q^j>fORG!gV_Z7r2?iv6$EbOaBttZh#rsTP*@;`O@Kc#sf4Lt9&c?w z+YKVOK*OUOY0X4*Y;~udq-II_5+2X-QHD#q7=v$ z^zzNIitHcS@RpYaX0>SFsz*s z#|_G(=IT|yKS=~6S17qFg=nT?LZ0w_Cxgu0_P=#3Bq49)MYK(ro^pC4?FH9nq+>lN zN61m?q%(e_+8dIUfL7AJMsOD3y8@|e@)@_vJ;0pHn63|Ho-^uG_=6Dbj1v|Nf+ zgeOK}S^4~lx93ihI+xY+$WEIz{A5IcB~SY(Ok3?gdW7XJ`v0|o7(zbK;E5vVp*AA)134 zOmJn16K>Qh&~D;o}hNQV2_#xr&h;@WJKFRNs}SZckO0 z`;NRfhH;7>sf0(Hc)yk7Y87}d?V+`dkM089qy-{&MmM-bs3tA-QMN<>?~HYQ79N*b7fJBQ0UfF_x2<%G)Y( zJp5jTx$BJyrw0EPQYqCtZQS?jT%l1~A)39iQWn&B(8ugT@>~fj>oC4xN<4|9Heaiq z%9_%Xnu+$rnn0RXc%6+^HXU$yr2UiENbn^<#30tDB+5%(X7zA_btT;|#Enh7*kE`4 z^@VJHl8*ZV-4tK3iK|b#L}}`R7Etd@i6#J!$t4&o>cpS*oSaH>9q*xd%u`ZRe>?(x zlee42HfX+8cAYZj5^!YixL>#HNk??3)J!m2-OA!l&iVdu?fR9(r8ku`VTz&SmSJ>M z>{RslBvc$MQWB9N+7jw`BBwWyO=@K#tTl=CaXlcbK@Z`njZG<)rx)o(#`#xP z)twBxR5frHw(lqA#4kA|S2l$uDnk!#e90kf6~Pl3&g!IVvYud40;R7=6)_;feY8*= zrYy2~aL&Qv6IH;HwJFTw&I>ARYhTZ>cJ z^VoPIC3IZk49%hcO4b@_cpjhh)dy@Km8V+~8q>zxSZiQFm6Qs8G~-)fv^5hk&mx8S z=!S`yrlB@sF+xKPrjZQ-xK-+ZfEPd|fHwdH#veF|f5pE5lmck^hfKtO!Xv;KP&iew z0h<2?U2x(5EW1KoFsgrIHH1|E87cq&Ny$}Qeg=;WE#21N`XpFrvr1#2|2SLdp^*;9 zPr**zbCN?Nw=)d2OLsL)*X^j*dryv}(!L;IV@K5))&g7F(6cITjXRl5*~R zcsT&w2RN!W2Tq$HJhgKyvx^Fo#D4VFdA<}mXUIWdpQE6kUxgO+Y-*1FaIMF!q8n+@ z&6Kc`61I`1Mf}30N&bVuFDtk@TFpv@#-~)8Ycp<6xG&rJGLc*cT?K;cSbqc!8pn*R ztopj`nw)|7fgR3lJ)%eTJ{M^`PGU`PZnSeSPUi_ySWiV@Ip%BG6Yl9aPX9>8FiOL{ zF85#S8>`e5CL0$ioHTL`kw&#YmoBR?n(ghn7G2bnqB&ajQW#Zo&2Rd2`yAK&4RpS^ zp7$~0C9tPs!o402{z5{+eBau56nJ>Sc4wpp5*}rrCxP=88O82xM1RyBYMM>UF8daj z{=2PpzSh5B-5P2jC3TGsO~;ImV$Vc)?+s^#uL%T9&j!8N`S?+M6#14%p5Y-xiTSA( z=ci2V#ywgfb%(bQJ>vXStnCf|qzY$=SJ#F{oVZE6g4NsEJkf+}D8jQ41iQm%)c})# z027x3!or-AM9$qVXue=?Pv1~w4G2tu+PWX3J~*Pyi%h;@I@_DS!|UQNj1qp+w6z}| zEmbl2=O($fWR)-XXNP(p(u=z%2A1zXPzgx3#KgxXcXKafq%@=6oWn$W;`CxlqBsb_b4WY(wewcVXAbK~R&kML8U_#)>wmv1w+8R!ZIw31eyQl3q^(_u#MTkuG z-l#Ke3@-n5&#Gw2>iFYLk$`jR&ZLi*{^JIgqMO#ocH)^ZXe!Eb2L3I$ssaN#SAR2q_6ni+ck9G|( zAw_KkAXJASRVTJBz`ts!lJ9S7OeYbXWi8Mjxpze#mL5EUQ`p%Z9INAEnIr?c-F+Pm zxMzD4`?U+sG6}8-eDEzBfd&(Ss$&A!tgOm6n>ZnB2fs`m^r)QDxRFE$&Y%0>6fn;qHw&*ysvF6|~?K_()rb@qpE8-rcj2Az%y0kiG5} z@xH43@)mgr=aOG>8Y4hf!G}D_(fzzI8h$MbS7pu>A=ujI+C48}vNe!zA>v7NbJ z#_S;(=MQ(|CZjYViP2Qz=RwxAs!v_%#NB=YzM!1+{8wnObrg zhO$*vyQERrbuS)_tr(dxy!zI7!pT|7xHgHHXg})_JyE};I;}1+mRP1od*9B!d*>i1 z?a9}tY4M*6LvIble7BP|^lbN%SRUIT4jbPomSQk5jNHWxaE_^wQ+;`7SL ztB-1?4%9#{whwELs5bMWBSW|6C4QJ3vutnkv!YTA7CPwB_wiu9$LbJRZdQE@z0EN8 zqkH>4z2{=Se?cM3)|vpBv^K)nJqW(l}ze*(ruK+@iPZ5GOWWv zoTg5`PkI_=dD{G|D}%Ckr>3+&2TBLJmXSsZedb z(9SY-HaFZF8+dk+b42?l2H(B5BS#5?Fw3EcfxD><*j;)*ZO+hz z<-FTpk@IvAo+cy_OV>-eK)S2*b3{)+|9nw|3?KXpzCiV(S2;iW!AHDPq(9q725gMP z0&!=A{#QbG_5|^C6R6_1E-21h{4Egs99Wg@E%++!MLO{kv(`l;I>UI()qFmfr2B{S z4VdS~Qhsmd6a6@{p8mBr(FcB*^23|jBtyPCdvS4z?n)D6WOmG|@RYb&0eSG+Z=9@^ zb&J%tLXXKHJiK>(#pO)piNIGrvHZP>x12JbQ1xTB!SSy(KO~g=j-8H2{QG#^R(zRv z8Q_FrT{O2tbOtlT?Po2XFc`maR~Gkvu^*>^Kmq3O0jrP-IrF55bp$i9=p1?5kQGO= z#6WrmFLMpmRuh922Z5&F?9Sve+>SZaHQ~*IPJvTCaq9Iwsgds}u3UsDA1Op#WM)lZ zBuDMUZy5SGS_qI$@h{n|UbRyhTYMr;ICvSRCH~&l^_bQOlz?ucI+#{t{X_o7O^S(! zWpM0GMe50hw0q}G{E)RhmQO11Jx3b>-i|LxbRe4dds%ajLMRjQtaTLoIfGN&uTgQS z#u|f3#fO?+FxQ0~VObB~4^8-`90H@w-SzZWK}1BReVO@oJCa0yYBBXAb&RFk?~A1ONZ2`94SpT7Lyluf*? zekU*#L!x<4G4xSg$ar=(zPN^83ic+8Y`&a?h9j#`n+M6DH12otP{0e?yo(O;iS!rU z#nQ$RcppZ|p5Bl2VxXS}OrJmxMGVKS(1QXR`i` z8?+kWkFg9v;K6n$*uapCVFFSoTo<};El;gd`q+WJNTBlT*l|H&5{5C}I4AI&g%0$d z$B^60u@BFM`bDz_mf>ZBJMn!q9amDKlt(|yJM23rwyx&jUMNoPWRkW|=)mPe3N`Ok z8RL7$`#KeKwfLX<5>PJaW;_62TUix2J(mPRvv?P@>if+@R_pz^{te`omu_!@!WYc) zmn&Fp%4|xL7fGrzC0(rfAI@&iKY9<+M5+Ft#08nov^9wjc+_Xe^}&E4HY zSDp6Hb&I)+O8)MJ{Q=1g`iRT<70JMd`1uSe#5~@vzDhiQI<&PsylKkwk{p z>wC04Wv@Kxe%bwPPD80t7eMMt=N-Gz9~qxj;cT^Lzw+&02H?)Lsx&VL%cthGh zkx308_%tbYKJE|z5b&M-7)#%w=ylc@cD3ZyJq+;+s(6mm>Oso8&CT_u(aaG?o_dy+ zpZzPz0@%(U0I(bYE7Yf%7)oo<+}VvYz5HIOB6atPI-FviqUn81=6r@^Ku6&KtjM&( z#N`@PS8nPWATlY8=}ceq^2t)(kAc$ykw6m-z2v7q3S&9TTA`kq(5E4|)$`dSNBHf# zWM9Gg8>EGHqq$pbw!}UG%whii!}H5tTUxwB775aYSL`5>&`046>$jcwH}?8b2tiT!GZ-+54n z?~=vJ#!@u!exOri07>=8{-$Ch$XKz$;)!4{<~!zvy2@E-!2?R<=|~ zsPH}&3^+W+K^J5rPCt4@xOQ!nNElJ-@*&g1TYt6zU9 zyM&lKoK4ENOxE>S6Vslc3whXU=N@DQkG&Tp=W9RDPF@GY^E~)mJsRjz4$#k96Tq$Q zE6GR1-0~%7MTbzA3{cDypFDqQ2!H!X2w}$Rhrn|*pl@IIY)RiPnl_!tma-L@=tEkjX`*EP=cyZoLjFaaS~X`!ijUlVkp{r}u-&Q7|hyN$zZ4X3ybp6hI1eSBfxJ-3bCQA#0=SMp8( zEj2rrY!ZT`UMxa0NUD_~GNpy8<4<-{Z$u`hBsH@><~rL4vPAtZY(`;;OS#n1aQ5PR z(^Ely4Doe_YuCItCq|M4#tRmey4FYOy=+NCucE}RHTC)n7^~4;{lvt{v1e`;w{~|! z4yPED0+SDiX>bMGC$_rHIADJQKgJCaa(On8&OCl>U?PXmrxSs^_<;0RCp0+CeLjYM zs4MqO>U$q2eA;KR!gRYMUPO)DUgJ@yZ9qqbXVUzUn%Sr+XZS8V-@Q%x9w7TEbrS1Xc<}Z{yctC z%#0&D$oTx~r-6Zi3&!Pxwxz{#!BzVMCp_?5KKkPM!xCJkvzYnm?ZV>BXO_!W`SI4t z`=89Sb!=mzN3XJ($wxD3d;=aK>@JB19T*ap{L1_pzEV=2sLe*^MNK z*&&*BRe~djeX2H{*b9+X zxnrHC6HWzVIA>rC#z)`s6uNT*ImiGYbiP1y1%o`$WMxAR)T$u>;^zspGyVDKPr&c5 zM?gj9>QO`*(4!?u^cR5lf4sTjKXZabjMgnd3nB_x4er%F{v6HuP$i@Jl}5Dcedb7GeFPoUdvmC-zQhca3la60Dys$U z&+#%(I3`X5aF>^^ss)#BQJM3F(Ecqw5rxM4IXQWSNd*$Gs_-(YFs>-5@jm?$Q2A~{ zL96_x^^#rli;JCI;Ipa+W|tS7r83vu&d(O9rS|5!>_(95z`3`=NPFp)7beRZpC7$D zOJRy6f6!kktjjX#L52^*4v*Nzja;5}7@L|1(KyskBuze{A%`+O-)903b zYo7YE=VusMT^k6EefD+n6CY~;`7`0$%SAwU6yTwNB5I_ew8;B#L;!Jh3q-zjNbeL_ zh{1;RBwxoEJ~t`VZm0=n3L$^Se8HTOOm7EI#$ud-YG^yuSf&Or|AKk~MF`GcJ@$`? zzRHx_9XyC`LT#maZClpkS$^2vEdzL*|K55C(H*-5Ccbil#D|TMX_>_XTo6cMuDrMa z;7}hw>!i4EoJF<}63U8>2pxick}u>m;`D^y7U^BC%r4e_V{41WNLy{HnNt2_)nYed zw1XEF7B0GE)BS`h1x^z#gzh|jJZ3YJF!- zo8G@?BekETyqRvlxsFPD>$oe`TRvje;GMU3K6}2K{#3m2H{I6s_w~$R4zpheBP!cq z;jc!aM_=m~b#M1A6#MLoowGbo6{xZ`mR06DT9EL%S1ot>yZb9*=OmP5|8#<>v__tQ z_}#Uc)7@b(0Bl!tp4#sHXiN3IKP_>-IHvIH3k z;~(#*>2G-VwpIaL?Q^*oowtAZ#61r96&nTc{xt9@{(1GiXW*yR>U%!!JzHt7+sQ;q z|9BN`;^h|k%JhwX)alAB#}RsL%bOOmB!O+%d(BYqfibT1+8_@wF(U?}b+#`C-6|R7 zO^Z;sq}DhON=#sz4g)rd97i;cta`J^HP6pts||haA>$2hrzNS|(RccXr!f=DAJ{rr zKO)9zp1Uq99vm4xFYrA)H0<9y+ea(7Y)Ki}5BC*v3L1uQ;}vF9Mm5Fk{h~_O{r>NhTq6wcB=HW#G)AIss`ePR+1VU6 z5YpeV^9{l%-!4`stMiAM!)=8Bs|5+ZwE5Sq+pyF5G(1s0@Z9Fi>}*s3MQg>uK;KRJQ2J*It`(Pp3}Y$QwR^#i;? z$&V(4hve_s#_z6rb1W-1iHGLmW&QzWd3+NngZz+7S#WXcu%DMGJq)jMu=)xh00Fb` zJ=N(00`Mji8Ty;NtccA9Oo@*b-mMFk12+ypw<>INp29(HcYQwl#? zo}@tIX*2JHgJ3kW`%YHcScmOn_RWp!Y;esLQfgW0F#ExPg@yFdq_%awasu1nV1- zo7O$4Fn1HgO!+*Cs2AvSJ6kkd;P?Hg>3*k)YUS2SB%thKAWK|VqEtr}f9F-z?AP@v z9m#`d6Vy7dX13=DhY10RzbncL^s$01atFIeCR8LIgo8kb(IC%x-W`VDB-n(L>iKeP z66M?t=|t{_@Qo}Czha0dQ#C7dji6lUBcNr0rp#!0AdCqqvq6Xsag90oIH%7}I30%O z)(ZJtCl0exB}`PPHg**M6lflvM8G`DF8ikz@hXx}Wt*_w_mUD2Nbn2oyC!bYm!u&A zZ~5?Ss$pD5kN^*c!-_@mn!Y9MwrQdY9)Es9wsI2@zSVrMD->fDIv^9nqghS;zCvV0 zN6;>PkVRae2WUQQ6=(VIrdzaS_@hhQLRe3K8(eT+&B*wEib&MWwEGTDwXf58`6_>{ zrp#^MKpwn-D=RB4s78#HaB3>Hm$Xxfh9p-^#?&=Y3EJ;JxuK;U<0*&75apO#XUYFz z((4QN93yjPueTcfL5y-$QxWG?wL4N3kFi3%`VPH!0B4oa>8FhAg-W>Ok_f; zTa#@sOw8Y`_8dzaL)MTPJF-)(a4f4UfWUG=RiS+TdkP{scf*8DrC_V1i8=R7F~qaY z$#f+3Q*u$Eo5`&YV%ec~(B9qTG^-jdn$}srK8`4kF-sRxm>6RO*Pw*->(yfpDr0iq zdVYZ>NgK__8Vew>McF`DDtwhv(&l|xU6@fjMg3i;!=mhOUJ@2Jr<=2ny?9Hg9{#Dr zP2^KzvfRs>qV9^aGNxPfiXmIsXH8@w6n%`62lWUK;b`=2_am zWtWcjlCT@#lg0?x-JX;17YD;zye5p+MN_Z(R`mkkvN#7V9`_waZSlof-DR6k@dlaw z9OC^>0G=G{A-yUm|Iod0J#t2&le2ldQ=v<1mI94c2+_%ol?(lYhrK^>#sx;0VYF05 z{mJ&eH)B-`RMM(c#jBPO_82SJ9V80Wh&?w1+&k6;oO%VX3{lvju*;Wan!GiEy^wA8 zD%uSBhfkTrOfmc}GA|H|%JA}st04U10TxA*Ht5;xCc8(k&}TAtEBSmucX~2>iFnK( zeOm*$_|c{uJQfRR@704}ZY1a!MX@ixZU9E_?vJ@g>*F=^<#*&Tm532mb4Wq0f1P2J zbhUrx^EhIJzd5KWQa>_e#LX?9TQTIukkI&5)&KR`qQh+L6d+VZ_t&$aEzfm<^7g)x zzR?l|k*APgk5r()HT5IUyXxSa!I1i?6!(ta+c0i^PRZ+9$!@>t1&LjkWU7zZ4n1}M ziE-MkKR*U2v$ZKI|7Tn1zayWkn+<>6X82dB9KgN*ch;M@dhWU;k!7?{C{qwCT1e=T z2S))=#~Z*%yIRne-n9G>5+4t`23z7)#lF-rB(*2J&O}8G2C8MBRD4+-r;2V-$`C zxN^HzthcF=5FARyK4HdjiK#{fvPlA=_S5g)cSWeLC-86cXS>$LMb zR!Z%}KmwWn;0my?^0Zq-k`;2vtF#%n<>VB;bpD(qzPFR$DYOhUK!kIfG=0dVQv$k2 zRf3=YD>%N&wXW{7050t+oBG#<7l!}3WAazy#Q(eiSb)E;vHX1j1+R58|7lGs&7kspV08T1^vtlLGDLTzZT(wYx%|QK!6kel`HWg{ j{q-Z(Pae2;_;*yo%LiUeB>>lz@YGebAAG%U9`OGFbqQNa diff --git a/content/en/blog/_posts/image02.png b/content/en/blog/_posts/image02.png deleted file mode 100644 index dfd14d7cdc9943c158cf5b380dd5d07b1beb4409..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30770 zcmb@tc|4Tu|2KS&HH6BRy@hNc>rjJG$~rWbVo(VoV;Hh8QB*?N#@5I>hKwSNeM*tN ztYsb9$-eK;IbGlHbzQ&bx$o!x=YGA;!JLl$_#B_@{XU)<8(m{%=3@o`fK~VUl^Xy+ zg9ZS|=OYZ@n~FC))&StWt$Rh&)TjSLXRdH^Duv_~hH7@{7@Km^JP)0|t;iJj^3@cR zo6y)zB~!Ipw_EF5jefgT){!|if5JcofRX2-kztk$}ufbxn3;aBXku;Ae; zp5gS`lksQf<05WcQt;m`?&zbd%wEjsxe1;^p7!;Pt*PDc?_uh^Be;r{`LIN zk`VNf|M>s{TK{_fM@_Jm|J#*s^(5kL6)kLoU9ul8sHLrssRsNi`sh7JBB=w&+L(G7JI{H_7WOx+@#^R68AQ!+;YFx z_n^FExmev|LVTc+mYl)BXD&aywuN0}JT(R7nNt|rsyf)$NTP?zT&glC_1m9MlroXf zeOzuf+FPdIH;64;*HYq5)-Fv!uV_-&buvMNMe;0sLG{O*GAtccsruE3AdUwG9ioN&7T=%Ks0c>Xc5OmwC@Z=Ovgte@LTWtMG}?o5 zn`=YbeRYOX+Ig^TR-Ai<4G?Vo-mqwSp+|ff*NZQKv8Eqd4x9IEJh%Lulr2= z?Lo)cfj-vYdDNme2Hu4#gRz2OovhQ=is)=>3)Z?U~#Ga2jn}FXN<~7^ee;!RXQg^R?$sXU=%{4cr zx^G+Cd1$TPRkreUcL1B%6p7L}5~BZt&P+bs`S`)q?36|9l17|~KsFO)k1jJ?bjd@f zf;2+DeY-r)@@RK6PtSRxWrD53jm(yflZ*gcNXui-5({5$#V3uc;y<^nrJcDDcaObn zQW#xMw>)R-WaVp`AU-7~s)&7&BF|vdn5}n08Vzn!Q>l>4VDe#wqzH}jp~A%R1g|OD z;5|aX*j}#d#Bq&lO<~T3k$#tT2bRe+0G9NqPfs$a?|vofq$umh1X`84rd!W+o(#Gc z>^ocAnuJyK=uK7L9`fr~eqcaH7PSsl&`h%I{bVg!>%G?;*<7nHv|`FhuIP<;m;Py2 zf6Dq%>Ir_Xvw%!`wz={ILn-pwy1)}Pj&|)@c7dm=bxh3lK{x8P5r{G9yBk#!ti-Bv6l%nm?p+7`?<-kNWv_Lee}C-DU2ojjVPrI|L1! z-EjWixK6^B>Z`i}}0rVw< zdzBcX!UjNgifVk`Zc1=36>qr~s$3eR&$b1&e^4Zs?|1C_K>is~_?M5Fd;IDQu2OaG zg#anwRRG77#1vOlBm!q*_qf*r=dOBIkX#|>iL!c>S)Cv=lDBg9FkgNm+HkBgIfXH# zrE*WE<=q|);heqbaf|8wUWK}fTh&P1{N%cg+#<$su!|7la)wT{tH{&}dz$d?`MH+T~xG4o|U z<%oCdXM)48?psG*XPJzhqsMSL$?LV5jl)GDsIma34$`cO3%WY@*_=cofPZ3AB|Ua3VYwz2Ph(fi6?ERM+J}e;I+YPb6)xzB{3H29{7#uW&RHR(>p@!}3q$Je zV$o=5=8L(n85|BP#ekz%nSFj|?V+*QA+ap(E-@Kk8(BY5dm1d}Qa zH|zG;c#hhe!PUzKjtZ-(+qHG;Lcr+^B+&rI$ijU}nZwa9XNO zhF|-@DO`p3O4; z5SA5FCST?W6*y~$0j~KZ*-5KvsFblnMI4{M|0PJl^`Ti6Kq{8Nrl{3@Mu?Zg@y^d* z8MJa>=|T{=er=pqMLVKOJ`OKWgT4k2I?Sxyi zrIMng^i6^*wyCv*q5!2W6xHA->}*OMg>ezDPrD_|B(|I72A18k z3Y_{nYDa5(UGL(GE+Q7YYU)yb6dXulLi4-kx@J3-j<0UgKk<7%I$i0VgLg}d7=87M z?sd%pQn?l>vVjBy5I$4ZGyc6qDnE3jYj_00O15XB+VtDjQwD_c&_v0Soy?Fm!|e-Vt6srPI_QDv_Juk}-^%f^Zh}%8T5f zr&41pk%?C!X#wMj(8Rq@7YMyCf)6&lQ+`YQRov^a0U8#(zjvG-?YCo6`>oDoX&fw% ztaJ_Bf8$tm9X_p4`ZdQz*8(?>Sq7+e1jtN@2gJFN)Y&iIS7_3h4?e8i`p9ZcuZRq@ zkp3(k_ucez{eUw+mrY^$uMMyFjahc+b1zNy?DYBsq!p%Z|LV5lv;2eUyqh6$jN_^a zUcOp|Z8EAg#Z%V-qtNrNh?^I(A><0z39)^t;O&H>24JD`Nx5SG4#l52gqGsLif%z! zbNFa+MXH2!eWRb?#b)iDW^v}7>zx;F+mxB+9nbq17Jdyi2jl9~DrCl1pImtF5yd`3 zflGVs2(Czi5@ioM8};7OHpJUIP?To8!T8Q^v!`Z1%uXeE#)lwdzg@KKb1mqveKBRp zJXq!)XViRKa(t)mz}2Xn1FS7dPw~W~1bIiIcJwaFjEYxmah-bfhE~N@ zDqVg5bEKoyuYis#q8b(%v21tg8%f3ot% zK7aX%%Ossl&D7aV??Sf&`5uQo*Y)7Qh`19*rmWW()~gKHiw?NOE3e?S7CBK~RLRc< z(h5S3{4zBU4C(UFwDd8Emb^G?ZWg=3c!gyR+4-$(C1z2ql>B59i#nRdZ!3AI_~p;I z=|V8RN7Fm!yvY&7MpuDj9RLmCc##OQEx=A4I)Hh$r**o;q z-BuwpB_S>HgU6z0!`)wUE0tYK-yV>ZYdT+>iR0!vHtX{#wgS1bqQTA&fV&$@hhA{A z7=8`Ud~;pM^y+o{i^#GeN0e3R`N#X&%w~ux;{@afjnWJM853s+g1*r!X9wWfc+Ku) zVt0K#xvsGG>~BJz4ns||@$U0b`3bm^a^8t?q3V~J;ZgwTgEv5oJgf=M&oEF_kItt_ zw35(fc8V>Ay`1dSYWRch0C00O5WS$}7}3a@clAuQx^*6aR0q=L^b*nq$P#jo-DaH& z5iKbJB8S`Xl5Hk&=e?BLv#9G@oa82#{`AV%(^~YD6($N29_0~AE8TZCJMn&SRr60w z*eD~o%)Xn#TPA(Zm7sX{3GxRW)6zcnNwsa;GNFL9oCk_W-y|?aF-WiCFK4b|E|mIa zIU>r-WgaDqrnh{P<26*KXW-m}VQO&YKQK}y_7!AH{?h)!0Fwt9Nbub!?()FP~Pi7wX*QM2+l>G*M!JllbmM6SVpDV0bf(q z8@`2v)pRg3gKTIWWw^3y_)(H_fPm@ANREd}G^HOk)x8RTk^jvl*;(Cv!1;9^-q?)g zdFr$&I*r{}GtOIj0CO$U@5Pxx-Xlc>?7o9TC zIaLf0v`@a1YJP=rPA_bQ??0X`=+lDotPK|ZJN5FPS4Tp$|2(vmKiL#&ssH77@a_Mp z$iM#oS4WLrU`V4*J6KZp!^Qk`q|F==Hc&g6A?6rY340_Oepa9vPG zn)iWkZr{}!gPG6E4hubfNj=^}t{V%gRO2}1H=E2;x73Y<^@zX?-{C9+%YE%Tj)S8; z3%CD_uwcaXSM9O_L^a6Za@L`f`P-Xz%#1uK%@vr|$wp|3T7+}kGU8gB@WEUdoAm22 zuDQhIPp9c~J%%f$CVw8zt zf;pJdh2lX9VGZZEk?{Hv{+yAZn@+Tlz4FN6({0ef1^$#3-v0KM)HNvq>sf&*S@}9Y z{fX{ySUq^JL5to#ik@2>zPGcsXxEFfv>(>kH;RCU7OkiDrKr=&IQ3S%GD$3~QBuJV z1RaO~eGmw0zhlAmVZ}$p^2HV5h}PJFBIrH~Ly}r)9Rua!DRQAL9`t+r&%xEen3SI^ zcxmq%;$l!}a&SFtb@y;xNaqOGLT8hml^4o%m5`T)ax|VHKvG`nwhF$G!=b-y z%I`Z%@h?jGSpU3r)@%+SvC*J4aVZys;H6HulsgU`1cQ4n0sv1&O-J3GowYeT-0-f5 z&Jm}@(W$V0gWUGEdbs-S9d?1+Y{EXmAuf$Txp&9M7m55ZE@{#HC(w3i0@eb8o)@5m z!g|tS38kR7jKyyR17uNNisvIgYtIbTy#d;jr$t`J+0+EbFJrRaAE9xG8U-l*!eqgN zbvJ{e;JGLG;%UMhs-pM&Ee{r;&j@0~Wy4zAh`7~`2;4m<35TFh`*7Gt4$FP2$eq^6 zvrT=Sihro#z`!mTdlCbnlf%jnp6oB1vcYm9!WED4hq%}RVm^o>dR(&!j!#}u&X+l4 zRnh0VT<+8Fl+mlMK`3WuVY!KkE`aoq4(&I45VEh288ZcL?7~wP!NR)kYNJK4|BW??wFj0FJHNTSS8DdnYuSOI86ZU@eTtV zT*Yk`^sa4C6>;$BX9UwyJnqxL9$6WX>43<5MT9jWo;4u?G3j`KbTJ`XDogjGj^EKs z15-NIgaoWL6L~_8?Bq-fOE?O5T9As;Ze71p2Oy=?%q|{>i%P@WDwsoBM9K5p6PgNz zcQ2lM>N~4ba%1XORYcTDhiHL|5c0%nvXlHzr;^-X8Faq+JJF&JV+~*s2csvQf?Ho? zWd|E!6~=YJdPX6{yMFdd%&afaM8)qYVtbU0SMm5x4uBM*L4L_8Su*_u%PCiy4om`& zLExJgsy>p>d}qHKdilE@CyWM)VO5Kgp`I!iV^}=!*v@Zj-R!Y=>G9@E>k}yz!NyJ}NB(w5gl=MlV zJ9}kqK1L{I6_ce&;Odg4rK}vGgtH1i!k?IxP;0$Vd`|?xIs(wQJ|Z|h51gN!_XSRt zkd}?dPs1YRG*g(4bGQp=KL^K3L;CnddkG$@mP4>USo&y7eWhYJ7#Vy5hkqUpnv!lz zolje9&7`xzvD2(aUi-V6$qTWkUr-mr%mZlYweC++%f_K%xveg}^B0Z1Hxq!8#a z%f_9n-CImx04m5$Q9N)Rh_jN@Y9-M`VRb*ff4em!sLE>#kW0x3=ty$XpNR*pT9!N< z@&jbx04O<^f}$g0aroZBZfs? zs+Oet#{sMuHTXQAz`v~}w1f#yk4zgGl-J;KN)21A;hVWC9m$W+4akG#dLdD*EwIwL z<+O1E8w*-A>zra0`NI|GUp<%gxyGqlXk^Y;b7{Kd#20qI^`q)>l2dI=i8Dzbh;;_( z1!WogGm8gdJM@RnBZqsh-Jk$Wth{3mb?an6FebX<-lyreX{ovDUyZ%C0{|Iz5aehj zDa%4rI^J3=3ey)w#KILB8oaXNJI=F=2V~9AuN5UWDWH;v%cVx{(-=!%TC^HDTAgWc zDTYEuoxt{XU2IcZ$oA1=D!RJ?9tv<5o2HjtZBV8}r7tf+uH$Y%tNf)C4jyjnbY9#U zm`HkM7XkfxHB@fuYPmrWf{jHxZ9`U8s$2rDt=}5`TM&cdz~T1 z5LW0e^Kdr$%(Ma}wnT|1IGH(U(=F)ieOHXm$_OTJy}n^X`7=oWWk|bMT>it%&?2JQh~nsY_WO(Km9p-1KVu#yyV|(7oxdlnD zK%0=}nc-}FpY*Z<(mN_+s+l~;2cjpUTv1*u-Wkps`9ssT28`-)XM#nI=yGJnaLPwRT93x z?rnR49>9K)>5d5=*nqZFem<>`7c#5MUwTxW=`qBJ>H3qAs7vUfg8coeM0!U)Y#!ZH z8jWCqM4~g!HbP;&$3jH<0ZSXCq>i>NJKn zc+AHfx{GsZ3g0UcuU^(+iqS`CeRrH?^D5=++n)hi-JJCMlq50JyNb0L6{Y-|cQAM% zC#jm1N&Q@p4J7p>qQ<|<-I9Rnomp@)_H7wjn|O`h%uk$Cz&;fb<@Ie1ekYE)#R(`m z!kju`IT$(D2Jrm;QRuhCp$njm)*CAuS~FeI@|k{g*+ANH6gkeWv}D!)sbx>%{hpTy z^hF*4r`L+es|%77^UbfPNxScR_}wD;m|SkEq}=KpY&GeYPg@jdwE^Vz_{=j2sE=e2 z+oj1zEy7>|k`t#Fgxk4lXtZJ)K~Dm+DM_v|ZjDFe;2q~2!sn{YQ@{idXej|oKK49x z;Gz19h!ejhN`DmDl!8Gu2ekpWqhJe`rILU89_)fw-=AzxjX8eTd8Wy?aqtSp=aBJF z-xP`Afu#Fzv-a0^8+$9K{@4VcS?A%OueV@kmWh9C*1~4vVQhj}z={bU=2g5{0GR7) z)h!0q!8AcLYzZ`)W-2PsAR3D0C_!t56ZG~X(^wX^(uY4DmBal0TvI3GLaROU!%i~S z0(1sw(5T^jBo_2w4=?w8{hsy5beH(f)|W3E-&pM~vTePEoBak*6GRU6`}}fRrs)Tj z_H8M9<%Z|9h_&Az_}>=;u-Aa{yFZ5<>g(s0jcGuWFO=_ADHzhC08YJo(*I9OQ9Y_H z2thtdwD9&zRJUU3I#L;4;9K1xx{pbGK=pQ>qq?v7Igcm3Qsu?dlI_U|&#ZF_@cjv> zm#_2Xc34ja#3;Wqpw@R53%?Vd#1j!=dpS+IaD+5`zRC+kZ5^qY0Z|UFcBmqx_Y%M6 z@cfy59LD?{Ksq{3n8J~@bz2ad!*PFHwgpP>plD@ZB<*EPCGgJOBCZ z-v-Kcn4kFrVcRhtx6YZKTjizbHBOe5+Hd4&D6?z4=VccqBlOT8UD`cv3T4Z2@KkEf{vRb_8~OP2~QIM?$pf8;#``v_GGut+`gLTBPDM zUl6o*prNLo-XbsOc~jSh7U{k`J?QUzS+lp975Cgi*Gad1GH6u^1iz~pRE9V_76=El z%cdt*x(9Yoyuybc6^`={kvbW_+n;gH;Jly)MB%eR(#?y$4<1gg^D2J*9X#_oEs;L( zFn$I*p+sgCP*L2(`K{0+EE|>{BmMLiv(x-1;2t*IHvTi#3Kg01MG(~O(Qo(02~17?AJ0@pVOt)V&gNf`9w=9xC_@b#ob+V&s_gu~O9Ht4t5pz@jX%NFJ-JyHuz zzN$W+EeeyBspjKKh80^O5(bkNY~CtUm|41tIAaGw;cR$JY}ulPTMg#E(Z!=)H&2 z$fsOQvLw>gf1WS2Nq#DiHXLGIdF5!+R!cPdCd5Ct>-ME~-0jR*`;ezgb;!PVe~Qg9<&CzCdE z!YttCzxHwT~fAtZZYjyIcJ(HBc>6t3E5&XKiO7DU+JI35Z3> zBqyDR57jhrvI#xK_kTmQ^qyyJm=a+c#szs4CErq&U4+OibtgTZAfyiqSC4SROTp03 z#^Q8-3xZu#46ar+X0Mnmvolw%g}D5yrP{?1PIzs5m7YZ74KzA$)6S`|7woUH7U zpR8Q?N#N36;}z@G%*@fr9`D0$8DN1&w)ReQi^I7VM3Hke>AjC+{h$28$5ifoH)P_f z*jW2=_siqFpwI#fGc|7Gp3zr9B__GlO=ofmvUdkPly1>m-t*{iUHiUpasEkjzTb)Aj~LmB;$ohvKU;mIQf})Pm%xlw`ZSU zD?-^jW_#vZe>b;)5%DXp%g`V3MtqLVj)|gLcIRHmYfSfa8Hjfr$QhY0oCr8xpmG@adu`~)EzK>k|o z{(ktj1>v3g6FW+noFYiIhWmpN8pFLm=-^D#5M$fZ<6|fM5qiaI`DvBArkM@*$HV9* zQ=s!t>gyDeB^yDgaE36&p7N@`@t4G2ycygnsDo#?H4Pt2gC3WFC6vI-R1~vt%_?Nw zIbpvu7p~oKQAx_$zdoh6)Z6c*9eWhr*`eG=cPs+Ggh=(Do$xl$w9a&z=Va7npkk)1 zAXaHF3+HlBry`~me9UDbrlHJ?Ff4xW)1sqh>p7B2386s#q+R)D=$A0BL9D7+dOhyK^bOGP1&c zPAqcFrt9;vzuqv4%WYZ!);KP4`sX9;rRC!(Ng(aAUXwQ^R%<#}+?yX30b=N}JM)3hA zwSd4U#P%704RZ=$(*QH6qaXxzhYg6vJWU6gY)S5d)Md^KC;~Pdn(c*%)wUY2){s|PT9wAtQNl2Ie zZx%*&rH+MuXZ8K9Jh{VN!N>hbH{rK(kwGhV3W2?EUl_hg;tYucQw~R!>-;9~tR4mi zJf4dx&OBQe)Or3UIw$#t`>h680%yqX891w4X;oR$ArCy`{LFMw`UMsq(_|-Xdpihp zGoV4X2)@F3m}~NIZ*u^7KAxk-w5sVSue|u#$?G&^>);kahmT6wyI7|j^ zWGLRay>#klo91RGow?5F&=SJ3rH>Be&Ih)T%CZK9o7C_ZU@qz+5uVBMVwi zYxf?r$#rbGhnmS5&-u)PFvxE(`HCS{HwgIo^7nCwYDE@R757jf-~FX)HLZ?^>*sjr zMYV)q2mEZ^Z_2D@AhG!G10Xa#9egmVv^us8rMB9=dj{+}0iVx}O!p7rS15bT5_%%1 zlPL$Z!W??n(1IqJiuoVTgHFI8$%G0WM+s`EZvs%eNmCG3j{oBY$y9ls={qi}m%?^B z9YV~^Oq+XpBF9M~m-auzM?ioo9n_rn-$cm2Uj5CV1lSlGiSkkJuB5DJOjNY?fNK2qWsk$@uuN@#E#gk9dUC z9R-IL>+LNj25cSoXXiToKI=ASdzD5-Ll@TVO=^1{%E$)pe9xeisEkbGGDm30UbjOK zqbZ8DKRgc`G?GO7+L@SN94xH#6s6G%@n$Uu-EDYr9+vtt+eTxLr)yrglA*=A`1?cD z_HW)XgO0_`3)Xc9r3@kFFjCA+;P=fAB7SjhaN8|((a1ZkA2t5&;G-W;z^2t?x|5-P8$;X@By95!^=USGOnkRu8^&vi=r~H#2)WPP_d@H1?fE znysnMz6b9FS7BC%SJ>u0L4Lk>&VQLbY#%B*`Zd+kqLH)QsC#+#)EJek;S<7_0bzfYmUGeDN{`UT} zk`@SN&{0gkni|)adU2iN@_$<>1? z$=#rwv+Yu$5(y^Q_{{1vMLQ#-d%PQ=C)yJd;yQ&2EE`ORC!RC_GP6EC&P5_Qhf>FN z%$~3LZUs=mP)xSGNC=As4V?!4zRUvvZ6a+gIp+M{Lls`j?0;u3=_BOAHVCdh-pfqm z@lMNsyp6+frRE9cs*FhTNR~TUm1avVJ}Ebwd$TGC^gV(Y-OkZEVKz-~I0G2P`g(G} zz2kU}63l;DmS)Dr7UVPv;^&6fHR3XQ40}vNy4>8;=MLq=6-tA%&~=Q|!ywP82^~Ry zA)B2-M&3-bNb5tkU2K+GI44{8)yn|=PwWb_A+z!w=mCwWgus?ZEp7Wx_v?5 zUTJvR-j^gh`EORN#)KPg%}t-oTu}!4K7w}tdX7~im!e6X!5PA3f|EsqM`Hpl+=F6G@5I7C)bK+&7|}96ryUme2)$}k5~V&(k%ofay(dCAvS)qvWzH0i?cU^i z>&e6pt_fcpZAn8VWyr4DZhdv{ks9C>#+5YFmNnJI=o%75MzeK*S_^0#H zs+fH>o$;u#C7OIJdMcsuL`n}pGBS`zsIO>EM8*w>d6ZUg!R_t9C2)hn-G{D?x~u5{ zlH-!u?zHMzCzAOck8M^BF6+@43YMp54=#E%m>0XZEmO}*=V_*(oKu3w2nIV72q9zp zhB~=MNZQB^9wGfKlt6;re9`-ez!X~a!E&?Jj?wmyh%1n{=>lFdD>XSlh&}^Z$xF>b zhu&4PB)@g5@a_!IzNzVRG_A~pO&4lDfb~nGvow*8sEz317?1urAzinu86`?AwGJ*A z{Twwx1$uwbOfmO6s| z@{{?!CzObWyT+aFg^%5>i>Y(y<|{{$-K{kbw2#(Wn5vgrZ+%Qt3BlM+eCQw%PYOn! zcaIA-7&Zt@ix-jT#<4RhMS0u{8$I#w6<6M@BD&*Tq`@lC zK}{wDE}2pWd3JCC`WeIMr1M{1Gu$3$*t=HRErHDw?eB1R2k^bl!F$eWVo2?lYs8fw;ES9h+$yqvU6S+jPoeJCA-B71hgm^6(LqtZ@WOnovPnL6D5vAF_vXp)01oV7HAz;)vksENIU78+1E{OW9;>$b= z??}e6wg0NLoKQXd_~rtc&`I16w<6c$zWLD*JWGW0u(~*xW`P%N5S590`$3v%lXbLU zuNa&N;+OF+69`vzg;vsngsH~p<0^GgQRyh*C9AT%1p%@7toNY=?xRpfbGub@B`(UK z+Y&1qXnzV{HCLW8veyghWVKdDTd&WAT$g6D94SR!^}X7X)!-4Hv+QIj-IYwI(s#Gh z#*!HzjlYmhh(Z3m!MoXLi07BWOL%>VlKeeNgzNfV-TqSan*3y9nsh|9PlX38KAZ>Xl;6rp_&S5*XDsu;3gbwpptq}Tv?kw%rHWjzk-&HPjrY3@yQ zoBd?&r66ZZD;$OuePg|t%VB7CuA)pFzw~foP!eOu080RwVBCEZgiW2vRsvMoPyhP{($w+4 zgy`>||KZF0&6@c)Tb%03|LwPffBaYC{{Je^0zmnGGPcbNDh);Cw#jKnBqX`##I~-ZB1(bwrLJ}rn z#@2f*Nn_J*73AWz9$<&%f_)C{9`p4qpA`l$gX5f-8MsnYEe=`*)R+4} z9Cf8TXT?+l_QZ0I@cb<`k`<2ujG$q@EQv`UuhzGfT(X&LcqX5@8MFU$u;gHB@>ear z!%WKVMNW{op@!4io&kg83>bvO!*(chKYt`(#BT7EN$ND1tiOwTTibP%SMJ=HG@~w^ zif_IK$O2y=B)IsNS$hnisMgFNX<=TW*#5>ZG0yCnXc8PssJLy2-k^p13A=m1o`ilAg$$y=rM}& zgP%^QLYps9Y@%9oy(r~^#eo!16D%eQI!NHO=D+?KJe`VK^<@ioz2~Ofj@EJ-ppyAj zTmWi{M11}v7+97nYk2DO^4vRV;@4IKOAf5r}q;`~Og7l_eJ5*l$a0Uv4|tX!Ae^fwAz zV+0TEO3DXz3!tH{fl6K)elfYd!3#V^UQh9pE4-Bc5<^+U_P0{V?pGKNp?O_x2J<)V*ExDRg5WWeCVr+}NHw zh;qXrmT!W{R&H?JC+^)3IRmq-LS4;{$+E%QEstb%*o0jUHe77oy3dBH`MNK?8J%QQ z^QW&PsA>4nQ13HN2$GpzdS-3hk;VdIR|V~m`h(QF2vk-vV<9K%@(Pu$ji}jTqtZv2 z$b{P2(DYsz!BkTlM;JJ5^{N1vYxbnUIwF(~qG)`c{ATc-oWZEylsMM!z zAjh)GdEEJUmtzo?U|IL!!I;_rAAs~3K)cx50p945l@*;( zGR1Igiq`lfMG<+~-a4I7y;Ep4ORPqXN`I6uEE%sHxLe*+XJTDOWHD_#4cFN|M~pp7 zE~pGRP~BwSSN@Cmr+`6K0BI5 zX4@6y-D;|)Yi#aN+!);@SKAc|7JnRzcPuYl%`f|2)?XCPPU4?J@lO99TvQfr;XQlw7Q@PyW#Xpp#?GlgCT_Ts^-1g81=Ud+# zO`1;SvzWn0W_H&Fck1e7KfW31*{NaiW2nne(;x%n2}W{7@m%GNzz{^F$4Qdn(aSBb z9Y0As9hB{`jWC85ymQ;*CN03FcapXl>pJ|hEK}y|g{;AwvSKk)dMm|4EnVCy!xGcy|&Y&H&b1(D_y4vW* z)E9O%|M%CrcErMu^@`e(z8Jr24?94jEOkVoKzs0P7bMO9OdFhrk5l>^$5*HGMm`~> zF0TY^#}wCnHg6iYAL?0rBrn#3U>P5%fO`uJVewCn1Y z$0yQ>p2TLypIo~gT6VmZz>k=#j5s-1)^9X0cQ`6zq4D!b!`^~zo=Df+fiVe69^NLVi-1#8rLDo-9Uax?LSNGm%o%-~*PSu<5QOOblH*SOMExA-- zHy>2=>wqp;c`)`T$tx^b8B@s%BmHKI1iYOrY8U*GmR-@!?jC=H|%W98s?B z&~_2>;Perx*RbB!x;r+J9Z{Ckk`5M0Ke0BizV`mtyOZgF<-E59`%{1~2A;Uh@`o&I zUT*L@HNr(maetRxF=7*Bz=Ap-TvBS{=-i_#MJ%%N5yAezZ5G|BY>-!Gzc=`MdmWPt zvH`;0hyODk;?JetVDjaX&8Q*3imrNHxLfxABbUg1jr9UP1=9VScTfG7zum$CpJCFa zNLz+Q@RagatMTv3?1hRB-YT2v!Ur#LUZASyzqsU)3Arpza|<5>z~4<96$IZ& zIfSaC%)bis;%tSE4j%*FUWEzz<@Fk*s)>q?s6F^`1T+4P>7Urln<$xU>wxp0=M9n8 z;!5ZW>$^Sis|xDoPd;C<09va5258<31KiAvTo4yIH>+9pH2(< zz>8@PV~M*!*%4`LGgoDG54aIX3Z0Suixm1_!6*6sVSI{-k-6mY_+ zUABLsK6|yaYu(&#EiP$Mv`8V#Iu~23Jw}gfu$4}eFsg@EHyIE2LqK6<`bQYmmBSdz z_c`)b-I5&h7jJe_6d27+fKBhEkV@M_o5R zEO81tA78bi&^ZvFNV8EsAf-`v#H$w)pM2cR%P~A9{#t>nar}L5?#8R?-&(BE_}fSR z5y#!FNT$0b7t&LN6w!sYI~IiwJFvkglHV}Wws%6T>U>?tPpXs?0hzrTzon1DV%|$N z7OTZ9>q~BIatwb9{^k>%=#v~|Lo3>+xbt4txE%|2ktYs4;&rum{ zoSYG}QDiBqM4~0N!fGec^sTO{y7Q1G6t!;g2z0jtM~2htLV zXLRhT<78@BLw%51YN9IFo29a1{g<}g%lvZ^l)*MFjJ{df4a*fm$UE9A$S?Cps) zp+2Ng{d16jw(g?*2T}WDutm>H9bW$HUdG=VAY4-L zl7vqM?AwOsIKP007fNWTVyLySQrU5W=3gBuI@|ZeK+2NvCQ?&vO--;fFLt-K=D;QW^s)ZFnWcqOZt1V#; zJ+$-)>@L#R#9DGcy>Bfj5RTPoE88h(GD1UXoGY2!fHK9}z55 zvU2=Fm5lczU!GgK1F&>( z2`5Hr4dFe$`;pFEU@YPzUutNz%eL42LO|{uFD~U4crv?(0aX86$<=1uSu(BO<1#m2 zkZ`p6hj8Rz{ptvOe!qu5_YgQGqvy}AF_L?%d^1Yz+4MzLSZV3{pQ!s3+4V~))vrD=YFT17_+oxnH zMW}=%N+rS&l|uF{dyzd`mciH~%9d=|Bl|LTgTaWT?EBcp3}S3EV;_v+xmBO<>-Tzo z&+qkmo`0VEbq_OV?sMkc*SXGh-q&@m>$|kF;^cRYi$jB)3cmeNQQ+$*IDeS>m4yZjTp^M7O7M{o3{;&qnJJg^s* z0y4$msj<`=C)E@4x1PV5PLh7L>!P#Sg>NxtXxVDk?O`|2AwQ|Ah3>A0Jq2)%p)0w; z^N$gOI&CXXpsTw`MiRt+Rj6TMnY$bC+p^BNF7(^|0&hL?{+G8(sjE+E9Iwl>hFSg1 zDTltBO1%zIsX&}Se`$leUX8cY5zMW(ry|Zd#m3x!!UX)6LmdiB+U7F3jCB;U|; z(wF%H(>Bp9s!%4H?*RMQx-~@k-rnKQxcpC4bTmDrqniAC#Kz4Y$>sqxfbm|qL~k{Z z@41ypPV!D9Khr?RzW8m)IES6Wt^f%L1oyv(OeEd;S$I(>ZeB_lgpcv>TqjX@dFEBH38MtZVdNLj2+K8AfTif053dlD;plxQ*}Q* zBqJ~7l|K&`=D96g7j%(BA5d$76as4?T|o`lO%cL`t(or}+)%h!`Gk~tO&RZP)g%m$ zSJ7z=iGHj?8%-d$jzP2(pLm9m4z93meEpcCd1)>v?q&N@NpX~kY9P;nrkI@%qc>wg z|IrDK#Va*dD(tUp`;)*zD&|-m{(1j5chY}Q{h!i5I0vY7eEw+zU>YDI)SogfTr~?r zbeD|Z?vCGZwI0ol(_+6` z?{ri*E!YSU=QfGdr`qv zdM?fEJC5JoepY2-x zLcPrPB#!%PV{}g6fyeAZhm7BB`-v5+ebKWM=DS{B{57W5?Zj#Tmkg#B*Eb5kBQaHB zfN#dBAqw}=^9ITT1Ghu(BU`{H;A4G*eWzi$^jd3M9zi$#i!KK=6~f>cN3^?z!&&(vgKuP=K+&A0qEM;!o_c4dFNEV-0~ z;bjbvrk551U64HY+iYF;tu^)jYH|vq&)@SPDxdnZ@oRkA2@vNA9>?N7bn!P@m*buH z{*4+pvgy}m|633r8UVxWH&m^pdEw6M$ku6qY?q8iXcNoikoRh>ME`M-7K2a!B-!`m zthZC8*_l)8`q>PVA9~1b?BTio9bK6dQbCc}q)0cWv5 zbhXRX-Ph=k6M|E%xyU)A;S0JjgjMiW!Y3`m) zWE6(uej3@>a(3VB07%tA_jj^>1dLYR_MJZa25h?wb*(H^IW$t>J6IYMU>Ewi>dGRc zre7G%4P{Yl`}WODUk?`U#b>i1r{x^hg$+Y?>Ov2VHJtp@EDEFI2)Gy-abhQWiVv6( zzeNEXKUF03=0$rI)|303Q;%wf3uUx3>j^{dAffw14FJ9cZeYcFWK0G>8I<(PXuOr> z>>}deY|Hls%`Z8LvS5K?!ZW{s;jOE{=>1{W_+b@+(@cHsc@@?$s5q=x&m~v?COudS zluu!zrNDk(4Rq^2Ll^fk!|9Ce-rUGAQ=JW}sgzpEFB343m?Fd80b=bh_ojUSg6P#y;5 z99=0k`Z2tH&30(Vbw!dMJGH(@M!N&u{8t=snif*?dk8Jt^-nq#+23q>d;e-t@!;3= z#Cn*1z-mfPVyz22=e~Gj#^;g>$y*HNw7$|xU@(`YL27efRJuba(Ll2N=+1FU20(H9 z&*^%K^*<*hR zwY~k(F&AhqT*cg%xlQ;->D-FhG0J>o7`pcx_>We`sF|w5t(d&p|JnG zXVfPrv)a}YV?Xfepw~>!D6`vLQ;imC#Xtb@@eK@L0ug{ja-T#N%&b&8{yAj^!i>p) zSvzxd7nGu|-hw!KUF1K8cEGuO8dzVuyTZD2t87vtpd_XSy&HgMo1$ahPpdCI-S361 zXFMXhyB=%2?EVYod6=-L?OFZ{ZGdKeuTTG>PTJ$&!~SRR|LENR*(m!heSh=UpN+OZ z3dDbD?XN8Vf2*uQ+d?rnU(zOTmqE~>r}q`j$v!B@(jO8sK+hzGlVl+N1ri|Cyh8w4 z)5H7oSz2JHz1YcfY#Z zweGY>O>I?Cj1zGRughL_%nPH5#jhS27FPe1;L9G116f3oAbla;g1%FNJsUI`$ZKF* zW8Q|a>wVO9Aw=UBUPF6mN2;eC+7_33G2fppu9u-^rTrEXUO?2RHU_2jP)qF0wukGh z+oP#k`F=ZQolP+p!|x(lrM)J(gW=Pj$q_KKlam5p#A*|Aq&%F8(-RZM=h2Gg1%Bqw zlRRq4BZtrtyF}Bo(bGF8CcDTbVPFE@vp#}()jvDB%daC`*H`rmxilT$gSbxpYR*41 z^IZOzUno@vrqcU%Y*xHr$Pgfww)=Ej^K@jaD!v|88DL(x=o_p}Ut>Ydbt9R6`zl z?#UqzhIIWnEriJFgsHUh4xRI^PWBIQd1a=Mq>=m`>kDU?S8K)AcSRZYNur2@NVP_* zouir^gvR7c_A|LzbGzND9O3grO%DSBpT?jcm zkcA|rLgtfO2IZ1Af{E<`Rvux%0ujzV}F zJT4hJ*Rm(ss(x3zo|48e@{i3}s;Uq|W3Hyzy^@TcFGb$clQ**sYSQvpHxgDK_sdhP z>7U+tOMLTlT9;Cntl_xfiZN!luU8HAH!p+}?2Hj=p-yRHUo$E?^v_67PkWjtOxH#< zITkl=?te~*cvQG6F2)yvb;?!qB6!JFF=oYq6K@y3og3EA3;Kf2av;L9d&+Uy5va>p z8q$*H<MOY`Y)ji!zBPE^w8?N4qR z5A5V}Me`faeeGm0R&Rio+GZ+&(r7C1tk2z>MOoS}U6omIB)y~O?Bzr2Y@GB(Xs`^qBrdwF=X16VL25rkidm z4MSZc$HL3vvWV!9_GIyQvj-hh(4s=uk#}->Z+GMnbsUC7~soisd>>0#8vEwu(5bLdtbmZi@%M!Nlv= zovQz^C{?TIP9)N@2ZvSm*OrAvx7A2(M@ePujE4FM<~>d4n@RTW*?KA^RzB~lRnWTb zUGxIwaigrHjIrkJy(0;FtaWZSqT96o8SgsJj9ABJsXXVia`ZGqi$^8-+`z*?H&Ue9 zSxtCYr>S({B&&3Wi}$oenOol-43TD;0s7AjQ(gqhvHsaBW5%|d^;2(>K=cHiTT7l1 zC@Rms=o+i0=sZp|#F@t6&yEgJnsuMnkMp>R7F+i2h3k^JX_8>t{ak&!SW1JaJVG@7 z-rDQWb1|jMGwCEz*1+nfWRSsmDlV|>6Jvl3PKGuse5m%7=qB+%H=LzwcV+Ih^C|dn zRJmO0$iu)*W0kpR+^5hsslXMEO;fGXaDsoOY%8-vqvg^o;yc2$=i9SV!L~>c4&!A4 zf9VJ$t3pFIwQjz(J2tRv*L_XMV#pOkW!kaH$J-j}<%XHa(!vPQum~j0z zUS8TT3E_LFTcVW1{jgzEy5y{ZS-i})Y1!qFO0jn-Ku^Cr(f8V4S>|*N)vS-pT_fHFL3x~ z6!9UqJ~y#$S9e9{_wvxde+r(1yzykT4J!>Lsf@^1EI7e>=<2d{DIkj24nj_K1S0VY zE-6A9UKOeh{oz5CQ7J0xbYYSUu}Ten8j;HALLfCc^{GL~6xc{s5~LB<^&_NqyRQcZ zNO~d{wD(5&q|xF2@5 zA#WLB(S}>_C39=|pa!FsT%T)Obg3mLm6%HU1~r-SP+_w>AF162&pkMLY!)#p3+`drSuFSoadq`zcgVs=Bh?E}hgO5?RD+8Qusaxv9o{6pblW=OZ zb+8@isj=7dZ?E`%v>%)2qjO2u*=7mfJtQHNP~(=y> z=zM$=&ZCA@5wm)P$XVvQy0o0m4FSuQBddzRNgdj_+IV)_fm7MiXV4PxF~NQ(UX!#+ z$u#XUff6|7hzoJ>tY{{1S#0Mm?js4NOF5VHG5l-|PvKij`PP;AoaM|C=|uZ6^bB@T zzP71wUuZkgr)I+}%SuFNU=Hi^II!&m@mw3yS+&5U_!}=ynE6Gj;-nKpdMaVMlKw%L zSa|*02%T*WFDEz|lU|gWeNpyQ83Np^GcDfHvu9w<9-L}WJeCndFbjrui3Qx8C4m=B zi|Mi)?zJKk`Eq*G&8#Be_`z^YEDNhxU$ae$QQpDp0gk@in;V}EzM~93xVAA!k8k=6 zie+*R7_GqiPOZeM5WPg2&Q78567DXz&-1>-Y1We_G6er0&mX2+PbAXzzw*U% zOWrq#sdAWLmK0TN*h@3w*1DZ2DfwP!{dNWX3KtZwrG5DTf%8U{-1~YTKc!?@c2Oq5 zeggVn4(8|@8~}yuQZBqLJ*h^2s~tf+|NgAP>Aedx@1!US^{(#_k8KSHsbJ8zK_ccL z7?exOS?vBzP3}O|S!ZYUU3N~$Pj(i7GkPDe2kx8_AP8q2iMTp^R90(UGUNH8EFe+; zYh`+0jjFD_a6hRILIS~lOkd=Dc;!GF(d`U?Q8*!zA#ljQxN$yR5y%(ene6`<$M9h( zqbL9!*ZW9=Xi~Sy?w@IIfG~#;xM=F?Fcaun!8)r$hrGP8dBQI9<>#+rO>3{tUTDDR z0Gx4s_z11j=&)c`KtPJXb-vkXfgsK3j@O939oc$Y7Gl~BfMQeQO`Ezn--iWhwv|A@9y}otomuhL;M2L%1Maq(=DOI*pZzJkQ!8g-M4Mw4VY;R zhr;QHC+rgohwE`;EqHB7iRs=}Ll*BP< zdCv~c^CD;^k^**EUGLdsD?LgVS`}{>i07GhMq80a&>8u+r`Ped0gz22E%)isaoTL6 zi*v!E70R!lW+J9Br*7)7={w;_2N89I-~_k^PT|qJv?!a9myq|k=y1fEyV0$pDcz^2 z%s|k@PzJB?zQXEN8U2HX6A~Hpp5Imx2vbxO3FSSgu&0gi%|&A#n;FVs4x75fAoBwX zW3tew(J@Q;xT1jK_W_Y2c=7L4CN|_$%tq&m9ns&V1-NjtTae|48Hqq&vm=Gbs}HF6 zsEO&ri<8PHGnmjA-!&O)zs9NkIX9L9%GE0Jz2`?5es_&EVk~)ez1fOVCo{5^4m$*Q zrdyib(K!7u6_j?=x}aM$BNCHsW`RD}vi&4|GISZ~HPJLV)P&rs(u^3$`}AwUtdW4q zhCqKhM5zI8CkOGRL7Za`N82Zppf&pEuuluDX5AUA5B&7eWX=S zQ)T`gy`z18$f`}_FYjw3Yo=khS6%Cm%Ki~Bu#CCA+`hE(>M|(+vg@NCb`kpc>BlX2 z>Wi2mQ(Q*jBBZZ2i~)JMNHs)qAr$7OW=pwk=Ad2YA$p)v5%cQal0{d4cz^SpY=76A z@&ZR2Z&EP5sLS3w5C>DV>Y4t)Y>I$(^Ze$kp;wTWvnu{Lp=?t$$AdAUU2WPUd_~a% z%#g0wQP;>w`pL^m3ZW)`dRo+MkR7-EEEY}J?5FUr3ItyakEAx4+E1pg!;5!uPX4N% z(s6w15CeUO;5$U{uy}PePdO}&y=3oUMEFIiT{X2Z!uE!+g|8bKrA?h|x;wHKiV|ja z*8)l2poMDvw110g*!5<>GAWkc`TpiB?~Y_?b;mC@XV-7{RNGCUM=AGC<70~gX8d9V zt6Thj)wm~yviU-pkJ5@TOBiB|cz7y^luR``ZTrbjQ%#1eQA|qua z(rD&yK{Mc!m?7;z)poCTQ;Y?)NA4u07C)*Am@M#2SQ7(dt0y{_cZ}_wr(J)pcre?{ z_?-!5`|6oGY(la+hgE-cVI@19;u&LiPwF&{OT^&tH&uL8K$6;B0kc^T#%xG|#Gv!u z2AqeLx-P~Dod0L+2SOVn;_}}NLdwQ!k1ogK+sfKIv2g*oD8O=GAV4D@i_Ih zdsc8&(#rVs)JfSXwS-E>@-ab`qYLhp&vnKD6khA1%Vp2weNzu}5+PZAnEZ*mR<%3* z!8^_Vnj0hKlO+yCFGwSWfSzcAl-TfEgr^f7C;5l9alJqD1VK|A2%=xqm5Xq6xGv?H zGQFE}!260^*UMYRP>I=-^sI%N<566?NEtoMqs87Y+y+Yx-zZ`Y2^sXfPbF3x+)MNQ z4HwT#6VygoBs`(Wv<_4GgmJiHremec$hnrngHYpaT<&s*`+mik!-!i&**g$?IMG}_ z8ZO>)W^84Sg;AJkFYY2k_RpCUV2ujN&bc0t6uLmeXRV(j(Cr;GO6@EVSoCg=hFL+N z_AN#En!i&%{;q5Mm`AUd%~o-b3(=!| zE59px&Ly_i+7QDaK)Ugs{Gh?m7kF=^F8Rz8yYV)#H z%yuB8Ra6LYk0=s z-l6}TeKKB?*(6G>vq}-?rn3mqgOJy;Uek-O=~#XHuw?z(d56KN zh_hbv(|Rq5-+g0>pNNtOTCSuYYot9~VHFOJ&VrivDG^cuk@8R`V)W_v39dEykteSO z{=hdjY8U%lP11y%I`}upd)U?5vy@n>X^_&8=Jg1`T>s5_S}o<1Gn@I$!D&$$W}z

mp}EqSyK+Q*(2>=Ymg=`97$;IcN6!AU#n%*cZr$D*-Nee z+S|EJ+Rx|ZZCR3;dYF~MF_igZM9pO>@k~gx4trPw=|VIJ`@5%$KeEPq!zAczq(Nh5f&nm;?OE#5rE}8U z#ZRH=@X>NGxGAhZ)1X9J@90zqd*NH6d^+_XI6N>w)f-SzZS+H~B2y-P<9Sh4sB9^` z54Hk023h#XXMb<0Pm)6GkiL?{-ABuuprZM)WLsEB2b#q#r(64fNqY$SGt4p`=w;TY2`nt=WNa%XrEo{9qUUnx_ZaXpXSey zBTj+0$7QyXxp9J52is`E@tp=*s29WL*lk(qL%b z{D4U$*vR{OibY3L2BD7Pf0W=s&DhiTDc+ldf?9vn!!thU_np$Lqt?iq96hT`p!p-C%gD~Q#Ul*)6z?5_RdTOI-am3Mb z{mH@`g7r?qEnr(xq2CH#m~Q&DN89Lxm2`%?X@LE}q(wl?^kDW+xQsXMkuzj~+bDW+ z4wW+nZNgMf($uLCCG{p9;a(l$t>o!u&q9CZrDDhlhtVJfx)#x_|N7>G1d6y2t;Rei!KaW>dR9q61#QpP>7?y2sz0 zVF0czxPS39D+6sX5t@%o-?Sc&i_`RRCPFf81c$EJ965j5bSJIC#M^;~bFDqWN-O+Q zp@y8M6@AxT0c+A7OJW1_dDkaub}ubJ?MB=@#iY39FS$V2zIppKGe}Y#ihC7z4xDd! zS6;k~V~e9x#yFph}M}^?QgnN9IF2W!Pai zmf+*_8fQVE=epRZlG}GKC(%09E9J>f87W|$1Z37{wq+bL?xyQvJ(*s=Wc!4;j^S2* zP&mD}X;{I#mmLD%**#yHE=~G{@HjPE_1!1RK4K#1EzT`DK1$ulwu$D! z_vZb$d7t+!d%E}>k@J2`JJ-lg;={K&>rN_b|Dv$d5)vNT>Fn*<4$s_Xn0g21-R_YsTkiAZmYTEKU4c|6VD+zj@%CrY`EeNpYC)=CyNe2ky@p7s zTb>M6f~@P@TdxFN4jWCLM=0Tdb>3yh#4qSkEg~_z#3!y`jAN7QD*7kpiqS#$1wNs-py~lL`>l z_KQyQq(1LSHII9YR>-bH_O)6i4uBm_*R7u5bKWm=e{gt3ScRoU2qthxwHXxMCpkjV{LBVQ+{MvUZe9B3%hng|FWbELG^=%-rJgf z$xN&Fsz7Wm{ zm>1SL_mG>w^ulGxIDK1Y^W=K+KX&`I=Z}&Gi z%!J9KrwdWK;j3fj@2847HF8Rl3I1xNLbI9CQMzeXVP;lKSLkkoG9g9p3uK*=Fbg$C z4kN3Q-P1XiSGK*+cz?ltk{8IbMYzEd#K$P zZW-rmy)|Yl2g(4K+^PDKy8T`bIS}Qk^yRXo!eRs4CrV0e%Ey-ai`8dHx@<31EDP== zCYa00+~(Ql5Lmj&fUdoAjk6A{{2aY$LH>w<4N8CL_uyJ?;1^fERh{Lzazg7o$As1A zN=I7dMwY?51h0}^xuoHtr{7K~PA+VtPX>GN4W?1;plDzdEUE_$$H9#Z=*6PQxZBU|Pr=jhQ7QCB2lo1OMWQ+=66f-jhzsE}ep-HjhL z@oImAu_;ZFPG_{>*qqSen#)^u!mlTPj$j`inG}|r9O+$_RL5?(TaM*3P3haFy6-}l zo)zmAXV`2?`)WSBapdy^sf?X|+b3eV-*m+#b$q+)VtSZiW6g53*N93j%pHv^2$D ze>8Y!Du>_2^_rV#NFQDAquJ}KU$|$Xjo{+E>twp1l>)grrJ32ST;*4_T+0)O?bIY` z6R1p9%RM=t?J&x@^HUdwIx_V-vX(dUXHP!ZJ319a*8NbEQSlidCN`h8_i>4nfTW-u z`PInnr@RXjog#PRTviv|jQ3VY8K%-+E_wXuYX5aL9A#fuYks=U9sl&B_N-1K5jwA9 zfVx5VRhV(AUaxOOjo&3dOHKM38Cc;gQu1y&bLZGRc%8CbStMmMPJE8cGtWO$*TqQJ zoA2VR$h{L!6bwiNh^fI@=U#=3wKWlbb>m69<_yk(mZ14Mjy79|-$l@y>usa@12Z`b z_xyt!TjRM-1H3WnxuMh)ML(ztNqc^y`mA7qLtG2qux8nlD+pTZW4csh!0+N%te*`F zsfLw!_tj_2w$pJ<%MtBqo$hX}w8m3Q-njRa`Nem}H05gW!Mj5=eBHPX=txUQoor+I zfTIW-kHd5cr1_$kns>rrT5Os}Mwa^KN2J|`v&Cq}T?$GoLv7pMu%^sx_7-~sZgsO> z=sW)Dd3dpZ`i#a&prOOerOB8jjA{xnnLyagAm@@5ohH>gfTa1~j{Bm`iZ>mrW<&&7 zj#?Z2CLm{c~_;H{-b-|8q>FX9D54xZqj&@>XaFlC*gnv+T!FC(yK^ zW?;NO9g}4rM)Y$o&lohU-S087Nzf2tmx@7&{u+W0%r&sNT}nR(&nNn_)a@xYdjp2j z$2W44mUTK2#YFS9IG~S#=AjYcnIEu^&+SE%o4NfbvM@0~oCvkkR&yCi%p?E3+u4Ub z0yooLurFXG(h&F0<-8q`r2x~bUC|py*R*7rsX48jeip@Xsq221SClpz%^Ixl9b6|y z>%HYJj-FT9Cb{1T{^38eO)N^vM`ZbATq7DVA-Fc0G`;TTAIo1G8-vS(jBE?q8@Cvq zrYS~kjKr3gENx11gVTJ6ilVIP-;-5ZlVV!-!k&IM6!v7PU=nqJk4xJm615 zPBG74Wt=K{*`F^Xe?;^+O)JJ(B1=F$rS*G1l>fnxw(s#}`6u@Dq5G1IX)&F9_J({F z2>5{TOQ6GnHf_1{jtZTJpR}>oSG3k=D77z(!9gv1SFzqZXdG7%_rzksjLK;#7B`ct zmgxX8shbUs$Y)B6k@9?V&i~_uV-}PZG2XGvxr*OjsC>;QmoMC5_NFB{Wd0f3Twfri zaIKV)7G^U-9dyNtBtZ?~r0tzcrw!WdgMS>nM;1@tyQzfTxfL|2yNefMm`V;Q&)qVa zmOHVBUcmjPXs8Am{lujAipIN>?-C+@RD%80d-B0UZ}xBR`{q}x2Q$FV%r5M7{8y&}_ zz8XcWV2Z{54S%tImXn6&@j@RMAaJ9^NFhb4%Kjw_@EPQP+n1yFG17PWPO0X0Y=f8@ x4N$m7uDtHLX*1$0{{6bzF%|xAkDGKybZkCbuVXd29v;_Gex&{oruaPge*rm~0^tAv diff --git a/content/en/blog/_posts/image03.png b/content/en/blog/_posts/image03.png deleted file mode 100644 index 443a6f2d671be341becf8e43dd2759dd742e3e1a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28564 zcmagGcRXC}7B)PiccMgZkwkP+M{hwQ7@ZJA2_nG=5>ZD_M2`|}5M_)`qC_1;38J^C zgXp67`tHej&Uv15-uL%?|80Br-PgL;TGzUkcRJeY3%ez~bf@4C-$&a4fbN}=%t&H1ZS&$*}E*-OY?l-n2>hD(aRk`6bCu3_!@{Cti zca{fvh53+`m=q}+MmwT&b=vJ+UEQLeHQeKO6}cSyX)Pfx;cQaJ&TlSqLczcIPWY+z z_B}r9AKD0&aL4>Q|DV6pM?sG-F;%5& z|Ga;gU$yD4uZ1|M$vzxOgC6gsHSHQdpM?lou;V=L-*5N!haC)kSeM}5HyL%A99v=7 zW-O%v3*J{|dAS#F?204LB$20_YkR1l-45NoX+({=Q+R`d&0heYMYZ(0#`C>wpuDuRUTf@9+y57^l1D%_^M_+tywwe=@7Ey1Pfv0d}EFO58KO zhQY~Z4bv9eHJDAiG>S;M_|_H7*4(6 zcu~lu0O31WYy!^I_|*t@(zZZIyOf`v`5$)#vr~ej968|3APNrd45x9Ae#h_GfjC7v zj}~QY>DUU=oQi|cu*s`8NH43`9@!oPyJ0a_Zj(m=ALxq{$$A2eA{l+|VE1u+*U}6z zP-ha8tHQ^YaW#D;Q!o{vvUVQYPFibsvlad{Psnr7)%^I{PqFDjEiem{>Ns*znikx< zrJS@FeHlH;q)5AV&~&*b-%XQ0YeJA2w$-kcVs*ieCWLZQ6#F=v1$oq6t_v0SmCJMd za00$y^U(0glt}`q!}m1vYx2?Ry}rXw&cS4y(kT03cYGA*n>lMGUa{1R~qeP z-Pw182|g6PpQG50WJYu5fA{3%4+}NQQ3KRs?{oiCV(4mvC`d}o;pXiIj~%zdnfF2% z+;5d~9la87-6{kxEp~fIJ=R;eDg?}rDJtc;MN2JkB$N?e2W66Z-6X}{IYCJSzyTMI+e_a`wWDDcnca)ie}2U;x2iz;W0Pl zMyXNb;KIq8+gPsNZDII|NL{i5Zuu!Iet0rG%mA_N9ykunNDkma<7o z<^&MA;{6oa36Q^}ZM@cGW$t+|T6deH!b2O;?sea2+lvb0Bfia~)JB|`R%`Wy1Dya$ zL%dYWRJxVqQEtv(*!@gck^rtjU&J5gF~Br0eglo~#KOY_0o__{cioMe@l={l!hoSt z;51uyO@-vi_rJ_pn9*_MTsV(of;Ab}1VqNP<<#@+dmuSyii(=&1b4(Pg1-d~;gW4< zm{C$QS?A62;5=4lGev(CMQ(NT=1FvYp~dE`F7A8bVVOYD&1_K=yfgR7O9 zol(pU^vR>`IH@RuKKE2cQL#*&zH2!*00EOSF{Y#@;&rA}hk#jvF~ORUQaZ5U9ZCTX zuF&!fZFunAXLh4Z@+vr)Ja842_Q&*%9?Z2tYRURV@%+;4^<3yTyZtzUZz-TuymxDz z`jun9+~t+TXw_mr|OLloJb zK)t+x$%k@1F_%p*E?vLwX3KyZ$AI%;oWMF?YWVF9benm8r*oV}^mPgxLeotKW01}L zSDe&nCC@+#F!_2*u=avW8Cg3ggmrd}{?(&UrR{|SV28}NgHkhc3Tc!;7K+{M2W(fL zz$3KiITtwRDE^nMyTJ!f&AH*C(ctwpXr&$=q=~fAbuZo%pPc5~ZLwfsGD51`j{sHT zwppQnpUxs)jE5y516fN=Bwd+@R-R;+lu|KR5mVk1>(u{9};*ZiKZ`IFzuD z+PH2!z`bflGk3@?rb*6bq^27=?1G^h5tW+u!Y8x%nAV9h2*iW!(~j^%4^DdZKXQx2 z?z!L-V^(+yZXq<>(=>?CR+pmZHHh!e9Q76-SK4%N9vtpa4znv!>E z0hRzI(6liVyj+DJ`L4-cOd z7{BycFqi^Sh)QRlt)DEv-V4;bI~L+*Y~Z?cN8GAwqoyZ+OC27B1eFNA@C@;Qhoj^R zFW#qIcV-~x;W4Byy)2#vO}%#I-9Sa|)6$hf3=3`>v~z2>;bVmTIHbh&(>?cY2;HDF zF19FFdTM(VePY7a1fGft;x5`Oq3ICZuGM!PN?f)yTz&R)t)b8P%Nuy%dwrLFeS`Xi z;r5zusE@cTGhqb!!xrap^idjbExb?O8ZKKuqfh{|$l`kzHYh@M-|gr}6MA>B%%9`I zjM6l|=OFVHJ5V2SSSZL=SB6+Lwcb35+Nf{&n2Wl=fo;>)YC)S+fmR1JBCh2d+6ScQ zN3I2u3$zKIzuzq8cZFM}h4Y{j3=ImL%ntXIb!T!=#?A)Oz2;MSAm^T79>=pK-}=UQ z?r9VQ&Dbc20y^!Sjl2`RKdjF@GR)A1`Uqyx8+>y?xQC;Q;N{|*EqLz)#zduE++SQK zpZCa>9s0tvH~fjNuZj`=dEP*Nfp*a~tt(0^Eu0Ze&SsGpWt6elmL+R=u#OzrzDwtX zg9#6fGV1VCf$Gl4r-$>XS>J`Gv4ct+Op>MBA*yd!krdx^7L{d=_@&&7q;otgqw;dg;%H87Ic=z}{6PH)v`QK~mRbVT)|hgUnRUW@ zDg`6i36k{M;M%8MWG*(jEIQa6=D>Zt3Tfx*Qg?Nnbv@8oe6a-<6z58!58jR3X8PcV z8;(N`+gQswzZv1uepUobm1Rk#vJyf^18=3FtLKQ%2>v=0NCrzzep4S6Um9%8MZk#J z4wf$ETI7RPieP83drH4!t<$JyHSksumjy|gb)M&k8-~Y(f%p=QjAEDPeE++H1zDG| zaypp^nvRmOZTgado9Mq@VoQ9(-7WmV&RxmzQPVr(7 z>OWcpCtRspOk{^Jr30?aL)#!Cgz5Qlfm$&8kbY z@e^`k%tJuk4yHW0TwTSP`z}ZBT)@7`64qo~ZP1UBTQn)jfYUtNSJfzlZs=fSxZxi> z`Vd7+ORECLy}5Ws>32b;*pT0tXrQ6lO>9Nrl~Dh;;$U-n4R_;kt^6_W+2S_=0^m>gOq|PJcRkOErUQ- zI4rUt_GaidL1GfdeSRFDv`69jVVI}x8 znrB13eT73u969+8J2)wtQ#4P~$Q`a?M{d#D6sM)~B=)WpoUZ{KoP`J6f8-)V^Xx^W z{$Cq@;+)~FSb2|AFsZ}kk)U5oJb=U4`y3wRA0szZ$KwH4IR|GxP-BXu1S_1glXE`M zc{^)s{!QLf*~@!d=rVd!ch96OBay;hctH>lD|%6Q>ny|DceG`;KYq1GrphZ8HItXZ zK8|P*ertxp0`?3e928jN%(KHX7w@!J4>-K^LtG1D=2O92Y&e z6~2?;VUChUbG}NZyp>s!jmTc7z8v^$1CsFtoYo`S^IyYSOaf@|%{G1QRyc6XHGp+# z2Z7pc$x^3krfaRX8-~W&eDX@)7o}X@WcjuKC7*0lEM)T*vBl|f9vCYZfZ4fej;e*G zWfHLkAL$@})Cy#LukEDx=%*aSJ0WO*NBe@3JljE570~4qd`v%DNjBEILGk0N4C`Q8 zd(6D3uX%%+;jegG#nCD9Mhu^9Wu#h+tS4AXj$a9A(#C?}m>P}+D<*0>PdP+>TvB|G zzi=&vRWT9V{<0grYO{Vb`-zdom#8vU2DHx7iTM;{&Ku)#66+{|qmrb2H5K5ad(jpfe}4LB(^`T>;O8b|h)h2jGH;FyY!a#|cg z41#z_6y|XLaow8Txs}Yh={H3O1lQ%0>!Ji(0w7xNWzgTXm@~Q~nU|DaDOx0w)Y-_G3E5)zB7*N5sS*yYh_C2nBv^7VwX7)a- zP?Nkrl%W}Ub7RO!SP~HHr2Gf!>1!PF>Jp}X?p$xtG`qv8Viqxx?Yo`#yVIObh4FpO z`Gd-(os0c8^fN-0mW*qP z!Nk0h3dVG_bop`D+6+vdF+N+HIomQc-`XtLtl6XRxWYXuJP)`qY~zZ~4}tO}XUuFG!ORCIY<4V|cjvGLkN_2j%1Ry>^@OU(^BdJPMI zpGg*mckS?^pm8Zu&k@qz6^Wp+C~(QricG=FwQMV>+5xggm~ z9>>;GCtx4~SR%5E&!CyA9rdvTH3dscu~?jhsKVU_Hd&i=<-86J%q*twaWPDoH?%r_ z=fLex`CT+qrg!TWcVR1`_BG4|82x+3qrZ#dOVv!j{w9a<-9=7A9i3m@F8}~Q+qJ+L z>cu6(HnHa-Es?y)BFYtp%Mt}$bkOEG)N@C`Dtm}RZA;Rl- zCm*kz+>qqLYa-a){o&Td{TO4{0$7a8JS|w;dyrPZCje#))5AVZb?*_2_p0(L|12KSbdbfT%6A}BoWLiD-1C$4hp_Db4W+B$0>Xp(SbX=XstHBuZljA44>n^TNg0@P?!Gby<`!@xzMdW^|K%Xw!p&l+@-Gh@d z0Nbmvrb2mgSNo!_+O@zOm!Ee z2M=~~lEyewifCzc+uJ3AfpLMzXF&flQs0I3kRQ8>&w3eR0onDM4<#WK(St*HHP^ku zr;&|^@!~@4*hO?~>SuJIuFi))Lq?pz6he6d?3PM2kjgvS>~Gg!jQ-Gdc5dWectr#g z00eJgm6P4O;EPYN^iKO3UJV?SwulP5-LK*otuJ*U_fG%!b~uTRNw!reP~iVYKR_^ zUHKt2nz2CJA2vnbQE`Up9_8A{gIwWGZChpg>H{}R(or^>?<1|$yB|RGR6uw+{M4)K z$kp-689HxJhYukmyEyhAzP*w6?yKYWmk6>N*{fB?LUI0yG8^;h*W7}Q1DHCS5(SylrnGBn(^|R>f{Y4K#|?1ky#bbo=_6 zbR!?5Va{#s()>MIJNu-&A`BZ5TSg_8b{C|xEYjtDrxcArk^|^K%jq?RKo--Baj0AL zq!6N|0cm@zgY#GsM6o5WHSB$Fj?0teQn9Cbc{}zSGm&55)y?b8l+^Dpiznq0o4&o@ zO+WtOV@g|2G1(&h-dMq+dRnm^McL~~jhqlb=OXPoEwfsi=m5L88Ls29ktY<17vrAKQ;lg#3&Y0_+(u`}`IkpapNXP2=8nd`Q%SlY-E< z<_#4<{Jt~Xc#J-w{&S4S8hsMr-fw27uZaxw`pP2^-y1KYCLj9-D2`CW_dX!wPWOQe z4G%LoR6#A0G=Z@SiJWof_$etjqnPeL^)vA9-bJ056*36moc>!)tD z_k8J)&OV2%`cxR|?4k%i*QR`jk%vn~x*1<%1A?slT%lvNWNC~yp9traS~1~qo%0FY z`ur4^W+mF#=IavK;S1+>!A;T51t8twLkWBIbH|0YnM<3|9!bU`LP+hS2@i5UX%_GR zh4tVhwn`B7FOeA&4U{uXthggEpL0ncmUSQ5n=*oInw+xJ`7PBZ)}Cfq2r* zD5-(Y6pf1`QinQzR?=e5f1z^ZAK=wwx@YC(cvHaQgcVuaYed22c_8gtoq_e>P2GqG zBOUGEr5UCvQJ2G9#5_+t3DMUs$W@G50R+TiM3L>-b;*m!cZ+}@Pd$F z2u#te?(zdf^|T(^uQzloa7#sv8Z#bZdh{z<$`&Nx`n)J-G$hZR`Uy^&Ejqnc zC-;my5wuJXu*WAdUi}HAX-KVZB%xX z8zX$eIF=;ZwS7k7zN=i!^T^&5cUmID&c7tb!B%;Vd`;X3?iJ2m|MaoAb*$7r=zu+A zwVY^uWoW(A)9Iww2d-3A+#!-LwIzOFm?pg)&h1(-jq`XUEqC;0c^cKgn+?P>(>}W> z|5phUWzI5SK@iAT+@;)Qja+V#-}Y8gv568^F#vzdi(Bs35u|eS*&mZob5Yue%KMR` z%~ex5XwKGn|BFUwLjE370mu-700-wJ2XJqCm6P`7))xLUZ9j)rIGZ|9S*@~vOdA#r z?&|8wy7su%_hR>LmALRs&=VE-^DN}c&ntjo?DBfqhpTW6V>*Yz)9h2h!gsmeG}ycC zJB_^G8@jOB;4rRwdC5_@DZaowW4C)EvuJ5m~Pb{b`lmIfLcrd`*KB@=id(FajYXFxFug0jI$E&N!)$u^2 z53qj(ko`x?ON#?Uf_tTNcb%ZzELxkz@{RlUIFI3=&GY1FLuY>qt?g{&_VCQR>RuWQ z4gui*boJ$ioiic5xV^6LaDQ0GOKux!9u--9@rJkR%P-O`Ji|*u`b^D` zGP*3%|0?{<{T#52cr_Q^Guzls1gB-6UoRYimIC23Ev5(psHs<{i>o6B1YIzwBZR!M z61Hel5rs4fLk5-!Xk-)P$&S8gQ(;V3QEB{MUEGL@nR~H=4}E=3W`Tb30k86>)P8)% z{O}~9oOfrhF-S5q^V`q|*oLINR6APLGs*Tw2ZKp-9yE^IZEfXW$SK*K!%_V(XxKV= zN6SgG{zohS#V&iudB0X4xfYat^M-eK_8y%ti8!sX zR33&PpAisRRovUdpWpE=aHuuC82*eH3yUZ^SHyj6XNNdKq{xd}9fTrYvj6&hFfOSJ zB)0*^w1z-juNKsANP2k>mwx zpL3Csfx=ev!oFFO;rCA00D>BM9>&XU+6vTuygh=KK}3<&tD)T+%Gy*E{;(UD9QNIB zhgL*kKfcza*{+AUcdC=5Fomm?pj**#pjc+%5HPLOeUf6eL#=Texo@YBH; zl?#}6invn+2STWXhraVD8~X)4mlYI|(AZ3kawMbK=KFO@t&+6DS1i)k0O)@0r;}rD z#=HUbf9{6-;kOo*GXBQ7qb0C1etQ~)1~ zT&T_Y$Jb`hkgxCUP*e4O*jT&q>*hxnl>5{-G%ibglESzpLi&wTZVs|v-*uN(gd!-6 zuAr-HwK1^z!rO5;fA;~9%IMa=@V$XGte<_WwoBm{E(6~ap^|Tx3sTiCkmJ?7Op!r zK%@oK(*aV11Q>QU>tjquAGw=^ls~5wckA6KLyD}R7`z-W9zHK3YcBa+de*D|%-3dt zM*);ZhYXF_wMF?6M=W!{JH{m12 z;Zg}P+Kb~sr$)WsR_TmtCw@;ou02IayIW^Gz3HCRn{J%jd$q|mYrd@2*HDz#^ElI5 zH|YWd>+helxmZPgpdbFm^P$+;(5v7C0fdB7-TDciZ$F~E;(uIYU@m>d4K$>eX#8}$ z&r9~U$4^RZ{PYcy(Tp(PilnLO8|Jb5KtdGkPopM1*HD@tpJ zdlGv?mn|wZ;}pu8#GCZ|tSlI0-}`0-sNxz#&&Um9vVy1^x8~K7&e-0u( z9;Gi-nE|2GhX)<9$P~VONqFi=rmpkF+q;rv7U?6d#zj;BAT(=3+$s$_GwcRMYvq<8 z3IH#7l#nV1M|{elUgPQ6Vtfb%@L7H+r}m;S5F8Q%5?mgsf0q#A@O#Nv;O*XX(y%25 znU=3O^_8zCs1$I4piZmK(76}n74E0enbQET9@Tcm*u&2RwSQM}L?J#Qsi~2A*6hf6 z99Y~tm6VBS8*+!Js;u9sM~_xlxR&Mh%1fI3fJpQ?Y`h(e$oDtOP|@$Xnfm%z8_xADMY;2OF99Vk2NC_gYs-g)!?LEu1GAs>+b2Po*Bo4 zjV1T9G!K@~rWr;X7niX$2{IX7Pa4&v0+l$2$EIxm$hvy++jl@9`6rs0hG5MVnwO3l*Eag5LA&!$E^#Q1ot_~>b6Eg4y4$BXvYnT;o z0InR+2f)lF4)hMoAQXT;5daHKj1Zuqt#IS8^l`}n+%pFX;4=jP)(l4gL^Ux&0B8L3 zrx$>M4zOV1(hMvj>J_d5LZ_c>`{#EgUYRDuCl&^NvNjZXNlTY|O3sjfd19}iAEM$p z^U2WpDxjduHh7|J{Yff5(9TaI=Vc3m_YNbY8}tNA5%A*OapgTk0LkRJTZcTWz9Vx% zX2Vx|8T9d|mPtIko3-wTbmDFkp5hfR+x@Ufpy(XOSIY11#RG!u%3JADD-GPZS6j0)%LjSX*O7Lzr!h?OUYb3!bfWOoi*Fj@ zu7h|7zx~aBwSAHuFdJbuR8w=*?3p8kr`9=q^>Xhr@e1G76d&mevqHLP$}SQDH+hQ4 z*EGZ@*+~7>Ikh+Zve;DYS!0~n(I=iemLGAGYcg@C2He;Sc=ljzjFA%WGXGV z4GM#d)V+oR{i(JRB||JhQy;@MJPwOl>xJWm#QaB99M3K#0dq-nTmI9GRAc1Whusjn zo{zTVPbfvuk=~x+(yx7Fu*v2h>5m2+p+IJLv*F4wsxQkYo7p(J@IkaTr5IYSzkyj! z^+fIJvZH!LCE+t*ypLWtMI%1>&>?yAdUx3{r<9$`_*p)=fl9c*k3l3aJz04yQI<-& znP6uO+7FvW7RxN_2Q0jlBZCmZB43s^kWy}sWb46Zg;mrdD8$eSiHX}lR(Nnbmg^t% z!2clQ|0Xv64@wS*&;Jdb|NRnZwR6p)ba$<4(Xpp9fLF93%*);S_*>(d>GYAMxlC?* z61ijMcRH1kwWmYFBb0^fiLDzary`V=hj_65%~*q)Us6d{F<#y57>Fq`%%vc0Ke zeI}Ew(y1A~vh&ZadKTK`UpZNrR#QY>4aaIlty@o00m40|XZ5Kom-Kc)%jIf;zjL_Z zvabJhJiOlj5IETi^s1tC{62*Yv89N$(@-<>xs5rj?!LuEn58*$|V^k6?wa zmtjYjwXs`e=Dv(X8L2Zed+mw0qiGIUb_mv=)H&1Kk!dsB!0q7Pp#TCL6Z%b|3Vv`& zI?mLXWbdQ?(Sv1$a$e~`Tuh!y*w%<41<2%>&D!v`0h6w2#^%~6`;;{!VYk(^_SNx` z$LvC067&$saV1}Z`mB;qcVnj3Ej;?B&5Kg2J{WW0ku^j~zfoOt zt+u>;XpX+-|EW265yQ2?%oeqjV4J^ga=qhMRm7Wmh>@r6Xsa*Y5e_`_t6)7P6#qmE zga_GTzPC-ec?Q36>^L67QrO#Yb*TNE5l!cl=}n@DSBe%Lgtsi4`+2Dcjl-YnG(9z= zN>pM^I+b@cLC|%V`ByJ|J2fG&XuI1|_clQLJHb#j#OT|t^V{FF`Gd{~8YRaX{1&yk zy_KA4frK%KfhGg47!`VYaA)VL26MO=3CW1Ks_ zx-9FDR<#ik1q+AI;D~~IWi1R|`9!?{H@05h%v@_Gv!mR7m_ZDiu_NBB-Rl_rG5!LM z=5t_C-6b)d%+DTNd$mhaAKo4z_T*cJT^=)-0?avBj&leyCSuY8rw#3NU8q#dfI8H+ zX($3?ne2hw)Z-7!v%R&+M^A@cJe3F`$eZ}{(fb8xs2A6R+3VDd(&%J_qMg}V-3N@+ zPR$1M%*(aku6qL3tn%tFaPV3(mbThmL3TewCVIgX38V?%P=ndx0pTYJ*OK(ZcNj6sNx|6fJRWW%4`>n2n#dc z8*Jl}d@fn~gz=ee%^OuFCI|EiIX^et&9Py<*i(`I(u2fk0K^jb>G=8)>1W;GrXU0l z2l`EhuiJZt{JOz$S6s+9VMRb*ra=H|cN!Pswu@JFjW|uG@1VrwnC8;6(932R#Ql!I zM-b#tLiv_#R~Vak$f-V@zDcLm9{cWQh=8Phb*gNzScUDaeP>Q0f$K%|s7Y!8Z_ya5 zEA8eV#F&^1+>djfx6qk%xI|(>k?zDaKqB)8-CDLS-VI_4C7pqbU}ljDcRTt5eOC1) zMh})5x3zF36}%3ssC8HUhaieGHDQ&pI5v}gwQHB7a%w;-Puv*CPCN1ja?u^iDHYJe z>Sg$i)EPu*CAAX(fL!Lq$NVW)#HSY^#y7}0Ty0R8%{Tc2O^(At`@w-46_)2~K0vC3 z82!~lC54V0tkm8;ch&iJbzKeMesKQ;oeP*N>PcC#D5gw87?Fy%ZKkIDl6V*%u)in# z3>yC>$uVBTn2JpzABIM@Ylwlu$Wp}YU@D2uY}2^TAa2CHD#c5+AT`+#4!D!rq((FW zCQUX3ldp24eP9hukRkK*7)z&F!g>6q`yCS2)N#T3j<6vL7<`)WnAzewG%E`ydd)m1 zHrVr1^nlx58IM@tzK8AObWp0k3oW>W6_=&fcZ$Ha)+6f^HMn+vROKc-C>iGVnpC4R zhLbA*ZbLx2LBDjjgO1Y^AYVq@LhmR!Cle`Vzo2h!e?}z%+^F=$I{v!R)3yB5wK}>A z1Hyx!9|7!?M<7OSnrHmv&ga|LtGN7L?mCEp2r1uZU( z>b6SligTlf$_Qkk)-~Lq1#!Zu@3m>f_D|5$%_?pEAQdfobSxLz0 zWDd>)9MJv6Yui(`=F{z(5IN4U-Y5Qnw%oU)z&9Pc0aSP7=WRJ=5o#?R-jz~ z!GTke)Q7`^(SvOI?RK~rWC~6+E!SyEY@2MMbkRXGza1$1b7?p(%u)w=P%c)^hP!;5 ztfu_}cUh6G9ZM}S{?=#PiJQd9 z9R`Ui*?2Q<>jTZ@_m_%MFFf2p+IHsXVdL1myPW9C)^|R-<*uJq@s$=;GMj6}Q0}n^ zW*4q7gqBcA6@cF`h~ zt}cIhUoG?UMqaXcM$AwIp2^JP-hISS5ry``L|Xj_3osCTc4PF12K@PQ>7cTz^823x zg^Pr9;~-NftH%k?ib8+r5I1`+=7v(7cWw51cGSVZyUX|k1G=eI00HOpuqfH=&2T3lPD`~1d< zr*Hr2!ZK+%Se;EbTHwl~aQ*Wb(Dcz4Ww}Zp!BT<6XK5*rXUdvbm%Z+MQOnRs7xnUN zv4FH`vh|2fk9g$F-aTdzr>E*0^Ky5O@*Q_0}x zm)1)F1i>OJzBH#XA|zyH4)9;`*3Vac3tWs|{Rs(ZeiurW0-%>+^;w-6)C6<)QFwTg zPKR(qGuO~^VYQ%NiM3%g04~%bHChn0k{hUi5d8X0gU2IHuozw@i|ar@1IB1V?`rER z9UAliq0&SkK((bEx&DhAo}FLC@i(>l52a*>Mejc%gDuhNNV$0Pr{tN$4lkiV|ipy~w-vAEj|wYdASNkVKgl7NEWsI4uRb}_#>D6}vq#~>j| z!NgD#kt?w&n-5TC!$I~AV*%B}MP;R-A-)L@EOm9g95;@6)=&p#BWCmp+*krmNx8P~ zb_g%4UqX+Rzrs&0LSwR$f=VHrDjH@o#mV3 zPNBLj^9O9$j2<+!G^X6Q)cv9VJ49#%8+$)DcG7E8Q9}VBrDOB1Wk+SeJzy{4ps5o+ zf5Q*Q!~wLXL=Pyh_mglAsvn>Rd8x>J9XV0!%zjSZzP%F9YCR{41CPJx{K zNUgXKiqQLZVMrkY&`1Ms{2Ixbv!JyR8LfuxDZY9sJ=Y-sc{F_0gi~1F&j#B3@mir$ z5lN(fk1r57-^sw8ic;Y&azB4(L+`1QGhYH^=Rd`@9tey! zx4VF<*U_#CgGeD?)m!&U8($h6Xm^LBN`XUKw1KzQLPQ;@69KNY_w z+g?G>w`E(+QkOWSX1W7F6tQ9iL1*QT@=0l>%l8Hh8_XMuzebtv4}W%|zz8pqPv`k4 z2J9;D6(Ayhe#BOHozP;sc=HO10a|vR`WuN?gqbvjg6vJ8sqwJ9d;n+JVbVvO6xeTl z`FM=IiRAGh?&(Vh>^;nEi>OhBy*MwQ)f5R_n0lXYK2TJ(GogoYoSsM;w(0iQ!iq5w zEGR4ql|8cT`8iuQgCq-@5dnOydMms5YptgfPpNN0TFL|LhXk;X$9Dlbv!rwvJueSa zuH(#w*fxmJGB--rW55|*xF}$2{9w{l+CAsDK4Skh~``Fa-RfgW>uA#*hfQzUCgsIkh(e=%`e$*D* zZ0-Oa-?dzs<8!N?Ef&}rCnC&CLW;Q`k-!~HPeCe+UR-oDW$Uu!#x!wBp;PzY9* zmrFD&!=%K5AkEH@NU0rMy0tDNsJ4CflbFgg0_3C}k%<6;JW`5!{ z;NDrB?aWK#ACH*yEuTy;i8;DQw>|{8-25JAhOe;MHjuLO{}rSfbBgv^oG=1U;PITF=@& zBY%kx0S#`)iI5gt{zf=QE^RAUfd$-&{MQ4z9=~23Jh*0dNyI|L(MiQY+HX+RJH-Z~ zqEZL!>KlAme)Tqu&#gzcG#HTjqRnm7M}kBLMq7FJjJY$3XWt?5F#8^JUz3Nf*Vq-N zgGzWl%ATK2j$R)0VX5guKm`@H>Jo<#Ho9@yRuX^JeGDqlmmHjJBB+A)uIf_+az7 z_*BXV!;TAZhYHqb09+4s5ceaoRgDoah0Xz%rA=OeTNBHL#<-a%BmF9Nx8-Ak4$1Kgtf#iwwp5Gexf3Nhv z^#zl-eJ1>}wrY&6xJv|aM^Y<(fYYz>o(HzaO+qA#T1) z9&XP~B|}XX?jJa2V7LtuBLp!8%^!J?KMX7 z5`u_$`d=We<_sfE_K?Tfc{s=APR{GrNs1V&_n_ zazJ;$JezI7_5Ou?8%YLurc6+l$;tdR4g~zrM^K;O9Vo@ih%#o3Z3(FS0}e*8AH>ZO zqKK_840__`u(ebBR2a@=jyz^AcDn!*k@+Pm-g-p%WjEEUQ<1-7r; zS0JE-wT!<479+*Y%%s;j)hu~TaoOL%wLb5S42@1z?-yT_xJs43*Ye%?$D142Nn_om z|MGDDr7C|_RRgE<-(=ta!WsXna{Dh1FYv*?+^zripnyHb;@?;_{I9BQ?6`lAbvc&h z+c8pc3tAXX{K3HQDuCA?4KfsYVNz%O%DKb~(@Z!bwApAj{bo76V6x$;Ah4y`asC_1 zaq&YF#jp$;@xMyIaUL@Rtg4&dKLdXv`oTj)0yxBSCz2AsdKF-4&PyN!B=vFf;^}bOie!ac%%|LO4@yj3SM-2a%zMJ19PJ83EQ6|z448{ggM;pz= z5g}(qheH3{uB_{VkJld1TTulk4S7omQcYKnnKkc?k-gO!WS(`x6Vd>Z;$hy7`Vs|}=)nr79>#g>0l^K0!#_t4fyY)7bbr8lFEsb_5Z zrR-1o*=_SZJ#YMDf6uf5lKT#OD_y9>h1@5&04QBCrEy~|lF#7T$y>EqD{Zl&&##8o zfF~SlbjHxxL;MYpe}(g~i*i}?z;jMg6A|}+9@pm##WtEKSArP!%OOhhS=-DY29W@` zhrjO z&r!FyvFPHzs~R5AJ8DeLW$#7HNT%HJ+*#{jR$}5Hn=W?7gjK78#?xwz?^0}@BkkAP z+p_qsXONQJI2)b1>*Uvk&dtgDV}t&6Kh}Kn$g)OAxx12RS~-i$nqcs(JL9Dy-C7}K z&eQ&eNvcClD(&Rv1xqR<083%C;f7{dQ|40)D31jIz5YF!5gtjQJ`W#Le#@>X`WD`iJ~B>Kl&SAf+OD(QDr1891`Ofc9)8U$6di zLHxC@tuI$2lFXVh4@aI4@i7db{c7`KMfo}aL z3(T|sTTm?!CSWZFz|bUtjef{yWOVgUH_}-=SpOm{U!G%&t>XCief0cJ{hjGm^H1?d zLSHPa?8;5g_V@m&MOng)yw&~`fU4C|IPJMdJkiUTSzCDMk1eOeYETq62M zO_m;eybJDlM}zb-;{Ny5oYn_+zCuVEoP2Y|lOW{8T~?)c^RsIFQ|9W7t;Umo6kZc%e@gS3?G;o65^-=|z^^-lN6WzkP<7ipO8Sn_bmWMBgnk2u+=h;j4i(+ZJ=ExQ=M!a5EiqC!C_)~9%&P%(B7m^w*RUoqE_1ry zdIc||6xKEaeS}7$bhsV;wob?iXwGim;y#;a&GZ`rjAGH)^ENkjY=O&^-;8}E79jBQ zobLhV|F(j7?OQo%Fku({MCCzdif(1Zzxpzguhx?Lx+`TlWeZf6mWcb}bC#618^oJ# z-Hz<2dJa_5JZShJwji2-k-?5hFh(HITe9n1WegZ!0y{jR{N1Hq+? z|6(aGS6!2fvzWO0jG>&@fllR^AoWzl&a;!C-69@&>x;~L(Fi$Tu*)c}0OQ5*nSaeS zPk2zNi0B>>WxP;)eEftX6gc26dl2CPGvr=_?|FrbvqWjbxmiN|)TCEB&0Mhs+_c`B z&q)70i{Du$zDuXi>0B-D4Y!vB?cW#xXfA%Nsgue39z0Vv{r@!gopDVy-?|APAkqYp zE+8U`^cFfIiYUEH7bzmW_dpN@k)|R|Iw-w&2!sGiuTlbm7?36<1}Q-ZKI&J*yawVT9L_v)#Bwy331eWa9 zx%jScGXuw^Qw)fLDM7%5Vn1_hGLC{ydNAW8wZFg57#>y|11fxWMhGf2Xxt3UHJxu* zp41ZHlQI%AKZ>oa*gHcSfKO`cKn+CbBz-}J||m_mR8)a`5?YnojPF;yip*s zaTZ|;%6VE-DJ~u)?FN+XI_DgB+FzJlFlb+3X|CF;>H!cznTeZ`P^Pn1XYks3FimpD zDP7xM%Lm!8oogEmrhO z;+Z|L%;x}tBPAJgB)4{!d)OJ`odH+>^5F6+6EJVVq-^V)hp+DkRD_d~M}coXpt)lB zZVSpx$6D|%(O?qekInztx+Cm6>)KfAhWS}LtfO+uQ{wequpYXjgeCuAA2U^Cx|95Q zf+Z9z^P`^ksS&^w#vZDXa`yJmPv{oB-fNAYK$PTUtjtAmpoEsL;8$igUDkL50#;IhtqN-Jmb8M)O$Yy))x1qBdD)TbCOtu z^gy)DbGnrz?^&K~JET*{^nK2iPeSB<=d(kPxdPQcQDT+tz?Vq{LiVM~v<7_&=)JMA zr`9UVF#kmMmcsMFyrhxtA(l`wnE{iOc3st|1S7tCyG%^84@P&0H2StZI(n_aX*&U$%yqj-B4G6V>NHdgb6|Cqa^LBXv>4&Zl_M!8zC%07Z$JMyy zfn*f4`gVS0dI*0V_cAmKiWlG{zCzcms#~;ZL{NIW>iVNo4%Bd`( z1j-MXv)B|@F7C}Gqb9y6LzCXLJOn-nM#8~lYQEq83_Qn0qVFa-SbM{(n1 z>w$z}dxoUc6UQ(qx827AR7^|`rC|pLIuq(ywGPh$`anq^(B?q-U@6~7D}8Nr=rNhc zs?wo;xkbH29J{>2QLw3U5!qY<`D*D2>GtGTDfZcG$V}po_R^%%>^g-nGWWii+umme z3b+!g3xV3{r^>Ab4_{xH_+6v2p1C+X(Rk{sSA6Q5mdB7+S&^x~v#Mj#HeU0D#4{A! zwA$!qAe+{=UcMf;p5ETp=P9Z42ZdwojB% zirS7tAUw5cZU|CKB7O0pA^_p0-{9a}MC-YxvZFg^N(!9@%D@Q2Nr}JAfK*OLPB<{K za8mC;fNJyXu}yd*2oey$DnfP-79;kDOat8YwO8+qBn%84Lc=Bd9N2n*B?u6%4F32? z|I;aDaryVS|HqR(qiy}Mx_=)#_YmLSW`IMt8gkAZ$L_gl9m8*>(3MEIyFAO$cxwGj zvhqzcwUvm&@KIdg&fs!|b)ZCvZEN6hY<-AkicSHC@4AvzeBT&2!trpcov4X=E<`B5`z*LnoSIIrvDpaL-f0M7iubvi>Qo zORFB>rIgOP6W{i0;&X$PwlpTI7qP`dBT80{gJV~O6^iF6*YS651BgBq1M-|Agtq{Z zWq_LKr(}Puc3!Dv3tp`t0R)An9KYl}ER6@0446V-?R=^81;A`knaXauItyth%{P60 zgV&x%Px)!KKW0@Fwc*;z2Lzk8P92|OduGY2EE^x6y$B^jE$-mnl=%J}=AI%zAM%8j z@eiDRO+!paqW*`EN)Zc6!h4F4Tg5+>Xs5?)V?yyzO!Dp-l*R@ zN3T+lyTv+%0&9;*K2Dn)C@`IU2Slj~-7m08fl|!vC2!r*jO_n;nLZ<)IgN&6!?v~u z?ND$i?&vZY=m1j~WUKXJQ||FQTVOUq7k{1x<#%&5@S0jfV2mt@ z0ShZPO|4t_v|x$9Cjn-br=qbWn%*G)z|t+N^JLlkV3tGr#g!XK&K_jFEsXThnh-5S z7JN~FSrhceiE4gS=w^}`NE^RTp?+2VGL4Ll1qt=5Wz6$9^KN>fw6vuBjaY|HHfpf# z77sazg~m(v@^Ko-hjhsL4YR4D6j-HthuUqu7LzLK_0|BYjjnVp%o5m5f7P7C!|~CY z(j8UY!Zr_$I79EK)FkdM;Qch8ry-e8Z2XmU9hSDmFLu9s@ZR>N+ZV4~0t77p#!F-q zAMIzIM|53$RXRR}uT%2(E;Z!R{pzS*S7n@+3J~sH!~%uH^a5&O1S?bN*M?CGzy3uA6MIh?LJJ)BGgSw9#7$S(Ept z$W&r=avA2$Hi{inBwkkNL7OxUvKUT*r>caGxKv55`tDt=K$^1{gSJRwWjm4+3+Yi? zX;~dT)f=iwTbZ-J80XQZ+m4qZCOx#0D(eoz6-Hv&wScZ(v2bT`! zK~rbUycDoC9UFQr2n(RPqegvY3@idng6oUPRzjL-VmPV>koo$v6GZnT`x3LgbRe~( zTK4l~gp6)vt_q2c4e_`3{(C^MEd(d|v9r(MRgwe2s@r8?luYu~H4Sx`a}Q`$`p2X0 za&`1nu?7RSi{p@Y7@#L{C%UmUig}1e-oEnuouQ*3-IjRIBr?OTTpakTHl3&XhWcsS zpG`R{Sygc=`}fCd(%M{@LJ-c>{?TTKl6| zaKSw9v#mZ-+Ox+s_%K`Abj>`%pf`3@7bUJExj@301i;d3fae)|6N|%rbQW0VH5ixX zx}70C29DD+d^zG+`hK@yTEXT; zG4DEn^}BWDio)pecQOyq9ZYb+>R@xpHQpa&7DMNyN!)rv?j|mr#gX#BEYu>S3zYP{ zGnML=k{`F-HkQyY|ZeQ%CN3RtAQ0>9>#B`!_wH1;N5xUwR7T% zeBRkW#()ZGLp<~<8h;C-$~hNyRgK5khTW~Ys5$H1qRToMPymzqe&iR7KxApKW!1E?dE8N34Uni;1>Tt!BWK$f6?@{`p)KEIV*mfK~-}xKx0w) zxyTz#axN&SkzyTAO(%+COhC$S<*hLn^2@!oA5VGqAk$Ais~{fbDws2_5OZcTWx!xR z1ygwr77rM#^CDCWhITJCEH2#)T#TTs7XHAmTuuGtQoPt#RnMWlOYzl0vCuyWmNT5a z`xhnyz5GYOpPc=lAn(6_)A<{TIg9Q5h3EhA)B($%EbhykkWcUOLV*~<_wRSCH2O^h zSR_S9LHBj^LdTJvB<=v+rq1FRe@f&9FdTE|TtJ}gvkrD@oy~ zqWh~t+n48up@&>0{@I3mtCa5Na*FO!RzB|BNF1IFy%GQ`RoV#`V9OZUczDohq8RLF zQmFL%D$ogZrG`!aeGiZa|7<{4cLr=_e)Kjc7-C~5iU7ShyAh3UkF~e0_qE1RjR$fFBmus8oOH_ zDLCv+(}DQW);kq!rfPsH-B^ywv5@fV`obr)|G}^UWQXq6p9A70XT*j-5c

R^R;N zgZ=f;0?e z6BYKKf1_M~b!u~2_G3}xA<7!F@8Ap)mXg4`DCm`Pp0O?c|)m0nuGUi%yTsX_<|;aIf(& zYjbil$B4xs^;gZRGjIf=gVc$r}91E`3iQTrgTW)Bq7B#xd()x^!OI+c3y|{ zf(B7=0LLCevAClxZO3;XB-0#jEdd&K!C~2^Xyo7AYgxM`-_#5Si9-!=dx@O!CWu=o zq(Q$r5&o7Il;R%NFdC`f__aeRp07ciCiSJjCN{A2I550zGedW&;+achI+U8Xe#h9R z(%R;zAXl3D$0})$iNmQfc3P>X*off&Obcf8mXL->cuTvgN-!CvhOs+E1ic*ao7MO1 zSD-@r)>ro3iE!fGd7<;b73BBVi$qA)KjSn(CUZJlg@yskJfgz=2 zf%4oBYg_QlhX`nf{Rb{NmYxzU#jf|Z_7C{C#E?kw%gF>2r!F%_dv0-XMLu+=!i@w- zbxF?`T|oL)xrajP)3e7sA_? z8-X<@2IR3`fPc0kK#6$&&0{K9YY|4?sxF1;u^w9K`pjvHT)M-l8@MK&(ERSu+$PM# zb}l6dO>CMxAr`b*|2km9W&KK8sV-o<7>qk?*Db9zsQ323oM4{1uvprk{uBt#hH$cF z>JLx%RL@%BZ{cL*XR(uxN^pjfk)PZ_j%`>IdK0rL<2rFxMPc zTMMIWqO|U?eX_#Dnsr7_ zmg}8Vs}rpB_dMrf7EfGa1-eZ@;x#s)6vxl$Xu|rfy&@3%%*7^Y;VI+14}N%bc+87v zRf?tfh6-hd>zY17dtMxkzE0l<+I&kGYP>#CHNeY&EdTnvPL{l)l+y(<1^V*rG>u%h z5257Qt6SEX;9mswreTHjW}1Nv(OP1%ow7e1DVha{k-?az=#46g2LtNS+UW2e+&+y| z4Ya=d@UW@CIX4n>{b{T>KZF3csOa+*LRwD}j-$!xOmI2*TwOH8{Si?Tjox840Ie>Q zzo>)2;0yGh2c+mYMTg;3vi>_s@)Najw2$>4<7*>uGw#&46U%UYy5p9i*O6%`ZDi40 zMORRB&k3Ry1DbK=8rOm=S=FSJ)~l4KcO6VShfNI;mbBnJSF89{$#%=?)N59}vgGd` z$jeWMrY{027+}*C4=tz7MWEhdoDOBdmRi%ENG5anaxy7-zz=)dqR_IanU^KXNAxtv z%sL5UHwbhF+eaC$8&#@}MpVdi)p(~f?S?fDN1~M&Xj=B4ZM4|iUVS(zrarw`^I{Z1fDou~R z)p;q7{gWrdf zr=g|RrZi!UTW}jL)o`e$wo1olK_bY)J_9w3R!Ts37{w=mP;#*GV?75{=DzArTmE!X z4@+12<~2|Rei|iiy_4coZ6XT`kpwY|E9hQvI=bC`sM{6AY!_0d6zU#W6l#%Ueq2r? zl@+PpB3_90-7UiH#nXcP;m|T|I@Sk}pN2$#p~jJXcI1P_)pyWVaV!4^7eEy;o?MS? zPfxE15g)OCSEUvd#3?a0&LYw_X{E9_ytH})JA1VWL#4g6n=6kp1ftTPyse%))IGK^ zeM@5VlKiDTEo7#g5a*rb$&05i!C$p>r--3igy%ODM9%emiLFa{TQOuuLwJ2Bt_P1{ zD+#-iBX_A;!%3@X;d38}C%fuiLUPJe;Ln zr+ZPbku>by3v9~DRwL5W97;8LRPJRtr1m`UxPNoQ$TrNsbbv5^XH4;4Mc1fd)QEi4 zI#-P>y;Ev+`76-$JzA7op)tglZj(kTBUF{mT3(f=uzPxKD|l#UY-~X$19TIM$nJyY z))IpK+1lHsmrit5-!5eTG`l?5N93$0KJ;aT87r^#%jSZwQjm1w6Ha0fVP!$s^{w(s#|JRoM zIeCyCE?{dU$}K0WHrr_cwfr7XhT(N-H+bKkuTk_Xv2N%9x?k1*yTAwV3~vI+GNv*{ zlaYrHy-!*rY576ozKH?fH%7m#=&Iwv8)5f9Lc~(fMSQ1YX?N!@jjd$!dR-U}4DL(& zmyB15pj91Mmx9r+!C0WwY4Dk^Ni4V`226S8Hl9INjrZpghHSSV4l*FrNM&GAny+I_ zkMmD(Cc= zH<23kVx9kz6=O;E`$u6kIuy864~}zM8N9~4XS8ZO)|`|}CcI2-Xp|1aOE&<(Z}?616r zHa@~ZvNsIc+;-_0W{I-tsc+6fJ7=3UNPeuZVjhKp&H*dpxT=H=O%zUdG8f zit5k%i3!s%Vb`vBvHsMmM~m3P30qnj6TgjxV!L&Zotp(VZ4m_><23R`PthAcDv2!)r(s6DfdwIL z7kmq$amlk_wx0Q2uWedF^8&`q`{ zEtnG(mS%kigh*xYI}1`i3U!XKLA2p@>acq3C1I{8L*m@^SuB*v>GxE$QSPDtvmm|X z>D6NxR9FtZ+8I)gR(Cz9lzdfLzwx$pVdaPaNm^AX3}vYo@K;4x&0D z%%N!ELjE$7IS>2w!4&;4SBh-m+;03)D7NpAJy2|5mwKFDk{K457Gkiv)v~bY^~Yc|wm`z;%S2~C zhxyoobe>h=c1>R5Rqz4S7WzqLL`r^<8#bjSUtnh2=Dx|z%RSmb3ByjScvL25vO0;B zy01b>CAE;EPjlKYmxSfE(S|B=l32z2(U!PYP3%LpE_7>)C?=43Xk10=_JUsSOH~ck zRi=>d)7HnixslK2PxW-QbdLSgw2Q zVx?3l*3I@nLfFcSt+XiE=NNFe&VWe@Nj#0X57f9^u8k~2DdEF zW-7=zPb;(87@PrQ+auEakjdCL?y6orfR|xR0fS@6e*}x-o3SPqf$=Z<NQJWvy@ zv_cZAX{ko;p&-gq+VFAup^T{ArQM2IZ9hU$At~jYle+<(8r*fOX&_SNI5SE$&L#T1 z^K$pEP-pj^wS6d436};-jg3iiaSga60xXpHo!Wz87mFIsjkTPBbLO9GqGDf?cWfuN zSW=jgcyIzK_HZrE0_I3IrAK0uCp6Ykb6go)RP)!;Cx>7yqyQXQ2KKpWP5?A0?;Tn( zI;BRx_YQy$$d=JaaTJIq{aqk5!XI>dqvDDpw~IM_g##tV?i?Z#zy$yPLU1OfeAgl~S+72XuomQn`l|Y{`&c zL_|XhbnN8h19@`x^lDi@?Kg;IQ#3#FfvE7;ie}j$Rt&e4js7((E%WWzpih_fiPgEEjuG&CO&Q^R52po&`P6{otLurFIN7V;8| zfyhYrLhn@f>~^V!3YOJVvm1yV`kE@BN!I#Qk#=(k1^g)A{h1p{dLTc0yXrG}a=h?t z8>UxnZML)qHRAspVcsj7@1FvVedPWzxw&}~93nSOw0eRc$XHMwI&rfpvXK_Pv=A_4 zj|sJ|X~PvuzxXlWQbwrS*XfkCoFBJ>^U+aeH5{9u?3^!}_Is>qF zr|h^(54!np-TUZ$L+ABK^|7(r#K_N9%RiHRy*n#bBDnko3f=;(BVj1~M7 zU>W{tJjm*^7g}|6io@`@TvEA2K7Wyk>D6nhyXQ#s^uli&a)00!5TK_d%OeTDZER@B zO;33ZxBy(bdzGH@pPRVpnJBLD*E>MXc~hh;I-$!j|_fO_#IY#>A*oI zJ`+Q{-dz20df@o;yJeg7OLbV0<0oAs)2Y&a9SKgP&NC;j&)WUJ$mHQX(+!1RS+1a> zHhO$*nGYg(kdwsE(>jqL=5&(mm3=V;d>K9)Sov*V7e)%YZ298jji(b3ZnzESW~6Nb#WA+9gJ#s{_aT$%q4ZHS>RfIdBqkq!*|Sr+SAsMYTa1_{>fP>ZxpAI&lhUgc2i z<-PoJBN3487Wl!z)4+ls#A1aWeWjk7)2krlRXL||d_>zY zsKE@a(1&Ei_qGkp`?zbvHabch!?JmP{frG3c}jou4ka_Br2OzrP#LfL`g`RThW83o}*h>aH+%%f~{o-Roe5|w17*U~y3Qyb~W?z)7=vV%t(FEUJjHB}W@Va@oDs|wt( zGYs|?4-^$U60RzKVZAGKA5|os5P7=0iNc z{aqu!)QNbmY!NfK<7jAFn0pl4WvDx*La1$tR=E5bbBPf~(m_W;y#5j6Xm|-OizA5! zk>JTm;8$MZ{?8}iWh5^n?7s^BtImxp(V+h-_^&z_RsL7C|Dzzu|E3$D@Bgu#{}|6) zMGwTK`EY#wrHLE;=IxWyWilk@$WoFH&s+1@XK6Oh5(+)t$Zm=nY%^2ECK`FGbG8Lt4OhZ2(qB`*6V=0D4nTo zJ#?DgJ?WNkNKz^Kg$iDE_<4l(<$eqO&TY8)EB$b}f3aw$>*k~nA~3bGGUMx?qQqY>*sd9-95%l;GbN>9SnF{MgKSk9pk)z!KE54?nk4mBD{?b z^Y~MOx}U5!sgmmiEj65<@b@J($+tPAySqotXXSit*6f#!Fp&EtvRHEVYdrmUlx5J2 z4c5RByLFY^E zX$(yj&q}uY7}6hG$kEfEMmVj2j21Xx+#0mcYnS{L{@O*OCvv884DvKB4!)q}=k1VM zO?D1;RPh|{$co8<9j;IHCMoMky`l_NRLMc3`r-VJH14uo&R3baQ<*_p`hze2v~{5& zp8O(r!@yEwyzfYZ$m0X%_{6y1dXhJfgVmKxE6Q@0vcEIJ92FW}C}|E-A1kE^;2EB4 zY{m;7H;{(V)4biApjr}LABhQ#-C*S6=ho}e6M$=%SI8+x>H3@5-yu>hS==h7z*P^f zj?`jnkh)#ry#YuroI&1too#|;*7ROSM1_*Q431e48K?cM{iNF!WH{ukVXAS-X*OT3 zAI)#Lh^+)?RiWes3XgZ)-FyOA)zMr!xPz>vhqN_Ro(w@@{WsuYJgalov~oe;8-%2T zeu$6eHdhlD`PN0OYF^pkdm1+t>ISW)O~x=q%hp2EMujBsum{ksVqKh#I-%tb>sz6P zTu(f>#O1^_6PmF(yd}a>46pBnk?LHzyE)1A3MV5woW;%V8zwp0{bzp_nYBlC(;jwP)L^z;X@D46AuT91= zHt~?j`e|C7SGb&DsInh4A+lVIyQo-K?`9m5Mpq$37}f#F;^4H7|Gwp`iniYE@j=M{ z9BEzQy-V9WtP|x0Rm^oM;P#WTCDu5!@ETey#mWcRkzWrU#=0&Gdwrc+NEaQlc7|2IX+Egdn>?va z;;U8Y-=pn>axucV5d?;dtSguPxdvZ?DP>KQ z10}PPO3fT?G@5gq+x1KAD7VEczufSm4J-|t#iCR78&@QMb(hajfI!J!VXP9l<#P33 zl^FDQ#hR7uZxZ){q|f!tM&pYlc={YJuO{+DQw0VP-f>#A<(5>`=N*Zuqrlsr=@-(=I70E*6H@wOOA0b^#w7@Mh4Yk&I6J_et4lWYrvZYa13|GQc@{QVu|_%kVv9kr{D>+Jp^%?-U>I@! z5=?iwOZPDWX@!DzQnJdWaPuOCDp$9jt&PD1%g#^UFLpqF^?`c=yJ)qeO`ry;Ny(>P zktq*gE-4haW0qH1eQv(iZd)FsRk-SDbE_ppy}olVPkOTq?*8MKhcL^ok;#xd8ZkBw z2Il*c=VB=VXM|$Hj$yAFes#ly2_^zY_b`J8`0LgD);)TQ3HJFF?<=@CIfa$M5~*W- zng#^>gOY+YhVJnC>%;Jqmrn$4lLpDse>_C;q-AVt#H0JdGE-9ns}k5eJ;fNR$P|@O zq(ez1FFLR$&cgnNKeFIk>l~IFtK^HprYyZzqU^$xFuX>Siw_Mvd846vNNYhkv#xah z>1^iN zT#^x+sN2ZuRP+9wFK|~g5v00w`$>##WV2gj|C6Cq9@o2CDJ7g2t(0owN#7@x#CKz- z4-a1Cciy-zeN~3#T2bn1NHK=%52 zPGqGM;w&LHt895qoZAygi(>Xzo|_pbEfw*2ZVTer zhhH+4D;MxP?BAfO2{lrDKE4cIqU&~xk z@41nWv)V@Xjr!%gzkMa-$2W()!kNqsAZc4%pESK!%fw7L`{?F8N9j*WW-Xl@jm`>T#rMgV%RQ&T%|}s-TY!do>@6JDx3JXf z-sBUplMw^L^V+2@^0_$WW#Vd(*c-&OuK8C}Xdl%^YURF;LACaH1dgHPBq1W@+1x=m zaK%fHa>K@sv`emQEwMUu#V|D2y5`wA76Pv!Eq;Iv%#?ZTO-e2aFgE##j()QSQD!C)-HwL9~PiaSU863 z$`SK8-x;7|gnksrJ7J~M+!Q-4FTwG4n6karXVmXDMKD{^WQ*YEr3G44P3g49k zF1q?*LCxQ_)5kEow{w^z|45aKEfd4{s%aK)U{|!u6m5zX=qW5@CyR?RrcTja5lwRm zG%=v!LWSco+vVp+bP@^`I9~zN5!?EmP0$dB4#~x4a96+C(64TXl$2tOP%OZ7P+S$eSo@JhyfB+R)ezirm` z;KOU2pK!wPPWdyyKIDB<4V`Q(;V;q^0j-j7iPz{A@l6=8g$!@$q3nE1^`JmqT0q4W?fjRV|DqQ!THXkxeACo%|tQ%FefyJ7(=~=QezIO&Sv!_@Ml{ac- zyK=)oGHWWkbgw{f1=c7$2;;5kO_-(RznGzgzs8lC;Q|M^^Ld0E%cXh~lZUvy>z<|L z9~nygytbPtcsz}FeRX>5?v;Ct0}5vc@hOK$6Zb83bX*X$MJ;pm1~nW{Cvf2n zpt9Lh+R_Y#))`#2DDCexdsAUREt50C3B9qi=njpZs&ZE?aEQaNYzZ4I@yTrNBL}bC z%68|Cn7zDVPRwsv0BaO-N=AY=>eYE*#nd=^S)G0t#F(X*Z307yoehnwiEYA0 zG+d!?2430y<)ZjyvcPmn*C+?2QWu2hgL)I`L5-?0N^3Di2}bPAYKFcUPTj0si=1Xk zH}%I-N;~2fkTT>)CScVJ3B9Lz4swc(a6)^Abr!!@!^X#ST3x+(8p?YIsUX*{yKjw7 z^v}oCBQ%aFFCW)$a_Oo5;D+HeZ(dv>+QTk)*-ylEx95wwKmvB%pr{ULpgs zGa_lLyd)ye)C3BSbv^|0J|&HjAK88T+x>bJ?Kk88tzBqiByNGAW;o!b=*D~%SlVkL zS9{dMT@YSX${WTk)3h{qH!H{EuiZ)$$P38QyyAX5=J^;QI+JM!p(kjRt;pwkc_N$A zzGZC%H-!&eEWxAdHv&tz{dNa#q~U}C0f$mx`Op;sjT!5?1zFNb+TFrI6X;xPU6?Zi z2R5Y(2seE?6Hd>|UGHBwTrPh9cy!=bfCc&I_m96u5`5w5`S$CcU@45S0(E1d zFlyv8eHMk}qxO`1vjchO+8sb;bYJSZ;U^|#E_%8aXSv;_7bgs@-Ev$x+Msb>#!FVN z)a_Zyw+h9@C7Dt28*e^V$J<+XVOQd)(?p^Z(&Z8j(>x9eZH!0o=0)3mqK%eCZtV}_ z;>5d34GzDph@CzMc7Jc~=OEuO=3RE&^X8h@?ME@?H^$_}%sL?aRmhp;z^v01*Tt4| zA9qENFUTJ8#7$-(b0Mc0`^T=Klcz_pJF!#-7o3c#mLPuZ>|#>JsR;4f39&7N_KY<` zj!s@srU^%t3AF}O2tv^7(!ho~XRlG@uSANgc23OuRE-eotX->8y68$` z&`gYgQakZGEhxM;D7w503Wz!4*vsZo{V(byo;iXpLq3LmzPmLvB2RgCnQZX9a{ACF z#M8sHjr_qi7iw%`9$C-Sjm%d#fwwp{%pjrXE|UY3g`ETD6sFA(C)us_{BT=CBzt5sl<`Z9Q##X=JYIM-dPGal!1T_k z-IF>}K{FPGz>Md3UTIU)^2(1HB)ea~2}unsXn}m{1?fLx+Q2x{u&|EU&la|uu6^I` ztnRr4DE7u9V$iqa(G~|N5v1}a6Iiq3ttu~(Dj#F-vnpq$eE0Gk%`I$0+a!*uW88Zy z&Ru2f&tp7qKC!0SOr2TTxg*M|nNyW;^|U@Ajw6}nvl_61tgx;0)1T%R7JzYOZl`5Y zNJB>?>KSXU^dvk8$gm!?dh{#r5G5{4=L1Ool)PJ%=ABlr-)#g7`!}QHudn(IeZ*Kq zV{At3H_W1C}Wd)}SvIrSDvq5)BzIpB6E6{KewW)@ZV%<*xkG8lR@OI}cwzNl zZ(gc>Xr#s3Gd}C{F3Kjg;VwKZ21CFTqUk9{L9@%@?>~ErHr%4@fL!lD7+3DhpgVLQ z3hF(-1Sj*{=77-t{f+#K?mf7z{}w!loASU zCzWS`Nmn>yFIjm;lkC23m{<5+8~-qh+qBaGJ_9m|$k z{mdZh!T#~voxg8}YaFf+sm$Pz@?FzP(;fDUMkI)=M5+nefX=wdF~&~X0M5rr8x3yB zV-1Ncu;nhFjVlgr+RQGCl8a9#2yrB>rxAlG_&BG+()8v$QvrX*_@*eo{6Dzb|ao{xVW~ zOn1}hKF@t>8p-3z@6D_+=u4JGUl|*{&e;Tr;Ew_pCifkGVtlB-$`-YWG(O4nxT>yO+0TCF;UQ|>Tphq|Etj}oljLWQNCa+ zjxtVk1H00YoiTKkU9=+axaq97q(LxWEHB)KH+56nVciUBrQ&$~A>Skw_x8*B(uCCn zsJdSbU8qhwyShA1z{2*5QXHCbS>#%4St!|S;CwGnMAQ`{&blavGqww|RbcWn4 z4-;^ZqN$U7ncC*sfue!V|>SIy}rDlh@E& zsLHVx!vo?bJu1p*WL1&-rH+fg0nuQE)tgKVe!^8uo4~`gOdXrnfIvkTVn-r*Q6%Sa zZ4#x&4oCbIr40Gy_3%hAPH5HAwvQ5QIGA2uX5^z=#vxpk`*q1xy`i1kY~dF(OgbhZ zJzbRTcqaTHwVm3M~!JfD6%CdBU~Q0+Yp+_P>@4N1R-`0ckm3MV#UZDX&f%!i6MJhz27cwUmQc$nURUH`S~OG z2P9mlRLA_G*fr%w28k#ijKla&+jd~k@K$qgDtaRATQrW`Q&K-Am+yCO zDlSjkDl`)Fm%e?X+O*LY|MYdKl4f|TSidX7h=Up%u9^{uxmFXI#dz9CBWxa$~o>D*4YybzxvsG*Im>QQ4eW?l!Zqk^6531kFl#qoGEfq*cOjnRI z6t8i+o|?5^YMZp71!K&>CylOn{-7slHqX!MYJ~ZU`Be**>ee^O?!AcX4}}7d`^eJC zaHafsQAix>WGG>Vrm}y-$H-!6OnmE$J4Qf+5>DoniDpEFux}5|Ll2_A8ILzeMVzUj zR|+;#tZz=uywFWv_c@Z#ZynuP6TVS#Lv3O_bPcy@e&9AdU3C*4)+PxX^{=8KNZ%(I zBXUJHee5k{V;6hF+{NjYE2`dyf8I%hmeXm?ls#74*ju&wZ!?57WyCyJ}cwWxbMbLql4wN%XGZ+vNlqC$$ zHrB+VFIG-_$GZYwxmYcQW%%mc1@D!O`sb9}>*30CKN-r20$LE@RvIin6t(h5zPtO{ z^h=8)7k`-$pu4}m5L5K@*v{B~&TKDo&9}?z-&;ScUDQW6_BT7wIv|u$Iceodvog4V z`E~4**H)15o7vfafFWR}Tjs)U6Ggr~O+ORX4_!Xew4!Lb+Xi?!2J43~mjx(C!wErg z8Dlfcdr<7cn~QCb5WPzVIde%eeNDMaY?e`R*ZKPRiwDpC=0AS30*NnI=r{d z-=WJYC~_T6mRi3ue8VM_s8KryOa7JT+R1$3UIpYI9_i371Wfw00 zC-4fFIMXmy1%q;cHG3u`l%vsElc&>6vuj9l*Q&oSOBZqkt(_}&dV)|=z!YQAjr^E5 ziQb8ffpydgH4nrb*OM-^3hZPbg{rCP^SV%jc6N4weR*mk)pgN2(U=JycBr*U#EWR*~UlD5{ED-`L zc1hsM{D3U(`K`?#SHfxj6f_+!CxdH`DkI(Nw%E?0wA96(27dpvzd2-46X3y{{yaV* z!5#=8JT?&Ld;(6;B<7!Iey^w~=Q0FTD@YtB8v92t>;wYPkltzt0= zQR$DS%14fhF@uLs)o)6ep;O4Cq#ih+x=LD-&;bEM8WaKRqPF-BWxh)SBbs+zkoAm_ z-QhYrK4FBc+HlV7jq93g8-*#5~<`-7^@6E zV@LiAHQcPF|E;g7KH#AK@Y#pa%|YC{7*@=DRdT$3cu#Fu%;;!HFyC@$L_^AN&-#XV z;{#!McV8a`hcfnnqe3X=#ADn!ak0P^s*sLsd4BD}FAwxXfW8k_MOa!z-8rqLqKf|} ztp1jU=a!V&FB5UA{EPw%bpdj za=-rP&d2obR`Hl3>tI)+g7eNy$WvZMDLxlRuL$#klG>5J+Rjp+KOu?{^<3ZxRU{1? zOqXe~YY2XV>7Q+!=U+>Tm=BFLp%}>4)8#=m?zmeVh+hS$$b)rZd8_bbLWd8b+ z6?Y~$y)RLGD z4bE6%uc@dggVrIjJrZO)1&(2Y6nF(9W*56%=ySxsHlC__-m`P(Lg zw@Ur#mK_J3M`lH^BPva7_gNfjZa_q{EeA}$q+%xH{Bh#PN*((3tvc#O%*q+$lFIFr zqF!?3i=w1qWWDoVn^9kDxxf#&3vP>nHRC`_5`@nP9?*p^gAE{E)+!o0^$Y~`%&E|l zcT)`Qwg{cw4sPb(8}P8Nbap;eu|$wMzVGmn?wJ0nk|0xv477FOyS@$+o`dG%_{4M{r3 zN$01r^JD2HOD3I)aLD1@ks`j!iwbJ0sX3nvV~22cPQ0;l zy@KhLlDRHfRnK&pMn6wrKKM`it;^L&|Mh-glAg#LGJK1iHe;WGde2~=VbX$jNmwk* z76REM^T{sTYEr6eW2f8zg%aI17n+kzLE+&)Ij!6i@7re|Zd%TY2+AS^L$c5ky+r%StF5VlM9+OMrfZ1iseC^_|Ms(cVj$ymDY|H9Q?;InjIvgzPrx zeCsEeSP0%DmvTgv(YJ-$AT7EjD890Sxf0N5+;O^K5?z9;hOw9k`|Gc!{#;M|a-D56%t^W}g;uZsE3}=0UybJgPjKzI!_DU5JTVVb3}z zzw1+=%Iv0)@I7HO%E7*g>?E}pUbma#mS4P9!~Hx1^@+6xXs!*uS{$6N>Ue5Y*Y~w} zLtYp6`qaMnA{6Sqc~=9)psJ6`0emi~#b}v!fO#-5Fgjv?CN2gy3%|;Gv}j#w zYRBmGVNt(sJR!nRTI?bMwiL>s;J0ns<(wb_ue6p5he*`1cQg0jqP2&xe+MRx`>dvnz{1hgFvpK6Oan3M7$N&^4F zrVgDL>q6r&dp)B&c;FouT*Ws4vg+W<#|E0_Fe_|HxKmK>*-3!M5&(j=(y00g1#vxt_P z?=%sQVj$M}W3vup4%4Q6-n&+KWK;_G(-KDQyG4d=y0{L?T{i9W)O0I{4AEcH%?`Pu zM}5s!iI{Zn^8?GHw8KRcg~imAJ@@2-OnB#G38M1O+K!*V6U>3a$BeY|+{3+NKY<$* zX+j6cw^s6z+#g8$5L;(Ai2I`47g16&HuY(YzW0`)z;6sMCG1V z3dznJq+GuNkfMh^f~+?p&Oxa=Ui~qCV?U_7Cmv#nFgcq?SKU&yN3j=hfE}aJ{Z7JR zhSG1?P);%tDH87fg3**@aO}aNxaHq3hR)WrwUF!)ukZy&+!Zq};5AtNdkrw-kSFNkBw4kvk8EQ#Qh(hE%Cvf%ab`exr9PXomK=7>ouXU56+A zyr12rnw-qN(by|Jf>+I*1oZGjZ>FDy1d0n~1D(4Blus*3`?18$My#<}>Rv=G=3QVJ zNOxWk9l9xAlej;ufKC?X!2;DVhSg6&bQv>An#7uVd%?N@Ks)q}u6q-L8C7c@ee4+` zF~j)sf*tCyvFp5GIK;zv-5GZcg{N_`nnV$TrPH;09i#MQtw|s?bnv&qqlE2+5t9GTNAd7#aP7fe33zM7sr4|viS)BqmjPxYl+mcUl*8wRl!c3%!GAG>Hq;* zRTWEdMMfbk8lXM8-f;Y>llfbhsu8D2lt(d=s+1N3s7LXASOCS9mVD@ZFQDQco&Rpz7EqTYtN`HR3hu!c@NKK$3YmvBq{{lq zSi0L)KAcJII{yTs)zkw7E~0f+%fNGYs{x3XxFW5IC9{Y1put$ zg}d-@l)p95zX9 znWZAAT&>FU;33*Oz`c%~Wo_c=xgZoC`)wFzens&YGQqYcKv};WJnRsM1u!41p<^3m zQe#zjjBG*rLDM>Jv~Th|Y3%Gd**{^)sca_#Z%$XB!;`|hcV3^Y+pca?DWwqePX0

@Ig$tMAK}m6u!y&&Z>chzl^`+pCl1defwb*UK_D^YZ-bdx>@gP??b21vva-frJ}@iU&ad5C)pPzZ2>7D+LSnx0U+Shmau>#RV;4fc z)}>+gOF3!argk|o4iu8Gsm^ZNcx&OH>@<6L?ZB&NtiaeMG}`O+t84Ns*oz>)8}j~U z%}N5gX^eZ6tqikQt;a<{>$nq{T3xpGK}^h(p;;5;!%=IMw|n}hx&(DW6!>4036eD# zbI^C{MudLyyveN?%Ok8NtVQ>5k9~w>kATvwY(B;|ZzY0)UiI^Kb^&VdE#->$xs4$s zwYsGvVS}^#?p?ZDvMkpw29S>_`nXy!m76ZP7aE_h9#2k?Kh%nv_FXCY%c;;65gPc7 zXen6Wy}WdR=_q*}`x5gHa(^if9SJt-XiHE6p|gn7d`b$h`}}o=nBTsBCNuD? z{7kPRulgB)c`Q%7&(4H))c4afAT6|?o#4V+vWycp@)YkMS29TVBQ^FoUIL>}e^i zq|=L4W(ps}!~BpHM0PG`6IzuG6MHbrbm~4>q9UEkVt|iJ<4i|<+IDP&>5X2=ZtaOg`eideti&orc6~OoOJqx6T- zxz=^oD8Q8BTxJ%>3S}+hx5e$c-+uuqoUS7-{-Sy#J`@RzL94>zLMsFfEPS)mJt z_;m`KB+yyNi0%P6{cH4Lx<(e3AN1{AfmP7KRVISU9+Z)T35eZe+588NNW@t;5<#aC z2g*j;h5vveTyzp+A_ff~uwk?Mx!g{%GGW1J`!c24t;)97xfHad;K4Qf4GQ3p1~I|* zXQADzhYOIVeJ1dc?{SbaH4S3~XzxyAR2wcVs&-?Y&1e1;U==Il{e zwOD`xck)bgDmc-H4>yl)3J>M~D(+pI1yD5cC!W1H&ofVe(e5ackn5yr8CV$4HHjqmD?>ll)h}jLH1%~{J$#q2X}pv_|Gq$v6}mZS zPYKM-i>jD3(#t1&Y;U=4_wA`ljmH!$&Y1E5n2sCdK7UHRB#><2hjj8=I?yZ8QNUDp zjny@caF>No^2*xX@xrcrA5)ULFexsOp8wad5}fP6Q%HHs2`ILMdVT)>P_dZ2{IeeS@;%DDFtS+6fI${zLr^m(@Ko!1j%4fia1ImwIEeq#k zutyzlyDm1&9-BxwjShGq^vj(RL|y(o0#4td^O<&~^OA9;$E22Et}lDtsxoS^PYDmr z6Lpz;L`=Lz%qQF7Gp^_{x3Eh2f6A4#r`fP`@ME7E*Y~9Pt^w#hak1)3_2<&ZPK3pu z4GO9|IqV$Z2=~4fVQ{rzKC4rLnI6*j+c=pw3rto4r}p?b&!%aOj|1$x@!4>X1zf@) zU}WX{3Dc&Dn;NtKATw~bE9U(Byh;2kT{EJlmqMIrx!-Yc*50FL22Z(|YSFF743m9} zo0{rKFp*595=o79l(SsJ@%s3th4!cZ0;ihlxAu%#xiP_Hb#Qzxr$2;xA<@=SZKqHv zd@)7c>IBf091bu9-gR-PJc}S()G&ORYfsQ!NXzJc6&1!XUXN*eNJX8pP?j!>3kIV3 z4;#E?+Onq9*(=WflCfLuDi*xGy(=o@1SAIm{&NpPfrk;iJe8b&tn)C&qV0>N>-{ml-@5@JhKnW95i z9RN!-FS&+afF=8#Jdfy$WIkc?qV{We^8p89*Ay^%Ht^wLOcy0_QRYebPgUpNSF^0P zWakrC7jl+#vqM6^Gr?@SMpI^4e?}M{Qa%D)qn;0)j++Wpl~x{Eag)Ql4dJ#X=r(=p zeonWKb+p>FDTYiil<*aSwRCJ-quBU_G%(bAXd}@xz+$ zrmvMc&A-O;$`b5{kLbTZ-qzKZlfuJdF~a@L*#8?$V&}0eJJbQ8b*g^HR7VuuQDvE2 z;luDMQzutbNJL^Rk}>HXx!vXwk3wpLQc5oPk{m|ZXNnA`G4i_~GptjU*RBV6Ad&ob z&t1_yzr$Nr{{c#7Dt=AYem`?cYQ&IFTV2H}@~#*t}; zrjeK<5h9f{+zWvEGkv6D^Q`gEp{vgLgiIEgDg8%c=&e3OQKsb(DxY9S!&J!TZTnpy zqGO6;*NNuI9Q3cI8@qZ|4^T_ZdwO`@MPj}tx_ZO5Q!Cy{SLz0w%vK#_v=u>OSYLp^ z^M;R09{=uFHFmgn!8{CSI1m7qgl$VKg@j5QU`a#+; z#|U0JqriT8{Y`dva>Rz=Vo2Sc462bisU-m~@w|#|l3w3a7+6Up2G#y*zTgAg$ya8p zsNu@eC^yV&l=r@JX-OIRsn5uWssKmi>3w)u%D)0VqrI%kn!TeVuThb}#UMR6RMAnV zN5fR>aim0Yqos*fcz3bSp@9baTR7TuD4*#mZe<3Tz(X> z*#A!puGM>sfBU0i@;H~EIDkmweACr_)d^WWMWfvVFcw#qpx3;j)}0*2Q#`~PCz(wv z4s7taQUu2;)ZGSFSuZ`vI0 z-!k0U0ALaX<&-N4sfr(nKi*t^p1IY^kLqs{D={&^FyY}>9}1ajFRbIBx`l$td>5dV z?G)6&+>OWkFT}y*q~FOkLA#&iea)`3xlWTN#W;*v33111o)+SKhztm<>Acmp!8SHQ08n=o(Ua>FsjP>?&dam)Tee`6 z(R3dBueMJX?IK8Xlo{|BJ4Q zo!fNo9Ymo!Ny+TBp(R^8|r0ibuMf% z)UIzaEYPDFH=_NoU$}y$4`JzE&_UROJGl-gxXa#8ekMm~U=IHV3&3GNH1v!U_nC9A zC3OB&BggK~>porIEd=B2DB7tQPf@?lW@gXr%f=&hG%!C_$!JBQUNcJi%syO2AUOZa#Flt-<2)~Wy1t0E6@BKyH^f!p1j@XS9$V*|i=#zj_HyB1d2?LN9 zJvTki{zR4Cfkvl1dEfn9G`;9>eEymJ_{zkf0AP7{cpvhZG@0X7`*W*radW%P_@^7Y z9z-8C1oMWIov_ovo%6{qx`*hgc{=A!dx(R$AGzUs`9w8fxS8Pt!04V>FHG|ad^KDF zq%2Rp&l}esV+}}!>N2hMb4r`bSGqplZTyUBgG~v|Km^%V0}q}$Ry&Tejj}f2Uumb2 z@2obQxAOpSw&;Pg&75>yWgW3AEnOzU0wo^_aFG_K)wESj_!%;8VV|7qtLY-T{jGcf z`&|g%@fMXb%;b0Ym8|M(`y#5t`{4E*r?%zDg`Q@$DmXY{Er@Y4#!XI$LFxrP2maM!-Hny=0xiNy+ zv9j9v&`rale7IF`snFC^VWu7)NK-4)jzQ1u!x0ZVga>sPcYoK@udl)+2DgKzG5TQ6>1bb*VKz$+%tATh>PR+ zy_JkBj!!}j{m#$V4eGuL=JhcF9Eth`M{>QUAAywGe7qt8Yt*#7DDJBjw=UWI^Z@*m zIe}*%P8OLD@0|(b%U;^mco^9os>`E#Tn}g&mc+4{1-bC_uuSDA|MxoU!(41T;L;iOP_6K#~356Z;F#*gYX2Z>m~p{@ndOq3BXDk zQf}T0s{Y!?;C%^w10Z~Dm7K{sHFca5sMS!xrI>53Lzn#@eJ4bzHNCeS5eD`s9>f6- zg5x`^99@%>h=WJ0OSir7yotp8II4PI=ftOS&%t?S002^i5Xqfw5Ti&SI}3&z8f-O= z+j+jfaUr-%5R&PDEn|^G@D7LykXzZ8)S1xl6!`2Qk2~{4U)qa(6-$F7dh|QeK zsg4$=_IN`E*YawdHLS(>xn^tQJ!rJu^2A)45#^g0X`Ghl#ycn1l`l0le{LjQyakFonJa3n*c$f#I-Tj*%so?%MKSE6<-l&4-#Sv@lCa>VT8WJmavMNEkk09K3 zo<#i%)$GE%t}RHgylBCG!v9|YiS#<@lV87=v(ybI&gN^rmlXrH0T&qN_dSG6a#_n= z8g*kN0BZG6D9{2>`&Wxflg;lmyMB2p#{^ETCVLK!iADqdV;OOh_uCQ&2wnSw2bU?P z%DRmU_%^^ffB#am^YCC@($8L0xoJqExO4HU=Tg~=&_Plsg0%QhiT1;hc&CK`QOnka z{=lg_yQdjSL_JBj_tw9NKk1CvVJ?@H@3-{n?$}$(dUe!smjUPnDd`(oiJKKmrT19) z>&!r6kwm;()%xj z#0}-LxH3-&>(olF8Bro%n7-hFPB^P>v1;q1`#0p)Tmbv0CshIb;A;Peaz43=#qETm6z-XZe={WGLO)X|&H>_QAG%Z~ zlNq_0QsPv(Se_Zn(Jyj?mP>I|QnwemqdLP|$-JFH=2MmlWMu;phO#DC+=z|>3wTQM z^=$5ORFh)HP)bPy;z3<1eTw0K!z2$Dx=L7{yF$rSWqsm`GG0!KlEpqd(Ve4R92K{n zyY49=Dxu)hs{Xj>OUoAkV^u3GrEz%#q`K~#XC3tU-Y+-Q{p=~xQkdQQ4@z<@PySbw zugL<)@aUNzyTPpeF1{qRi|bP#5HRV5s~-+_jsmGHs!H20hU=>mUgn8`LzXz7a;xE? z7UoXXjb@oChjIDK9oTLuukrXOWN6Q&ik+_2Q2<<98nL5>95^_73Qq-Am;I9S8*i!u zGCc69`>ymcM`>0g!pV{8W#(ToNbkR$iay)B9ix^`1PO>58OpU0z)3*WtG{(KBbwGl zsmcv|pF`XlV`GRl#BB1I?u0}Jo4B3GboH2mAB(GT?mfkyb5cq!G*HNr-xGTH|1n|n ztLKb%m+z5U$-K06C08bJH^Swb>#^sadY-HMQEc|{?~B~$TXH<-_+?!9u%+Ca1GK25 z?x2a;-FGbx)Ax2AVyb$7v!MCzDTPyz6(!Pby*+0%o&#snwsb$6W^I33G`h>;bqR1p z#45YYBrHBCfAc4yZj;97{oc@ly^+b3Imv-HfhUN6{Fl5-~&I8S0R z+vF9`$xk26XT4gb={zrRvD@tI^?$`b_vY7}jw!u*{mj?v|8H2@0cYmsK3u zqEgsDuUhlUYTcN3#uxU@1Qr=5LgyESiu{a`t$OM5s8(!t@&2FQU!M5Bu`fF(Db^F! z&t}yBY?*7?%Tw;Yt8V3mJoP=ZFSjvrcN+72YnI8Y=P%O>51eWqSRcJPa`&tlM~3Lb zJo2f1^L}fj=9)kKq_*u{RYo@pO$6LD-@mVbNle@pMMnB z?p!td?u<*ISy#K$p{6^VzRU}pBVf*(v(zX&bE3I5({l4Y#kU>p96ThW4jXLUcJEAX zTuRX0w8c94+b)FMH#vM~YeMD<@sKv2vsZOro&+u?dAok+taoo1firwlbAYp7#Y^Xe z^zB$0d+&{uMQp7ja3_4Kv|xw+>}&504sUjwxz#_<{AI@a!0G0vC(XCZ+Eo1MrG%r` zjUPvfL9@=A%O3Y_d7||SG+!tiu6EICk7ciJ^Syc#k;>Qi+@2iSea@~Rie>fXg@TqZ z&MTdNR@psA|7zQ^-E(;tUcT!0SSkO}+4BE~ubj&$ILEQ)`@E?sT)*b7{yG2ot7|6D zwe&!9_Aj4su}#yAo#=AOY-QRk%c^(xeXaIt2~-xxcRswhe4cr5-pte@ck$VOnv9J1 z?KuUUvt)g-Y|ENUBGaeL{&-&KE(l7Jo7fv$oF591fd(-JFK6BrucpcBJ zS+v;2x+$*MRC!0tQO|o-PhM%}uUipXkh$y=_i34|<#)5pY|9HSgsXHe*~92E)8dr0 z!|}Jbqk7<(GRjh@!KG>&?xraI(f54*SREsXLhChIl>eZ5d-gC70QUpAzjKR~@&t;ucLK6T<_H{l0 diff --git a/content/en/blog/_posts/image05.png b/content/en/blog/_posts/image05.png deleted file mode 100644 index 6d80447d094a3d99f2df7d87fba025d981e88e08..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18311 zcmeIaXIN8Rw>GLMs32kk1e7MdskDFyhzKYMh?E#YQF={)&_Wdy6r$2W2t`Cf4?=(t z2w($gp%{7z2q8$<(4?Q0=Y8IH?`NO0f1UIFJ3qLtFxRYOjXB2{_qgYZxo@P$&dSSr z=+GhdyLWCsICO{!e(2EQyT@685|VVg#-T%cMt5&(nFWq4Wd-}1&tz>nm!+?7SZ65> z2F*O+%)f=Wg_duDa_$+J65T9*@`Qg_au;CE2}qlhP(Gm z_J@&W!%tBf$66ht<4Rrkqy-K@g^yZ;FM?ZrW1m|z&N*XQ^`*PKt%S&Jk}|`qU}i_o zaFreO!L>u__aHMr4}Rm?er_@zn}q*X0H-L~t77AS-D%WW) zql{?W%GXfLmbo=`h1RIbW{9^5C4sPw10gHh0vYvAH>e?5o2*E-XMSoN>3XT9buR=q zyOj_AbWdD`XKsu14AR@t1(||B3AU7gmz&3sD3LOOaO7P~U*P^We81%-T3-Fb;O}j< zCixhAx2yc}!u_Yc3lA&2xmP&pTqv9Mfbav3hz+R|8P80nYCQH%i92tXAH{wsjbse^ zytzbKRbaUUPxS8{6gB0oD7l}6dU#-aVE)1EqnlM*oZ9+U4vB4aqhL3iTS=KzXWsuh zU~=y!W$FpNcb2yAbQAVN=kh9^a$qdZ-Tr)Y;hW}mmI4CP>3V;9FDp3i0E(ViZHM|e zH5GHvqlh~Ej^h0{zb=`#zrcg%Cax@L1U1{_?^dr9IW#Lir#t-G@5JIMem31R@+VrM zBTkKt`}lBor>^kb&x2-f-EM$<7v@5Kx?3!E1q;*`suoJlIV;ldo0QDeHis-Vg^X@B z>WT!FijA6=gmp zztE-niJb_A&tNVw<*|TXK|M(CO+C##E6jE4Bz;QXTSfa{lX=F!DT43>od%r-eO!B+ z0wp5`5u8Gwp|y}A>B(ayV;(;@C7*6L$|DIj`C-a+SdE#@m_{c$pG;Pp>DZ6f`DkA$9Hr*$lYjxdj8}!^&Lt` zTdUS9Ep~}5cgwYq%}%7G-Ue&(K>r<dJV0_f^tbS%HZ1xNkvcDwczt|jFlC; zJjo+V0E^2%;dYW@s=f^YQxf^=(F%AiSDFlRJkBR?IC7&DX-e+(2=vKgqRQ$QQu<4X z1?nkm0m7ejoJ0EJ9;{1U3J*0_H=ZCjjb|3l&bb#EMYu5uZ!Mk|gy-gx-@1tvafP}i z|G3m&AL&oHSvz5B>Wxl7fhmfpw=Maz`ntABt-tw&;3mW=vvi-d3nFf zGl_Ex%O{cYb%O62+$oA3Wggb9>{5d8nq12F1e<*O(ui2whIY=+H3Ki*-hRyhQDuAB z&MO+k&+l9xe>aSymb>cSyPVmwk=8e-eEYf{wS^sh88Bymq z6LUYjm@IewRi1}Pw~zF5OJ{^A(iG9NveEvqcwN=AZS)=ILRKeE19&r2%&ea5Km9iD zG<*xD-NBL47RzW`*L3rc4cl~!h2cM|v@pzIE%i7=Y-7N3I zraoGl`Bp)oNRX7%t4E5Jy}tNOC6RLVJEEzS_(OE3oqII{BZaPk3NzWa`XA{a@5?1~ zkT8U&?>pMim2YpI;?T{PiSmei zoBYOsY@3vj-bne9^}@uvhE6{TfyN2=k1p3bsg&4u4> z1zO{ogacb{%|gOY#RUG6^d~h&<3TyLTjm;sM~9Jl;QVFtAdh03nJnwU?gge3py%yZ z3AY#~;@{TdzkUIQ|7Ah`daV7wRs*Pia-V(n$p@4=` z(&6no*V;`jh01OJB(xr8l1|{=H^*AAB$r6Ek2VvEMZrB{?Pjj7`RNbC;~R#nAB9Nc-8#>X$g?nL^skwYGgYnUOmaR<8uXJ&qbGdwjS*6)7}Y*b!2m zaan9Ck>m-PSG3fXodmIQ%P8q+w`XyT-Ae?KUZ1>Z_DC5!s&nb4gO6i9QI*i%CJvrd zo9)`Cr2mkUD*L4lc1-|fQNAbRY*gTa@1K(RHZm5!9>eBDe1K4VuNNFVu|ll9QXn40 z4_LWAnRLc_q=Ll3kf*Jjx8=Z%9ix479M>^zd@Nc=Zz(W`WzAQZQ_cr6=RiD;%YgGa z8)w=rkfvD0Ln7eQ%~o4da8%ByOUuB(s;<`Cf}ko3@h-Z5$Il9*U&tmQ3Cobqlrp+5 zR`8dVdSYw;)3_l~IB)aMM2KwPv|*%R?})x8_ExnCrH0={$=n?OTSNO7K}uqMM#_wD z#&a~{9L7zA5INa)MP^kL+}Jis4R*@t?*RR#-IM@#xMn$Cobhb^0PV~T6-VX>Wm*}! zy~rn~oGJrmW`wm-wQJdqm$s!E&U#h*KLN3PzO`fS=VyFrl)CgsuL+^dGP4&YG25;T zET;W=g7oogU{&kxlq>X_pbnea`3IEq$JsooC2H5c-lg!{?!Uv$U0%8JEWL!&x--R- zzVTJwhgoR0STdpHRsH8Q{D^lFsSzCa8*|ge<=Y=gcEgH9(9&90iq~m>tk{IQf(2a} zlWG;gOsI2q`a{Ps2NH!&i`C7 zmOY}U_qk6d)eyY%yYbf0+KL_4BMD@k)wRQ#JL`56_J?j^Y0-v0oSVnT^zAOiH%Es8 zKCTYF>}Dqlr?PrHaxnJGf*(OWdk9_e7~h8#SS@bl1CPC$s32e#o73~?)S_zd+@Kj43g_w!t<;X|ZP;1MNC1qXhlK|2Q_sGy1t3~YECx**Bdfkl1%8@DDK)&b+rF&FKu|Y_i9qV=C@>y zOT)Pn`Geyr8Zku2TB4|ibf@Otv?e;~n`UERsQJ4K+|@D*eJi^g_(peS?0A3Cw|Q@{ zwco3)ou-}V{u88k>p!2rHTgoD8Iq-5kh$}A=2t`2@6*m8{F{8^8*s6~3HSZ&@&kUmJq>WjRvc z+zMcC@N&m)nfBW5j0|2pqb~}Vy3LsxOGgiebx4j-=N`XOY206L@@u55d^?h9(g01) z;7WZl7rb4kD@{o-^-=h~FhrPns~~SOUHg3j{^rIPq-<7JI-^_PR&;RegX)!RjrG`M zqji_bTn`DTQosF`-BO=CjzC9UD!Wu&VdMJ6pqF8~O?hR#VbITov-i4!i_wsF7bwuR*RY)*ip5{doPWU*i0!SJwxcK78522Meeh3W#&($67^rqoc#p z23Xz)$?{gaRIR+euZYOCGoE!71q`@B8c|Zf?I_ItE|=tt&U)EyE$%Sjz&Q7-j8!{r zzctQ=q^=baYH`O#Zd$QR_Ule9BgD#)W zv9>`!^wW}gqRQTAiZrmUP@^f@k~syUvGV@8fP5^lW^2}GvF4Z0;2$Q<7{2=CDKF>7 z&%5XHxTfaAo&9Xy)Yu24+fCu?bi=L#?z4&{Y<`ezP}tq~I|oXUKAJDa$Odv0S65)} zI77-Be5OS`G(!enjOY(8{)FOv6()x~S{~I43zg3MOiJ!q(f8BRE>?oKmt?&0nPbgB z1z22UsLp~pEL^Hnz$b<%I_T%4`JkNF7POTZb3|6){>uG<>)%V(aFpyfZn8D#60-mz z!(*9X%4PC5&JpW?(z|P2#BNRu9*YWSh~Qsfxf$1gusIca3HuY-yI`V#K)*l*WHAAI zdRPL!+)w81U-&}d6av3X6ew6(@O=$kXj8CnPlQfvmxpzsi>MLZDFZ~;=ucTQMqMfR z-QEyeQ?>BF@MLzim3PzVW&sD&%YpcM% zORc$CtH6|zIB=|9`VOV{WkzfxM%erU(X?ZknQhrc45?TiULf^wkKZsLc#!52z2d`G z0w@v5A4!&}kcMK97hxLQudgfJ%?`7{`L!8J5E_+&1%=kI`FE>t*2^MIr|g0oqqR0Jmu*4$UKs>vNCV8+*d33a7?V+PV$Ayv6lCi_-0kj*c@MU>nBeXC)g84J> zes0u@R(u~+%;?$VJa4GuZXIEC%=IXhD|g_FD0e<@vc|nN+~@Ig;wkU*>l~onWlsRD zf|||9ZfTEm3caGG@NM2Y5;{~rdaAZSH-*zt*H8pWLbE&wDuwNr9+V%rMPE0+?3DxB zI$vV+NeWJZ90!yG^`P9xxhoMrV`3;R-JUjDVz$70x(F;fdt3F+G9oC`>C3oHzSaY% zx=w-Cdhq&c#`~UbclOLij9)-0pdjC0^G0um5Mxc;506O)nf@S62HQvG$9 zk$k~Q(U0T!3W?hpCS-e^V(CD$k3o%QY2XFq8gIGpk4PJBDEBP@7=gy$`S>6vp)vKw zXnx(`NxT!wZF;|Zd*o50^at8KrCL6r?&`16s-aC~o_@*N>dx{R<77njY$S82a+P!R zDPSU>f0`!}sGjgK!~7IBJXjPyn8%|squX%Do%U>Wwl8h%;L2c0V($|JZu;Hh{!!&G?B*XOE$GAlXz&-N z^#5G)gt*mQlV%HA?QXd0!5ehWzNzfyj6T72TM z58bD8ad8qc3ee5xBC!c($8g(3fDZHo=WmqI1H>`YFlYnaik(UYG$tPu(@5?4s{B$~rbhoc5BND$WcGjYFr_(*9%Xw)p zt*!@Kt<}=D3^5-Hhc1Rr14s!1iKsNY_Rc`*oSwfxXfyRB|4+piUHhjk1`4*!pIr~=h^MO2pjMBqLNHs^UoO>c-sTtIf z?zy^9dogtLZCe}Z8XB~yL$Q;B8>Oe(sX zR5?$njE0S1E}vhD|bW zeE~hrQ&Or+@kqqC%#pfog)n(ttGgvW{%vM$cI8djUgfQvL|sZq`-ABt4wB9Nu#5(y z0Gwxr+83Fy5LG&tJyLCh2AMs1W{T@U3r$46#)X;g^ptpM2=N(sYeI+kecKHdSRw!Y@ce%0%((62K~A*r!#anCEy*y8_t!}5@=!CAtBclB7cNehTc54}n zCN|%v(A3x7xMr^k5-Nr4m8x5zZ|F7qQJ1<>1&@AII9G#sg=^4`@80?Zrzk=85b^jI z?W5b(qvs$Qx594h=**cJ^>jm3gPy5Yt%%UME+cm)*WmXoER+eAqV{=G+N6gq9Z;lB zJ_4J0+Ug}&XtcK#?@H5^s$sy3K|y#;v{D~*9+$BpNc@`Z@4UHZEPfe&4T#+8WW#8Z zK8~g>Vuo6j&m+%^cT;~H6>QFiCE~AU@2nMw8&cj1Q)TVwGzVAeRr<6w-?@)W^EEx= z_Y|Q|i8{6vUuJxlzE?t|BaifjiP5K^8e8>*}l(gD8C7unas~}uDXZVp*ca{Oxqx0b%aDa0aaV}N(=;Q-m zgM_O&;iK70P9uHktpTa8wH@celss*yP3z#(Q@<5c?SDxFIN z*f`!s9A}@Pda-|$FQz3<NYBm(ZRUd5hPn|;&-8lQUP+x)wA!mqfc7`GJDgydQ|RaT#z-LcvgEa^Va*RWfwtT0~u+U ze8~1}MED(Hcvk0wyIMh;`vwlQmSvDib-4_oy8LC)wOZsEH;rZfs5WHpP2H^|ATWvf zf~H?9((MOZBT|xv3uWK?=YtjUTA{u(vd)u_R_6q|@~v<7J*^s#+b{cMWMi@%l`V1? zvHHLi)Se(4x-vSkHxTu*$XL?e5`|fj1TEXUeT(xJJ1;lPCG|Vl7b?9jy`KHr1LR=d zq8q16nX0Ugc;7a9dQFM>OV*h?Tn*vdv*C58lc~D7IR~F!O*Hkq92-_s*VX#1QO=w2 znc8V{X`=r}x+0V*m!sruxrz?udV2n+xdz8L-8BlEaG!*TU%N>#UAXUp@2VSsbZdr~ zvOlY0j-VHk6)vTK@V_D6IyN?^7%GU+I7V*3JKcJxD51YU@;RR^lSet=-KV|nE^X+Q zwG(YnpO)lvF&R4z)Von;4R;#avl7Uw(hDYN7cco#wc~)sRnjY~8q=58e4uO>`yY-$ zo`xNPH3}kZkO;l2Foo~;GWSbZQs!BksBeoT{noPKFa%rH6;1i5E$w*x@m6S^yROps zzzZvaiD^%`Qf1n+qPh2ppqWpe5{aNtTOce->+sQ=Az@mkLI#B32`Qji#8r~gyNe=M zq<_(jC5|QDAoUQOeTj4o1caXrS#r0nkFS7-EsS*7^u3Qr41?2M2k$s~{JBq+m8h)5 zecyI+lNwTs^%^`OlB4WrJY1U>XGW)WwM!laVDS$wB6;bcJ*c5UoGDp96;zhDP)LmO zemYU(M9CyGx1!hSsVPa6jNUtxVDt3)0nx3hBA?_Nc}poEC6=r(zZt`cb3vsn+S^&M zJpHi7u2!^?!qD|o+ojr+ccCMSBqiZ9K56Fz6;eY+%#P$!@17_px>|H)`K5j!@FXot z+ILB|l*`kwm(Asre$M}_dr%^;>~#n@7MR%_=d2ydOkO&*5!qy_H(G*6%oMm28_eh! zo@1bs6sPx2`C@rJk`H;;e{R;s4GymFybJvFxh;!P|5ow}4<_BskFpT1#jm^9hW|izr@pU8AVR(_K?aomFP5m9xxYBX`Y?mJD zlX~BeS*PQBM9VL&x1++cyonQ2Ww95am6o zM1sO0-VavTWwp!jAPqj4Ft$5+qyjLq&t}RzT8|@^@pBVAS_+W53tBv;D_)|V^)$nC ztQnsyLZ#hdEh47vnWITS{?4{3{@P|ha1J=#cPA^LMa`0j4@G1qbVE;X-cAa1_4cDD zf5;xJ*L)RbWJ?YoPl&<1O+uNzYTL`+AGi21zqz2QhC607v>99fVZ5#86~=o+hAMk@ zhiHE6fkOLJ5hc09_RsOlhQLuX}gzR;5(b3{;99yacBBfTtdR2H1qIWInYU z(kUlgQa?pcS3dfY=QNxMtp6z0({kFkGTDygu2-XeXI64n$M^=2+f^3Z#y0@i} zc%W5Bfexy>n-=eJoQp)6T<}nlo0&o$Cz4OMYYE|3Iej)ar)1 zrOL9Ec^py4hSIJt-8p@K#nbLccx)@8os)k2?Wd%D+`iK}r2tC0M6GYTbG!Dq*F!1X!21%+~QT(@Ka*Gw|~8V z;&5o7IH6L~{!4UJYRStqJQ4xDmF4eveFW$3Z|7TNB1Y#@N1``>Mv0i1Y9@#kzk`Y8Bn+=SlQ8I#-J7Y@Yv0F@!-}dwjxa4(?+ky(_QNk6Xu7Jt7-;T zwUasl(Z_w%-{PQQHbxQ#9P#+I2as#+7gc`qGQ3g352sF6B%cORs*fXW7{~P{%0NKq zD1TKz8Vtwz6)_i^TUzRY*r0Of_G@i@D1O+6-p~GsQw$D!q;!t+bwl`{&t<||lq zedkK9WMr7wYlj{vy<3`}15kHaH8ssl8PBn-*6()!=e8S?kuzYb#_Pd(O7iOSjjSr& zvE-73YigbuS?Vfl)0PkKe;&B;*s)Y(_vNL~>6bjFYS%)LWYmWrM+&@yaIP5$0ij<# zD()Gs8ETi+eQGyFoi~47HBO5iAB}rczk(FBpkpvR1I+@jUU7(2Dn8v6$wiO4i>X zYWxlfgw)9<UubHK+xPwQmtIr>L->-5iC@me638Fb4{iI-B3Qb_d z%LS(S3zxjf$Ku&n>gQMb3c`VRxup3!p{R(T50)Bz3WQ3P715?>b-dkgTr%wcGwEvz z@_*+ys3L znfr#Vkh*VXaYe!g+3Xqy(VVKmI({<&=ef8uhB-9s@@pl*m!-b>ssgDG^lAnj5QaPh z;LwkKoV^cf;xsvWBunAPuh}R>3KXGVT2weGgCJ6o6~~Y9?a?X*u-S&jizx;NZUgc z!oRiV<;=;89+drG_>ER{?Rw3LF7q7aOA@m8%4HQq;Td ztrnR~nSN)|0eE_bUmDoErbcL(=i0VE(LNLJ$Lyx{_C^4Azwr-hE@1Vhw3F%;2T2pQ zR~(9%()4@Qc?tkZf-};?9rbjYn^ZQ{%%@PP)byy;G)sk5vIVxTPUi)Q?oS(R??2?; z#V^wBZ-l;1Ufa1FPQFo|{?bp;J|@gubE}|huiCTo&48t{`k%rp1iCqevOkIMB3MBO zw2+W`DIwt`=Qs2pcop!SRIpjb{Cvhs=bpDW$PK|9t63{ygI2*yAd(?yZ%9=m<+85o zHv=+JxiRN@eu0wesIBH1DDqslXP{D_^^dj(JB7u(DW`E(t6yN_rcQSak)xQs0amCF z*4NT|bpKpM3aG-ZA(WsW#!&^QQMX+kN*j1eKB(F6*r?MMul<49O+LyZ#*D+W{V~j* ze7!e!T{4wBIz1tFyT#Vf8NFuOLqQxSq?4AB*UOe%g=!K=yFD0id==;YPF)xQ1#bab$QV%T6QiB3}n&bXFz z3=g7|;Hvsn9BhA%ka2zdz-m|qNWOh8R>=HKQ65*Z0FSfV89` z3|4_)mPn5@nw`&Hqe)58fo&f!Hm7I@sdcCY(y$1PJK!ebplWr(^7hL_A|Z2ox0^P{U6XMQpVOWE znlC+4nkFT*Ujl04x`TZu0_UD}VB5i9P$v4 zrL#Oz`v;BgAGAL2|3T{mNErYs_`gv-{%;h)V@Li6|Kq>&Hm0BaAEO9{RW6hT7<$in zA^vpE{@z!M%f7ptHADMVf6%q_`d_R>sBd#wa5u%NKa!b-v1zcE_BxBFjMt%Oxo)@x zt#6%Sdj6k!OXFE;qysfX3z6VtD8t;viQn{J^xE$N4K%#?_%|Ps%hMYXgZ_wYCNTrB zBOsurf$@_({5u@v2E8z3Sm>F&Fe*vj8-@N@EeZobMlQ21{pgb&FS0GZT7K_L25}w4%XdiD=!-*l{RT zksV0Zc%zCx`k9T%tD^uQHYfwXFP~o^rKXkMw?~bmg(*(IVQ$w4^Pl*7#fBC zU1wO%D=Y>o=5z*`SCC&);8^*w*@$x8l4>Hm@tR8&koT}0tcEpnVk*@Ya+~tgb#Bo$ zUCOH>5#Pmy?2B!{>AVJnU4uefZU%SOv$+#r133yHlyfykwvt6fRaS zXA8qNZu7eD8|7wPh=&54j0sP!@(~B4(0ylWSysA~Ovz6lj9gl^``KzApN`a<&%{(w z<6*y!C@iM-M)Xbe+~%qaJ=lqef!>Fllm?rm;NSY_7_Wkyq1l#pjo$tJ5xZH%F*y#i z>7!!zuEohfI13e7gy9TgClAQ6ZmaF1l|t@5KW@1hO!}d{l5VWAo5Wz>_-8QO=j`#I zjDUdm-s6Vd6bz8Tw+_m(o|#;pj_JLNUn{GJ)OW4T^@G-$R+$9$dqe2GIh_>sZ=^45 zsc#~lvSvoF4(uGn5JRJQS3(h7*99*6CGmDh!+Ap=4AvhGW5 zQ*oIxVetba!IysrY@G0Fk!_U;qnl$l7LlW!&|hYc2(gJo>(>68Gj7EYTv(cFW^4bq zm}$&(+H$u>P=>9lWln3aA+Xx^zE67vo#%{88Du3x6W1t6O_*NQ%u*+J__a-qKQC5= zDt&2``XgTOs>_b6%c?{eFhW6+dVuC5pJUwUG=G6t*+kcE zulr#!Nc!(tVJP)6ckq23e$EKBfQ?OBc62%|)dXKKlWcsRy0o0W+WBTwJbg24OW?3_ z@~>vuaVBR}ZB#-t8-X_~+e$}^(lzU5_lWSWfVrRi`HTv9N=mQq&0SMxm2;4Uc#v$( zqNd>B;;5etU5DSI;+Qqn(vA`c<}6 z25VE%mM-f8n4XsamNegpLFR^L_mZciz3fana2S_6gcSnanxZ%@=56pTg0-aesochg z*cTf6QdEMa+~CUE8e}M$O{ArRi^D)fa{1GCl&7nN3gq5esB6a^hN&?ZfEW zEg7#G6!Z;KK{`{Dk7EguIZHaoa!6e^B!b+0GHj+DjI3YHPIqcnxYAtHO156CIgd;} z3j48}kNF+JT#<_3KZ3|wKlcJ1&cw~>%_fav(6Pv9?yr^<-)M_l+?x#{g8XFYq!L`OwQpyR6e|a-_0g` z7F>~xFF4Z4sZOfN=E8oQ;ZEL4l%AlLZ4Y=mZEjrVF`c1txgEZt zHBZ=Uc=%~Okrd%c#|$sF$M(KJ8#8hHd9z(c)kzeV5GoBRr*i)4WR%fK0V=XC17J{j zuG0oP_g*0su&wo$izbqEiQ`rbZ>o5`yhWc9wz0;GD|FBu5YfBu3~ifn)8R(Uposmi>&c73>koCgkj^M|Hyv30N6*o!iF>XtK;ZJK zDC^C}^w?z~*bZ1zfGyY6U~)zuwer`ssSys@-Z_ zK_LXWy5O@=$c5BHsTb?Oa35Xw+JF5p2j;L3dCEix*~4Uqc9wJSzs&y#gb@ms2CjU+tdO?&=!Y)ZIPuliA^qvoeyk<_MwOk&HU@ac*x{h?nQv5)gPv{q!oRi6^s%BjvCQGfPwiQAVneDPP*&vt=c1>v-@abHtLk7ozP20-H^t=|ZW37DZfT_Gj=bE*Pb5^J@7>fri zwl}~eNdyG}XOTVim0dw=0!WR@9-$391IpNuc|(#uj09EmMwWb)vOV_P1rM5zE&#*^y+yh1dW~KPA5%9-5_?Xg=Q_;j;@=w=R zzGkBYAO({KIx2?)7LZraOO6M{W(n&tGEc{?XfOEay2g10Lh$oyI_5sn z0MB%CiXq_t5=^QPAcW@sA-|_o*(7E;x&K-*zpyyo)KBQ|)|X2*<&X6lZxyBEA2kj)EC>RGoa(k3&)}@Q+L;?JEO=cKlbbOb`cl9VU(M{mnSoM)@cae#qh8I_3`JH;)5-!*lC(Z(t!KOw8(@p5$!ci`Q= zsMgp~8itJG(VRJzl=NrvjXIE4+uA4xTy;{>k$!CpBa8O!-<4a{>@-K!M0)HNO$m(tmReo4^QKVp z3WK%KG4DLkRLhC%xQ}GmW~Y$`g?nuu>~LYT8Pc;7PnqS+jq5$I128?xJ7Ivsk)tqk zS@#Wq<7Ltki<*~ZeMmJaV0MJA?|hLmmfq7*ScF5yhb^lQnbGhf@S*Qm>HCYhUpIgKf&7}Py2{akmU}-?_1!4 z6FvdNQKqM<2sv`>jps3&uAg?BT*w7IxGK(*GW{L$x-qyXJ%d7g%t22*Odr@DI3^~o zR${Qe+7i=%3D|_u8ZN2Xr4vAdrQRC$$<um4*Bl)+IsAqAC31~ zwyk9k{IHstg{A=fY6i^#UqczYC+QDQa9uQ##YmWlFY1j7tB3tpVo;TCx z#h0gO1s`2Eugtbw;O-Me6fNV4hm^cgAPhC= zU^$|R@5ZG0)3B^#0C773f1w?nBz+=?dtTi1E6F&59Q3(8;c^20P5bCPC%R4smOetR zi%cqk7RQ6o)H3hwBVD(4H?Z|Z5;89wb4%`umDjswDwU0Y%g1Hz?_MI8+aB~3yidR@ zQr%iQ<@${=G3XqoLB)V*$nIBME3}ikG=z2rP{WAE7wC?2-3~^^3sOc{e%sf?VL|w@ zjcl(0_9}e(T3fsyh_@*Dq_loXEul2r@l{cM=vfA_K<(GaXjeb{1yqHpfQBsz@;l5$ z=Mq3lOP6`oRR3DQZBC8`-^+oOiZ0#9R^l?*KE&hWXUaYlwnA>^%uUf`H?+2fm#)i& znSl0B;Q&94Dl0~(S(%KrZMXS)ua^A&$&>)#)MZ8O|>-m1$3Lwc;+W7!sS5iFXqNm za+B)8){9z*kXK5c?i=m$112Qf!l7jF%d6ignlEda?=LRpR?Ic_I8DTZ1Q1H^reX=V z#vf^owI$ufr5o&wGSMJ%s&M>m2 zx{$E=Q99<=#$qI+r1mY(5GwHbMw@giM=dFR9Cx3tp+Ra;zdMDR=oU??u-R zi+W9q8_Cz>1+*x_Qt(%sO4iE28h$~neo=4W0)HzojF9Jy0$(P&^D@kTT~Qn?WZj$9 zOHWNwJMN9UKUuRh?1VAQiXrNlQG91lekEaLuH$P;N^df{`-}RZJ0`Q>YJ`>26@+VD z+4pb{KK)hJDne~@C!B4jPm)g4#&+$3pVu4%9R3bFo5%cgE)!%&8~@8%$OzdNU_-FQ zldddzE5FqlTRdO;Z|=iZe2HN>rjv7uD~!v1UYBv4X;Xy7042)2>gRXvG{_*>k~DN1 z%b-Ir*bf7u^>UYC8^^~M39FqW=FkNEi#90v2x)1AbkUnl1BJ@=yk?Mw1!gSr$bZrdRVA$~zEKfhSEZ{VL5?2`f z&<2)vye5@M#u75E{^mYZ3cWx#GvQZ$X^^iXSs0C_C8f2-WJvFp{jW8K z3IEj{a8aTwY+r85$<-J@=)3`~A1e!HJp)29FAvS-9NQO)Ap%QJWco__E;HX=JD&j% zlkDiJ3SJ+!);MMv%g#tD>CFR33V4N^V=`Q|U!R-*O_NDgP~r2&si3-)=V%aLEYF6Q3bYS&GPUU5@_kmjtM7E7Yo)fpB$xn7XyP@buUj6{@Tm zlB~lP%+Uxa+z^>%79i!k;A0Zc5D025wP^$xFHeZgkg=eRMJF!xQ9IZ?`5b_G-Ach@ zg3*TMeDL=LkT-cS%|BX`250Offz!2$d3h|n3GP$5%HR{~QG^BI>UEp)JK45;2Ht8t zb%RB1q;k&}`5((3_SosiMhn4HJ4Qc^e>2Pxke_b3DbWJF3LV=?3#|2=l{IIJq9T33 zo00O`UaM;h3Ho_QrkNxPV+wGar92DVR0g)}O^Qq^NN-HVJoaNVgfb-sa})bAT?cp? zA33?W&_GU82!6(Lm5V#V-C%4=%u6el4;aDG%$V)|q@57ln+u=a&}bPzu6JH!hE2a_ z|0}tuJvnk2=6$C2Z8JL{aCvM2vxt{P>nIT9SVIHIWogL0Ep9sqmAkAy=5dAQ4o(kn zK7%_SiFxiD?n|jJ6(T@6G!A0v&vT>>r4r z``zbh;HWEbX6Aei#aE3qoc42jpnIiP7dUs+cKp-%8|u}fiEGOfAj(}_iLNGx9NJ6<72m9GRQth}R!)8l44~yLcFb2GxF&8LH zu8c=X*~}YY5f01cP$~>5mAEg%;Sy2|)DM;MF-j8F-rqVOn59fQApr!qU&~})H+r|c zEXumjNSxETI1(T~joQ=K;ZFm4!;^xQuh&0+j;u3dTmc(XFH`+@Ca{Lh3fwF!#l(#P zl5cK5|GG!RPLFH{f-!G{=522OH!bNL&#%CZ%PAfJfAFC$cW^N3#(ZOCaq>{aiC>g{CI+Dcv8B2W2Vpb zFPM_Q+Ag5kbKPaBQ7JRMeSV8#HkNGBJ9_Lh`UkwSx zg%`;)R>NZOF86j04TgXXJs6p`KXZz~)TQd2CV2aE3rDkxfa=z~2a9L%%P~!ejmtPg zBO@#Xxc9>OohIK1tAx~Vtv_ssb|iJD50hkb1*Sh3=VG8^9%q0tmO`+YH%d_3xO`&u zjSBf>Ac6&=#oJvUKhc&r!ky!YF;I8$3{Uii^X-Zqb7~FuPA-X$WDd4<+}_wrK#Z1r z;OenmcB{q093SC`S&R$wz(u*TH!~*y`Uljwzc-oF&fa~2)Yk%OS;?q| zP{wpkLl*)O2QLX-MP;?DnX*%v++y=rKxHLk2uUy zcQme6YE62a!~34dG@Pn&+v>mocN}qn{g^6hyFU^q1YWBOTvr@VzrER$^}VNWW=~H$ z0nax2;$rZ`@G&FQBR}zIiE(}DQelp|Ax}<_EZu_`kh+?C9XWEOE4pW{m|WP34qVtf zP^_L7R|6l_SbKyjf13BoxL3;Bq5-VZhfVFiwji;-=SOU(OxLzv{NeH&NS8#QC{%W2N>mN=ku*CY&@lx3Mln zN7((~V*3Qr1lw{ftwf@2REH4s30+cra~_q=m6eUTH^ZH(+DGv&fb7vJauW$(^^pe0 z7X10;8@6{_L$)zZc_{$_%ZlUY@}6458{A0~!}zp=1(?yI`lGt_Mc{uPF_UD_Lgmor zH(JscJ+;hge)S+749j}J#@smm3uL!>Ig}^8TB4l+n>eW@q~U$f*LyMmRK2lyU~9{{ zO$|UII?VnJ8p+1n;j3Hs_x?`(TpN-$2xN^F@;@^bIaG0OT3I zpEn+~5fjSp)IJgOFOUl=(H!P9TT&rGm*!-^SimPoey_NCp8-%d`hZh~UMMOOX#1}R z=>U)}0t3*pr9``l&CeqbW6}5_TWfp*0Er!n$G_RE8a%5m`x<~DwJGwxvR!W-cddaQon}-tjB<&!Il&0w35pbXUjdHulz|$o~tLmn*9P diff --git a/content/en/blog/_posts/image06.png b/content/en/blog/_posts/image06.png deleted file mode 100644 index d40b2eb0b6838ef8701a4dcfe2e14fccb4ad98ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19689 zcmeFY_g7O*+xN|>VgV^4MXDgZs`QRXmtG?zfQo?B2%&|hSWtQm2%#f25~PL_Kt(B` z_Z}1ydeKlq=iQuFx$o;*>v{ixXFWe;tz>6r_Uzd+a~$(MK1YO!kq!+N8x;iw1&yAr zrWpmrIVc6iUwW4>0#|xOajFy)2A}mb)hz-h*E0|Otwx&4$v4*JvmUoMR~I=?Iov+1 zF7lh2;qq|2uB+RDxS;!cRJk|@WnP}mk)7VY!;l3CtanGQRSKpk zslyU{;Z8EET1DwRG;wdX7<;#cQ1{KG=(+noduO+$o!5j`tmswWN>UWYP>iKh^$^f)xKhw*l_`zuWnb z{``#W25;x=k9Bj{`*AK`K002aE~_S4sj?HqYdL}zXRkZCx~^yDHpdOTbujE&unjiq zV6Snd&p6$?y|BTL>Q+1bd2ws@QniBK$=22!J6`cBwGnq?<4n5WX`X*hU z^yzQJsgs@3;p)BFt|(}R@2i|Kyw?(6HXVssUr{0?(7Mnfp!xTY+v&@&g-g4tv@OS? zLpy&IwAz~Km~J%1?M&_dvF3rgn4(U0H&0aiZTQub?mU>b!C(ml12z8V`}~f?y;k|awQB( zC$IehiGfL8#p>%o$Vr#wY0VhLUnFK0z6WEW^WfOzi58dPd%PF5<^>fe2G(oGi;GvG zvq#ypUV(eKZ=VhtgM7S`nS6bGZW0GKePXh)CHGvu|7?E8HD9HUe-h;y<%-Zo%<$sb zBC2X0>{r%TSes-`*WjEq`iicu_&HW$YzhKp>kCK8vc#yqOK@t&h(xL|GPM@WXPWn- zD?vnHrD^=0nBB1^=A|&BtO?bt#--v%%Q8oH_A2~uJDX#(-kp1m{lU32X!&#+=FR7Y zix2vKuim_OZBvBL@9CqF>&VQ-&qc8o$?li*;HDL3wG5reNQAge;3p~$7%UzmZ>(<; z^1C<2s;F(I)}zeK$(`BV@Xx4-W1lU`=OWHFZdwRAEI<&KECQBj+lLqRzY4noX+^Gz z+}mhz8QjXGPTk*KV-o5?C0t#Ro$2c}jwdWbsSsq9C55-)WpA@|e$8#EsRR?(3#ADgGZ1Ap?4!?B`j_PrqLjH=--9;KWwNKb9?JGx~ zW6R7jS@oxd*Y55-52Svhz2#Hd9^$H-@3m495EV`p7ibu&BGzO36LudzS9ec+eQdKj z%4lAQeZ$m~XtL#w=7WmR26~o9@AvfS@j=~2pXUk^H~i)xm3p7_(}_2V)>b5YnA7h@gR2 zlBoUSL_Hd(rJ6m11k4xR>gC+eD%akspISML2%9&G^m?9KNfC|7o5*#GQ5-jwq14dm zvAZ?|TUIy<|0*MygJ(oW-hz7NVR|&zbt{?0b#s!bV?Ndk-dp{%F_a_gygO@ilrxt% zq#5{0`y4}N!%lNE2^C!V=r4G_)&7`>%4oS2TxZPpg?RjD1&i=brZbE1FXIsY?{C0G zo_`yRfBgM_EC~Gl9|q%p7yR#b{-YoT>HppE-y;qT#hY7DXSJg$xYJaq`E0+th)AiS zkzX#WTJyJk&M#)FxwxlKqb27bi1uVCA-jkAHl>`^4x~s6t^0AOPh>Y0-B%W8oCU1# zPL;3@#_cTBpX|j_@f^qxQ+(h;X*xvxesam&oVOHeSE9=E*lO#b?{+~Pwq{e=T_72h zhMj&gG+g(6Z5B425u861{2PDsF9vV)yhc^fA*ygcizpx^H8$Lx?ndi>L&x_CBX@S2TlH!Uaje0Gcn~(5& z@Y|}WaMDdu^6`V{aF8v>s5Zf$aj&>|gYAS0+N9pA_L1cxrKoAj4g{TkKkMCb{e$$v!C$?Pw zAOgF*of4k0)3#5+!?ZRX*#Fco(cekJ6wOZaygxa9I$< zn%}|nT>_++imBh)gc&5!U43==jW$)HZ>eA@sql;FUi1@w?1 z_W|`0H#J#=Nh=wt2{xf$D&guG(Q}43!bui_d%<{mM$x1XsA3#@{i`pqokt+_M$jD_ zQlVXRO1Ens*G+~C5errV4ot3$M31oqT60q`(QK&;*(mJX$4~cT^04cw-jfyvd?0$| z`h9-APRl_;*#`Qp)4ciA{&Vi+5$|;8v{;g?({#_mQm#~RFTD}kX{@LK%M>Rr2m*fC zDSH+Q@PPCZ_Pao@UqR%`>kzp@6J8WjS2SU5BEvLyvq*ko!Cz3WBJxSa-hB-;!4`p6|H|24;vXq zpga!0rD9f=eHZ$>1out2rK=r|G7bu>6Xn*odmqSORs#9@OBjg4k!FTy?sN4GZjbj! zg;xTCx#5N~B`DrPRxTlMN?PFs5UOoHKB(alWKQeM-?2|iZKP>ck5}NL@et2>Fp~?oz{#a+r|HUCW zq&Z!yJ%V~cX|}l+-gkBzSsECNV*Q{obZT_=5-9^WcW)9CV%Wr@KiV;1)Rv6IbD@q9 z(P?*nlA{50pgtl39X&8dT{K(d^TZ`^wba22T;NZWN>G{3&A;E?-gQATdzj{*W`~XH z0p2}~EuJl>`C~aBl*%xVtI;ZToVXlHDx5-Fh`Db*Xv*ERKW$ZwX|6%L`JX#(y?JyQ zI4#`bh2mZ|chp3^fRYMtqVF&bqdWN6&b;h=8ji5hT}@#qih zH`%rX*%IH!1KuZ3>DLm|(Dih;Ozl-JE~~it6;uo1C}ccNN4_a=N=Qab?AnmWL1UmO zJ}as5#&XD7$S>a_=e%!T1!Q}B%oq6W^PFkG5ej3`fmG2dsf8M62WmYi3LO4pBAZ-R ziYu8RTY7F&mUq9;Fr`$P#Mo)X4TNlSLr;oN&&TVX4)Xlt#Pkm=AZtruvh*Nz z(v~(oTnUySGzcn)D8s9;T(1T^sfx)@7CTrk8XlDrz41ejOFj%Lv7IJGvpt zU<`85GEyuFen};fCm)!7vc>m`D@p~Qu%p0uiyQ6<6~zI?v+*7g0cVq7?GORzhTl3` zr_@r)bwz?*zJL3#5n9JGwVDUyol(t}`f*uMCxhI5fRmrS*L-v*^r&GEJ6i476zq44D95t7IUCW&kcQ$JFQy zuLCa8WA>or5H3aB*n_zABDHU<31O5#_-}D%)jnAsl&Di4HbLFuH|9CRYHPZCi*_5! zFg8U06^s3$%a8I#7D}xHKlLCs&dLi@zu~;rA79-(Ud9a%k`fU)K(dv5`^jljOU>*K zZJR-kZHCfV^_}^)(;P9scVsKrfW?+40cGQ_t#s7V^_WyDGJlaoSUzDZgR$`VAC4}C zz3Ep?a>JU})@#r24i{JEbr-*{Tf~;| z7PPO-^&MbBUEfTW(~pn&WiQNkI>yD9yeKCnHM}^Q&5YhO>jjh z3F5*UdzZb~XfQPoGCyj{XcejNU>jO43j4h12^*S~-i%cMHWHla*WX!jActWHMPr^0 zjeSy)<(=Un)7O4|?E}Yrum(d)o$+sZ^e&o)aZI720mCnM0ztz#b(YZNr}ig9bUsA*>|a%e`Cb0+b`^f8?&Isf|nE&c4>`Hv!i z9`xUp=>C>}&MN&~0ch#Jn)vq>3b;OVqd$ji7gs>XFg~Un8Up?7Z+}eQ#&rI*lvl}kIkzABt4p^T(ya zEy-_Tn7=AufGX*(_nVTkY?n!Y=p7wJ+H`WLKR)<(5w%8&REw%3bGf@LYkv44S8DHU zmdrb&bt!iLPjXT#g=wEM#Hs@RcrOOrf@+xKdovoAs>7nYuT~P~2#xIq^LZBq^>_UH zmSyxuqzXTLS~GPT!RZ(Rj&6b+q3n`ioNsy?C3R->eht6mXk4^Pwd^3P20k#?as8s; zg|a@h6wB#mqejD@Mwf~0;_I(YI9O)N*Y~ZGlCm9A#I4_AY3{<^;A<{P30f)Gv}X?= zLL=LlK8Od&k@vkslCxL}A{d#$(4t@}V&k#cx-mpOG>Fj0)3_OkD zT4sD7rut5sTKa*Bxs=o(?}ZMq>%{5Y-6R>;Wb2`ZNcVzU4nxqpGUm=&5)K{6cd^%5 z4BFl&IFS=LFMQEzKZGjgdvq#F2;$*0PaU1pGQo7Wpe`a%qZI8}ZhbE4gW?1V-|)mg z!HOeEQi^hwfHGMSJL-!xG}SX5oMd3tkFQwwux0IvvS^E!V>x)l=c&kIL=m{F%IDds zDiMg!m>x*DCQlIWv|eG5tMq}J4lMar>hNIJgGL!+g#C=Nd7v5^cs@@X>Gt=2Cagid zj7$hk0idI9y)U~17e&EWUM@;I8>#!@V!NErrMm4l|5hZW2#^+@maIzg>!-%W7vvTD zu$1jGn}@>qNwOCR9%m-Vvg1AYGOK90CO(f38ncDWjM%%7BWRPGv@-9#k&o&5{FpVD zPLCI=hclaRdfMq+gVZiJw{=IRNZfI|8A=7$cYrJHYZt-7FMCl7JLoq^);g(>m?_~( z-ao5_*}$3<%1gr3$ea9rr`+f6{-k&^Y+Tms&ALr33jAA$bsFS7_Nix&eSu7$?_Mwhsue-N0XZ|5or zk0;U(Gnq!!o3fbSTNu_e!SyxZV82t>AuK`Fo9S(|r6n))F>Wfw+0MK60uM!2!sjMr z<~rxA;2yFny#49bzWewmB2e%o*~ig^cK^>BOM$t9dW+{i;mX+=q65sf0$JpI^y50|f8`S~sCiHX; ze)l4@E6EAGyfPfoKVl$l1IT*z`07_+Xa0vD=_{g%F|^_i8a8;g#o1NJd^A0z`I~h2 zyrhFu*+k2tJc*eX?gr`COi^C)e%^2beOTTyH{j^B&T9vAaxEssumCwELkh zs_|Vlb~*9Zg`eI&dI@*zUy#F83LkIWCx~m4QV>a6e##d3z$&Ro&K2UCKr-+k+|a}+n3Y411|cvB{fOmSD9 zo_w(Sru*J?)!9$3`dH9TnudC{ao}01c<72=9Ta9i4G!N7jTX>9i0o^2&Kz?K@v}WO z28Aon_INLi^O)OX03j|6OX=mJP~%$@|#ta%&9vo2K(b79@iy+?2W@;`B)|d#(h|c@@T=Qe+(J1 zn97jivz_o7`V37lEw&2ob?}rR#bWfOzvez`;auH^Q^6uP^Kj>R%RLbZNYK zjP5l8wm5&o#E%nLixC5F@7$^8lODH5sY1nqXcvie&%+h9(3-=$fbEh$PZi->u##aX z=o!V-ASDZzB&A4z*>%&gl14KkV`}C{M=XB%E>$Gi z*YZ81fb;gsaC2zfM=gH(gG#EvAHpQrFJdu9>-<4{rg+M0B8-WO#NylVYa#{>0W}CY za%#D~jxM~0H~64QEh*pQSJd}NY=#&?yp6OFOjj5P?U7(LFX}kPSc;2gT$5Gme{|YR zUP5mu^1xxv2G+^N6I0W#fj%clidC$ZW3?P}zIHy`6(WcWJ$RsRcJO%e7g6&%hvmby z;;)~t$@z9=*wfjo+yF-pYYZ4O3AHeVZse2*?U2)Le#rF0`YWPQ>tm_k9 zx3av5=H8C-ai$tu+BJ91OCe)C&_Ajk>2tETu95tV+}oy$3`b18=^g|S4fDPIhONIj zNeN7tw`G_|XFI!h;d_jVbO(!jlN7`JJ}UXKvMZc}Q@ek?ujeXgjZ!at94Pp(^m2eP zvUK3V?h(O|msa*F{>jHD1-J71g&(f$EWy|2cgQEZPI;A&esv#0r543yshQ0h)ytd5 zo7Nxly5l|^R+lK=V`KU}N9>8Je7rbQXN;s$>5a_j-&{EoYTeM%@M32&x%=r;sV06- zm9*PVHV^gk_kA^eo`#LXWbuazH+L?qk8=ly%xa~z0V+>Q=?1DI?-Iu%Gh8#WFltfE zw-6PbN%6pDc8yTbESDetNxQ@@>bqgf6qCCr+#IS&1LiT&7!t;6HJbO_CqQN@8rTZ1uv2 z-d9J4H=E4B2aK;|+vbYHtKMpGGPn4Z&*ij_A5IKagk*-&zm0mos#Fl@Dnnv+fQzz9 zeT5v^uJ~*B>!vXC3OST=T7AR9@X|Rs*W7EMjy+BWaMJ~MLm*BE@^@}m_5W6ud-3_C zv<)jV5=&X%2c{p>ObT%Eu4yEr$96W1^ADGy2QEPZY_I&}jF<$1kfFxI&v%Xjd&1Vj zuJ6{&HMC-bkS^{I-(Y_{$;ROBO-7TrO<{dKrtIYA+Dab1t%V<8v1wk4=CSePV9lu~ zN~R%`zD+0-t;cjC$7-MZz*u5gsduS@*FwOfpp+87I?RztBRQ+rqH%C`n)S~@YbGao zi=VA&#kl!aY6Uz7nC;Tt&^2=Lu*p4MR#SNiQ7e_FHe)rs(7W-N@V0%xS?YDm(_&W7 zGw*fdf?tJ4RbXAJNW1In1Q8r7xQZ4|9l2|$l=18F{r>B3d8pb8s;j3n6nw9oy@47G zlviIh|ILPi=HuLvFdbUp`~#SvAp3xs;hDRs5G@6Va_n0y<&S!d+WX$x8`ti)jsaaz znArcy!b?T9pWn+B%oE8@O8E#cP=G%bDP`@tBIuxDg@5(onIZQ@dE?*1 zkEQYpY2i^3SOsfbpMa9;_0m>saGXG_5^5y@Gu{Q(^GaJj+G8PzB(k!<#!?#gY%d>Y z@hMnH?6EI+x!6^9xCgr`ojDz-w7W54`|d4w;#a8!VN{x1d4~xY>3EDIxm>o49o%Wj zDCIZHKoEcCw<(|eJJ3yo)lijgweA_Ehm}+8nOc43b<~o7fI1WV7~k9>=MDPZ7mPb) zfa{-!D~HYRk!l9;@gdkV4F-81bndFwLoWIOCha`aCXAZ zD=DB6p;~`rVva-#XB&S|JccpW6VxYr7`G0%KDhBZuUw*QN7qzkfv5(jpQHR_)m>N6 zrdg8#K|GXHyB7AYBe734uF1nF9p}L)>p32YEl9v}Dm0*c zn-R<`u}7nGspV*%rMOiU#>aK*e6eVF1v6(j}D~69sGY0An zj4!J=_qpC+1AGK$aQ3v4*)rpu$dYI#~ zyw}&K(H&1u!2PVqP?U9}Q^->NB&l29>Fc*kfT}MjyWtc5D*v~mc}SP^P^Fi+dflfx z23`_hkZYFrANRqkq-7-TBC}_+tw^;`d{KaoY#w0UP$R03>`27aU1j%5+LB7^Lm4>_ z&5(5-OuM7ACPINxU;k?PYXK+zyfvK0I1qfx;fiE!ou~sfv(7D$%{XRpWawAd)Ku!p z5MePk;=B4lMrLB7S=B;BHp7L!=M@rKl?$B+Jw?lgAJ^nU{?hsd105cs`p{8Aox6Z*4ABJ+ zzPW2K*^iQ=9A(jtS8vJ`%VtQ!VOAL_R@zSJ@JOQ>V>SavOUL3d!>GJ3BRweJg)cPQ zbxR7an;$$%1!kEeuE|Zk7SA@3S-JxR6xN>Pkb6TIyi+zL5#}NtiO#S4^1LW}?Mz?8 zV~@TCSM4=l;uZG9jl3ZVzwWghCiK9K>w(eeNi$D1F#Yv#2)kxiiAaoQ`tYNNPD=nZ zHAzlOtMS*;!Q1UyI~VJJzMcuPDlhciniuQwt8mk|(iv{@-SzspOh_>_>}17-Pv zWCc3V5=_3ncra<&{b9jIc7|+X=8dp`2dy^EDlB^EOVZ)=55py{m4F{TXrEpC6fY(& zDu+N=YGj$_dw1`Qy_gN2N6t!*LAGRI^D%0EvBvvv^m_jc-hncz8D|a-Wn`$lGcGE_ zMigP1IabiWpO!!^EDTl{s@%7lUNqz7^hI!um=-qc$FRi*MNIxJfr=hPL-%opCP=;7 z`E{fcG8&f$`5BVz%|Ye5iG8 zOT6YfXN=SOima;tWg$?Upg71B0mLq09ZVOhWx9Wh!H%YN^xPWtQspXq$J}DSX!%~C zoipB2>FTLYgBeOrc_b4Ji;UY5mfBCE&g;%YWPFLPR^BisN4Pn`Vt?nL$dW$ zZO_NNCT&~1n7)7PsJafYKe}*cgVD~L^0+4%t(#XXRqtwJwupHv08NRF%2rGr9IP>^xoXQE|6s+Z zOo2jQ&#PWoj1B6&rJr$oFh+@1u}IczMs=PKno@Q1ctZK4TSAIh+b3BtuU+^vvVMi( zG)Bq(3LnTT5tCIP7zX*uB*h=sxjt8$K9#B8NwXWtmyj&?Qs}x4{QP@GFW_tO>wkj- z|BYC1fc`B2{{z(fhhFMnr=hs6fgZw+^9u5ltARe-$#y`D=y%q_)MOOH#Lm^2Y*TJYM0`futb2p@O zxW#GV1K7FuE&v%mif!83T;~5fWS*w#+`K=QFyZ91Zo4qCXeZkqdop-*@GZOh(&q8C z96^?oXQ9U@+1+2y7j>T(9ZgHfCuO|aT4+2C!-m9CZO-UhDcEz!RITKi<5H=NGz5^cT=dn=%vm)ID2= z#@~(c(!uf~m=5KG-$TJU@r)9FCts&6Tl%M!mur(VND;5GGxg8Y(!8hKx4P_)Qe^+? z>=_{p6GUi9yVhu@1X116?+D{!$*M((=NBuV@IVjo!R=RXCV0@UD|NoRtmh2KvW9*} zxyyBX%khbFnR7P%+Y}xYW=W`>smy^_z?!BN1~#V+PaAFam&u=h)+elZhU`Nbh=pt^N`BONb-C9Izi@-qQIHS3H4Tsc)p? z6ZYeRNC~Ns$G-;(=(f@~Z(Lw_|1Iy29(!VJDRy1nXOOUv`a2DPJqC5vE4`U4;k+RF z0CZ$mV#*oqvFhVY-E32C0Yn+m<}&WGju$$2bHS0~G5HY<+(;mMYIlZZ;Uh7FnFsiHcV z-h4n!>e)6@pwC0D&EO*<^D`AHYKw+k`=9%Ljm7)?)x zXc^+G6*B?oLmNEuMgscd2Ql~ty<++spoo>3av1PO-^iO%fHVQ>KYF6&yGODb*b&&f z!|g62fKRs~6`tPw8xnafCTW_4!67@DxS=k$p{%V{!61R2;u0x<_@G%*&~G>5_?;EN zqICMmTbt=a!z4T;>xQ+Y)2pr1+-%?&$_dxN-`9ehvF~nF6&Ms}(h?L9T8`e_K@!U{ zGeX%oLulMtT%aUNvbDbD#NLK$;tded$sH>Nbl9CT%k#FjD6y;6y}nud%>&(Mb( zI}590p+iT%FR772Uf|oMWyMx{pP0`x6fK-#5G~695dyM5=7-JIY=tB?H0Ok4%R**f za!u`ix7sqEJ*cf;DMg_JTvGGj^~!ICOdO*!N4`)c&Yn{L9kray^(Knv;oh96n(%PE z-6vu<0>C}OB{AH zq4tZ|T7t3lV8PXvBRc!LJHFI`h#!yDxPJL$B^#>ftak?jfqV9c+@U_LpS!4DgYd&j zpJ{rHONU#>Ja(gI#i1)6OS28n%RD#Nj~SVwVXunE#_&!ck;MIahrlDl)7w!<&@JF; zivc8RkEQvyR@I-9^(GuryKdXR<(dQ2<(a}KL_sKn=uDRGx4`Rd0BNF8G`30pAl`>1 zF!GS!zod%Ru08AjnhpQ7$B-yAq|CBZ-wwX&R%XT}J-5Iq<7PXy;p4Nd9GdTWVDvm2 zF-UG5L)S9{uk#}FWhUAX>qf^;aezi$;VI_ULl4W3=z5q8BTdc^r#5Q$oNPc(o56U- zb1E1*_!l%8R+z_5sH8{Md8=~;axDAn$2#Xuvr;!jsg4lAN=MQ}fY&pX(a_$v> z)o+{D7z89R2fr6Q&ptdPBivmiqN&2f6>A?Z6|q%%!rtk?CK#^*#K@gukYkW=;`fto zoqm|8#qkw7V8RqUzz?5TYP$PP-zvC8=tXXzAmeb4gZut0mmK?X$mZXc0qzUQ8cdu_ z*_wbOH?B*MK%FoKgvb?}Bhc&!Ug)24ODjB|l<*45vKAx@X zyoaby*pjlqf(M+Oj;O+h-AcD+5jor&@0Bu3T*S!&Cgw9u5-EjfBET7ibEUX$ojTLCccf$Lf9hr?T1s(*1vn)4n=8O68PRS(?F-}lAg~x zMMY@elx9TdjZID8JnOXveywFBWn>FODr_m42?QmVdWgm~WCvZawyq=Qbejfn?;KZPvdpwA-TyS;1V&+_&bw!>mRarhAkF<4dOEE< zEy~x1FXobe_SSnNp7jc!1=^M9H?nGjZWRSy-Y7cSMCFsPV?_kQf0Wdt6Q<2HpJ- zJ_HL|%offLo!a6{lw+dZTn#GX{&=V>OG;6#!Hr%%-pJbeB<5m=e4|JHlXbCXD9|E* z`)K1YiYBYPGPPt&@-XEl<7>^al#$E94RKF-p*)n|X!WMd#do9ZXe@td#(*{Y{0d^Y z+>F02`R}+72duxM%)}=^O62&q8~wk3?S*blMc(5v{c;qJ-f3pM;K4MfDlXlxvH$=M z;!LCk)eu)LoCI5cagBHWTwcP3DZN2qu80h-qG{n5R`_jbO;Akra)s*_fMSpqdf(Yq zVX|7X?Bh&RX5#e6&O+2DH8>oy%rbo;-=@GQw#<@2+P}=~l2%%^uhDDzQe#|#qWFQz z0F~Q_SmS%+TZI`espnN%X16Jk@W#uXnW!+&&bQbKy-mla3pCeJ@8(V_Q@CBW?|n7l zo{k$xZOYQYEK}RRZD(eX@y;9rX|Np3Oo2^XQwjUx(k4H*r$fQQOZ8axcv`0yiom4N z9~Ec{ee&_PKUrChi6zKYD{p~o#;U^9s{&vUMK=dCAGaf1y*p!Zco-Hu<AoSO25P-#JMCab>>s?{msorGF(G{Z|wJ>;J?b8@88vDw#wL zJZ6jy^jso@Gv)CXc(c2NPEnkU9Ns6te&b-Zd#nZSF}5l9VGyZrxSo;i!&y#rI&B_1 zvjbjezneaj5tG#Q2cr4ZI!N}=;Me;bH0A- zyV~LI;!HjCK_aRi*5F+;$f5Y01FnAwKJ*sOa@*QTQc}vm*pjSVZ+_<&y zTFv%GWyb!Fa8-a~Rp$+vX>y&<$?hiwAJ^=OI)|pTyUyvIxK3#x@GRwwKVSk@pThdR zbspX*NWH@T#@(U1;;)^S$1||N9?P9*VcU9~B+5!{dX=&{0h7xM1ytZcX*?`0Nwj5G z1AUNyvW)xCTh#(2lL$e@S0^**t5q9&AxABnw99Rqg^ok{Mwbs@5TBOXMX;e!Xsvav zI{-YqA(j=qh3pgaA3tsqW&=H{U7ML+Y%kgS1otjrO}X3c4)wtV**D;l5&H`{fR|-SFazsWsSC>^k$rysME_n^ir>DxL0W`vyZ_+VE zpC}jeB-KcW-Qw_n!6#8?@CmK-6BDVBK!P?kpaxf-7B~oqD!G{eH?idb4LNCg`mn>d z`n~2}b>2|;wx(M7nxbwg;MCB$THLyw_+6=7Nn;3b>+j*iUt)c+`&m*uFCjac`EhJa zChbD3eMLj63h;bh%V$8_1Cx~#j4|Q{HitZ*CW45ZoLT*SJY?y0LftY%{;#Ep!}vP> z!@4E7<+Hq7&pW`8-oCqXwsYF(Bfx*COj4n$_n56JEDnf zM0H34oJy z04R$8*+f=x4ft6OR;};m2-)!105qD0s7=uHbAa$z8tK)0@y4d zMr3$!b-qqoc7}6u2?*$XhZqX+f>iU*yG@N=IB8(xFK#;aJ8P!U6jM{?)@hdPZK-rd zni$L3ZpKhRnv0MCd^e^_{?!NMUI1xr4)CEY&w}gi3H+PNX%qtHg7-nP106IZW@GqJ z>3Z}pYcaF;HfWOf{jQPt6ii#&+g_R8%U_A22Cd2xQcgXxhhz=_ix7dv_{$}T(ltnD zpeZ>&5l;r9>0kzL-;>kND!LbxG_x}mdw@(k5vX-qey6Dg43P@Nt2%oZ>Ew$p0WY-r zdMIP8zUi;=FLLWz_{4bB`t2mq?BjpoCNfFgo%1a~m`wnxLz^KhBO}#mW&u0X@XjEV ze=sMgGuS4`=ZrZi_$PDH{Wo*sO_aK=4TJrQIk6c9qJT;NWKKS4uq{JWb^-Kdoa0qz z(gcqCE5|DUeDP@X{{S2?|1OCu)g#~<_kBEk`D8^$ivwGa_Rmmoq*%}7pE~GdE5Ea#KUYqr;j{_KklLjhS zuk>vDUd8G6D5JQX*&?a_^J1J4qK<^6iPjmFNB-z! zl^~>&(&WYV8Hcf!W3Ks`#2in%vQFU+zrds889vMw##y5;nQA6_()#Z{V%;m3kCrm; z?TVXcC5w{pQ}`PWa!5IL;um^jx#qcijkj-XzImb2o9Esx39549g?_bd5L6>xKP_)O zX`BHI6muU9VOofnGb0UJza4Dk<)B!XWR*w-UBSib<8DzR%Cc80gJ%k}&j^j?@+PnL zz^xj$^5xK&6FUY`1{PLTa@Cg>z61;(p;YKLF^Il&W-2x+R7B~30ThJcWCkD;|H;g; zovJgt;5S^sDmy2zINPwkI*T3MZ;T(_Umr!ogK-6<(sM;eKU~aw{d3O(^1w8(=G8&c zooqdERq86*RoKD#K`b|GGUtT)(#F29Q+x6--SIBAA1t*6Mbzb`+T2Q|K)!*^J3eNteumHIq6;M;4jzTTD$KtA%OKSSfR6e~L~T7F9(biZ zAZ1c;)-QO?Bajv|UT-XZ;Bf>{#H3HtyUxSn56;sVp{v(#keILX7w`+;mW(heFU~D} zJoVU-i8?5780f<#2kQ9HF^AKQ&C}X3H=RJrit^$+TDR!4o1p-5&V4rR6D#MES<*Dw zDNs!)jrm?g;4Y9Ik@qwzD%1vjGB~1AC{mZe{ouRwtz08Zj;QFZrgzvM5AYeAUl}pf zMBW5tVVLWdEQqbf>Rl(GwRt-$Y1!}1u>=xuw=rx#`=@0bdsSRn8X++^qksjW?WFAr z(nq^D8=!|fMqZ~rbn!7bt=;ljdtL#;TRd$(Lfp2OB^53XOkarM9*1Ar-ClVFY=r*< zoKz&90VfL&>i_t8THIOdKxTU9;IWNGn?Szk=Lu5<^5rn2u9evXmnZ~z$34_RF1>6% zXHx};6BikpH^~guz3S3V;s?;>QawLhn30$3Mp0872#?M8-dG&`^4i-OMnotCQOTs# zvjKpKerm*sYyV1FouEbO82aK0ljx4J``zC-Wa!v;gL3S|7k;+4H;MuH=j*pL@~l%Y zn1e;^W5B%IPWMiMhwj?2VRC92#**U8sSd~yB{ zG&!^{5C3xL1smw#RgWznOocDJYgqhPQl?wG_UasxlW4gAnL70E+6iv{=xupJ8j{4F z{^NC_+FpyDA}1{oP&b_VIhVTKN|2Omw1}8%j^bN~AhA$@-dMKL7-%S0*~xD&Hkq(g z+=U*LP%JY#=TVbrE%14gYKu$D`GtOG5U3O!=ErXAH5IQr!Q_Pt6VL$jz`Bru`+kGf zJ<*|)mZi1vbwziC%0t;-h#&(qLe@{Z`4 z=I1UP$=D$?V8OlptAV}Xq%eHbwj!@(**Om(7uXVFi~dDU4>+mqo_o^0&QJkW}% z#WKnG-TYdQBA&X_h4qQmrX}dcT0qNhet3lEmeYf`C^gZ!HEfp9nhyV8j@g*KPOv3$ z_{%Q5YNtJu_qbe9m-2QZ6CavoOrlVBz5NN@D$Ck_0oypW;Tjr}{n zL5$&m`4}AZh5;z}-}Uj$nGkq?-YqiU=I4IwB=0+;TW1Tg={DALW#fZ>N?tmu0S)0) zjyWSnV(HAA%hWjr==~ULsXIG+>6$yZp|AT!`T&x|PwLt&DwtlT%HBCox(;ySLZZ`R zQhxD*h+qbU#u`rTI&d!R#us9o5vMGWQxcE4fdi!SBMxtZRJ~%i{{FMj#Hw!qLR5ft z8Mwa)?M)~US||;;N@A9l8FJbV0xjlxf1SM(r@wVfM*i(p0mfQps3=L4&gRCgBp1XaFz_`vPYyIbF!4l<8a+~0euoMWEOb8&0xT{ zH_*8Zvo9TtnE$6G7lEcPBxT!crrb(J{aGMX7!ky;`WRLUtubAfnP$(us0R*0RyBwu z-+GPZ-v)+fbejA2Ir+iU&rvLo+Y3EEHD1eIwzb5D70konL{>G%L%7m zrO;)3HO{0O+d80r8#2B-IQpf$6v(y{0zqGV`9^uur=*d4@x1H(8YwEaZ_)r`yALFn zO6uShHB<89MKf!t?}8HIF@NSo?<-L}`4d8yZYfU3aE@-Wyvwdgf(Xjn+hY;nMhw z9#8_H#lh(nAo)-yUNl?yin{yL;O(`?=F$Swn^p;#<{zv`%zo%^VE7R7^5LWRZE$j0;SVzX|0YMyPi4%swkq3-0MP}{4fkh&=5Ic5)$NZ? zA4KL1GfCoS;oNLlj&?07?YA&)8Tn*erMlf`S6>Q9|9Jw?6~fz06p%8I;Ub`8)N()T zRrg5sGlFZE)a|skw{*Y|#zCi-uTs7Xxu?04yjnE#^ z9|56**QmS%(JRvctY0|)t*^tJzP`b}-i9Fp;_`TQ17!T}Hk30}RR&)FBSBs$y~x}Q z;xww;ZN|n zB0^_XPZ*E@Tw`qMuoY|$oBE_1yl{VZ6-d^G0i8fWZSPnj209S6bOYq26NlRgMHUMy z_%<}8C)NNh86L-G!ETg}jqoZ!B*^=^QXATOye`R5p0K9WyKy4&+;_`0Q{nmfrmT#t z{Mrq{SPRX^-+Oi+2_}PR6t4mqxs5lLnqb+QVjx7WUECelXH|bAB+aJhLu2)!X-BWl zRn_h1b}-%`mVOGx<(%ci!tmyvKHk2-CTVUV-z$CFL%_JR*K!O7WQq6@T^*l#*OY~1 zr|l<&Fa3Wzaa-HI`udc|H$9(vb#><58>cSih(~|RE2}cS)v|T{ao6wJcduk^ z+O!E+Xn*slJqIlQe14xhoT-_wEnjOD6mMf&w6!y)GW>Mrp{?tt-|>)xxL?3m#^^pC$lWO!8^b z%KROdIUDt=MO!()+`C%PeD{>XsZVA99`BzqyREn9jK=fSov*UQdwt96r)h5!DSY4w#2ezgJWgTK)U&xzdJk-Tclo8Ub2GqOQPM#>h9&SJEde##$>KbyG2i!5xA?4Sys`PcAIZ;u086;CzUgh1i+@zU zGMVRB&r|j&9k?)Ki|zjNAuT(!YxWn1dact^^D8^@ruzQwd*IGuYhYvcHc{1X!(Ekeiwh>B9h0?=O!Lp z@ijx?Z$?f;%CvPgcOZ*Mj=ZrJo3SLSD$b(-H=NA zp;7$i4{RYx>iO(*8&X5uA1M4YG1oP}oOagqWX^WrB9FPxB*V>4Zn4?bd0swf*6LX| zzj~exo$~bU=9th)+8%Ga9_bZcJ`iS9JXNT2vvNVzVc=lS6XupB3oUtqf%VLtW3Fm9 zbIneDO5+ADqBA~Y%RQ~F$}!dBa?IUnvtnE$@BN##`sF#fgeP2+JZ}p}&((gaQhY@B zDQF$Z%U9e{tx{8ST@?1^#O%Mzb6z6#n%!RbN|L%?J5HWCwKSQ zz12J!dgUo_H@Mb1pGB1wz{yI;dXj5oKH3H74`{u=UO2$Ves)dCxmjuTI<#Ba; z`Q|xxesxrwz_hJH9KltSH^x1Ta5dYnGbM6!tTnJa`T(XDJWv0a`X@@JFi-Am8#AZK2%S9CP z(M}3MIpO0tc*h}P=}B)hBXH^oxMBtCi6ey$@V$yy4yHnL33O8<(yl~=HAthzU_q26 zEGT{(L>69(y_Y9r_UJm71n?}cj6X;CCr75VrT6}`mkF16^+aUKp4n$&dQZuIb_bpQ g)t3IO2qy85}Sb4q9e0C>ep2N1cZ98s)ij=V%^1d|+_y z90~N?x$}=MkpWKzg)j=|&goS?dZ29NH@Vpa^D;KhI+%4bu0#Zv`tR$F`O3(%jZptp(SihnUcLP<}0zN*V(qx!gf-lhCk zzJK}Q>Dh7PaCed{YFa5$V#9N5(JWgPO_29hv=2G(`|bMN#@W*bGKzTlQexE_|Nd;K z{Vb0`EwFu2qdjve;rq5(dg;qA17d!VM@WX0cOHA&^rPZu$IIJim2~t&A8g}}TTD#E z{p2~o^jkT>RpQ|9uq0sYw+o^9=qvYOJTLKTmvC?CFl=!`xoCatrQH8``p-|?$)NlE zP`P?w?0@_Lc*;S2AND`f{F&>oKfKf>3~J`egDG{gWvnJTWMB`Sj{YW&BK|Y*=YRor77C%6x=$Z_Sw$ohev5vC#4z9 zzf$tEahtxCC0uh&ha4@vv#;>WkKVVO{EpSiT{!BSJ!{fRzcW=e?7q-3t1-7?m=d!q zqt`a)={z~pOPupf!y4z=YBqa$Dc5s4wb$mGu_}W0Kd+Lcy-V#Q{(jjBv7vwKu;FyO zwt-zdg83e035ssdh@!0+2g$yBhmP{xhb)r-|FL@kpsE<3@7h*nHt4Z+T}O+WmtTs z9?g~fiu-C|>w+gurK|1o>w-&_<%dBI`+VblbSLk+4_%9fJrP>;7`ThPmE!TMkocpN z%KGU@lIZA<>f6C(doq{+@O$BGDje?F;RR`YVTT34;ODx4Ce9{NRZ8F4Ou=_$!RDOX!F#p zZ`$PTbsJI|b~!Q>wIqZ=1FNT;+cVddHnEGyMZ=iCh!mB6E_L48RnyJT2XkAgxyA@{ z=`J^5ddVzlSvyaAmuA)kzt}HQM&HN23fw)+Maxh=pkPwop&<428jvUBUnMqSwsx5nBHDXo@=1|E-_)u5-L8SZWpipRXZ8v66u{sulBIzR(3-^?nU%@ruZW% zE&t%)-_`LV0T(F=j$GkL+Bk{jTS|C0w);#ctUK?|&3rvq8SX~h8jR_#+wGXJXf8iT z62Zv1Tn@Rf@gUDG*QJ4?oX?(iS%=}%Ifxv2PpsZ*4rm&pe+2_iQ-Y~mFv)F1L0H%+ z$XIrjgBVLH+IM?`iU@9;zok6FWN8r**GN>!U&6ei^{|i&9XUl>_qBgL5aK}^LoZFv zkUX7!rO{OsLmh^rX}VN|+_AKCV|x(Was@Z|Mo1??z&bjVb%_FZ&+jsv`?+rtXxeoz z9^MxA`d|pAA_l7Vu9PWyi7#X;GM6;~&$Hy`J~Otkz4X-FQ*A_VAp?Yg=Tk9$)}&ht zrbgr(4g|9KcYbb1nR9)Hzsltxe5UhX{QQif;;!Cq&NyR9L6C*9^sxz&qkTTbM7+-)OGTGY)~eWM5j_74{%jJ?k*{795T+jcc`6Xm zKpBESREXoPvl^ak$@c`*8m9p_zaZyx9`6*sbjsB1N_WW4VZq;)d|A;wAkUYxxU?)e^pxDsa*oSYcWR|a5FAbaw2Y#$n+Lk(%8Ru1{doZY%L|ZD(t$vt@1D5VOi6i zP|{k*>&hto&j~hP1yxc^H+8-xS*b=_oGy#um$&!?1j^$xG6JLCK1$u&yS@+NCi`v* zqrY-RBwyUawwr*%cUP#Per;OuL%ET^XRK_eLKzIi*Hu}Q?&&{!mM-9N2q`#wS2}nZ zhk1klDhodH#|Z{x>;;|d21Li2`uQvVdS^ox?91M7`x_ZG+M*tJIVih_6ZSyu?dam6 zQlWuijjh=TRdw)~jtpTW?RNs=SJ17SLicp(7Yqx~&QW*S<;1SeHVIgT99-ab@hCrH zbNJ?IAh>L-VPk`4)Pf7#dLN<*Q|Uh*ZR!8{XzT(_&P@R!3;tJ!*-r}}_IMcowyr27 z_t>9@3P9B=_;kjk!1nJ&>q=hZ5P@@)Dql&zRx*=n5 zmo_MLy#xtUo@>1dI{V;{OfrxV8BP(wN6w3Q$t;EwK0MrbGCO2V&ni7{#(F6eN7Esr z@~W#hVr=5*n8DmLf-EcrHEIgXcO$QED8+c8@J*{JudC%1=|&G*6nZe@G*4VR>rxa$ zsFH4|v5I^HhK(d{uI4ISI<@qRR=-CPg~srynS|0b1(Z4~+`^;J)Dzjhwi(9@EJhL5qXc%M~*V#!ta+al0x(- z+)3)!BkZR$#t`!bp9(eaJK%;s^Ott0QPt@(dTEPJZTnC|6lTEN3Y%y+3d`fE2v4fc zUZN>I{n<#5V4eIU-s4Oi_Zsy%F9uL1>1B9DqbRi4;k#Vlm5Dae(9@exgDgdl~>H`vGxO}6H*vywK5Z}>HKMHoVO(>}G$(mQ)c#|nG+a2(}4Rds0xlTXJ9TLb5sQ2Z2?oO4hFx6=< z25Ag@@Zt~?o)B^r0YUBAt{)kzxOZ z^M8nwpEL}z0}RQai-!FW7}eg=X3K`~U6xGf8cK#P%I}&|=SB7u$IPrR#hw(r?Oofm zFGNRVqhZ?9TfA!^y#V@}=E7!XvHm9MLcP1UHti7YE|z1CdL(NP-iZ47%It(Sfe78j}kkC)IsM8Kdvnha@J61)ttn$z>C9GRmq;UEx-9db0 z(s@DJ{D`<{5+-tgDjziL;i(Hmz0z*qWdD66I@crg32O?OrI7W~yZCHUd6(y#Y)yeb zP%nDF%alC;Jecpi&|dOu_qunAO+Un6O@=P@{6MLF2XVYni@h^J{q~yYu=oes`}TWY z))(CGX5DwW>5Bdc8)Czfvr=azO_&UG0{4vBmE_Q-63Ynqz@R0XTX<#N%MPpxlKpIg zWI!CEgbPi*$7g3vMMVfI`lRQOBi^W%gzmF^#IW=g)(%aw zqR+(m8Bon3JRqW2SiK>Atcs*1mxo^lY-D&c1#XT3iBraCokyD#k<#QYqq!@eyK$#Q z=CEEoj5Y*f^GqgWLX|sS5nSnS@B$qzx}XO0)WgQ)UbFuBMHsr=!;*9B+~MMbD{^<( zh9w1=JpNG2>$_3K6}mH*6jzAHk5E9qDSO zpu-q$O|_%$7C(qv;Q&o`z#vQoiX7>888UY3(gn`cJ5_tV9`mjCE+vE|8u)mg!X*4J z-VTgrXykslY|=176*+Y~FuaSj?G1YjDOJ2Ox)~)I*P>GXplkg}5x<}7W&fXBQdRGX z5?EkAYdZI)a^L44Yx9(KXS+!orw+{ooL@-uoh1tHyNXxGsP2pKIcj24Ma+s77+z@$ zKpCZpi<0B*sI%V(Ah=ABH1bc7oGA>EYIn#lcPO)%NWP>d%ntyumGZnzSPuzx1esep%f1uF+?%`zGbMXzD+Z?#aa6=OHK1xpae6-9tL2B!T8kn`NGw+i!H5J`+ ze7gDVN^xw0R|tXEB1Y4)xq)zux= zNwZ7Jz_H3uTog`bnV0f}Ud71!BS*$)KEWzbh|46ew3KT<(R( zZ{hqd3UvQoB@OhVG`dsuR`2SsO-4}Mn|~aB#O-)s{SaWfugcdBvf^#UnuJw{6@(mC zd3bs5sd)6Pyp*TtqY&RI4jrxW)rI|-e`PZ_7n+!mQXI<=3m%*I1#ArBtLfyS&bshh zyHq8JL57^n&qu2xr9HFN$(aIO!}7O``@xvq?nB^PMfg-feUpSO@rjH&*;qmA?0HOG@Ia4h5jW@d%-`=HxD_(wonI-WCu{7u=^Y zUos-xIohC|z^yCJQkqu;dOu$&#xPzy~GaxjU^b z0GF_`6*e*zM;>gx5*PL;f;h=tH(5A%jBpiqn0$F)yMe1SvyZ|1Ty^~Fs?S|C(`?!N zQ!zWzo7B3{ZvC3^h}RNw&K>SMuHpQwOxV0S49{$bH>ug8iF*AkHB3TZw&j6=8gE~0 z$JFGhoRM^?Wo0V2%B$X37jmNQuH(wxi$Eoci=$?2i;oD0?P@@96BM8z`caINpxw<6 zb=j-8wXxCc%UF3la&f=jb}7a{q+y8Ts$6AWHH1DX5!b%Do(#(Fhhq%vOwuqhoDK~; z^$)sez)TP9tu8c?{@~^3e^=s7dmcAv`5voqOMhgfy2MBIRllC}rHuEBUqAXbc5TMS zXv54X`LxGGt&4uu={;M(6x&@(ac9{X_Qqk*exlp0BE zof@XJ&Er4ynWcJR|MnLKwyTxpfa|i6?{g~k=#l2CfvNb#Ex4z>fUAqG^YF4#H+?OJ z1t;>H_i`tV^?aF)8A5@})r9jZ+SNHQLyANlRwJ!>-rYKNixISEX+A@RYIJY>5cANu z!yb6C@u1gi%!8)-squy_6dA?~2Z)#^UI77zNt+nF-Fu01HQ2);9YEZUjH*X>>eb4-2nxSLX}ol7r-fUasIaqp(dEW(%-cF(MQ;&f|O-_+`A$+7H`Dtm*9< zd=E6KJv}r0D%r_r69JzN1Mk6Y`lKeNXLF5K8;qM=)Y(h6jzH^}4efN-zTIqR_7xRz`cWYA8sE?$iX*fumtk~J~bh|d^!J9o&yb^NuxUivpncseh6r! zE@4|+aZPUjnXj&vxSz*)_BlV&pJCu}OEgWcPN1?H$n*kzK5p;8e|C!i#014L+S$r5 zPbbcjn32^ooBLk5D%feiZ6_k2MiOs!66L<^aHT+N=-QP>8WVz-xIxe)M%YC@GY?Qc{25kL)s@fgR2Fb9PTtf^T zh$aNZhYZ}OntZ&!MZqmdUJ;*$Z%o0AXlSbw9cX;ICgzbJZpL{9j#*;NZ`IU(OfaRa zQh%nb-@Km9108*g^4sFW4R+uon;57l3k|04GD)XY`9ck_8F}cy+)3^COoN&CYS-N$ z%wnc|`{tK{k^~nNMqxQG?@ zu_e<1F%~P1DGva0{NJvJxi2EimDvbhUecnCv>OT0u#{vFZLhg2_OYFSU&m@*poo-^ zny&xq#&_U(KU;;I=X^eug&_G@mPouYpQV;gTbWEU=;Omj?DM@Rcgm!~>6lWq zTx8Ng7JlX$D`~Yau%h{R6{nfN?4t$j?LdO(ZPV)V^|AR=M?~fJPq%`8!VGn6_B~PF zE+B=<5f8d0D)Op2)+0{%Dd9PUc7lI>8O;U#M#3#E#`0CtDyyV?6Xao8ggi-woX7_TAc`)32VUpAY{vyW32sBDb|M z&QLFs=~ONey|pxVPGr}qV-LjovMBeK0`4T z7)X{TC)(u$!oOuxhY|X}lEoL!QJzQS;_vuh9LPH)Se>9%u>&a=InEvrtD6h*n4X)b!L_p}&Ws`-|E>(N-jXj>m)#xk6{Bg-)Xr6+Q791OG4w0)T+ee#A<@C&74zWQ)c zvaG*_YQBeO=C)#5=g%v|8wQ`TC9`cPr+JaUc2{pltdLTv-Q`xZZhLJ-76Z4(X=joPxSCNx%zixt+}H;X)V}$*mlX+v<;mJS z=cr*4H~U$t?}!G_YhYt6WsL5=yYPyVq9gwYtYfyQc1O<0%nsHe*AR<=vt1o=EnU&3 zxvGb)O+&oq59rqlp8t8z#Pu4XZ=j>FHvy%7?=klFNz^=81?$Gy>N&#F?sP-3)lbp6 z`pv}Zj}v_t|4;7M-750Yuz+v17g`Q6bt#OXSEQnV!-RobQ!$TjWZJyt)~x*+e(O#? z{Ur`5)nQT_v1D{^Pa8`ZMMaFBX>Ou&+&Mw64Uve6J6Um4yZ)AtPi{B@t<1`x1Qqkf zOwQG&XKgqZdFS4Duy2esTm$TqokG%Ly=2#9+d2x*4NE+@JO{lS9RHM%$6dLPe){-N3!}`Po?N{U3M1 zD;u~PgS&Mv@yq97Z?s?61?|#e%`ksoCt>!KLXxOzf0FJbar1$Y}$ zcOTzR3An_PWi2#kP}TW8e#w_W#G$|x}n+p^Fo(4IJ7ad>QOT2)*;Mu zp&^`3Oshf)f!%Fx3s9PON79^KD?jV7r>!kHr+0_-aDKZUsKzK@Yb1`fP(kwDshc|2 zd^Am0?_=P%4q*M>9|pK+=_CoMocxPz+i#tfM#Bj7=*-yAqkS_A>%$%nmkaq<#w#o) zsJb1ihS{1g9Ks0oXdF%_kqSICyYGvwcjD<9CRp&j>u>-q{u15cUQE{2+OT=Uh?Fk<{`D)YK5 z;5*tkKP=&9F}Yi=9FgDQ-q+ik3i!z%h;yy2`vPpW?9FrvimcNUJK($n+$nH_4%ZGipx-@S)LP0fesyo7oGj{ne6+Bf4Oogq zLca2h#z*-r1q9RE6z2o??DIO#Y^CzNZZ1)9FzMylk0IHX;(onPz!HTK=z@Oc!)_QqE8%RuWst+Du28Cu(CYON>@lOrxPSEw?(FKhjObobH(%`M;qRehEucuw_ z^4qC8I?xM3WA>7W^%BcOEblV7tGlbenP*C#+wG7ZsLy2lQV0sr44%LE{p6SV*0LZ^ zPY_J=;Y06zuGTD`NLp}{cA2x87)R?qcc!#xgsP#1>C;Ka#y9V7TJ^Ct#|fE6U+)%< zco#X>sn%^@nH8V>#mJ?Trn8gpSOYuVYrKOQ>oR_`)H;$0X#W6W7Tf-{d5}uxhZr|S z>)j}at$w!l&4bWy2*SH1h0KmNTiwPMU4c`Km zWUC0zc()MTEt=6Rx=8ql5`+Gvlg)4g!jp$biKw#YVf^Ng1#y}y-*{g2_!Q4{%v={G z)g~YACx{8}q@Z(0y_|N{O}~dJr7quAo1Z9Ec41GLFOmDcIHyiYuP5 z>qs5x>ZZPLs4H{cDq`s_rP?uTq95Lbxsu^*HEpIU_^kTO$1!HbrcL8b!rGT?9E55Z zh22yl?qk|>0}Ce2 z((zR?Z|ZuPm=DBrhaJ1LU)VMK>mD08L0d1Z08KU~Cy=L17hsh{& z&5GvJn#!fkF9qfm%YuH#@jw)8Z0cb?S4I!AxHu&2YRlwrEBx%aR~qJ%Exyns!*0n@ z0(>DwXIG*i5uUcbR!8B@h+HsLb-WC*kE628|AAUf1i9uh%e$&kN!=5w?0@=M8ynN5 zM67M~oBAsQ$(P%j%J3hBF&YNPRo*yB`XmP1-8cPpjG!e!p@6_N&jL8Qc$jh9wPFBB z)E5BWd~&oc6@)?hPen>y*C|^s&UzrmYuyP8o5N&gHg0Pmq zpLD5oU-4tOcNuCZPyQ{wuBnGKuERYhIyQB3f2#IZCKXk5LGY}*6~=u3J&5F1mg8Yf z?dB$DjZud%Y5D!G`(zNFDxPCx`Qt3Y@6mA~!SsmLt#!y&doGr`Fucf8>aF-`_Fq=nv zh1Q+zB`0XN>nnM@wD5B!M9_}A@^I^X4fzKflc@I)bN%0m>kL6ETX4oCVUuJlcHHJK z-;fk;U7Cr!v>IN`&}zN<9ATSuiV2uo0YUQ%M0&5YF1Wj&jzgB)MWeQ!i;wp$~RG9qq272>?NMZ?KyM#Qb( zIhp=m=uce&%5}$cHHt-55Q}AHQo|Nrt{YvRH+h0;&A8JO(nU}x*i&6Eud=<>F)+v0 zSM9BqGqAYZ+!OD>>YDD7#RykR_7Gj_)u%c`Z47BaLeROlnd*=krYuuOP8k{N3(uH& zn*0wb(s_+AGyyt}Le9⩔n@VmuzP-Y~QL?S|I5m;k*PPT$JSjMMlGeFFbL*y|wy7 zHaDGOs5e};4`JwIp6X!l%+!|EU7oAs9N{1m8uHC0n!8N_+TK)aU-P(s=a+fq3YS}o znWR`);Aj}-!9XMnl)!baW5%6QaT;KlA?kguZ^h+}=ZS5nzsyDDN_s>-_euxtRp-xs zs z3)EC})oqs@tGdqbx>sN>px)cLRCuL=mMGtBdJUzFldPm7zw^2^yn&nPU#A0kIh&>n znJ;bn&`$R_UNT0=xOGlFxvZiboRB==uU{rwBij$xABkaCRK=_ak1XfmIu%jx8GFYE zzJFcOu!j*eDuD7GqHEAGbj)H}3i_?M;)nazB2;)d#RYFS5ZZBrtnB{yYNA*td~UM% z%9W97AA)ODK6*_QN~URoE)>n|io8;F7YzNDBs7B?Jg3qnyHZk0^6?GQl-Eu64Vww_ zS@4u6FQEZ_RwRVC!>Zksmsi&g>+*OWeSNj}rtUyn+oQUlM@OS4xWoDs!^&PZzwNQJ z#1nxoAR|*|@p7S7$-J9rC$o`ef)|x}#aNT4g)*gstCV;2R zQf5qaZd#J4|2vOu9!e-d*73RlrJ}!Knr0@Xp|^JyGvY~T$6@Zjlk(-Q0wcRfWIAaq zP`&v#@cI8|>IL9J|G%Q6#O<4#XEg3|N`GCmQVh%nfX9bwTNfPw%Hnr(GH5v(l}D^i z5UfZgy!>L&mZ#BsA=0MuyZZb&SBjV(x;44c8WPRu0lc`^3%h5!# zN1NQ8@V<3MbDG~R8N}W>=tSZx=APf`zy&4Ok?b#v=(mL^C#;cmt+BjJpu3P9_4H1Z zc<~RIX`^i%j4U1f?mxiRp&UW!)OLzxbcUTho|tbDF3q zW_&NC1Ipk_mFF?&1c+O1dM-WxUgvMosM9MV@3udMi~6mr0eE?4DSd|OVwx@3Q(Sj$ z2m3FD%4Bfwm#}9d%^hIH<+UFpv^J*WvOOL+UYJwgxZ9Frq4U%T{|X7}m$=)s+lg1- z_|~}7y?3STp%9uccfqH!8Kq1(s{KCW9On!fn6XJE_67h&@d`*9I{IG6*Q0ZtJior_ zAAj_0#r$nD{iuVF3pBa6$PZ@QQU56oq20)qQd@2Y{%Cul=wJK~Bl4aHle8EuLI!M^ z07@x+0gN#J7*Pb^Hw}~!`z~$uPzg`mXd1qqKy{N()YM1PVkZlodwg3~LG>&87a1=6 zu@Bxq*QPm!X(d6oHV|_cLk?mlStqI5q3w)%{K94cwLJWdhZ-m^_0fPsyPy z2uvC4y5ji$E9zQO!juEi0c{X;wF@1S1ji#c%H+}&JYz*X_nZ?zS_;=?h1Fs~mLI(6 z`irNK>d&4f6&$_U?$E&A?v(s^!do5!cOV_cypN8skmX*#;bsyAZ=*{32p|e+`-9_! z@+FAYwbNR!G|-Zx>yh}N=YC9Cx!*`01SRxmD8aDMA+ywYyi#-YHFduyz#A3QT)41M zIQ30-_bO0@N$TzO)4Y&dgC495G+rJuFKj}V3tBztqxk-ZG@`cC$td#%GGR=%MB9rv z5+n7H1c$hr8!OB%Ku|nxCU4+vF;RxK!{a+e7d1U?i_y9L?1ZVE3;MTTGlV|Wd4wp_ zGy}t?0#5P(o8xe5E@vn`=J7X?RMXm2`@G;Q>OR0dp+=3d==f55s%#)zKTR*)*f}Wd zHn}1wIaUoZ6@(5q-?ox+`KDUM6SWvCt_ zu}2_TV|0g;Cnlg{Ah5Bdcw-)B0v~zhMy6-oT{la(g!lVFl!w~f1~Vuey(e7mY3%Z} zLO2OToz~rX(~$r5;?2&Fm_AxX02>?!5*yll6QMj!UvB(?nC3z%EK{ay_07O{ z0bI=kpxl2MU@gbwz;uYjsHE$dlqQmuWI%sm9%)X)*s{_4bXME>0Ghl z9w6A7mKBCAva&)lgaJrzBU)tyh&|0(L=K70WGWp z0xJ8R|Bw#xIwocvFGbz}Fl#k65thq_$y@w+k)MYv7>WXGb(z`{|{IdwB3uYE=az#aE*x| zXxksl0|ZlG%PGFn5+}e}uX5lSrx<;!(XA2cX*f+CjP!bxnE$M{?71LD$d*u{X4N-^ zd0%d}Ot~N>B{GPEnDV%}M#E*mkFKpa8ea7Th%cNj%p{g`KFhUJ{@OuCGAl0*SYY>P ztai#no!q?!Z~^H+Go+VMg2BzxV7ydtSTJf-nFv*}cWV0rP?i7F=3{)}v4_(_$n6kb zdHEh-(A^tmrPhH@)RSUj-j3ykt(LT|-gI4#=CkY7lPMoD(d=canD}uE_U5xNnC4#W zbHHPw$j~9~d=GkKuiw~C6fhVs{sy$X=n=_9&%JMa!F`xECAw79!M@8eJ|q+4M5OF- zWdT!8bZ6Rs6ymP9!|TpkDY!96bmIJB3T0gLU<|E*=3 zdpIIAr3(O10hW_EARVz*BL>t2%gIDDWfYfuiLEauT&^w@8CScsZvn_`1cK*UsGM&89;T!%2yEH6F2GIRoG` zn^k_0XD6>s5c7?~6-9->5?h!T*r273pkp~-11T!Vc2I2F?rbmaZ2AV2P6{n`Q^?UTgZ>`c zvS>Jrd000Xe^}BLPF1~F#m?^0weq;pLjoXSe88W?M(31N7rQt}!VylEN#xsSDR;NO zFMi*C!9nu67M@fl0G+Sc6o4wEf~Y0#ci%R0IQa$? z2Cm;FP@MOi-+A0?GPqXmF&ork)Z~Pu9gUpmY>5|`w%=;&fJi-rn1in<4X2NolRw(% zMc+e9z1+5J3LsK7Ru_l*e{cbi3T1|(C$0i;maiv+PTpoo^fG+t7lzVF?&XjUJB1^0 zabiaAw%5Ps$0dUHe$4NerdH@zlHUFjg{{-eCT$8pa%Pj~q|wePTw=8xr`4lnQ<6p-JHQ~ z>L<#&fV6ZsUo?xdrf!p&RZ8T9?pC3{-P5jQ0nh>l7_^hryq$lIk(9N z`o+~RBb6CnQF9cE_Ot~7pw}PLNjn>&Ki6&uWS>QQ;dcU~r)Xe8V&-|wH*$Qb@e!H7 z@7$$?wR>`V(21YCo!8t;Wu;=iZFSirqFnCWZ#CWYqtlopmv4MGe-ctsM2*B>A5u2c zW>e)wmEVtfYWn%R{4zguUWiBo0Wk~Qlcb0GWgPL~CqIDvezQdCRu#3PAD;{Rf*3x> z>;8jRLXq6y)kIrRJRf`CG5GKj02@L7 zYnEit?K{wD=knl)@4M;$IZHN%?O=Q0x{R4mxn&Ra(Zm0o=RcvPzokh9)wHr>f09Yy zF8tU0X`n?%Irot$kA`6KuEE8>L%h&@V5xQ`K&;;YdSy=TAnm4=YHa*Se^(+gHGz-S zdX)EYHRIDQZeB|=1xY>sgH-`q?||Dohe?TdWL2W=*XvkCBWr zjAF=G!>oF5eS~M@;$^0R#hFIU+227Q)GK?;aV5wlnc@;l?w+{JkAPOX@3|iE9kKmM z*s|V;@=akq#`xfyFhEC<<9fR%It->cU_ZE5XAp;(sP{5F@uiM3!FzJJL0Wv|n_e-? zYONbLcs>-_lzag>Cm0U)EY&DN(kjmag`83Nab8o11^1N^6%~#&_!-PN*2*!tp$Y!M zt=lFl`fYNTW?EkgZLlVw>We*I;OVY-Zv)x3dPB0q?WKAP=0DLRwrCajj|{V2ttw2) zQsLkbpx&E@9xt0MyRX+6)D*ZIqCn$hDq&dT{Wh^a_WD&hS3-qZly*#lZGVXX^rqSi3igPN-i@sH=NZ@9M+1O@wf z@5np?a^5;HDjuNQ%16z_?qh%4EUC_5*Xmf~%z_=6Jo%41{O`n78?hu9c+*{W*<1Q| z^fjbp&0O1+=Zq?T9k?Qp5d-5(_c#3!0#N06SCqpoR#{xldsdkctj82ndg7V9uff8k8!rHZw)O6wOE?DpN;*7Nd4ROk4iQn=Z+kKZ}Se245kjfX@*qYIoh~Cbe0mHGRSlIQHhHkhep*=+&E0 z+nTEY$HZ9wHCaTs?iBe1Hx|#WVzHfAi2fd5DnoD)Qgm`E;T7U;8%r5TY6|Gbs_fTa z&&C&c+H9^(02V6=@n##F3}nGw(&OG@3-2=T+olP7A6ET zekb4hPJnZh!CT(LV);lJJ{n>JZ$dv}kCe)TYx2<(kJO*fCu|fE&<$=JMp| z0wzXK8mSJ>@#G=~4#(G9Pbt@VpSKk$y0x(*)e}c`@HOi7>kmW+!^au=so>G5{<6sFuvE%xQU{lEP$@31hHITJnAbBNZoFTspvocI>xd(tY*6Dx z9bCzCGGI3O8Mh`7W-`;_M?46GJ)Q({K62Kg|HJ6W&xm)&TR8)%lYaJw_4xN%R2%eD6|1^VY{{j?9rCl0`0Spn|N*NSE)SkS2 zgZwVwX6%&OH(AF96gN{4gKRz}ZwAa|RCyttgF1l6?%|;7`?%lSNb9bx$VGkk4vV0( z$b6c=h^Bmeq>#Pd^d7ymw?pHG76z`4quKs`vPzy>o;rZx5O^hpW;RK$Rsp`le_*Bq zu7AT!x@TLM&jPr?Fo?z3Y^n<_eFd@Gql>TG04pa5)lYCwjh@OHFYORF^tp6BQbz!K zxf(j5FDdu%sS}M)`+p!!a8lHyXJ;awVQazeRV$7uCgshFBG62#h1Fqi5+)@z6mWe{ z3Bi6)su-I2fR<5(9o?Vud2%5BP2TL)Vzu9@c=0TtUzhK|gsk*NaCjABV~0tTrHG^? z95|FDL`i{@4dH9WFn6Z7q*c9bK^J>}9k9yX9jLktg{F5?CT z3`W(yiErDyk2 z{4j;;SOmiOO-@Hgis-Pfgmmy05^%%*Y~NtifFIcU^$H*T$9DmK8xRU;(E_zJDZ7su z0dJ*H8_2);u4X-$y7l{TV=Ujc|78tWx0ff*2YjErSTNISRGjuaSfVvZZ5XU-ixQ;k zu2bdq0$fwF{+^(phgDf|k%8NQ#aaT>VrTIj>`gaQ0kCoRZ%m1KNsu>`2t+-#g}7RZ z61UGRDW}u?ds|F4r4&YG8ILO{7>IpK+`g+Y&}}=#S}S=umZH6_>FLkhLA;%Fs^6^G zwo?MgGme$xObCD?-c9mx z2=I(``{1&)J`2|y$~;2l6y0w~EWT(hcIg4!srb|cyV5|Vj8CCf=FiBvB_fr`X<*g1 zes>>Z&-sr~1JsV+mf|Tv5xYN`&=&U1^0rZF1Ul#pb+bDzhj^_<#epkzjVeQSTOTXO z3tf|Y=sspPH#({g5OX(fh__?M6UUi+dzt(PD}s)5>4o-U=g)RQAE-OZa+hd2N)*q! zhPMGWX#I&O`*PE~feCV$#d0$mJ@4(c3N!2=C5mqT!02L^wsolCiabF0MQYy@U#lrh zTv-0ps!GkN1|zh0t{vz++qjn#cMa?OeLze=ZBU(xxil;)9qRM)QsW)PM~vJl%LQxy=G3iDfwqXsx2Iby4D zonAUcXb9%%#-V&aW0NH8siCmhi}- z*NzrC6>9f-@7#qU4BJIRa@f6qqcTQqkw2I5i|ZmINQCqC2XC zv3t1=1gzYrVvp4vk9I7!umUvVJ9J9wpli*E-t;sR@_o6J&G?!(QARc?p~_j~*<}=d zljokzgTGN4l-a*f8dXxjk6?y$FpI=$s;SijwOQUj-?>%GEgPaZVs7ui0c$^1iAd_^ z&Tk2Wlj(uR=qcJX(W;Zin{E+Zp%C^)EdcRx8l3g=GY@w=r&2EKe3hOp=|>=u4e_r~ z`1LCmP$Oe1b6+?DFQ<_M=&|P`Yx+Mr&8X}iiqahB={OR0)jeROy;$?shfg{0@XN|T zEl(q+0O`Qz_Kv62TudrXH=G`735`^LvNqt>q`~5<4ZW^T%?NO>m>8J%FHD{EO?>;ouG@^Q*c3 zSIxtBaff>qyUAT_Ru7PjU(dhbueAsNbc1-W?X^eDyz-F^HYUZuwNbxzoLGN;QDXg( z8(k3RP3?TCIuvDg*)8}m32hs5eQ~l1{c}q@>aAwH{97pO{@fPOjmaR*>)Jb>Oi_*cii6lL zA;q9T9<_qw;y2IEPxF}#8Ohmg86b%Icog7s1icp$(lXyJNL~W^G!Nb7!5y#q;K#F{ z&E_VBh#6Cr={2^so3<|-S4`*_;2T0P&wJpaI zSIg51OXrH9?QG6e=k*P|Tp;o~XnVs5KyCt%ccB|M!66~F&l_dY`EazVTK9TF=|@i^ znNj^YS)lonDudg#M@&-VnMN4kwI!?rx&8IZr(^sy9w{t}_jN23D#}~Eji$Y>0#X@O zBr*^aVHLlBU)8Zn1CpH6X`sMM2wF0J_$vM2Gvz+CzMe=% zV*mAU8WE+@b7-jhneSa8_#v_r8he~v6;~c9wP?NjM+yd~k#AEz{*^I(C)5gL1^GAyx9}Y5C zDA4c$tt<`aDht2{muj!iWUbigS=K+mPLBhpT?#Q3k}O560Y`94Q+vkDtTjt-V0R1fgPci2nmQeMndq1rWoeRuHuEHCmWZGxB|6%A37b(-yy7rwR}n}}B*5F|oh&Pv z7)52ghL_SznY_-6)$TU4f9~__|A)hS&Uqd<@9+EheguSDucvj*i4N;k)8aFq42~n> zXPA`71i01h2-Xv!|9u*Nx2qCrmTuRrIUNSVk_# zop*<({jd~0rhCs;vtzQ|5NuFWxe=d$LqFSz`509y_D#0QLfAnen6zS_(h@6MC{qzJ z)D1n<(L3-bUW{L6f!2lD1N`U(IGqkU!o{fc!pDi79SIhg+?uuMer7n5ceF-srp~Z|*wIhEAb~cjl@~t#b zWtV`Af#WbWurB|8Uc4GZ4Q(TZUEF(_BCC11PBkYl6|nQB3_!aa;D3v#E$zyKs2~k( zQ|lZ(?CB0l;K@$#!(v=j^Xr zHPyz*Pc2g*$G_Uk;JkK@V?UQk1urR6%7dzTC8}q1{MXF1go|*0-s%hfLc39jV|>5| z>`ffH94Z!d80D=fjQz!-Rf78Fx}w&guj|+6u=7qla(;;tYER|K)_ zmDgqAfFc)u>nZ_=oDeZI&q{UX`Ymh7_DZ$3qLN@qhV<;*_+;JN-yjh}F#e2kkQ*r8J z;q&IuDVPidN@#@$N)pilskzvS+kcM2lYAa-1Fk2W=V(n5UBQB{bdNyYAi3!s>Xl2{ z$uAb-Q;PyeQYHEPrs}PNrEW-!318IdWIy5m~ zUz)sU(v}^_Kxj?Iovxbtb{Jj2gCz~+n1)!YruVh9TyGmncY5+-TUAAwJsj>OCFir8 z%1|8d8JcOF^59%yNG``^Y%Q{9auAm(*MS|$j18?dqm`pZwsH!itWp}UdUj5BL4C`i zGyo@@ouOsi{8ZDlwZ6e%lgPiMM%Q6a9u63cK=)$+7IwQ_Zv+G+Pde=C=7A1LI_ zsi_lPhKAks@#4Cp^>xeFHOLR!`U_wS`q5Z-^1zP)BX{E2aXq>Bi^AEVm@Nu0!jYNA zydyiL8^-iC{sGNc+EJT@Y2wztM}eNYDkVFKYCR1sgx-`eTOjQ7S{qUWM5X{7+RU|C zn*~R1b>P#n*#G$05t$6U$t8kcpcGB38>{rzt=&az+;D+B;rEt20)HVlU0a6+2|Ne? zM(_Se%apiKXJ_TYyh#J@edYHZPxOED-~5kApbvaRzq__S=zn5xI$&P>&YK4~qCZ&b zh9b;C$n?QEXM}p1?B*@b_1_lVa>ncTvb&+i?e9+m3P4&Ze_i}&J!A*V!^3R$J0|l% z*Dc{{r$z%PzjpU1_rZzCrBpMG2JlOJnBdLorhoJgnrt`DNCyVPbsla Date: Wed, 14 Jul 2021 01:30:31 +0900 Subject: [PATCH 084/175] remove duplicated sentence --- content/ja/docs/reference/kubectl/overview.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/content/ja/docs/reference/kubectl/overview.md b/content/ja/docs/reference/kubectl/overview.md index 2ddd60b62e..c5855ce4af 100644 --- a/content/ja/docs/reference/kubectl/overview.md +++ b/content/ja/docs/reference/kubectl/overview.md @@ -457,8 +457,6 @@ error: one plugin warning was found cat ./kubectl-whoami ``` 次の例では、下記の内容を含んだ`kubectl-whoami`が既に作成済であることを前提としています。 -The next few examples assume that you already made `kubectl-whoami` have -the following contents: ```shell #!/bin/bash From b0a48af6f7dd2310e0cfdf16adb4049b25b93aaf Mon Sep 17 00:00:00 2001 From: James Strong Date: Tue, 13 Jul 2021 13:17:32 -0400 Subject: [PATCH 085/175] update for a draft and dates --- .../en/blog/_posts/2021-06-21-update-with-ingress-nginx.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md index be79f3bdee..c95c96a82f 100644 --- a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md @@ -1,7 +1,8 @@ --- +draft: true layout: blog title: 'Updating NGINX-Ingress to use the stable Ingress API' -date: 2021-06-21 +date: 2021-07-15 slug: update-with-ingress-nginx --- @@ -39,7 +40,7 @@ versions of Kubernetes. {{< table caption="Ingress NGINX supported version with Kubernetes Versions" >}} Kubernetes version | Ingress-NGINX version | Notes :-------------------|:----------------------|:------------ -v1.22 | v1.0.0-alpha.1 | New features, plus bug fixes. +v1.22 | v1.0.0-alpha.2 | New features, plus bug fixes. v1.21 | v0.47.x | Bugfixes only, and just for security issues or crashes. No end-of-support date announced. v1.20 | v0.47.x | Bugfixes only, and just for security issues or crashes. No end-of-support date announced. v1.19 | v0.47.x | Bugfixes only, and just for security issues or crashes. Fixes only provided until 6 months after Kubernetes v1.22.0 is released. @@ -62,7 +63,7 @@ In the meantime to ensure no compatibility issues: [v0.47.0](https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v0.47.0) * After Kubernetes 1.22 is released, ensure you are using the latest version of Ingress-NGINX that supports the stable APIs for Ingress and IngressClass. -* Test Ingress-NGINX version v1.0.0-alpha.1 with Cluster versions >= 1.19 +* Test Ingress-NGINX version v1.0.0-alpha.2 with Cluster versions >= 1.19 and report any issues to the projects Github page. The community’s feedback and support in this effort is welcome. The From b1573ad31463dc25ec77cb1cbee7684ae944d7cd Mon Sep 17 00:00:00 2001 From: Shannon Kularathna Date: Thu, 8 Jul 2021 21:41:31 +0000 Subject: [PATCH 086/175] Create garbage collection docs --- .../architecture/garbage-collection.md | 164 ++++++++ .../kubelet-garbage-collection.md | 98 ----- content/en/docs/concepts/containers/images.md | 3 +- .../working-with-objects/finalizers.md | 80 ++++ .../working-with-objects/owners-dependents.md | 71 ++++ .../controllers/garbage-collection.md | 184 --------- .../en/docs/reference/glossary/finalizer.md | 31 ++ .../reference/glossary/garbage-collection.md | 24 ++ .../use-cascading-deletion.md | 352 ++++++++++++++++++ static/_redirects | 3 +- 10 files changed, 726 insertions(+), 284 deletions(-) create mode 100644 content/en/docs/concepts/architecture/garbage-collection.md delete mode 100644 content/en/docs/concepts/cluster-administration/kubelet-garbage-collection.md create mode 100644 content/en/docs/concepts/overview/working-with-objects/finalizers.md create mode 100644 content/en/docs/concepts/overview/working-with-objects/owners-dependents.md delete mode 100644 content/en/docs/concepts/workloads/controllers/garbage-collection.md create mode 100644 content/en/docs/reference/glossary/finalizer.md create mode 100644 content/en/docs/reference/glossary/garbage-collection.md create mode 100644 content/en/docs/tasks/administer-cluster/use-cascading-deletion.md diff --git a/content/en/docs/concepts/architecture/garbage-collection.md b/content/en/docs/concepts/architecture/garbage-collection.md new file mode 100644 index 0000000000..f5f8c9c38e --- /dev/null +++ b/content/en/docs/concepts/architecture/garbage-collection.md @@ -0,0 +1,164 @@ +--- +title: Garbage Collection +content_type: concept +weight: 50 +--- + + +{{}} This +allows the clean up of resources like the following: + + * [Failed pods](/docs/concepts/workloads/pods/pod-lifecycle/#pod-garbage-collection) + * [Completed Jobs](/docs/concepts/workloads/controllers/ttlafterfinished/) + * [Objects without owner references](#owners-dependents) + * [Unused containers and container images](#containers-images) + * [Dynamically provisioned PersistentVolumes with a StorageClass reclaim policy of Delete](/docs/concepts/storage/persistent-volumes/#delete) + * [Stale or expired CertificateSigningRequests (CSRs)](/reference/access-authn-authz/certificate-signing-requests/#request-signing-process) + * {{}} deleted in the following scenarios: + * On a cloud when the cluster uses a [cloud controller manager](/docs/concepts/architecture/cloud-controller/) + * On-premises when the cluster uses an addon similar to a cloud controller + manager + * [Node Lease objects](/docs/concepts/architecture/nodes/#heartbeats) + +## Owners and dependents {#owners-dependents} + +Many objects in Kubernetes link to each other through [*owner references*](/docs/concepts/overview/working-with-objects/owners-dependents/). +Owner references tell the control plane which objects are dependent on others. +Kubernetes uses owner references to give the control plane, and other API +clients, the opportunity to clean up related resources before deleting an +object. In most cases, Kubernetes manages owner references automatically. + +Ownership is different from the [labels and selectors](/docs/concepts/overview/working-with-objects/labels/) +mechanism that some resources also use. For example, consider a +{{}} that creates +`EndpointSlice` objects. The Service uses *labels* to allow the control plane to +determine which `EndpointSlice` objects are used for that Service. In addition +to the labels, each `EndpointSlice` that is managed on behalf of a Service has +an owner reference. Owner references help different parts of Kubernetes avoid +interfering with objects they don’t control. + +## Cascading deletion {#cascading-deletion} + +Kubernetes checks for and deletes objects that no longer have owner +references, like the pods left behind when you delete a ReplicaSet. When you +delete an object, you can control whether Kubernetes deletes the object's +dependents automatically, in a process called *cascading deletion*. There are +two types of cascading deletion, as follows: + + * Foreground cascading deletion + * Background cascading deletion + +You can also control how and when garbage collection deletes resources that have +owner references using Kubernetes {{}}. + +### Foreground cascading deletion {#foreground-deletion} + +In foreground cascading deletion, the owner object you're deleting first enters +a *deletion in progress* state. In this state, the following happens to the +owner object: + + * The Kubernetes API server sets the object's `metadata.deletionTimestamp` + field to the time the object was marked for deletion. + * The Kubernetes API server also sets the `metadata.finalizers` field to + `foregroundDeletion`. + * The object remains visible through the Kubernetes API until the deletion + process is complete. + +After the owner object enters the deletion in progress state, the controller +deletes the dependents. After deleting all the dependent objects, the controller +deletes the owner object. At this point, the object is no longer visible in the +Kubernetes API. + +During foreground cascading deletion, the only dependents that block owner +deletion are those that have the `ownerReference.blockOwnerDeletion=true` field. +See [Use foreground cascading deletion](/docs/tasks/administer-cluster/use-cascading-deletion/#use-foreground-cascading-deletion) +to learn more. + +### Background cascading deletion {#background-deletion} + +In background cascading deletion, the Kubernetes API server deletes the owner +object immediately and the controller cleans up the dependent objects in +the background. By default, Kubernetes uses background cascading deletion unless +you manually use foreground deletion or choose to orphan the dependent objects. + +See [Use background cascading deletion](/docs/tasks/administer-cluster/use-cascading-deletion/#use-background-cascading-deletion) +to learn more. + +### Orphaned dependents + +When Kubernetes deletes an owner object, the dependents left behind are called +*orphan* objects. By default, Kubernetes deletes dependent objects. To learn how +to override this behaviour, see [Delete owner objects and orphan dependents](/docs/tasks/administer-cluster/use-cascading-deletion/#set-orphan-deletion-policy). + +## Garbage collection of unused containers and images {#containers-images} + +The {{}} performs garbage +collection on unused images every five minutes and on unused containers every +minute. You should avoid using external garbage collection tools, as these can +break the kubelet behavior and remove containers that should exist. + +To configure options for unused container and image garbage collection, tune the +kubelet using a [configuration file](/docs/tasks/administer-cluster/kubelet-config-file/) +and change the parameters related to garbage collection using the +[`KubeletConfiguration`](/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration) +resource type. + +### Container image lifecycle + +Kubernetes manages the lifecycle of all images through its *image manager*, +which is part of the kubelet, with the cooperation of cadvisor. The kubelet +considers the following disk usage limits when making garbage collection +decisions: + + * `HighThresholdPercent` + * `LowThresholdPercent` + +Disk usage above the configured `HighThresholdPercent` value triggers garbage +collection, which deletes images in order based on the last time they were used, +starting with the oldest first. The kubelet deletes images +until disk usage reaches the `LowThresholdPercent` value. + +### Container image garbage collection {#container-image-garbage-collection} + +The kubelet garbage collects unused containers based on the following variables, +which you can define: + + * `MinAge`: the minimum age at which the kubelet can garbage collect a + container. Disable by setting to `0`. + * `MaxPerPodContainer`: the maximum number of dead containers each Pod pair + can have. Disable by setting to less than `0`. + * `MaxContainers`: the maximum number of dead containers the cluster can have. + Disable by setting to less than `0`. + +In addition to these variables, the kubelet garbage collects unidentified and +deleted containers, typically starting with the oldest first. + +`MaxPerPodContainer` and `MaxContainer` may potentially conflict with each other +in situations where retaining the maximum number of containers per Pod +(`MaxPerPodContainer`) would go outside the allowable total of global dead +containers (`MaxContainers`). In this situation, the kubelet adjusts +`MaxPodPerContainer` to address the conflict. A worst-case scenario would be to +downgrade `MaxPerPodContainer` to `1` and evict the oldest containers. +Additionally, containers owned by pods that have been deleted are removed once +they are older than `MinAge`. + +{{}} +The kubelet only garbage collects the containers it manages. +{{}} + +## Configuring garbage collection {#configuring-gc} + +You can tune garbage collection of resources by configuring options specific to +the controllers managing those resources. The following pages show you how to +configure garbage collection: + + * [Configuring cascading deletion of Kubernetes objects](/docs/tasks/administer-cluster/use-cascading-deletion/) + * [Configuring cleanup of finished Jobs](/docs/concepts/workloads/controllers/ttlafterfinished/) + + + +## {{% heading "whatsnext" %}} + +* Learn more about [ownership of Kubernetes objects](/docs/concepts/overview/working-with-objects/owners-dependents/). +* Learn more about Kubernetes [finalizers](/docs/concepts/overview/working-with-objects/finalizers/). +* Learn about the [TTL controller](/docs/concepts/workloads/controllers/ttlafterfinished/) (beta) that cleans up finished Jobs. \ No newline at end of file diff --git a/content/en/docs/concepts/cluster-administration/kubelet-garbage-collection.md b/content/en/docs/concepts/cluster-administration/kubelet-garbage-collection.md deleted file mode 100644 index 65353892b1..0000000000 --- a/content/en/docs/concepts/cluster-administration/kubelet-garbage-collection.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -title: Garbage collection for container images -content_type: concept -weight: 70 ---- - - - -Garbage collection is a helpful function of kubelet that will clean up unused -[images](/docs/concepts/containers/#container-images) and unused -[containers](/docs/concepts/containers/). Kubelet will perform garbage collection -for containers every minute and garbage collection for images every five minutes. - -External garbage collection tools are not recommended as these tools can potentially -break the behavior of kubelet by removing containers expected to exist. - - - -## Image Collection - -Kubernetes manages lifecycle of all images through imageManager, with the cooperation -of cadvisor. - -The policy for garbage collecting images takes two factors into consideration: -`HighThresholdPercent` and `LowThresholdPercent`. Disk usage above the high threshold -will trigger garbage collection. The garbage collection will delete least recently used images until the low -threshold has been met. - -## Container Collection - -The policy for garbage collecting containers considers three user-defined variables. -`MinAge` is the minimum age at which a container can be garbage collected. -`MaxPerPodContainer` is the maximum number of dead containers every single -pod (UID, container name) pair is allowed to have. -`MaxContainers` is the maximum number of total dead containers. -These variables can be individually disabled by setting `MinAge` to zero and -setting `MaxPerPodContainer` and `MaxContainers` respectively to less than zero. - -Kubelet will act on containers that are unidentified, deleted, or outside of -the boundaries set by the previously mentioned flags. The oldest containers -will generally be removed first. `MaxPerPodContainer` and `MaxContainer` may -potentially conflict with each other in situations where retaining the maximum -number of containers per pod (`MaxPerPodContainer`) would go outside the -allowable range of global dead containers (`MaxContainers`). -`MaxPerPodContainer` would be adjusted in this situation: A worst case -scenario would be to downgrade `MaxPerPodContainer` to 1 and evict the oldest -containers. Additionally, containers owned by pods that have been deleted are -removed once they are older than `MinAge`. - -Containers that are not managed by kubelet are not subject to container garbage collection. - -## User Configuration - -You can adjust the following thresholds to tune image garbage collection with the following kubelet flags : - -1. `image-gc-high-threshold`, the percent of disk usage which triggers image garbage collection. - Default is 85%. -2. `image-gc-low-threshold`, the percent of disk usage to which image garbage collection attempts - to free. Default is 80%. - -You can customize the garbage collection policy through the following kubelet flags: - -1. `minimum-container-ttl-duration`, minimum age for a finished container before it is - garbage collected. Default is 0 minute, which means every finished container will be garbage collected. -2. `maximum-dead-containers-per-container`, maximum number of old instances to be retained - per container. Default is 1. -3. `maximum-dead-containers`, maximum number of old instances of containers to retain globally. - Default is -1, which means there is no global limit. - -Containers can potentially be garbage collected before their usefulness has expired. These containers -can contain logs and other data that can be useful for troubleshooting. A sufficiently large value for -`maximum-dead-containers-per-container` is highly recommended to allow at least 1 dead container to be -retained per expected container. A larger value for `maximum-dead-containers` is also recommended for a -similar reason. -See [this issue](https://github.com/kubernetes/kubernetes/issues/13287) for more details. - - -## Deprecation - -Some kubelet Garbage Collection features in this doc will be replaced by kubelet eviction in the future. - -Including: - -| Existing Flag | New Flag | Rationale | -| ------------- | -------- | --------- | -| `--image-gc-high-threshold` | `--eviction-hard` or `--eviction-soft` | existing eviction signals can trigger image garbage collection | -| `--image-gc-low-threshold` | `--eviction-minimum-reclaim` | eviction reclaims achieve the same behavior | -| `--maximum-dead-containers` | | deprecated once old logs are stored outside of container's context | -| `--maximum-dead-containers-per-container` | | deprecated once old logs are stored outside of container's context | -| `--minimum-container-ttl-duration` | | deprecated once old logs are stored outside of container's context | -| `--low-diskspace-threshold-mb` | `--eviction-hard` or `eviction-soft` | eviction generalizes disk thresholds to other resources | -| `--outofdisk-transition-frequency` | `--eviction-pressure-transition-period` | eviction generalizes disk pressure transition to other resources | - -## {{% heading "whatsnext" %}} - -See [Configuring Out Of Resource Handling](/docs/concepts/scheduling-eviction/node-pressure-eviction/) -for more details. - diff --git a/content/en/docs/concepts/containers/images.md b/content/en/docs/concepts/containers/images.md index 1cd678e4a8..55534e502f 100644 --- a/content/en/docs/concepts/containers/images.md +++ b/content/en/docs/concepts/containers/images.md @@ -316,4 +316,5 @@ Kubelet will merge any `imagePullSecrets` into a single virtual `.docker/config. ## {{% heading "whatsnext" %}} -* Read the [OCI Image Manifest Specification](https://github.com/opencontainers/image-spec/blob/master/manifest.md) +* Read the [OCI Image Manifest Specification](https://github.com/opencontainers/image-spec/blob/master/manifest.md). +* Learn about [container image garbage collection](/docs/concepts/architecture/garbage-collection/#container-image-garbage-collection). diff --git a/content/en/docs/concepts/overview/working-with-objects/finalizers.md b/content/en/docs/concepts/overview/working-with-objects/finalizers.md new file mode 100644 index 0000000000..14c7535b35 --- /dev/null +++ b/content/en/docs/concepts/overview/working-with-objects/finalizers.md @@ -0,0 +1,80 @@ +--- +title: Finalizers +content_type: concept +weight: 60 +--- + + + +{{}} + +You can use finalizers to control {{}} +of resources by alerting {{}} to perform specific cleanup tasks before +deleting the target resource. + +Finalizers don't usually specify the code to execute. Instead, they are +typically lists of keys on a specific resource similar to annotations. +Kubernetes specifies some finalizers automatically, but you can also specify +your own. + +## How finalizers work + +When you create a resource using a manifest file, you can specify finalizers in +the `metadata.finalizers` field. When you attempt to delete the resource, the +controller that manages it notices the values in the `finalizers` field and does +the following: + + * Modifies the object to add a `metadata.deletionTimestamp` field with the + time you started the deletion. + * Marks the object as read-only until its `metadata.finalizers` field is empty. + +The controller then attempts to satisfy the requirements of the finalizers +specified for that resource. Each time a finalizer condition is satisfied, the +controller removes that key from the resource's `finalizers` field. When the +field is empty, garbage collection continues. You can also use finalizers to +prevent deletion of unmanaged resources. + +A common example of a finalizer is `kubernetes.io/pv-protection`, which prevents +accidental deletion of `PersistentVolume` objects. When a `PersistentVolume` +object is in use by a Pod, Kubernetes adds the `pv-protection` finalizer. If you +try to delete the `PersistentVolume`, it enters a `Terminating` status, but the +controller can't delete it because the finalizer exists. When the Pod stops +using the `PersistentVolume`, Kubernetes clears the `pv-protection` finalizer, +and the controller deletes the volume. + +## Owner references, labels, and finalizers {#owners-labels-finalizers} + +Like {{}}, [owner references](/concepts/overview/working-with-objects/owners-dependents/) +describe the relationships between objects in Kubernetes, but are used for a +different purpose. When a +{{}} manages objects +like Pods, it uses labels to track changes to groups of related objects. For +example, when a {{}} creates one or +more Pods, the Job controller applies labels to those pods and tracks changes to +any Pods in the cluster with the same label. + +The Job controller also adds *owner references* to those Pods, pointing at the +Job that created the Pods. If you delete the Job while these Pods are running, +Kubernetes uses the owner references (not labels) to determine which Pods in the +cluster need cleanup. + +Kubernetes also processes finalizers when it identifies owner references on a +resource targeted for deletion. + +In some situations, finalizers can block the deletion of dependent objects, +which can cause the targeted owner object to remain in a read-only state for +longer than expected without being fully deleted. In these situations, you +should check finalizers and owner references on the target owner and dependent +objects to troubleshoot the cause. + +{{}} +In cases where objects are stuck in a deleting state, try to avoid manually +removing finalizers to allow deletion to continue. Finalizers are usually added +to resources for a reason, so forcefully removing them can lead to issues in +your cluster. +{{}} + +## {{% heading "whatsnext" %}} + +* Read [Using Finalizers to Control Deletion](/blog/2021/05/14/using-finalizers-to-control-deletion/) + on the Kubernetes blog. \ No newline at end of file diff --git a/content/en/docs/concepts/overview/working-with-objects/owners-dependents.md b/content/en/docs/concepts/overview/working-with-objects/owners-dependents.md new file mode 100644 index 0000000000..a981745ca3 --- /dev/null +++ b/content/en/docs/concepts/overview/working-with-objects/owners-dependents.md @@ -0,0 +1,71 @@ +--- +title: Owners and Dependents +content_type: concept +weight: 60 +--- + + + +In Kubernetes, some objects are *owners* of other objects. For example, a +{{}} is the owner of a set of Pods. These owned objects are *dependents* +of their owner. + +Ownership is different from the [labels and selectors](/docs/concepts/overview/working-with-objects/labels/) +mechanism that some resources also use. For example, consider a Service that +creates `EndpointSlice` objects. The Service uses labels to allow the control plane to +determine which `EndpointSlice` objects are used for that Service. In addition +to the labels, each `EndpointSlice` that is managed on behalf of a Service has +an owner reference. Owner references help different parts of Kubernetes avoid +interfering with objects they don’t control. + +## Owner references in object specifications + +Dependent objects have a `metadata.ownerReferences` field that references their +owner object. A valid owner reference consists of the object name and a UID +within the same namespace as the dependent object. Kubernetes sets the value of +this field automatically for objects that are dependents of other objects like +ReplicaSets, DaemonSets, Deployments, Jobs and CronJobs, and ReplicationControllers. +You can also configure these relationships manually by changing the value of +this field. However, you usually don't need to and can allow Kubernetes to +automatically manage the relationships. + +Dependent objects also have an `ownerReferences.blockOwnerDeletion` field that +takes a boolean value and controls whether specific dependents can block garbage +collection from deleting their owner object. Kubernetes automatically sets this +field to `true` if a {{}} +(for example, the Deployment controller) sets the value of the +`metadata.ownerReferences` field. You can also set the value of the +`blockOwnerDeletion` field manually to control which dependents block garbage +collection. + +A Kubernetes admission controller controls user access to change this field for +dependent resources, based on the delete permissions of the owner. This control +prevents unauthorized users from delaying owner object deletion. + +## Ownership and finalizers + +When you tell Kubernetes to delete a resource, the API server allows the +managing controller to process any [finalizer rules](/docs/concepts/overview/working-with-objects/finalizers/) +for the resource. {{}} +prevent accidental deletion of resources your cluster may still need to function +correctly. For example, if you try to delete a `PersistentVolume` that is still +in use by a Pod, the deletion does not happen immediately because the +`PersistentVolume` has the `kubernetes.io/pv-protection` finalizer on it. +Instead, the volume remains in the `Terminating` status until Kubernetes clears +the finalizer, which only happens after the `PersistentVolume` is no longer +bound to a Pod. + +Kubernetes also adds finalizers to an owner resource when you use either +[foreground or orphan cascading deletion](/docs/concepts/architecture/garbage-collection/#cascading-deletion). +In foreground deletion, it adds the `foreground` finalizer so that the +controller must delete dependent resources that also have +`ownerReferences.blockOwnerDeletion=true` before it deletes the owner. If you +specify an orphan deletion policy, Kubernetes adds the `orphan` finalizer so +that the controller ignores dependent resources after it deletes the owner +object. + +## {{% heading "whatsnext" %}} + +* Learn more about [Kubernetes finalizers](/docs/concepts/overview/working-with-objects/finalizers/). +* Learn about [garbage collection](/docs/concepts/architecture/garbage-collection). +* Read the API reference for [object metadata](/docs/reference/kubernetes-api/common-definitions/object-meta/#System). \ No newline at end of file diff --git a/content/en/docs/concepts/workloads/controllers/garbage-collection.md b/content/en/docs/concepts/workloads/controllers/garbage-collection.md deleted file mode 100644 index 3e8bf31c1c..0000000000 --- a/content/en/docs/concepts/workloads/controllers/garbage-collection.md +++ /dev/null @@ -1,184 +0,0 @@ ---- -title: Garbage Collection -content_type: concept -weight: 60 ---- - - - -The role of the Kubernetes garbage collector is to delete certain objects -that once had an owner, but no longer have an owner. - - - - -## Owners and dependents - -Some Kubernetes objects are owners of other objects. For example, a ReplicaSet -is the owner of a set of Pods. The owned objects are called *dependents* of the -owner object. Every dependent object has a `metadata.ownerReferences` field that -points to the owning object. - -Sometimes, Kubernetes sets the value of `ownerReference` automatically. For -example, when you create a ReplicaSet, Kubernetes automatically sets the -`ownerReference` field of each Pod in the ReplicaSet. In 1.8, Kubernetes -automatically sets the value of `ownerReference` for objects created or adopted -by ReplicationController, ReplicaSet, StatefulSet, DaemonSet, Deployment, Job -and CronJob. - -You can also specify relationships between owners and dependents by manually -setting the `ownerReference` field. - -Here's a configuration file for a ReplicaSet that has three Pods: - -{{< codenew file="controllers/replicaset.yaml" >}} - -If you create the ReplicaSet and then view the Pod metadata, you can see -OwnerReferences field: - -```shell -kubectl apply -f https://k8s.io/examples/controllers/replicaset.yaml -kubectl get pods --output=yaml -``` - -The output shows that the Pod owner is a ReplicaSet named `my-repset`: - -```yaml -apiVersion: v1 -kind: Pod -metadata: - ... - ownerReferences: - - apiVersion: apps/v1 - controller: true - blockOwnerDeletion: true - kind: ReplicaSet - name: my-repset - uid: d9607e19-f88f-11e6-a518-42010a800195 - ... -``` - -{{< note >}} -Cross-namespace owner references are disallowed by design. - -Namespaced dependents can specify cluster-scoped or namespaced owners. -A namespaced owner **must** exist in the same namespace as the dependent. -If it does not, the owner reference is treated as absent, and the dependent -is subject to deletion once all owners are verified absent. - -Cluster-scoped dependents can only specify cluster-scoped owners. -In v1.20+, if a cluster-scoped dependent specifies a namespaced kind as an owner, -it is treated as having an unresolvable owner reference, and is not able to be garbage collected. - -In v1.20+, if the garbage collector detects an invalid cross-namespace `ownerReference`, -or a cluster-scoped dependent with an `ownerReference` referencing a namespaced kind, a warning Event -with a reason of `OwnerRefInvalidNamespace` and an `involvedObject` of the invalid dependent is reported. -You can check for that kind of Event by running -`kubectl get events -A --field-selector=reason=OwnerRefInvalidNamespace`. -{{< /note >}} - -## Controlling how the garbage collector deletes dependents - -When you delete an object, you can specify whether the object's dependents are -also deleted automatically. Deleting dependents automatically is called *cascading -deletion*. There are two modes of *cascading deletion*: *background* and *foreground*. - -If you delete an object without deleting its dependents -automatically, the dependents are said to be *orphaned*. - -### Foreground cascading deletion - -In *foreground cascading deletion*, the root object first -enters a "deletion in progress" state. In the "deletion in progress" state, -the following things are true: - - * The object is still visible via the REST API - * The object's `deletionTimestamp` is set - * The object's `metadata.finalizers` contains the value "foregroundDeletion". - -Once the "deletion in progress" state is set, the garbage -collector deletes the object's dependents. Once the garbage collector has deleted all -"blocking" dependents (objects with `ownerReference.blockOwnerDeletion=true`), it deletes -the owner object. - -Note that in the "foregroundDeletion", only dependents with -`ownerReference.blockOwnerDeletion=true` block the deletion of the owner object. -Kubernetes version 1.7 added an [admission controller](/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement) that controls user access to set -`blockOwnerDeletion` to true based on delete permissions on the owner object, so that -unauthorized dependents cannot delay deletion of an owner object. - -If an object's `ownerReferences` field is set by a controller (such as Deployment or ReplicaSet), -blockOwnerDeletion is set automatically and you do not need to manually modify this field. - -### Background cascading deletion - -In *background cascading deletion*, Kubernetes deletes the owner object -immediately and the garbage collector then deletes the dependents in -the background. - -### Setting the cascading deletion policy - -To control the cascading deletion policy, set the `propagationPolicy` -field on the `deleteOptions` argument when deleting an Object. Possible values include "Orphan", -"Foreground", or "Background". - -Here's an example that deletes dependents in background: - -```shell -kubectl proxy --port=8080 -curl -X DELETE localhost:8080/apis/apps/v1/namespaces/default/replicasets/my-repset \ - -d '{"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Background"}' \ - -H "Content-Type: application/json" -``` - -Here's an example that deletes dependents in foreground: - -```shell -kubectl proxy --port=8080 -curl -X DELETE localhost:8080/apis/apps/v1/namespaces/default/replicasets/my-repset \ - -d '{"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Foreground"}' \ - -H "Content-Type: application/json" -``` - -Here's an example that orphans dependents: - -```shell -kubectl proxy --port=8080 -curl -X DELETE localhost:8080/apis/apps/v1/namespaces/default/replicasets/my-repset \ - -d '{"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Orphan"}' \ - -H "Content-Type: application/json" -``` - -kubectl also supports cascading deletion. - -To delete dependents in the foreground using kubectl, set `--cascade=foreground`. To -orphan dependents, set `--cascade=orphan`. - -The default behavior is to delete the dependents in the background which is the -behavior when `--cascade` is omitted or explicitly set to `background`. - -Here's an example that orphans the dependents of a ReplicaSet: - -```shell -kubectl delete replicaset my-repset --cascade=orphan -``` - -### Additional note on Deployments - -Prior to 1.7, When using cascading deletes with Deployments you *must* use `propagationPolicy: Foreground` -to delete not only the ReplicaSets created, but also their Pods. If this type of _propagationPolicy_ -is not used, only the ReplicaSets will be deleted, and the Pods will be orphaned. -See [kubeadm/#149](https://github.com/kubernetes/kubeadm/issues/149#issuecomment-284766613) for more information. - -## Known issues - -Tracked at [#26120](https://github.com/kubernetes/kubernetes/issues/26120) - - - -## {{% heading "whatsnext" %}} - - -[Design Doc 1](https://git.k8s.io/community/contributors/design-proposals/api-machinery/garbage-collection.md) - -[Design Doc 2](https://git.k8s.io/community/contributors/design-proposals/api-machinery/synchronous-garbage-collection.md) diff --git a/content/en/docs/reference/glossary/finalizer.md b/content/en/docs/reference/glossary/finalizer.md new file mode 100644 index 0000000000..c44386fbf3 --- /dev/null +++ b/content/en/docs/reference/glossary/finalizer.md @@ -0,0 +1,31 @@ +--- +title: Finalizer +id: finalizer +date: 2021-07-07 +full_link: /docs/concepts/overview/working-with-objects/finalizers/ +short_description: > + A namespaced key that tells Kubernetes to wait until specific conditions are met + before it fully deletes an object marked for deletion. +aka: +tags: +- fundamental +- operation +--- +Finalizers are namespaced keys that tell Kubernetes to wait until specific +conditions are met before it fully deletes resources marked for deletion. +Finalizers alert {{}} +to clean up resources the deleted object owned. + + + +When you tell Kubernetes to delete an object that has finalizers specified for +it, the Kubernetes API marks the object for deletion, putting it into a +read-only state. The target object remains in a terminating state while the +control plane, or other components, take the actions defined by the finalizers. +After these actions are complete, the controller removes the relevant finalizers +from the target object. When the `metadata.finalizers` field is empty, +Kubernetes considers the deletion complete. + +You can use finalizers to control {{}} +of resources. For example, you can define a finalizer to clean up related resources or +infrastructure before the controller deletes the target resource. \ No newline at end of file diff --git a/content/en/docs/reference/glossary/garbage-collection.md b/content/en/docs/reference/glossary/garbage-collection.md new file mode 100644 index 0000000000..ec2fe19af7 --- /dev/null +++ b/content/en/docs/reference/glossary/garbage-collection.md @@ -0,0 +1,24 @@ +--- +title: Garbage Collection +id: garbage-collection +date: 2021-07-07 +full_link: /docs/concepts/workloads/controllers/garbage-collection/ +short_description: > + A collective term for the various mechanisms Kubernetes uses to clean up cluster + resources. + +aka: +tags: +- fundamental +- operation +--- + Garbage collection is a collective term for the various mechanisms Kubernetes uses to clean up + cluster resources. + + + +Kubernetes uses garbage collection to clean up resources like [unused containers and images](/docs/concepts/workloads/controllers/garbage-collection/#containers-images), +[failed Pods](/docs/concepts/workloads/pods/pod-lifecycle/#pod-garbage-collection), +[objects owned by the targeted resource](/docs/concepts/overview/working-with-objects/owners-dependents/), +[completed Jobs](/docs/concepts/workloads/controllers/ttlafterfinished/), and resources +that have expired or failed. \ No newline at end of file diff --git a/content/en/docs/tasks/administer-cluster/use-cascading-deletion.md b/content/en/docs/tasks/administer-cluster/use-cascading-deletion.md new file mode 100644 index 0000000000..eb72d68de0 --- /dev/null +++ b/content/en/docs/tasks/administer-cluster/use-cascading-deletion.md @@ -0,0 +1,352 @@ +--- +title: Use Cascading Deletion in a Cluster +content_type: task +--- + + + +This page shows you how to specify the type of [cascading deletion](/docs/concepts/workloads/controllers/garbage-collection/#cascading-deletion) +to use in your cluster during {{}}. + +## {{% heading "prerequisites" %}} + +{{< include "task-tutorial-prereqs.md" >}} + +You also need to [create a sample Deployment](/docs/tasks/run-application/run-stateless-application-deployment/#creating-and-exploring-an-nginx-deployment) +to experiment with the different types of cascading deletion. You will need to +recreate the Deployment for each type. + +## Check owner references on your pods + +Check that the `ownerReferences` field is present on your pods: + +```shell +kubectl get pods -l app=nginx --output=yaml +``` + +The output has an `ownerReferences` field similar to this: + +``` +apiVersion: v1 + ... + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: ReplicaSet + name: nginx-deployment-6b474476c4 + uid: 4fdcd81c-bd5d-41f7-97af-3a3b759af9a7 + ... +``` + +## Use foreground cascading deletion {#use-foreground-cascading-deletion} + +By default, Kubernetes uses [background cascading deletion](/docs/concepts/workloads/controllers/garbage-collection/#background-deletion) +to delete dependents of an object. You can switch to foreground cascading deletion +using either `kubectl` or the Kubernetes API, depending on the Kubernetes +version your cluster runs. {{}} + +{{}} +{{% tab name="Kubernetes 1.20.x and later" %}} +You can delete objects using foreground cascading deletion using `kubectl` or the +Kubernetes API. + +**Using kubectl** + +Run the following command: + + +```shell +kubectl delete deployment nginx-deployment --cascade=foreground +``` + +**Using the Kubernetes API** + +1. Start a local proxy session: + + ```shell + kubectl proxy --port=8080 + ``` + +1. Use `curl` to trigger deletion: + + ```shell + curl -X DELETE localhost:8080/apis/apps/v1/namespaces/default/deployments/nginx-deployment \ + -d '{"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Foreground"}' \ + -H "Content-Type: application/json" + ``` + + The output contains a `foregroundDeletion` {{}} + like this: + + ``` + "kind": "Deployment", + "apiVersion": "apps/v1", + "metadata": { + "name": "nginx-deployment", + "namespace": "default", + "uid": "d1ce1b02-cae8-4288-8a53-30e84d8fa505", + "resourceVersion": "1363097", + "creationTimestamp": "2021-07-08T20:24:37Z", + "deletionTimestamp": "2021-07-08T20:27:39Z", + "finalizers": [ + "foregroundDeletion" + ] + ... + ``` + +{{% /tab %}} +{{% tab name="Versions prior to Kubernetes 1.20.x" %}} +You can delete objects using foreground cascading deletion by calling the +Kubernetes API. + +For details, read the [documentation for your Kubernetes version](/docs/home/supported-doc-versions/). + +1. Start a local proxy session: + + ```shell + kubectl proxy --port=8080 + ``` + +1. Use `curl` to trigger deletion: + + ```shell + curl -X DELETE localhost:8080/apis/apps/v1/namespaces/default/deployments/nginx-deployment \ + -d '{"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Foreground"}' \ + -H "Content-Type: application/json" + ``` + + The output contains a `foregroundDeletion` {{}} + like this: + + ``` + "kind": "Deployment", + "apiVersion": "apps/v1", + "metadata": { + "name": "nginx-deployment", + "namespace": "default", + "uid": "d1ce1b02-cae8-4288-8a53-30e84d8fa505", + "resourceVersion": "1363097", + "creationTimestamp": "2021-07-08T20:24:37Z", + "deletionTimestamp": "2021-07-08T20:27:39Z", + "finalizers": [ + "foregroundDeletion" + ] + ... + ``` +{{% /tab %}} +{{}} + +## Use background cascading deletion {#use-background-cascading-deletion} + +1. [Create a sample Deployment](/docs/tasks/run-application/run-stateless-application-deployment/#creating-and-exploring-an-nginx-deployment). +1. Use either `kubectl` or the Kubernetes API to delete the Deployment, + depending on the Kubernetes version your cluster runs. {{}} + +{{}} +{{% tab name="Kubernetes version 1.20.x and later" %}} + +You can delete objects using background cascading deletion using `kubectl` +or the Kubernetes API. + +Kubernetes uses background cascading deletion by default, and does so +even if you run the following commands without the `--cascade` flag or the +`propagationPolicy` argument. + +**Using kubectl** + +Run the following command: + +```shell +kubectl delete deployment nginx-deployment --cascade=background +``` + +**Using the Kubernetes API** + +1. Start a local proxy session: + + ```shell + kubectl proxy --port=8080 + ``` + +1. Use `curl` to trigger deletion: + + ```shell + curl -X DELETE localhost:8080/apis/apps/v1/namespaces/default/deployments/nginx-deployment \ + -d '{"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Background"}' \ + -H "Content-Type: application/json" + ``` + + The output is similar to this: + + ``` + "kind": "Status", + "apiVersion": "v1", + ... + "status": "Success", + "details": { + "name": "nginx-deployment", + "group": "apps", + "kind": "deployments", + "uid": "cc9eefb9-2d49-4445-b1c1-d261c9396456" + } + ``` +{{% /tab %}} +{{% tab name="Versions prior to Kubernetes 1.20.x" %}} +Kubernetes uses background cascading deletion by default, and does so +even if you run the following commands without the `--cascade` flag or the +`propagationPolicy: Background` argument. + +For details, read the [documentation for your Kubernetes version](/docs/home/supported-doc-versions/). + +**Using kubectl** + +Run the following command: + +```shell +kubectl delete deployment nginx-deployment --cascade=true +``` + +**Using the Kubernetes API** + +1. Start a local proxy session: + + ```shell + kubectl proxy --port=8080 + ``` + +1. Use `curl` to trigger deletion: + + ```shell + curl -X DELETE localhost:8080/apis/apps/v1/namespaces/default/deployments/nginx-deployment \ + -d '{"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Background"}' \ + -H "Content-Type: application/json" + ``` + + The output is similar to this: + + ``` + "kind": "Status", + "apiVersion": "v1", + ... + "status": "Success", + "details": { + "name": "nginx-deployment", + "group": "apps", + "kind": "deployments", + "uid": "cc9eefb9-2d49-4445-b1c1-d261c9396456" + } + ``` +{{% /tab %}} +{{}} + + +## Delete owner objects and orphan dependents {#set-orphan-deletion-policy} + +By default, when you tell Kubernetes to delete an object, the +{{}} also deletes +dependent objects. You can make Kubernetes *orphan* these dependents using +`kubectl` or the Kubernetes API, depending on the Kubernetes version your +cluster runs. {{}} + +{{}} +{{% tab name="Kubernetes version 1.20.x and later" %}} + +**Using kubectl** + +Run the following command: + +```shell +kubectl delete deployment nginx-deployment --cascade=orphan +``` + +**Using the Kubernetes API** + +1. Start a local proxy session: + + ```shell + kubectl proxy --port=8080 + ``` + +1. Use `curl` to trigger deletion: + + ```shell + curl -X DELETE localhost:8080/apis/apps/v1/namespaces/default/deployments/nginx-deployment \ + -d '{"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Orphan"}' \ + -H "Content-Type: application/json" + ``` + + The output contains `orphan` in the `finalizers` field, similar to this: + + ``` + "kind": "Deployment", + "apiVersion": "apps/v1", + "namespace": "default", + "uid": "6f577034-42a0-479d-be21-78018c466f1f", + "creationTimestamp": "2021-07-09T16:46:37Z", + "deletionTimestamp": "2021-07-09T16:47:08Z", + "deletionGracePeriodSeconds": 0, + "finalizers": [ + "orphan" + ], + ... + ``` + +{{% /tab %}} +{{% tab name="Versions prior to Kubernetes 1.20.x" %}} + +For details, read the [documentation for your Kubernetes version](/docs/home/supported-doc-versions/). + +**Using kubectl** + +Run the following command: + +```shell +kubectl delete deployment nginx-deployment --cascade=false +``` + +**Using the Kubernetes API** + +1. Start a local proxy session: + + ```shell + kubectl proxy --port=8080 + ``` + +1. Use `curl` to trigger deletion: + + ```shell + curl -X DELETE localhost:8080/apis/apps/v1/namespaces/default/deployments/nginx-deployment \ + -d '{"kind":"DeleteOptions","apiVersion":"v1","propagationPolicy":"Orphan"}' \ + -H "Content-Type: application/json" + ``` + + The output contains `orphan` in the `finalizers` field, similar to this: + + ``` + "kind": "Deployment", + "apiVersion": "apps/v1", + "namespace": "default", + "uid": "6f577034-42a0-479d-be21-78018c466f1f", + "creationTimestamp": "2021-07-09T16:46:37Z", + "deletionTimestamp": "2021-07-09T16:47:08Z", + "deletionGracePeriodSeconds": 0, + "finalizers": [ + "orphan" + ], + ... + ``` +{{% /tab %}} +{{}} + +You can check that the Pods managed by the Deployment are still running: + +```shell +kubectl get pods -l app=nginx +``` + +## {{% heading "whatsnext" %}} + +* Learn about [owners and dependents](/docs/concepts/overview/working-with-objects/owners-dependents/) in Kubernetes. +* Learn about Kubernetes [finalizers](/docs/concepts/overview/working-with-objects/finalizers/). +* Learn about [garbage collection](/docs/concepts/workloads/controllers/garbage-collection/). \ No newline at end of file diff --git a/static/_redirects b/static/_redirects index 0d95cef56f..5c6c1bd0b9 100644 --- a/static/_redirects +++ b/static/_redirects @@ -89,6 +89,7 @@ /docs/concepts/cluster-administration/device-plugins/ /docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/ 301 /docs/concepts/cluster-administration/etcd-upgrade/ /docs/tasks/administer-cluster/configure-upgrade-etcd/ 301 /docs/concepts/cluster-administration/guaranteed-scheduling-critical-addon-pods/ /docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ 301 +/docs/concepts/cluster-administration/kubelet-garbage-collection/ /docs/concepts/architecture/garbage-collection/#containers-images 301 /docs/concepts/cluster-administration/master-node-communication/ /docs/concepts/architecture/master-node-communication/ 301 /docs/concepts/cluster-administration/network-plugins/ /docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/ 301 /docs/concepts/cluster-administration/out-of-resource/ /docs/concepts/scheduling-eviction/node-pressure-eviction/ 301 @@ -145,7 +146,7 @@ /docs/concepts/workloads/controllers/cron-jobs/deployment/ /docs/concepts/workloads/controllers/cron-jobs/ 301 /docs/concepts/workloads/controllers/daemonset/docs/concepts/workloads/pods/pod/ /docs/concepts/workloads/pods/ 301 /docs/concepts/workloads/controllers/deployment/docs/concepts/workloads/pods/pod/ /docs/concepts/workloads/pods/ 301 - +/docs/concepts/workloads/controllers/garbage-collection/ /docs/concepts/architecture/garbage-collection/ 301 /docs/concepts/workloads/controllers/jobs-run-to-completion/ /docs/concepts/workloads/controllers/job/ 301 /docs/concepts/workloads/controllers/statefulsets/ /docs/concepts/workloads/controllers/statefulset/ 301 /docs/concepts/workloads/controllers/statefulset.md /docs/concepts/workloads/controllers/statefulset/ 301! From 442a1bf172315c17c7b0d9879436cd22efa4be29 Mon Sep 17 00:00:00 2001 From: James Strong Date: Tue, 13 Jul 2021 17:03:00 -0400 Subject: [PATCH 087/175] rename --- ...h-ingress-nginx.md => 2021-07-15-update-with-ingress-nginx.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename content/en/blog/_posts/{2021-06-21-update-with-ingress-nginx.md => 2021-07-15-update-with-ingress-nginx.md} (100%) diff --git a/content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-07-15-update-with-ingress-nginx.md similarity index 100% rename from content/en/blog/_posts/2021-06-21-update-with-ingress-nginx.md rename to content/en/blog/_posts/2021-07-15-update-with-ingress-nginx.md From 7aa360d29f6ac6d041c537dc3139d0dc62bfcc60 Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Tue, 13 Jul 2021 23:41:05 +0100 Subject: [PATCH 088/175] Revise API removal article --- .../index.md | 49 ++++++++++++------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md b/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md index 632a7a808d..337d9f2342 100644 --- a/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md +++ b/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md @@ -38,6 +38,9 @@ API versions. * The beta `Lease` API (**coordination.k8s.io/v1beta1**) * All beta `Ingress` APIs (the **extensions/v1beta1** and **networking.k8s.io/v1beta1** API versions) +The Kubernetes documentation covers these +[API removals for v1.22](/docs/reference/using-api/deprecation-guide/#v1-22) and explains +how each of those APIs change between beta and stable. ## What to do @@ -94,12 +97,13 @@ and explain the steps you'll need to take. `TokenReview` : Migrate to use the **authentication.k8s.io/v1** [TokenReview](/docs/reference/kubernetes-api/authentication-resources/token-review-v1/) - API, available since v1.10. + API, available since v1.10. - In Kubernetes version v1.21 and earlier, the Kubernetes API server - [defaults](/docs/reference/access-authn-authz/authentication/#webhook-token-authentication) - to sending _beta_ TokenReviews to webhooks. See [Rehearse for the upgrade](#rehearse-for-the-upgrade) - for some specific tips about switching to the stable API. + As well as serving this API via HTTP, the Kubernetes API server uses the same format to + [send](/docs/reference/access-authn-authz/authentication/#webhook-token-authentication) + TokenReviews to webhooks. The v1.22 release continues to use the v1beta1 API for TokenReviews + sent to webhooks. See [Looking ahead](#looking-ahead) for some specific tips about + switching to the stable API. `SubjectAccessReview`, `SelfSubjectAccessReview` and `LocalSubjectAccessReview` : Migrate to use the **authorization.k8s.io/v1** versions of those @@ -150,7 +154,10 @@ removals before you upgrade to Kubernetes v1.22. To do that, add the following to the kube-apiserver command line arguments: -`--runtime-config=admissionregistration.k8s.io/v1beta1=false,apiextensions.k8s.io/v1beta1=false,apiregistration.k8s.io/v1beta1=false,authentication.k8s.io/v1beta1=false,authorization.k9s.io/v1=false,certificates.k8s.io/v1beta=false,coordination.k8s.io/v1beta1=false,extensions/v1beta1/ingresses=false,networking.k8s.io/v1beta1/ingresses=false,networking.k8s.io/v1beta1/ingressclasses=false` +`--runtime-config=admissionregistration.k8s.io/v1beta1=false,apiextensions.k8s.io/v1beta1=false,apiregistration.k8s.io/v1beta1=false,authentication.k8s.io/v1beta1=false,authorization.k9s.io/v1=false,certificates.k8s.io/v1beta=false,coordination.k8s.io/v1beta1=false,extensions/v1beta1/ingresses=false,networking.k8s.io/v1beta1=false` + +(as a side effect, this also turns off v1beta1 of EndpointSlice - watch out for +that when you're testing). Once you've switched all the kube-apiservers in your cluster to use that setting, those beta APIs are removed. You can test that API clients (`kubectl`, deployment @@ -158,16 +165,6 @@ tools, custom controllers etc) still work how you expect, and you can revert if you need to without having to plan a more disruptive downgrade. -There's another setting that's relevant if you use webhook authentication checks. -Kubernetes v1.22 is the first version of Kubernetes that sends TokenReview objects -to webhooks using the `authentication.k8s.io/v1` API by default. However, you can -switch over earlier to try it out. -Add `--authentication-token-webhook-version=v1` to the command line options for -the kube-apiserver, and check that webhooks for authentication still work how you -expected. - -Once you're happy it works, you can leave the `--authentication-token-webhook-version=v1` -option set through and after the upgrade to v1.22. ### Advice for software authors @@ -237,9 +234,23 @@ are documented. ### Looking ahead -The upcoming **v1.25** release will stop serving beta versions of several Kubernetes APIs -that have been stable for some time. The same release **removes** PodSecurityPolicy, -which is deprecated and won't graduate to stable. See +There's a setting that's relevant if you use webhook authentication checks. +A future Kubernetes release will switch to sending TokenReview objects +to webhooks using the `authentication.k8s.io/v1` API by default. At the moment, +the default is to send `authentication.k8s.io/v1beta1` TokenReviews to webhooks, +and that's still the default for Kubernetes v1.22. +However, you can switch over to the stable API right now if you want: +add `--authentication-token-webhook-version=v1` to the command line options for +the kube-apiserver, and check that webhooks for authentication still work how you +expected. + +Once you're happy it works OK, you can leave the `--authentication-token-webhook-version=v1` +option set across your control plane. + +The **v1.25** release that's planned for next year will stop serving beta versions of +several Kubernetes APIs that are stable right now and have been for some time. +The same v1.25 release will **remove** PodSecurityPolicy, which is deprecated and won't +graduate to stable. See [PodSecurityPolicy Deprecation: Past, Present, and Future](/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/) for more information. From 9296fc4ff038274a88d79c82463a4254665b9f6f Mon Sep 17 00:00:00 2001 From: Toshiaki Inukai Date: Wed, 14 Jul 2021 01:36:20 +0000 Subject: [PATCH 089/175] Add description and modify title --- .../ja/docs/concepts/overview/working-with-objects/_index.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/content/ja/docs/concepts/overview/working-with-objects/_index.md b/content/ja/docs/concepts/overview/working-with-objects/_index.md index d4a9f2e6b6..10da27655c 100644 --- a/content/ja/docs/concepts/overview/working-with-objects/_index.md +++ b/content/ja/docs/concepts/overview/working-with-objects/_index.md @@ -1,5 +1,8 @@ --- -title: "Kubernetesのオブジェクトについて" +title: "Kubernetesオブジェクトを利用する" weight: 40 +description: > + Kubernetesオブジェクトは、Kubernetes上で永続的なエンティティです。Kubernetesはこれらのエンティティを使い、クラスターの状態を表現します。 + Kubernetesオブジェクトモデルと、これらのオブジェクトの利用方法について学びます。 --- From d59c77f6478659deb810098d3b0b73c49714cb3c Mon Sep 17 00:00:00 2001 From: Taylor Dolezal Date: Tue, 13 Jul 2021 18:53:44 -0700 Subject: [PATCH 090/175] Clarify v1beta1 API webhook behavior in 1.22 --- .../2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md b/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md index 337d9f2342..bfed8d9407 100644 --- a/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md +++ b/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md @@ -102,7 +102,7 @@ and explain the steps you'll need to take. As well as serving this API via HTTP, the Kubernetes API server uses the same format to [send](/docs/reference/access-authn-authz/authentication/#webhook-token-authentication) TokenReviews to webhooks. The v1.22 release continues to use the v1beta1 API for TokenReviews - sent to webhooks. See [Looking ahead](#looking-ahead) for some specific tips about + sent to webhooks by default. See [Looking ahead](#looking-ahead) for some specific tips about switching to the stable API. `SubjectAccessReview`, `SelfSubjectAccessReview` and `LocalSubjectAccessReview` From 30c6cec1ba2a60dd731550e11843c27d7febd460 Mon Sep 17 00:00:00 2001 From: seokho-son Date: Mon, 12 Jul 2021 16:08:07 +0900 Subject: [PATCH 091/175] Update outdated files in dev-1.21-ko.6 (p1) --- .../configure-access-multiple-clusters.md | 7 ++++ .../tasks/administer-cluster/certificates.md | 5 ++- .../highly-available-control-plane.md | 38 ++++++++++--------- .../ko/docs/tasks/tls/certificate-rotation.md | 4 +- .../configure-redis-using-configmap.md | 4 +- .../ko/examples/policy/restricted-psp.yaml | 9 ++--- 6 files changed, 40 insertions(+), 27 deletions(-) diff --git a/content/ko/docs/tasks/access-application-cluster/configure-access-multiple-clusters.md b/content/ko/docs/tasks/access-application-cluster/configure-access-multiple-clusters.md index 8d25bb7ca6..b3997580f2 100644 --- a/content/ko/docs/tasks/access-application-cluster/configure-access-multiple-clusters.md +++ b/content/ko/docs/tasks/access-application-cluster/configure-access-multiple-clusters.md @@ -20,6 +20,13 @@ card: 반드시 존재해야 한다는 것을 의미하는 것은 아니다. {{< /note >}} + +{{< warning >}} +신뢰할 수 있는 소스의 kubeconfig 파일만 사용해야 한다. 특수 제작된 kubeconfig 파일은 악성코드를 실행하거나 파일을 노출시킬 수 있다. +신뢰할 수 없는 kubeconfig 파일을 꼭 사용해야 한다면, 셸 스크립트를 사용하는 경우처럼 신중한 검사가 선행되어야 한다. +{{< /warning>}} + + ## {{% heading "prerequisites" %}} {{< include "task-tutorial-prereqs.md" >}} diff --git a/content/ko/docs/tasks/administer-cluster/certificates.md b/content/ko/docs/tasks/administer-cluster/certificates.md index 44159fb22e..076f09faf4 100644 --- a/content/ko/docs/tasks/administer-cluster/certificates.md +++ b/content/ko/docs/tasks/administer-cluster/certificates.md @@ -116,7 +116,10 @@ weight: 20 openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key \ -CAcreateserial -out server.crt -days 10000 \ -extensions v3_ext -extfile csr.conf -1. 인증서를 본다. +1. 인증서 서명 요청을 확인한다. + + openssl req -noout -text -in ./server.csr +1. 인증서를 확인한다. openssl x509 -noout -text -in ./server.crt diff --git a/content/ko/docs/tasks/administer-cluster/highly-available-control-plane.md b/content/ko/docs/tasks/administer-cluster/highly-available-control-plane.md index ae6f79d690..56cc5b3d9a 100644 --- a/content/ko/docs/tasks/administer-cluster/highly-available-control-plane.md +++ b/content/ko/docs/tasks/administer-cluster/highly-available-control-plane.md @@ -1,7 +1,9 @@ --- -reviewers: + + title: 고가용성 쿠버네티스 클러스터 컨트롤 플레인 설정하기 content_type: task + --- @@ -62,7 +64,7 @@ HA 호환 클러스터를 생성했다면, 여기에 컨트롤 플레인 노드 HA 호환 클러스터를 시작할 때, 상속되는 `MULTIZONE`이나 `ENABLE_ETCD_QUORUM_READS` 플래그를 따로 설정할 필요는 없다. -다음 샘플 커맨드는 기존 HA 호환 클러스터에서 +다음 샘플 커맨드는 기존 HA 호환 클러스터에서 컨트롤 플레인 노드를 복제한다. ```shell @@ -89,39 +91,41 @@ KUBE_DELETE_NODES=false KUBE_GCE_ZONE=europe-west1-c ./cluster/kube-down.sh ## 동작에 실패한 컨트롤 플레인 노드 처리 HA 클러스터의 컨트롤 플레인 노드 중 하나가 동작에 실패하면, -클러스터에서 해당 노드를 제거하고 동일한 영역에 새 컨트롤 플레인 노드를 추가하는 것이 가장 좋다. +클러스터에서 해당 노드를 제거하고 동일한 영역에 새 컨트롤 플레인 +노드를 추가하는 것이 가장 좋다. 다음 샘플 커맨드로 이 과정을 시연한다. 1. 손상된 복제본을 제거한다. - ```shell - KUBE_DELETE_NODES=false KUBE_GCE_ZONE=replica_zone KUBE_REPLICA_NAME=replica_name ./cluster/kube-down.sh - ``` +```shell +KUBE_DELETE_NODES=false KUBE_GCE_ZONE=replica_zone KUBE_REPLICA_NAME=replica_name ./cluster/kube-down.sh +``` -1. 기존 복제본 대신 새 노드를 추가한다. +

  1. 기존 복제본을 대신할 새 노드를 추가한다.
- ```shell - KUBE_GCE_ZONE=replica-zone KUBE_REPLICATE_EXISTING_MASTER=true ./cluster/kube-up.sh - ``` +```shell +KUBE_GCE_ZONE=replica-zone KUBE_REPLICATE_EXISTING_MASTER=true ./cluster/kube-up.sh +``` ## HA 클러스터에서 컨트롤 플레인 노드 복제에 관한 모범 사례 -* 다른 영역에 컨트롤 플레인 노드를 배치하도록 한다. 한 영역이 동작에 실패하는 동안, +* 다른 영역에 컨트롤 플레인 노드를 배치하도록 한다. 한 영역이 동작에 실패하는 동안, 해당 영역에 있는 컨트롤 플레인 노드도 모두 동작에 실패할 것이다. 영역 장애를 극복하기 위해 노드를 여러 영역에 배치한다 (더 자세한 내용은 [멀티 영역](/ko/docs/setup/best-practices/multiple-zones/)를 참조한다). -* 두 개의 노드로 구성된 컨트롤 플레인은 사용하지 않는다. 두 개의 노드로 구성된 +* 두 개의 노드로 구성된 컨트롤 플레인은 사용하지 않는다. 두 개의 노드로 구성된 컨트롤 플레인에서의 합의를 위해서는 지속적 상태(persistent state) 변경 시 두 컨트롤 플레인 노드가 모두 정상적으로 동작 중이어야 한다. -결과적으로 두 컨트롤 플레인 노드 모두 필요하고, 둘 중 한 컨트롤 플레인 노드에만 장애가 발생해도 +결과적으로 두 컨트롤 플레인 노드 모두 필요하고, 둘 중 한 컨트롤 플레인 노드에만 장애가 발생해도 클러스터의 심각한 장애 상태를 초래한다. -따라서 HA 관점에서는 두 개의 노드로 구성된 컨트롤 플레인은 +따라서 HA 관점에서는 두 개의 노드로 구성된 컨트롤 플레인은 단일 노드로 구성된 컨트롤 플레인보다도 못하다. * 컨트롤 플레인 노드를 추가하면, 클러스터의 상태(Etcd)도 새 인스턴스로 복사된다. 클러스터가 크면, 이 상태를 복제하는 시간이 오래 걸릴 수 있다. 이 작업은 [etcd 관리 가이드](https://etcd.io/docs/v2.3/admin_guide/#member-migration)에 기술한 대로 -Etcd 데이터 디렉터리를 마이그레이션하여 속도를 높일 수 있다(향후에 Etcd 데이터 디렉터리 마이그레이션 지원 추가를 고려 중이다). +Etcd 데이터 디렉터리를 마이그레이션하여 속도를 높일 수 있다. +(향후에 Etcd 데이터 디렉터리 마이그레이션 지원 추가를 고려 중이다) @@ -152,14 +156,14 @@ Etcd 데이터 디렉터리를 마이그레이션하여 속도를 높일 수 있 해당 IP 주소는 마지막으로 남은 복제본에 할당된다. 로드 밸런서 생성 및 제거는 복잡한 작업이며, 이를 전파하는 데 시간(~20분)이 걸릴 수 있다. -### 마스터 서비스와 Kubelet +### 컨트롤 플레인 서비스와 Kubelet 쿠버네티스 서비스에서 최신의 쿠버네티스 API 서버 목록을 유지하는 대신, 시스템은 모든 트래픽을 외부 IP 주소로 보낸다. * 단일 노드 컨트롤 플레인의 경우, IP 주소는 단일 컨트롤 플레인 노드를 가리킨다. -* 고가용성 컨트롤 플레인의 경우, IP 주소는 마스터 앞의 로드밸런서를 가리킨다. +* 고가용성 컨트롤 플레인의 경우, IP 주소는 컨트롤 플레인 노드 앞의 로드밸런서를 가리킨다. 마찬가지로 Kubelet은 외부 IP 주소를 사용하여 컨트롤 플레인과 통신한다. diff --git a/content/ko/docs/tasks/tls/certificate-rotation.md b/content/ko/docs/tasks/tls/certificate-rotation.md index 037f99d87a..eadec87b4f 100644 --- a/content/ko/docs/tasks/tls/certificate-rotation.md +++ b/content/ko/docs/tasks/tls/certificate-rotation.md @@ -27,10 +27,10 @@ kubelet은 쿠버네티스 API 인증을 위해 인증서를 사용한다. 기본적으로 이러한 인증서는 1년 만기로 발급되므로 너무 자주 갱신할 필요는 없다. -쿠버네티스 1.8은 [kubelet 인증서 +쿠버네티스는 [kubelet 인증서 갱신](/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/)을 포함하며, 이 기능은 현재 인증서의 만료 시한이 임박한 경우, -새로운 키를 자동으로 생성하고 쿠버네티스 API에서 새로운 인증서를 요청하는 베타 기능이다. +새로운 키를 자동으로 생성하고 쿠버네티스 API에서 새로운 인증서를 요청하는 기능이다. 새로운 인증서를 사용할 수 있게 되면 쿠버네티스 API에 대한 연결을 인증하는데 사용된다. diff --git a/content/ko/docs/tutorials/configuration/configure-redis-using-configmap.md b/content/ko/docs/tutorials/configuration/configure-redis-using-configmap.md index c1b21d1404..fb1ac922fc 100644 --- a/content/ko/docs/tutorials/configuration/configure-redis-using-configmap.md +++ b/content/ko/docs/tutorials/configuration/configure-redis-using-configmap.md @@ -55,7 +55,7 @@ EOF ```shell kubectl apply -f example-redis-config.yaml -kubectl apply -f https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/pods/config/redis-pod.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/pods/config/redis-pod.yaml ``` Redis 파드 매니페스트의 내용을 검토하고 다음의 사항을 염두에 둔다. @@ -206,7 +206,7 @@ kubectl exec -it redis -- redis-cli ```shell kubectl delete pod redis -kubectl apply -f https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/pods/config/redis-pod.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/pods/config/redis-pod.yaml ``` 이제 마지막으로 설정값을 다시 확인해 본다. diff --git a/content/ko/examples/policy/restricted-psp.yaml b/content/ko/examples/policy/restricted-psp.yaml index cbaf2758c0..4cdc12639a 100644 --- a/content/ko/examples/policy/restricted-psp.yaml +++ b/content/ko/examples/policy/restricted-psp.yaml @@ -5,14 +5,11 @@ metadata: annotations: seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default' apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' - seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' spec: privileged: false - # 루트로의 에스컬레이션을 방지하는데 필요하다. + # 루트로의 에스컬레이션을 방지하는 데 필요하다. allowPrivilegeEscalation: false - # 이것은 루트가 아닌 사용자 + 권한 에스컬레이션을 허용하지 않는 것으로 중복이지만, - # 심층 방어를 위해 이를 제공한다. requiredDropCapabilities: - ALL # 기본 볼륨 유형을 허용한다. @@ -22,8 +19,10 @@ spec: - 'projected' - 'secret' - 'downwardAPI' - # 클러스터 관리자가 설정한 퍼시스턴트볼륨을 사용하는 것이 안전하다고 가정한다. + # 클러스터 관리자에 의해 구성된 휘발성 CSI 드라이버와 퍼시스턴트볼륨(PersistentVolume)의 사용은 안전하다고 가정한다. + - 'csi' - 'persistentVolumeClaim' + - 'ephemeral' hostNetwork: false hostIPC: false hostPID: false From 397edb91f7436c7633e412e53b0153f347093ba7 Mon Sep 17 00:00:00 2001 From: Kunal Kushwaha Date: Wed, 14 Jul 2021 19:26:27 +0530 Subject: [PATCH 092/175] Add Sig-Usability-Spotlight Blog (#27809) * Add Sig-Usability-Spotlight Blog * change date --- .../2021-07-15-SIG-Usability-Spotlight.md | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 content/en/blog/_posts/2021-07-15-SIG-Usability-Spotlight.md diff --git a/content/en/blog/_posts/2021-07-15-SIG-Usability-Spotlight.md b/content/en/blog/_posts/2021-07-15-SIG-Usability-Spotlight.md new file mode 100644 index 0000000000..43488fc11f --- /dev/null +++ b/content/en/blog/_posts/2021-07-15-SIG-Usability-Spotlight.md @@ -0,0 +1,65 @@ +--- +layout: blog +title: "Spotlight on SIG Usability" +date: 2021-07-15 +slug: sig-usability-spotlight-2021 +--- + +**Author:** Kunal Kushwaha, Civo + +## Introduction + +Are you interested in learning about what [SIG Usability](https://github.com/kubernetes/community/tree/master/sig-usability) does and how you can get involved? Well, you're at the right place. SIG Usability is all about making Kubernetes more accessible to new folks, and its main activity is conducting user research for the community. In this blog, we have summarized our conversation with [Gaby Moreno](https://twitter.com/morengab), who walks us through the various aspects of being a part of the SIG and shares some insights about how others can get involved. + +Gaby is a co-lead for SIG Usability. She works as a Product Designer at IBM and enjoys working on the user experience of open, hybrid cloud technologies like Kubernetes, OpenShift, Terraform, and Cloud Foundry. + +## A summary of our conversation + +### Q. Could you tell us a little about what SIG Usability does? + +A. SIG Usability at a high level started because there was no dedicated user experience team for Kubernetes. The extent of SIG Usability is focussed on the end-client ease of use of the Kubernetes project. The main activity is user research for the community, which includes speaking to Kubernetes users. + +This covers points like user experience and accessibility. The objectives of the SIG are to guarantee that the Kubernetes project is maximally usable by people of a wide range of foundations and capacities, such as incorporating internationalization and ensuring the openness of documentation. + +### Q. Why should new and existing contributors consider joining SIG Usability? + +A. There are plenty of territories where new contributors can begin. For example: +- User research projects, where people can help understand the usability of the end-user experiences, including error messages, end-to-end tasks, etc. +- Accessibility guidelines for Kubernetes community artifacts, examples include: internationalization of documentation, color choices for people with color blindness, ensuring compatibility with screen reader technology, user interface design for core components with user interfaces, and more. + +### Q. What do you do to help new contributors get started? + +A. New contributors can get started by shadowing one of the user interviews, going through user interview transcripts, analyzing them, and designing surveys. + +SIG Usability is also open to new project ideas. If you have an idea, we’ll do what we can to support it. There are regular SIG Meetings where people can ask their questions live. These meetings are also recorded for those who may not be able to attend. As always, you can reach out to us on Slack as well. + +### Q. What does the survey include? + +A. In simple terms, the survey gathers information about how people use Kubernetes, such as trends in learning to deploy a new system, error messages they receive, and workflows. + +One of our goals is to standardize the responses accordingly. The ultimate goal is to analyze survey responses for important user stories whose needs aren't being met. + +### Q. Are there any particular skills you’d like to recruit for? What skills are contributors to SIG Usability likely to learn? + +A. Although contributing to SIG Usability does not have any pre-requisites as such, experience with user research, qualitative research, or prior experience with how to conduct an interview would be great plus points. Quantitative research, like survey design and screening, is also helpful and something that we expect contributors to learn. + +### Q. What are you getting positive feedback on, and what’s coming up next for SIG Usability? + +A. We have had new members joining and coming to monthly meetings regularly and showing interests in becoming a contributor and helping the community. We have also had a lot of people reach out to us via Slack showcasing their interest in the SIG. + +Currently, we are focused on finishing the study mentioned in our [talk](https://www.youtube.com/watch?v=Byn0N_ZstE0), also our project for this year. We are always happy to have new contributors join us. + +### Q: Any closing thoughts/resources you’d like to share? + +A. We love meeting new contributors and assisting them in investigating different Kubernetes project spaces. We will work with and team up with other SIGs to facilitate engaging with end-users, running studies, and help them integrate accessible design practices into their development practices. + +Here are some resources for you to get started: +- [GitHub](https://github.com/kubernetes/community/tree/master/sig-usability) +- [Mailing list](https://groups.google.com/g/kubernetes-sig-usability) +- [Open Community Issues/PRs](https://github.com/kubernetes/community/labels/sig%2Fusability) +- [Slack](https://slack.k8s.io/) +- [Slack channel #sig-usability](https://kubernetes.slack.com/archives/CLC5EF63T) + +## Wrap Up + +SIG Usability hosted a [KubeCon talk](https://www.youtube.com/watch?v=Byn0N_ZstE0) about studying Kubernetes users' experiences. The talk focuses on updates to the user study projects, understanding who is using Kubernetes, what they are trying to achieve, how the project is addressing their needs, and where we need to improve the project and the client experience. Join the SIG's update to find out about the most recent research results, what the plans are for the forthcoming year, and how to get involved in the upstream usability team as a contributor! From f65f046cb5435329cd3341fc67cfbc6b36d71195 Mon Sep 17 00:00:00 2001 From: ruffrey <1668190+ruffrey@users.noreply.github.com> Date: Wed, 14 Jul 2021 09:45:25 -0700 Subject: [PATCH 093/175] Fix incorrect mv command for kubectl-convert Looks like the `mv` installation command for `kubectl-convert` had been copied from `kubectl` install, but not fully changed --- content/en/docs/tasks/tools/install-kubectl-macos.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/tasks/tools/install-kubectl-macos.md b/content/en/docs/tasks/tools/install-kubectl-macos.md index 2b214f6396..1cd6cb043e 100644 --- a/content/en/docs/tasks/tools/install-kubectl-macos.md +++ b/content/en/docs/tasks/tools/install-kubectl-macos.md @@ -228,7 +228,7 @@ Below are the procedures to set up autocompletion for Bash and Zsh. 1. Move the kubectl-convert binary to a file location on your system `PATH`. ```bash - sudo mv ./kubectl /usr/local/bin/kubectl-convert + sudo mv ./kubectl-convert /usr/local/bin/kubectl-convert sudo chown root: /usr/local/bin/kubectl-convert ``` From fcdb99341c831a8eeb4c86514547327f48be1b3b Mon Sep 17 00:00:00 2001 From: Arhell Date: Thu, 15 Jul 2021 02:53:24 +0300 Subject: [PATCH 094/175] [ja] Updated TTLAfterFinished feature reference --- .../reference/command-line-tools-reference/feature-gates.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/content/ja/docs/reference/command-line-tools-reference/feature-gates.md b/content/ja/docs/reference/command-line-tools-reference/feature-gates.md index 01a2c289e9..caab3b8c11 100644 --- a/content/ja/docs/reference/command-line-tools-reference/feature-gates.md +++ b/content/ja/docs/reference/command-line-tools-reference/feature-gates.md @@ -136,7 +136,8 @@ content_type: concept | `TokenRequest` | `true` | Beta | 1.12 | | | `TokenRequestProjection` | `false` | Alpha | 1.11 | 1.11 | | `TokenRequestProjection` | `true` | Beta | 1.12 | | -| `TTLAfterFinished` | `false` | Alpha | 1.12 | | +| `TTLAfterFinished` | `false` | Alpha | 1.12 | 1.20 | +| `TTLAfterFinished` | `true` | Beta | 1.21 | | | `TopologyManager` | `false` | Alpha | 1.16 | 1.17 | | `TopologyManager` | `true` | Beta | 1.18 | | | `ValidateProxyRedirects` | `false` | Alpha | 1.12 | 1.13 | From 060953606448013f8fbf8d8560cfecde3f828200 Mon Sep 17 00:00:00 2001 From: chenxuc Date: Thu, 15 Jul 2021 13:21:22 +0800 Subject: [PATCH 095/175] add chenxuc to sig-docs-zh-reviews --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 8dda114690..a4286d1a9a 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -163,6 +163,7 @@ aliases: # zhangxiaoyu-zidif sig-docs-zh-reviews: # PR reviews for Chinese content - chenrui333 + - chenxuc - howieyuen - idealhack - pigletfly From 047357392447bc10434ad79895eceaeda1ae480d Mon Sep 17 00:00:00 2001 From: yuswift Date: Thu, 15 Jul 2021 15:21:45 +0800 Subject: [PATCH 096/175] fix ko wrong numbered list Signed-off-by: yuswift --- .../participate/roles-and-responsibilities.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/content/ko/docs/contribute/participate/roles-and-responsibilities.md b/content/ko/docs/contribute/participate/roles-and-responsibilities.md index 897d638435..c01a97c6e1 100644 --- a/content/ko/docs/contribute/participate/roles-and-responsibilities.md +++ b/content/ko/docs/contribute/participate/roles-and-responsibilities.md @@ -146,12 +146,12 @@ LGTM은 "Looks good to me"의 약자이며 풀 리퀘스트가 기술적으로 1. `kubernetes/website` 리포지터리 내 [OWNERS_ALIASES](https://github.com/kubernetes/website/blob/master/OWNERS) 파일의 섹션에 여러분의 GitHub 사용자 이름을 추가하는 풀 리퀘스트를 연다. - - {{< note >}} - 자신을 추가할 위치가 확실하지 않으면, `sig-docs-ko-reviews` 에 추가한다. - {{< /note >}} - -1. PR을 하나 이상의 SIG-Docs 승인자(`sig-docs-{language}-owners` 에 + + {{< note >}} + 자신을 추가할 위치가 확실하지 않으면, `sig-docs-ko-reviews` 에 추가한다. + {{< /note >}} + +2. PR을 하나 이상의 SIG-Docs 승인자(`sig-docs-{language}-owners` 에 나열된 사용자 이름)에게 지정한다. 승인되면, SIG Docs 리더가 적당한 GitHub 팀에 여러분을 추가한다. 일단 추가되면, From 484dc86ddf1682b2b5fb5a28fc67250bce07463f Mon Sep 17 00:00:00 2001 From: keashem Date: Wed, 14 Jul 2021 17:40:50 +0800 Subject: [PATCH 097/175] Fix a translation error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the english version is :"no pods without the toleration will schedule on them"，the Chinese translation in kubernetes documents is ：“没有对应容忍度的 Pod会被调度到这些节点”，Here the translation omits the keyword "no"，it should be translated into Chinese：“没有对应容忍度的 Pod不会被调度到这些节点”。 --- .../docs/concepts/scheduling-eviction/taint-and-toleration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/zh/docs/concepts/scheduling-eviction/taint-and-toleration.md b/content/zh/docs/concepts/scheduling-eviction/taint-and-toleration.md index 7dfcfb110d..a2949c330c 100644 --- a/content/zh/docs/concepts/scheduling-eviction/taint-and-toleration.md +++ b/content/zh/docs/concepts/scheduling-eviction/taint-and-toleration.md @@ -312,7 +312,7 @@ manually add tolerations to your pods. 来表示特殊硬件，给配置了特殊硬件的节点添加污点时包含扩展资源名称， 然后运行一个 [ExtendedResourceToleration](/zh/docs/reference/access-authn-authz/admission-controllers/#extendedresourcetoleration) 准入控制器。此时，因为节点已经被设置污点了，没有对应容忍度的 Pod - 会被调度到这些节点。但当你创建一个使用了扩展资源的 Pod 时， + 不会被调度到这些节点。但当你创建一个使用了扩展资源的 Pod 时， `ExtendedResourceToleration` 准入控制器会自动给 Pod 加上正确的容忍度， 这样 Pod 就会被自动调度到这些配置了特殊硬件件的节点上。 这样就能够确保这些配置了特殊硬件的节点专门用于运行需要使用这些硬件的 Pod， From 7146318713c76725273607b50877bbb8d198b878 Mon Sep 17 00:00:00 2001 From: howieyuen Date: Thu, 15 Jul 2021 16:28:47 +0800 Subject: [PATCH 098/175] [zh] optimise translation of taint-and-toleration.md --- .../scheduling-eviction/taint-and-toleration.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/content/zh/docs/concepts/scheduling-eviction/taint-and-toleration.md b/content/zh/docs/concepts/scheduling-eviction/taint-and-toleration.md index 7dfcfb110d..6ac19a680a 100644 --- a/content/zh/docs/concepts/scheduling-eviction/taint-and-toleration.md +++ b/content/zh/docs/concepts/scheduling-eviction/taint-and-toleration.md @@ -6,16 +6,16 @@ weight: 40 -节点亲和性（详见[这里](/zh/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity)） +[_节点亲和性_](/zh/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) 是 {{< glossary_tooltip text="Pod" term_id="pod" >}} 的一种属性，它使 Pod -被吸引到一类特定的{{< glossary_tooltip text="节点" term_id="node" >}}。 -这可能出于一种偏好，也可能是硬性要求。 -Taint（污点）则相反，它使节点能够排斥一类特定的 Pod。 +被吸引到一类特定的{{< glossary_tooltip text="节点" term_id="node" >}} +（这可能出于一种偏好，也可能是硬性要求）。 +_污点_（Taint）则相反——它使节点能够排斥一类特定的 Pod。 -容忍度（Tolerations）是应用于 Pod 上的，允许（但并不要求）Pod +容忍度（Toleration）是应用于 Pod 上的，允许（但并不要求）Pod 调度到带有与之匹配的污点的节点上。 污点和容忍度（Toleration）相互配合，可以用来避免 Pod 被分配到不合适的节点上。 @@ -312,7 +312,7 @@ manually add tolerations to your pods. 来表示特殊硬件，给配置了特殊硬件的节点添加污点时包含扩展资源名称， 然后运行一个 [ExtendedResourceToleration](/zh/docs/reference/access-authn-authz/admission-controllers/#extendedresourcetoleration) 准入控制器。此时，因为节点已经被设置污点了，没有对应容忍度的 Pod - 会被调度到这些节点。但当你创建一个使用了扩展资源的 Pod 时， + 不会被调度到这些节点。但当你创建一个使用了扩展资源的 Pod 时， `ExtendedResourceToleration` 准入控制器会自动给 Pod 加上正确的容忍度， 这样 Pod 就会被自动调度到这些配置了特殊硬件件的节点上。 这样就能够确保这些配置了特殊硬件的节点专门用于运行需要使用这些硬件的 Pod， From 131632c58e4780b26ca4b216094de05a31f483be Mon Sep 17 00:00:00 2001 From: Jihoon Seo Date: Thu, 15 Jul 2021 15:18:16 +0900 Subject: [PATCH 099/175] Add jihoon-seo to sig-docs-ko-reviews --- OWNERS_ALIASES | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 227f4e9e05..7622b37fe4 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -132,10 +132,11 @@ aliases: - ClaudiaJKang - gochist - ianychoi - - seokho-son - - ysyukr + - jihoon-seo - pjhwa + - seokho-son - yoonian + - ysyukr sig-docs-leads: # Website chairs and tech leads - irvifa - jimangel From 9d0d21381461ff744843ecac3bf00ca45f94e751 Mon Sep 17 00:00:00 2001 From: yxxhero Date: Wed, 14 Jul 2021 20:10:29 +0800 Subject: [PATCH 100/175] fix incorrect field name in event API deprecation guide Signed-off-by: yxxhero --- content/en/docs/reference/using-api/deprecation-guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/docs/reference/using-api/deprecation-guide.md b/content/en/docs/reference/using-api/deprecation-guide.md index 73a4ae2a18..aa5ad18fee 100644 --- a/content/en/docs/reference/using-api/deprecation-guide.md +++ b/content/en/docs/reference/using-api/deprecation-guide.md @@ -53,11 +53,11 @@ The **events.k8s.io/v1beta1** API version of Event will no longer be served in v * Notable changes in **events.k8s.io/v1**: * `type` is limited to `Normal` and `Warning` * `involvedObject` is renamed to `regarding` - * `action`, `reason`, `reportingComponent`, and `reportingInstance` are required when creating new **events.k8s.io/v1** Events + * `action`, `reason`, `reportingController`, and `reportingInstance` are required when creating new **events.k8s.io/v1** Events * use `eventTime` instead of the deprecated `firstTimestamp` field (which is renamed to `deprecatedFirstTimestamp` and not permitted in new **events.k8s.io/v1** Events) * use `series.lastObservedTime` instead of the deprecated `lastTimestamp` field (which is renamed to `deprecatedLastTimestamp` and not permitted in new **events.k8s.io/v1** Events) * use `series.count` instead of the deprecated `count` field (which is renamed to `deprecatedCount` and not permitted in new **events.k8s.io/v1** Events) - * use `reportingComponent` instead of the deprecated `source.component` field (which is renamed to `deprecatedSource.component` and not permitted in new **events.k8s.io/v1** Events) + * use `reportingController` instead of the deprecated `source.component` field (which is renamed to `deprecatedSource.component` and not permitted in new **events.k8s.io/v1** Events) * use `reportingInstance` instead of the deprecated `source.host` field (which is renamed to `deprecatedSource.host` and not permitted in new **events.k8s.io/v1** Events) #### PodDisruptionBudget {#poddisruptionbudget-v125} From 25de96863dd96ef67a9819cfa4fba05d19215054 Mon Sep 17 00:00:00 2001 From: Sinan Kahveci Date: Thu, 15 Jul 2021 11:53:52 +0100 Subject: [PATCH 101/175] Fix Missing Punctuation This change will fix the missing punctuation in the first paragraph. --- content/en/docs/concepts/architecture/nodes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/concepts/architecture/nodes.md b/content/en/docs/concepts/architecture/nodes.md index f8c0092d7b..0e954e0743 100644 --- a/content/en/docs/concepts/architecture/nodes.md +++ b/content/en/docs/concepts/architecture/nodes.md @@ -14,7 +14,7 @@ A node may be a virtual or physical machine, depending on the cluster. Each node is managed by the {{< glossary_tooltip text="control plane" term_id="control-plane" >}} and contains the services necessary to run -{{< glossary_tooltip text="Pods" term_id="pod" >}} +{{< glossary_tooltip text="Pods" term_id="pod" >}}. Typically you have several nodes in a cluster; in a learning or resource-limited environment, you might have only one node. From 9693458ae21b1a683d38ed35f1f327ba6cfc229c Mon Sep 17 00:00:00 2001 From: Sinan Kahveci Date: Thu, 15 Jul 2021 12:55:09 +0100 Subject: [PATCH 102/175] Fix inconsistent usage of term The term "control plane" is written as title case (e.g. Control Plane) only in this document. When it is referenced, it creates inconsistency in other documents. This commit will fix it by lower-casing the second word. --- content/en/docs/reference/glossary/kube-controller-manager.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/reference/glossary/kube-controller-manager.md b/content/en/docs/reference/glossary/kube-controller-manager.md index fa4205292c..78c22b32c7 100644 --- a/content/en/docs/reference/glossary/kube-controller-manager.md +++ b/content/en/docs/reference/glossary/kube-controller-manager.md @@ -11,7 +11,7 @@ tags: - architecture - fundamental --- - Control Plane component that runs {{< glossary_tooltip text="controller" term_id="controller" >}} processes. + Control plane component that runs {{< glossary_tooltip text="controller" term_id="controller" >}} processes. From 0e06c4f6cd34a44e59466ade0141a47a5736194a Mon Sep 17 00:00:00 2001 From: Anubhav Vardhan Date: Thu, 15 Jul 2021 19:13:34 +0530 Subject: [PATCH 103/175] Update _index.html --- content/en/community/_index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/community/_index.html b/content/en/community/_index.html index 5b65292ea7..b41323c69e 100644 --- a/content/en/community/_index.html +++ b/content/en/community/_index.html @@ -13,7 +13,7 @@ cid: community

The Kubernetes community -- users, contributors, and the culture we've built together -- is one of the biggest reasons for the meteoric rise of this open source project. Our culture and values continue to grow and change as the project itself grows and changes. We all work together toward constant improvement of the project and the ways we work on it. -

We are the people who file issues and pull requests, attend SIG meetings, Kubernetes meetups, and KubeCon, advocate for it's adoption and innovation, run kubectl get pods, and contribute in a thousand other vital ways. Read on to learn how you can get involved and become part of this amazing community.

+

We are the people who file issues and pull requests, attend SIG meetings, Kubernetes meetups, and KubeCon, advocate for its adoption and innovation, run kubectl get pods, and contribute in a thousand other vital ways. Read on to learn how you can get involved and become part of this amazing community.


From fe7771aeac323f2bd59490fdcf06e283eca67106 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 15 Jul 2021 09:07:47 -0700 Subject: [PATCH 104/175] Remove annotation for Experimental Windows Hyper-v removed in 1.21: https://github.com/kubernetes/kubernetes/pull/95505 --- .../en/docs/reference/labels-annotations-taints.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/content/en/docs/reference/labels-annotations-taints.md b/content/en/docs/reference/labels-annotations-taints.md index 29f3a63a8c..cd966f6150 100644 --- a/content/en/docs/reference/labels-annotations-taints.md +++ b/content/en/docs/reference/labels-annotations-taints.md @@ -200,18 +200,6 @@ Used on: Service The kube-proxy has this label for custom proxy, which delegates service control to custom proxy. -## experimental.windows.kubernetes.io/isolation-type - -Example: `experimental.windows.kubernetes.io/isolation-type: "hyperv"` - -Used on: Pod - -The annotation is used to run Windows containers with Hyper-V isolation. To use Hyper-V isolation feature and create a Hyper-V isolated container, the kubelet should be started with feature gates HyperVContainer=true and the Pod should include the annotation experimental.windows.kubernetes.io/isolation-type=hyperv. - -{{< note >}} -You can only set this annotation on Pods that have a single container. -{{< /note >}} - ## ingressclass.kubernetes.io/is-default-class Example: `ingressclass.kubernetes.io/is-default-class: "true"` From d4c540fb31ab0a08eaf6e485ec165dbd5c8f33a3 Mon Sep 17 00:00:00 2001 From: Carlos Panato Date: Fri, 16 Jul 2021 09:46:08 +0200 Subject: [PATCH 105/175] releases: update patch release to August cycle Signed-off-by: Carlos Panato --- content/en/releases/patch-releases.md | 5 ++++- data/releases/schedule.yaml | 27 ++++++++++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/content/en/releases/patch-releases.md b/content/en/releases/patch-releases.md index b44604bc3e..504c821c0d 100644 --- a/content/en/releases/patch-releases.md +++ b/content/en/releases/patch-releases.md @@ -78,9 +78,9 @@ releases may also occur in between these. | Monthly Patch Release | Cherry Pick Deadline | Target date | | --------------------- | -------------------- | ----------- | -| July 2021 | 2021-07-10 | 2021-07-14 | | August 2021 | 2021-08-07 | 2021-08-11 | | September 2021 | 2021-09-11 | 2021-09-15 | +| October 2021 | 2021-10-09 | 2021-10-13 | ## Detailed Release History for Active Branches @@ -92,6 +92,7 @@ End of Life for **1.21** is **2022-06-28** | PATCH RELEASE | CHERRY PICK DEADLINE | TARGET DATE | NOTE | | ------------- | -------------------- | ----------- | ---------------------------------------------------------------------- | +| 1.21.4 | 2021-08-07 | 2021-08-11 | | | 1.21.3 | 2021-07-10 | 2021-07-14 | | | 1.21.2 | 2021-06-12 | 2021-06-16 | | | 1.21.1 | 2021-05-07 | 2021-05-12 | [Regression](https://groups.google.com/g/kubernetes-dev/c/KuF8s2zueFs) | @@ -104,6 +105,7 @@ End of Life for **1.20** is **2022-02-28** | PATCH RELEASE | CHERRY PICK DEADLINE | TARGET DATE | NOTE | | ------------- | -------------------- | ----------- | ----------------------------------------------------------------------------------- | +| 1.20.10 | 2021-08-07 | 2021-08-11 | | | 1.20.9 | 2021-07-10 | 2021-07-14 | | | 1.20.8 | 2021-06-12 | 2021-06-16 | | | 1.20.7 | 2021-05-07 | 2021-05-12 | [Regression](https://groups.google.com/g/kubernetes-dev/c/KuF8s2zueFs) | @@ -122,6 +124,7 @@ End of Life for **1.19** is **2021-10-28** | PATCH RELEASE | CHERRY PICK DEADLINE | TARGET DATE | NOTE | | ------------- | -------------------- | ----------- | ------------------------------------------------------------------------- | +| 1.19.14 | 2021-08-07 | 2021-08-11 | | | 1.19.13 | 2021-07-10 | 2021-07-14 | | | 1.19.12 | 2021-06-12 | 2021-06-16 | | | 1.19.11 | 2021-05-07 | 2021-05-12 | [Regression](https://groups.google.com/g/kubernetes-dev/c/KuF8s2zueFs) | diff --git a/data/releases/schedule.yaml b/data/releases/schedule.yaml index 78fa73a6a4..003af71a3b 100644 --- a/data/releases/schedule.yaml +++ b/data/releases/schedule.yaml @@ -1,10 +1,13 @@ schedules: - release: 1.21 - next: 1.21.3 - cherryPickDeadline: 2021-07-10 - targetDate: 2021-07-14 + next: 1.21.4 + cherryPickDeadline: 2021-08-07 + targetDate: 2021-08-11 endOfLifeDate: 2022-04-30 previousPatches: + - release: 1.21.3 + cherryPickDeadline: 2021-07-10 + targetDate: 2021-07-14 - release: 1.21.2 cherryPickDeadline: 2021-06-12 targetDate: 2021-06-16 @@ -13,11 +16,14 @@ schedules: targetDate: 2021-05-12 note: Regression https://groups.google.com/g/kubernetes-dev/c/KuF8s2zueFs - release: 1.20 - next: 1.20.9 - cherryPickDeadline: 2021-07-10 - targetDate: 2021-07-14 + next: 1.20.10 + cherryPickDeadline: 2021-08-07 + targetDate: 2021-08-11 endOfLifeDate: 2021-12-30 previousPatches: + - release: 1.20.9 + cherryPickDeadline: 2021-07-10 + targetDate: 2021-07-14 - release: 1.20.8 cherryPickDeadline: 2021-06-12 targetDate: 2021-06-16 @@ -46,11 +52,14 @@ schedules: targetDate: 2020-12-18 note: "Tagging Issue https://groups.google.com/g/kubernetes-dev/c/dNH2yknlCBA" - release: 1.19 - next: 1.19.13 - cherryPickDeadline: 2021-07-10 - targetDate: 2021-07-14 + next: 1.19.14 + cherryPickDeadline: 2021-08-07 + targetDate: 2021-08-11 endOfLifeDate: 2021-09-30 previousPatches: + - release: 1.19.13 + cherryPickDeadline: 2021-07-10 + targetDate: 2021-07-14 - release: 1.19.12 cherryPickDeadline: 2021-06-12 targetDate: 2021-06-16 From f91fa01d100d38b50e7f5980f4a54eb3d9488d2e Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Fri, 16 Jul 2021 10:07:22 +0200 Subject: [PATCH 106/175] Capitalize non active branch history headers We uppercase all other headings so we should do it for this table as well for consistency reasons. Signed-off-by: Sascha Grunert --- content/en/releases/patch-releases.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/releases/patch-releases.md b/content/en/releases/patch-releases.md index b44604bc3e..35dc8bf6ec 100644 --- a/content/en/releases/patch-releases.md +++ b/content/en/releases/patch-releases.md @@ -140,7 +140,7 @@ End of Life for **1.19** is **2021-10-28** These releases are no longer supported. -| Minor Version | Final Patch Release | EOL date | NOTE | +| MINOR VERSION | FINAL PATCH RELEASE | EOL DATE | NOTE | | ------------- | ------------------- | ---------- | ---------------------------------------------------------------------- | | 1.18 | 1.18.20 | 2021-06-18 | Created to resolve regression introduced in 1.18.19 | | 1.18 | 1.18.19 | 2021-05-12 | [Regression](https://groups.google.com/g/kubernetes-dev/c/KuF8s2zueFs) | From 4d77af2dc4bd89745f680a0075d446047e8fb319 Mon Sep 17 00:00:00 2001 From: Arhell Date: Fri, 16 Jul 2021 11:48:02 +0300 Subject: [PATCH 107/175] [ja] Updating --cascade as false is deprecated on Kubectl 1.21 --- .../ja/docs/tasks/run-application/delete-stateful-set.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/content/ja/docs/tasks/run-application/delete-stateful-set.md b/content/ja/docs/tasks/run-application/delete-stateful-set.md index 659dabec17..81142068ed 100644 --- a/content/ja/docs/tasks/run-application/delete-stateful-set.md +++ b/content/ja/docs/tasks/run-application/delete-stateful-set.md @@ -37,13 +37,13 @@ StatefulSet自体が削除された後で、関連するヘッドレスサービ kubectl delete service ``` -kubectlを使ってStatefulSetを削除すると0にスケールダウンされ、すべてのPodが削除されます。PodではなくStatefulSetだけを削除したい場合は、`--cascade=false`を使用してください。 +kubectlを使ってStatefulSetを削除すると0にスケールダウンされ、すべてのPodが削除されます。PodではなくStatefulSetだけを削除したい場合は、`--cascade=orphan`を使用してください。 ```shell -kubectl delete -f --cascade=false +kubectl delete -f --cascade=orphan ``` -`--cascade=false`を`kubectl delete`に渡すことで、StatefulSetオブジェクト自身が削除された後でも、StatefulSetによって管理されていたPodは残ります。Podに`app=myapp`というラベルが付いている場合は、次のようにして削除できます: +`--cascade=orphan`を`kubectl delete`に渡すことで、StatefulSetオブジェクト自身が削除された後でも、StatefulSetによって管理されていたPodは残ります。Podに`app=myapp`というラベルが付いている場合は、次のようにして削除できます: ```shell kubectl delete pods -l app=myapp From da677ad55d1ab06c66c1b7cce1fc5799397d8881 Mon Sep 17 00:00:00 2001 From: Jeremy Rickard Date: Fri, 16 Jul 2021 13:33:05 -0600 Subject: [PATCH 108/175] Update content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md Co-authored-by: Tim Bannister --- .../_posts/2021-07-12-Kubernetes-Release-Cadence/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md index 7c86f985c5..e7dd6c51c2 100644 --- a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md +++ b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md @@ -1,6 +1,6 @@ --- layout: blog -title: "Kuberentes Release Cadence Change: Here’s What You Need To Know" +title: "Kubernetes Release Cadence Change: Here’s What You Need To Know" date: 2021-07-01 slug: new-kubernetes-release-cadence --- @@ -81,4 +81,4 @@ Join the [discussion](https://github.com/kubernetes/sig-release/discussions/1566 - Read the KEP [here](https://github.com/kubernetes/enhancements/tree/master/keps/sig-release/2572-release-cadence) - Join the [kubernetes-dev](https://groups.google.com/g/kubernetes-dev) mailing list -- Join [Kubernetes Slack](https://slack.k8s.io) and follow the #announcements channel \ No newline at end of file +- Join [Kubernetes Slack](https://slack.k8s.io) and follow the #announcements channel From 7095fcce17330d122d6bbb2cb64dc4ae1a738033 Mon Sep 17 00:00:00 2001 From: Jeremy Rickard Date: Fri, 16 Jul 2021 13:34:16 -0600 Subject: [PATCH 109/175] Update content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md Co-authored-by: Sascha Grunert --- .../blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md index e7dd6c51c2..0ef377ebc4 100644 --- a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md +++ b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md @@ -55,7 +55,7 @@ These proposed dates reflect only the start and end dates, and they are subject The only major change end users will experience is a slower release cadence. Kubernetes release artifacts, release notes, and all other aspects of any given release will stay the same. -With less releases to consume per year, end user organizations will spend less time on upgrades and more time on supporting their Kubernetes clusters. It also means that Kubernetes releases are in support for a slightly longer period of time. +With less releases to consume per year, it's intended that end user organizations will spend less time on upgrades and gain more time on supporting their Kubernetes clusters. It also means that Kubernetes releases are in support for a slightly longer period of time. ## What this means for Kubernetes contributors From bef5681585cfd1a764eab699d2c6c815a7daecca Mon Sep 17 00:00:00 2001 From: Jeremy Rickard Date: Fri, 16 Jul 2021 13:34:31 -0600 Subject: [PATCH 110/175] Update content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md Co-authored-by: Sascha Grunert --- .../blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md index 0ef377ebc4..7acaeda4e3 100644 --- a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md +++ b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md @@ -12,7 +12,7 @@ REF: https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-pr On April 23, 2021, the Release Team merged a Kubernetes Enhancement Proposal (KEP) changing the Kubernetes release cycle from four releases a year (once a quarter) to three releases a year. -This blog post is a high level overview post for what this means for the Kubernetes community's contributors and maintainers. +This blog post provides a high level overview about what this means for the Kubernetes community's contributors and maintainers. ## What's changing and when From 85d8b843d3e52556aebc5a3b9643946edcbcd7e1 Mon Sep 17 00:00:00 2001 From: Jeremy Rickard Date: Fri, 16 Jul 2021 13:55:09 -0600 Subject: [PATCH 111/175] Update content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md Co-authored-by: Sascha Grunert --- .../blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md index 7acaeda4e3..aa31edb80c 100644 --- a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md +++ b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md @@ -65,7 +65,7 @@ With a lower release cadence, contributors have more time for project enhancemen ## Why we decided to change the release cadence -The Kubernetes 1.19 cycle was far longer than usual. SIG Release extended it to lessen the burden on both Kubernetes contributors and Kubernetes end users due the COVID-19 pandemic. Following this extended release, the Kubernetes 1.20 release became the third, and final, release for 2020. +The Kubernetes 1.19 cycle was far longer than usual. SIG Release extended it to lessen the burden on both Kubernetes contributors and end users due the COVID-19 pandemic. Following this extended release, the Kubernetes 1.20 release became the third, and final, release for 2020. As the Kubernetes project matures, the number of enhancements per cycle grows, along with the burden on contributors, the Release Engineering team. Downstream consumers and integrators also face increased challenges keeping up with [ever more feature-packed releases](https://kubernetes.io/blog/2021/04/08/kubernetes-1-21-release-announcement/). A wider project adoption means the complexity of supporting a rapidly evolving platform affects a bigger downstream chain of consumers. From d77ab6d2b96dce5a7e2938365408730189c630fa Mon Sep 17 00:00:00 2001 From: Jeremy Date: Fri, 16 Jul 2021 13:56:10 -0600 Subject: [PATCH 112/175] Review feedback Signed-off-by: Jeremy --- .../index.md | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md index aa31edb80c..f2badbfdc1 100644 --- a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md +++ b/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md @@ -5,9 +5,6 @@ date: 2021-07-01 slug: new-kubernetes-release-cadence --- -REF: https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/ - - **Authors**: Celeste Horgan, Adolfo García Veytia, James Laverack, Jeremy Rickard On April 23, 2021, the Release Team merged a Kubernetes Enhancement Proposal (KEP) changing the Kubernetes release cycle from four releases a year (once a quarter) to three releases a year. @@ -16,7 +13,7 @@ This blog post provides a high level overview about what this means for the Kube ## What's changing and when -Starting with the [Kubernetes 1.22 release](https://www.kubernetes.dev/resources/release/), a lightweight policy will drive the creation of each release schedule. This policy states: +Starting with the [Kubernetes 1.22 release](https://github.com/kubernetes/sig-release/tree/master/releases/release-1.22), a lightweight policy will drive the creation of each release schedule. This policy states: * The first Kubernetes release of a calendar year should start at the second or third week of January to provide people more room after coming back from the @@ -28,34 +25,36 @@ Starting with the [Kubernetes 1.22 release](https://www.kubernetes.dev/resources * An explicit SIG Release break of at least two weeks between each cycle will be enforced. -As a result, Kubernetes will follow a three release pear year cadence. Kubernetes 1.23 will be the final release of the 2021 calendar year. This new schedule creation policy results in a very predictible release schedule, allowing us to forecast upcoming release dates: +As a result, Kubernetes will follow a three release pear year cadence. Kubernetes 1.23 will be the final release of the 2021 calendar year. This new policy results in a very predictable release schedule, allowing us to forecast upcoming release dates: *Proposed Kubernetes Release Schedule for the remainder of 2021* | Week Number in Year | Release Number | Release Week | Note | | -------- | -------- | -------- | -------- | -| 35 | 3 | 1 (August 23) | | -| 50 | 3 | 16 (December 07) | KubeCon + CloudNativeCon NA Break (Oct 11-15) | +| 35 | 1.23 | 1 (August 23) | | +| 50 | 1.23 | 16 (December 07) | KubeCon + CloudNativeCon NA Break (Oct 11-15) | *Proposed Kubernetes Release Schedule for 2022* | Week Number in Year | Release Number | Release Week | Note | | -------- | -------- | -------- | -------- | -| 1 | 1 | 1 (January 03) | | -| 15 | 1 | 15 (April 12) | | -| 17 | 2 | 1 (April 26) | KubeCon + CloudNativeCon EU likely to occur | -| 32 | 2 | 15 (August 09) | | -| 34 | 3 | 1 (August 22 | KubeCon + CloudNativeCon NA likely to occur | -| 49 | 3 | 14 (December 06) | +| 1 | 1.24 | 1 (January 03) | | +| 15 | 1.24 | 15 (April 12) | | +| 17 | 1.25 | 1 (April 26) | KubeCon + CloudNativeCon EU likely to occur | +| 32 | 1.25 | 15 (August 09) | | +| 34 | 1.26 | 1 (August 22 | KubeCon + CloudNativeCon NA likely to occur | +| 49 | 1.26 | 14 (December 06) | -These proposed dates reflect only the start and end dates, and they are subject to change. The Release Team will select dates for enhancement freeze, code freeze, and other milestones at the start of each release. Feedback from prior releases will feed into this process. +These proposed dates reflect only the start and end dates, and they are subject to change. The Release Team will select dates for enhancement freeze, code freeze, and other milestones at the start of each release. For more information on these milestones, please refer to the [release phases](https://www.k8s.dev/resources/release/#phases) documentation. Feedback from prior releases will feed into this process. ## What this means for end users -The only major change end users will experience is a slower release cadence. Kubernetes release artifacts, release notes, and all other aspects of any given release will stay the same. +The major change end users will experience is a slower release cadence and a slower rate of enhancement graduation. Kubernetes release artifacts, release notes, and all other aspects of any given release will stay the same. -With less releases to consume per year, it's intended that end user organizations will spend less time on upgrades and gain more time on supporting their Kubernetes clusters. It also means that Kubernetes releases are in support for a slightly longer period of time. +Prior to this change an enhancement could graduate from alpha to stable in 9 months. With the change in cadence, this will stretch to 12 months. Additionally, graduation of features over the last few releases has in some part been driven by release team activities. + +With fewer releases, users can expect to see the rate of feature graduation slow. Users can also expect releases to contain a larger number of enhancements that they need to be aware of during upgrades. However, with fewer releases to consume per year, it's intended that end user organizations will spend less time on upgrades and gain more time on supporting their Kubernetes clusters. It also means that Kubernetes releases are in support for a slightly longer period of time, so bug fixes and security patches will be available for releases for a longer period of time. ## What this means for Kubernetes contributors @@ -69,7 +68,7 @@ The Kubernetes 1.19 cycle was far longer than usual. SIG Release extended it to As the Kubernetes project matures, the number of enhancements per cycle grows, along with the burden on contributors, the Release Engineering team. Downstream consumers and integrators also face increased challenges keeping up with [ever more feature-packed releases](https://kubernetes.io/blog/2021/04/08/kubernetes-1-21-release-announcement/). A wider project adoption means the complexity of supporting a rapidly evolving platform affects a bigger downstream chain of consumers. -Changing the release cadence from four to three releases per year balances a variety of factors for stakeholders: while it's not strictly an LTS policy, consumers and integrators will get longer support terms for each minor version as the extended release cycles lead to the [last three releases being supported](https://kubernetes.io/blog/2020/08/31/kubernetes-1-19-feature-one-year-support/) for a longer period. Contributors get more time to [mature enhancements](https://www.cncf.io/blog/2021/04/12/enhancing-the-kubernetes-enhancements-process/) and [get them ready for production](https://github.com/kubernetes/community/blob/master/sig-architecture/production-readiness.md). +Changing the release cadence from four to three releases per year balances a variety of factors for stakeholders: while it's not strictly an LTS policy, consumers and integrators will get longer support terms for each minor version as the extended release cycles lead to the [previous three releases being supported](https://kubernetes.io/blog/2020/08/31/kubernetes-1-19-feature-one-year-support/) for a longer period. Contributors get more time to [mature enhancements](https://www.cncf.io/blog/2021/04/12/enhancing-the-kubernetes-enhancements-process/) and [get them ready for production](https://github.com/kubernetes/community/blob/master/sig-architecture/production-readiness.md). Finally, the management overhead for SIG Release and the Release Engineering team diminishes allowing the team to spend more time on improving the quality of the software releases and the tooling that drives them. From 46987cf0e9aadc563b9843ae30d3e4d1459ef7d1 Mon Sep 17 00:00:00 2001 From: chirangaalwis Date: Sat, 17 Jul 2021 13:42:22 +0530 Subject: [PATCH 113/175] [NetworkPolicy][EndPort] Fix incorrect grammar Fix incorrect grammar of the valid range of `endPort` property --- .../en/docs/concepts/services-networking/network-policies.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/concepts/services-networking/network-policies.md b/content/en/docs/concepts/services-networking/network-policies.md index 2c9ed4a90e..ec29f1d813 100644 --- a/content/en/docs/concepts/services-networking/network-policies.md +++ b/content/en/docs/concepts/services-networking/network-policies.md @@ -255,7 +255,7 @@ The above rule allows any Pod with label `db` on the namespace `default` to comm The following restrictions apply when using this field: * As an alpha feature, this is disabled by default. To enable the `endPort` field at a cluster level, you (or your cluster administrator) need to enable the `NetworkPolicyEndPort` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) for the API server with `--feature-gates=NetworkPolicyEndPort=true,…`. -* The `endPort` field must be equal than or greater to the `port` field. +* The `endPort` field must be equal to or greater than the `port` field. * `endPort` can only be defined if `port` is also defined. * Both ports must be numeric. From 3dd33ef1079aaea681e90bf7ab7a32339355484a Mon Sep 17 00:00:00 2001 From: edsoncelio Date: Sat, 17 Jul 2021 13:48:55 -0300 Subject: [PATCH 114/175] feat: add contribute index translation update --- content/pt-br/docs/contribute/_index.md | 90 +++++++++++++++---------- 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/content/pt-br/docs/contribute/_index.md b/content/pt-br/docs/contribute/_index.md index 86c4d92967..510bbe490f 100644 --- a/content/pt-br/docs/contribute/_index.md +++ b/content/pt-br/docs/contribute/_index.md @@ -1,31 +1,41 @@ --- content_type: concept -title: Contribua com o Kubernetes docs -linktitle: Contribute +title: Contribua com a documentação do Kubernetes +linktitle: Contribuir main_menu: true weight: 80 +no_list: true +card: + name: contribuir + weight: 10 + title: Comece a contribuir para o K8s +--- --- -Caso você gostaria de contribuir com a documentação ou o site do Kubernetes, -ficamos felizes em ter sua ajuda! Qualquer pessoa pode contribuir, seja você novo no -projeto ou se você já esta no mercado há muito tempo. Além disso, Se você se identifica como -desenvolvedor, usuário final ou alguém que simplesmente não suporta ver erros de digitação. +*O Kubernetes agradece as melhorias de todos os contribuidores, novos e experientes!* +{{< note >}} +Para saber mais sobre como contribuir o Kubernetes em geral, veja a +[documentação para contribuidor](https://www.kubernetes.dev/docs/). +{{< /note >}} + +Este site é mantido pelo [Kubernetes SIG Docs](/docs/contribute/#get-involved-with-sig-docs). + +Contribuidores da documentação do Kubernetes podem: + - Melhorar o conteúdo existente + - Criar novo conteúdo + - Traduzir a documentação + - Gerenciar e publicar a documentação como parte do ciclo de lançamento do Kubernetes ## Começando -Qualquer pessoa pode abrir uma issue descrevendo o problema ou melhorias desejadas com a documentação ou contribuir com uma alteração e uma solicitação de mudança (Pull Request - PR). -Algumas tarefas exigem mais confiança e precisam de mais acesso na organização Kubernetes. -Veja [Participando do SIG Docs](/docs/contribute/participating/) para mais detalhes sobre -as funções e permissões. - -A documentação do Kubernetes reside em um repositório do GitHub. Nós damos as boas-vindas -a todas as contribuições, mas você vai precisa estar familiarizado com o uso básico de git e GitHub para -operar efetivamente na comunidade Kubernetes. +Qualquer pessoa pode abrir uma issue sobre a documentação, ou contribuir com uma mudança por meio de um pull request (PR) para o [repositório do Github `kubernetes/website`]https://github.com/kubernetes/website). +É recomendável que você se sinta confortável com [git](https://git-scm.com/) e +[Github](https://lab.github.com/) para trabalhar efetivamente na comunidade Kubernetes. Para se envolver com a documentação: @@ -33,30 +43,42 @@ Para se envolver com a documentação: 2. Familiarize-se com o [repositório de documentação](https://github.com/kubernetes/website) e o [gerador de site estático](https://gohugo.io) hugo. 3. Certifique-se de entender os processos básicos para [melhorar o conteúdo](https://kubernetes.io/docs/contribute/start/#improve-existing-content) e [revisar alterações](https://kubernetes.io/docs/contribute/start/#review-docs-pull-requests). -## Melhores Práticas recomendadas para contribuições +Algumas tarefas requerem mais confiança e mais acessos na organização do Kubernetes. +Veja [Participando no SIG Docs](/docs/contribute/participate/) para mais detalhes +sobre funções e permissões. -- Escreva mensagens GIT claras e significativas. -- Certifique-se de incluir _Github Special Keywords_ que faz referência a issue e o fecha automaticamente quando o PR é mergeado. -- Quando você faz uma pequena alteração em um PR, como corrigir um erro de digitação, qualquer alteração de estilo ou gramática, certifique-se de esmagar seus commits (squash) para não obter um grande número de commits por uma alteração relativamente pequena. -- Certifique-se de incluir uma boa descrição de PR explicando as alterações no código, o motivo de alterar um trecho de código e garantir que haja informações suficientes para o revisor entender seu PR. -- Leituras adicionais: - - [chris.beams.io/posts/git-commit/](https://chris.beams.io/posts/git-commit/) - - [github.com/blog/1506-closing-issues-via-pull-requests ](https://github.com/blog/1506-closing-issues-via-pull-requests ) - - [davidwalsh.name/squash-commits-git ](https://davidwalsh.name/squash-commits-git ) +## Sua primeira contribuição +- Leia sobre [visão geral para contribuição](/docs/contribute/new-content/overview/) para saber mais sobre diferentes formas para você contribuir. +- Veja a [lista de issues em `kubernetes/website`](https://github.com/kubernetes/website/issues/) para identificar issues que sejam um bom ponto de partida. +- [Abra um pull request usando o Github](/docs/contribute/new-content/open-a-pr/#changes-using-github) para documentações existentes e aprenda mais sobre resolver issues no Github. +- Leia sobre o [guia de conteúdo](/docs/contribute/style/content-guide/) e [guias de estilo](/docs/contribute/style/style-guide/). +- Leia sobre [tipos de conteúdo de páginas](/docs/contribute/style/page-content-types/) e [shortcodes do Hugo](/docs/contribute/style/hugo-shortcodes/). -## Outras maneiras de contribuir +## Próximos passos + + - Aprenda a [trabalhar com um clone local](/docs/contribute/new-content/open-a-pr/#fork-the-repo) de um repositório. + - Documente [funcionalidades em uma release]/docs/contribute/new-content/new-features/). + - Participe do [SIG Docs](/docs/contribute/participate/), e se torne um + [membro ou revisor](/docs/contribute/participate/roles-and-responsibilities/). + - Comece ou ajude com uma [localização](/docs/contribute/localization/). + +## Se envolva com o SIG Docs + +O [SIG Docs](/docs/contribute/participate/) é um grupo de contribuidores que publica e mantém +a documentação e o site do Kubernetes. Se envolver com o SIG Docs é uma ótima forma de contribuidores Kubernetes (pessoas desenvolvedoras de features ou outros) terem um grande impacto dentro do projeto Kubernetes. + +A comunicação do SIG Docs é feita de diferentes formas: + - [Entre em `#sig-docs` no slack do Kubernetes](https://slack.k8s.io/). + - [Se inscreva na lista de email `kubernetes-dig-docs`](https://groups.google.com/forum/#!forum/kubernetes-sig-docs), onde acontecem discussões e + decisões oficiais são registradas. + - [Participe do encontro semanal do SIG Docs](https://github.com/kubernetes/community/tree/master/sig-docs). Os encontros são sempre anunciados no `#sig-docs` e adicionados ao [calendário de eventos de comunidade do Kubernetes]https://calendar.google.com/calendar/embed?src=cgnt364vd8s86hr2phapfjc6uk%40group.calendar.google.com&ctz=America/Los_Angeles). Você precisa baixar o [cliente do Zoom](https://zoom.us/download) ou usar um telefone. + +## Outras formas de contribuir - Para contribuir com a comunidade Kubernetes por meio de fóruns on-line, como Twitter ou Stack Overflow, ou aprender sobre encontros locais e eventos do Kubernetes, visite o a area de [comunidade Kubernetes](/community/). -- Para contribuir com o desenvolvimento de novas funções, leia o [cheatsheet do colaborador](https://github.com/kubernetes/community/tree/master/contributors/guide/contributor-cheatsheet) para começar. +- Para contribuir com o desenvolvimento de novas funcionalidades, leia o [cheatsheet do colaborador](https://github.com/kubernetes/community/tree/master/contributors/guide/contributor-cheatsheet) para começar. +- Leia o [cheatsheet de contribuidor](https://github.com/kubernetes/community/tree/master/contributors/guide/contributor-cheatsheet) para saber mais sobre as funcionalidades de desenvolvimento do Kubernetes. +- Submeta [um post de blog ou um caso de estudo](/docs/contribute/new-content/blogs-case-studies/). -## {{% heading "whatsnext" %}} - - -- Para obter mais informações sobre os conceitos básicos de contribuição para a documentação, leia [Comece a contribuir](/docs/contribute/start/). -- Siga o [Guia de estilo de documentação do Kubernetes](/docs/contribute/style/style-guide/) ao propor mudanças. -- Para mais informações sobre o SIG Docs, leia [Participando do SIG Docs](/docs/contribute/participating/). -- Para mais informações sobre a localização de documentos do Kubernetes, leia [Localização da documentação do Kubernetes](/docs/contribute/localization/). - - From b0a27053d224554ae7911104f564cc1074362910 Mon Sep 17 00:00:00 2001 From: edsoncelio Date: Sat, 17 Jul 2021 13:50:52 -0300 Subject: [PATCH 115/175] feat: fix typos --- content/pt-br/docs/contribute/_index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/pt-br/docs/contribute/_index.md b/content/pt-br/docs/contribute/_index.md index 510bbe490f..b0e8b09ad0 100644 --- a/content/pt-br/docs/contribute/_index.md +++ b/content/pt-br/docs/contribute/_index.md @@ -33,7 +33,7 @@ Contribuidores da documentação do Kubernetes podem: ## Começando -Qualquer pessoa pode abrir uma issue sobre a documentação, ou contribuir com uma mudança por meio de um pull request (PR) para o [repositório do Github `kubernetes/website`]https://github.com/kubernetes/website). +Qualquer pessoa pode abrir uma issue sobre a documentação, ou contribuir com uma mudança por meio de um pull request (PR) para o [repositório do Github `kubernetes/website`](https://github.com/kubernetes/website). É recomendável que você se sinta confortável com [git](https://git-scm.com/) e [Github](https://lab.github.com/) para trabalhar efetivamente na comunidade Kubernetes. @@ -71,7 +71,7 @@ A comunicação do SIG Docs é feita de diferentes formas: - [Entre em `#sig-docs` no slack do Kubernetes](https://slack.k8s.io/). - [Se inscreva na lista de email `kubernetes-dig-docs`](https://groups.google.com/forum/#!forum/kubernetes-sig-docs), onde acontecem discussões e decisões oficiais são registradas. - - [Participe do encontro semanal do SIG Docs](https://github.com/kubernetes/community/tree/master/sig-docs). Os encontros são sempre anunciados no `#sig-docs` e adicionados ao [calendário de eventos de comunidade do Kubernetes]https://calendar.google.com/calendar/embed?src=cgnt364vd8s86hr2phapfjc6uk%40group.calendar.google.com&ctz=America/Los_Angeles). Você precisa baixar o [cliente do Zoom](https://zoom.us/download) ou usar um telefone. + - [Participe do encontro semanal do SIG Docs](https://github.com/kubernetes/community/tree/master/sig-docs). Os encontros são sempre anunciados no `#sig-docs` e adicionados ao [calendário de eventos de comunidade do Kubernetes](https://calendar.google.com/calendar/embed?src=cgnt364vd8s86hr2phapfjc6uk%40group.calendar.google.com&ctz=America/Los_Angeles). Você precisa baixar o [cliente do Zoom](https://zoom.us/download) ou usar um telefone. ## Outras formas de contribuir From 6493a185ff602e6957af8e6944ce33a79ce117e1 Mon Sep 17 00:00:00 2001 From: edsoncelio Date: Sat, 17 Jul 2021 13:57:43 -0300 Subject: [PATCH 116/175] feat: add analytics page translation --- content/pt-br/docs/contribute/analytics.md | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 content/pt-br/docs/contribute/analytics.md diff --git a/content/pt-br/docs/contribute/analytics.md b/content/pt-br/docs/contribute/analytics.md new file mode 100644 index 0000000000..fd129223d4 --- /dev/null +++ b/content/pt-br/docs/contribute/analytics.md @@ -0,0 +1,29 @@ +--- +title: Visualizando Analytics do Site +content_type: concept +weight: 100 +card: + name: contribuir + weight: 100 +--- + + + +Essa página contém informações sobre a dashboard de analystics do kubernetes.io. + + + +Essa [dashboard](https://datastudio.google.com/reporting/fede2672-b2fd-402a-91d2-7473bdb10f04) foi feita usando +o Google Data Studio e possui informações coletadas do +kubernetes.io usando o Google Analytics. + +### Usando a dashboard + +Por padrão, a dashboard mostra todos os analytics coletados nos últimos 30 dias. Use o seletor de data +para ver dados de outros intervalos de data. Outras +opções de filtros permitem que você veja dados baseados +em localização do usuário para acessar o site, a tradução +da documentação usada e outros. + +Se você identificar um problema com essa dashboard ou quer solicitar qualquer melhoria, [abra uma issue](https://github.com/kubernetes/website/issues/new/choose) no repositório. + From 30c671fdc776b3573ee8250efe801891b0a5722f Mon Sep 17 00:00:00 2001 From: edsoncelio Date: Sat, 17 Jul 2021 14:01:11 -0300 Subject: [PATCH 117/175] feat: fix new lines --- content/pt-br/docs/contribute/analytics.md | 1 - 1 file changed, 1 deletion(-) diff --git a/content/pt-br/docs/contribute/analytics.md b/content/pt-br/docs/contribute/analytics.md index fd129223d4..305e0804cc 100644 --- a/content/pt-br/docs/contribute/analytics.md +++ b/content/pt-br/docs/contribute/analytics.md @@ -26,4 +26,3 @@ em localização do usuário para acessar o site, a tradução da documentação usada e outros. Se você identificar um problema com essa dashboard ou quer solicitar qualquer melhoria, [abra uma issue](https://github.com/kubernetes/website/issues/new/choose) no repositório. - From 5f4a3fff9ec19bc11564810b2fabb4df81692114 Mon Sep 17 00:00:00 2001 From: jaron <77253497+jaronnie@users.noreply.github.com> Date: Sun, 18 Jul 2021 07:45:48 +0800 Subject: [PATCH 118/175] Update connecting-frontend-backend.md ``` Error from server (NotFound): deployments.apps "hello" not found ``` chang hello to backend ``` Name: backend Namespace: default CreationTimestamp: Sun, 18 Jul 2021 07:33:39 +0800 Labels: ``` --- .../access-application-cluster/connecting-frontend-backend.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/zh/docs/tasks/access-application-cluster/connecting-frontend-backend.md b/content/zh/docs/tasks/access-application-cluster/connecting-frontend-backend.md index 42ebe8d250..93a230d365 100644 --- a/content/zh/docs/tasks/access-application-cluster/connecting-frontend-backend.md +++ b/content/zh/docs/tasks/access-application-cluster/connecting-frontend-backend.md @@ -82,7 +82,7 @@ View information about the backend Deployment: 查看后端的 Deployment 信息： ```shell -kubectl describe deployment hello +kubectl describe deployment backend ``` This page shows how to install the `kubeadm` toolbox. -For information how to create a cluster with kubeadm once you have performed this installation process, see the [Using kubeadm to Create a Cluster](/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/) page. +For information on how to create a cluster with kubeadm once you have performed this installation process, see the [Using kubeadm to Create a Cluster](/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/) page. From 782e6264de97857b38a3225cc948a0dfdcb78846 Mon Sep 17 00:00:00 2001 From: Shubham Kuchhal Date: Mon, 19 Jul 2021 16:56:08 +0530 Subject: [PATCH 122/175] Add Note for deprecated PodSecurityPolicy. --- content/en/docs/tutorials/clusters/apparmor.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/content/en/docs/tutorials/clusters/apparmor.md b/content/en/docs/tutorials/clusters/apparmor.md index 32f25ba483..8907768089 100644 --- a/content/en/docs/tutorials/clusters/apparmor.md +++ b/content/en/docs/tutorials/clusters/apparmor.md @@ -348,6 +348,11 @@ node with the required profile. ### Restricting profiles with the PodSecurityPolicy +{{< note >}} +PodSecurityPolicy is deprecated in Kubernetes v1.21, and will be removed in v1.25. +See [PodSecurityPolicy documentation](/docs/concepts/policy/pod-security-policy/) for more information. +{{< /note >}} + If the PodSecurityPolicy extension is enabled, cluster-wide AppArmor restrictions can be applied. To enable the PodSecurityPolicy, the following flag must be set on the `apiserver`: From f4a8ecf31d10ec7e9f5ad2d9884bc1816a60e43b Mon Sep 17 00:00:00 2001 From: edsoncelio Date: Mon, 19 Jul 2021 08:44:32 -0300 Subject: [PATCH 123/175] fix: fix typos request by review --- content/pt-br/docs/contribute/_index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/pt-br/docs/contribute/_index.md b/content/pt-br/docs/contribute/_index.md index b0e8b09ad0..e5a227c2f1 100644 --- a/content/pt-br/docs/contribute/_index.md +++ b/content/pt-br/docs/contribute/_index.md @@ -57,7 +57,7 @@ sobre funções e permissões. ## Próximos passos - Aprenda a [trabalhar com um clone local](/docs/contribute/new-content/open-a-pr/#fork-the-repo) de um repositório. - - Documente [funcionalidades em uma release]/docs/contribute/new-content/new-features/). + - Documente [funcionalidades em uma release](/docs/contribute/new-content/new-features/). - Participe do [SIG Docs](/docs/contribute/participate/), e se torne um [membro ou revisor](/docs/contribute/participate/roles-and-responsibilities/). - Comece ou ajude com uma [localização](/docs/contribute/localization/). @@ -75,7 +75,7 @@ A comunicação do SIG Docs é feita de diferentes formas: ## Outras formas de contribuir -- Para contribuir com a comunidade Kubernetes por meio de fóruns on-line, como Twitter ou Stack Overflow, ou aprender sobre encontros locais e eventos do Kubernetes, visite o a area de [comunidade Kubernetes](/community/). +- Para contribuir com a comunidade Kubernetes por meio de fóruns on-line, como Twitter ou Stack Overflow, ou aprender sobre encontros locais e eventos do Kubernetes, visite a area de [comunidade Kubernetes](/community/). - Para contribuir com o desenvolvimento de novas funcionalidades, leia o [cheatsheet do colaborador](https://github.com/kubernetes/community/tree/master/contributors/guide/contributor-cheatsheet) para começar. - Leia o [cheatsheet de contribuidor](https://github.com/kubernetes/community/tree/master/contributors/guide/contributor-cheatsheet) para saber mais sobre as funcionalidades de desenvolvimento do Kubernetes. - Submeta [um post de blog ou um caso de estudo](/docs/contribute/new-content/blogs-case-studies/). From 51c621a6baddfa93185bf9a8981d9654c716d98e Mon Sep 17 00:00:00 2001 From: Sam Wheating Date: Mon, 19 Jul 2021 10:07:51 -0700 Subject: [PATCH 124/175] Fixing improper sentence in HPA documentation --- .../docs/tasks/run-application/horizontal-pod-autoscale.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md b/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md index 5e94027423..ed095fa0cd 100644 --- a/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md +++ b/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md @@ -198,9 +198,8 @@ The detailed documentation of `kubectl autoscale` can be found [here](/docs/refe ## Autoscaling during rolling update -Currently in Kubernetes, it is possible to perform a rolling update by using the deployment object, which manages the underlying replica sets for you. -Horizontal Pod Autoscaler only supports the latter approach: the Horizontal Pod Autoscaler is bound to the deployment object, -it sets the size for the deployment object, and the deployment is responsible for setting sizes of underlying replica sets. +Currently in Kubernetes, a rolling update is performed using a deployment object, which manages the underlying replica sets for you. +A Horizontal Pod Autoscaler is bound to a single deployment object - it sets the size for the deployment object, and the deployment is responsible for setting sizes of underlying replica sets. Horizontal Pod Autoscaler does not work with rolling update using direct manipulation of replication controllers, i.e. you cannot bind a Horizontal Pod Autoscaler to a replication controller and do rolling update. From d4052254154fc81eb851108a5fdb5acd1dc3e830 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 19 Jul 2021 14:08:14 -0600 Subject: [PATCH 125/175] Change date for blog post Signed-off-by: Jeremy --- .../index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename content/en/blog/_posts/{2021-07-12-Kubernetes-Release-Cadence => 2021-07-20-Kubernetes-Release-Cadence}/index.md (99%) diff --git a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md b/content/en/blog/_posts/2021-07-20-Kubernetes-Release-Cadence/index.md similarity index 99% rename from content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md rename to content/en/blog/_posts/2021-07-20-Kubernetes-Release-Cadence/index.md index f2badbfdc1..3f563cedfb 100644 --- a/content/en/blog/_posts/2021-07-12-Kubernetes-Release-Cadence/index.md +++ b/content/en/blog/_posts/2021-07-20-Kubernetes-Release-Cadence/index.md @@ -1,7 +1,7 @@ --- layout: blog title: "Kubernetes Release Cadence Change: Here’s What You Need To Know" -date: 2021-07-01 +date: 2021-07-20 slug: new-kubernetes-release-cadence --- From 8ff3213727f595c6ecbaa2a9af3e58f968a77c4c Mon Sep 17 00:00:00 2001 From: "able.lv" Date: Tue, 20 Jul 2021 21:14:36 +0800 Subject: [PATCH 126/175] fix some typos --- .../concepts/scheduling-eviction/node-pressure-eviction.md | 4 ++-- content/zh/docs/setup/release/notes.md | 2 +- .../docs/tasks/access-application-cluster/web-ui-dashboard.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/content/zh/docs/concepts/scheduling-eviction/node-pressure-eviction.md b/content/zh/docs/concepts/scheduling-eviction/node-pressure-eviction.md index b7b7b3987b..5f163fddfe 100644 --- a/content/zh/docs/concepts/scheduling-eviction/node-pressure-eviction.md +++ b/content/zh/docs/concepts/scheduling-eviction/node-pressure-eviction.md @@ -430,7 +430,7 @@ As a result, kubelet ranks and evicts pods in the following order: 1. 首先考虑资源使用量超过其请求的 `BestEffort` 或 `Burstable` Pod。 这些 Pod 会根据它们的优先级以及它们的资源使用级别超过其请求的程度被逐出。 -1. 资源使用量少于请求量的 `Guaranted` Pod 和 `Burstable` Pod 根据其优先级被最后驱逐。 +1. 资源使用量少于请求量的 `Guaranteed` Pod 和 `Burstable` Pod 根据其优先级被最后驱逐。 {{}} -仅当 `Guaranted` Pod 中所有容器都被指定了请求和限制并且二者相等时，才保证 Pod 不被驱逐。 +仅当 `Guaranteed` Pod 中所有容器都被指定了请求和限制并且二者相等时，才保证 Pod 不被驱逐。 这些 Pod 永远不会因为另一个 Pod 的资源消耗而被驱逐。 如果系统守护进程（例如 `kubelet`、`docker` 和 `journald`） 消耗的资源比通过 `system-reserved` 或 `kube-reserved` 分配保留的资源多， diff --git a/content/zh/docs/setup/release/notes.md b/content/zh/docs/setup/release/notes.md index 99eeefa42a..05ef6a7088 100644 --- a/content/zh/docs/setup/release/notes.md +++ b/content/zh/docs/setup/release/notes.md @@ -1161,7 +1161,7 @@ filename | sha512 hash ### Bug or Regression - Add missing --kube-api-content-type in kubemark hollow template ([#98911](https://github.com/kubernetes/kubernetes/pull/98911), [@Jeffwan](https://github.com/Jeffwan)) [SIG Scalability and Testing] -- Avoid duplicate error messages when runing kubectl edit quota ([#98201](https://github.com/kubernetes/kubernetes/pull/98201), [@pacoxu](https://github.com/pacoxu)) [SIG API Machinery and Apps] +- Avoid duplicate error messages when running kubectl edit quota ([#98201](https://github.com/kubernetes/kubernetes/pull/98201), [@pacoxu](https://github.com/pacoxu)) [SIG API Machinery and Apps] - Cleanup subnet in frontend IP configs to prevent huge subnet request bodies in some scenarios. ([#98133](https://github.com/kubernetes/kubernetes/pull/98133), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] - Fix errors when accessing Windows container stats for Dockershim ([#98510](https://github.com/kubernetes/kubernetes/pull/98510), [@jsturtevant](https://github.com/jsturtevant)) [SIG Node and Windows] - Fixes spurious errors about IPv6 in kube-proxy logs on nodes with IPv6 disabled. ([#99127](https://github.com/kubernetes/kubernetes/pull/99127), [@danwinship](https://github.com/danwinship)) [SIG Network and Node] diff --git a/content/zh/docs/tasks/access-application-cluster/web-ui-dashboard.md b/content/zh/docs/tasks/access-application-cluster/web-ui-dashboard.md index a9d3793cda..470a378bab 100644 --- a/content/zh/docs/tasks/access-application-cluster/web-ui-dashboard.md +++ b/content/zh/docs/tasks/access-application-cluster/web-ui-dashboard.md @@ -172,7 +172,7 @@ The deploy wizard expects that you provide the following information: --> - **容器镜像**（必填）：公共镜像仓库上的 Docker [容器镜像](/zh/docs/concepts/containers/images/) 或者私有镜像仓库 - （通常是 Google Container Registery 或者 Docker Hub）的 URL。容器镜像参数说明必须以冒号结尾。 + （通常是 Google Container Registry 或者 Docker Hub）的 URL。容器镜像参数说明必须以冒号结尾。 -{{< feature-state for_k8s_version="v1.15" state="alpha" >}} +{{< feature-state for_k8s_version="v1.19" state="stable" >}} The scheduling framework is a pluggable architecture for the Kubernetes scheduler. It adds a new set of "plugin" APIs to the existing scheduler. Plugins are compiled into the scheduler. The APIs allow most scheduling features to be implemented as plugins, while keeping the From 27e5f9fedc97fcfe9d1ab20fbba3ab02053841d8 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Tue, 20 Jul 2021 17:07:06 -0400 Subject: [PATCH 129/175] Complete details regarding CSR garbage collection Signed-off-by: Monis Khan --- .../access-authn-authz/certificate-signing-requests.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/content/en/docs/reference/access-authn-authz/certificate-signing-requests.md b/content/en/docs/reference/access-authn-authz/certificate-signing-requests.md index 9cbbb6cda7..6d4c97df10 100644 --- a/content/en/docs/reference/access-authn-authz/certificate-signing-requests.md +++ b/content/en/docs/reference/access-authn-authz/certificate-signing-requests.md @@ -3,6 +3,7 @@ reviewers: - liggitt - mikedanese - munnerz +- enj title: Certificate Signing Requests content_type: concept weight: 20 @@ -56,7 +57,9 @@ state for some duration: * Approved requests: automatically deleted after 1 hour * Denied requests: automatically deleted after 1 hour +* Failed requests: automatically deleted after 1 hour * Pending requests: automatically deleted after 24 hours +* All requests: automatically deleted after the issued certificate has expired ## Signers @@ -78,7 +81,7 @@ that case, the documentation for the signer should specify the meaning of additional certificates; for example, this might be the certificate plus intermediates to be presented during TLS handshakes. -The PKCS#10 signing request format doesn't allow to specify a certificate +The PKCS#10 signing request format does not allow to specify a certificate expiration or lifetime. The expiration or lifetime therefore has to be set through e.g. an annotation on the CSR object. While it's theoretically possible for a signer to use that expiration date, there is currently no @@ -185,8 +188,7 @@ To allow signing a CertificateSigningRequest: A few steps are required in order to get a normal user to be able to authenticate and invoke an API. First, this user must have certificate issued -by the Kubernetes cluster, and then present that Certificate to the API call -as the Certificate Header or through the kubectl. +by the Kubernetes cluster, and then present that certificate to the Kubernetes API. ### Create private key @@ -211,8 +213,6 @@ kind: CertificateSigningRequest metadata: name: myuser spec: - groups: - - system:authenticated request: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQ1ZqQ0NBVDRDQVFBd0VURVBNQTBHQTFVRUF3d0dZVzVuWld4aE1JSUJJakFOQmdrcWhraUc5dzBCQVFFRgpBQU9DQVE4QU1JSUJDZ0tDQVFFQTByczhJTHRHdTYxakx2dHhWTTJSVlRWMDNHWlJTWWw0dWluVWo4RElaWjBOCnR2MUZtRVFSd3VoaUZsOFEzcWl0Qm0wMUFSMkNJVXBGd2ZzSjZ4MXF3ckJzVkhZbGlBNVhwRVpZM3ExcGswSDQKM3Z3aGJlK1o2MVNrVHF5SVBYUUwrTWM5T1Nsbm0xb0R2N0NtSkZNMUlMRVI3QTVGZnZKOEdFRjJ6dHBoaUlFMwpub1dtdHNZb3JuT2wzc2lHQ2ZGZzR4Zmd4eW8ybmlneFNVekl1bXNnVm9PM2ttT0x1RVF6cXpkakJ3TFJXbWlECklmMXBMWnoyalVnald4UkhCM1gyWnVVV1d1T09PZnpXM01LaE8ybHEvZi9DdS8wYk83c0x0MCt3U2ZMSU91TFcKcW90blZtRmxMMytqTy82WDNDKzBERHk5aUtwbXJjVDBnWGZLemE1dHJRSURBUUFCb0FBd0RRWUpLb1pJaHZjTgpBUUVMQlFBRGdnRUJBR05WdmVIOGR4ZzNvK21VeVRkbmFjVmQ1N24zSkExdnZEU1JWREkyQTZ1eXN3ZFp1L1BVCkkwZXpZWFV0RVNnSk1IRmQycVVNMjNuNVJsSXJ3R0xuUXFISUh5VStWWHhsdnZsRnpNOVpEWllSTmU3QlJvYXgKQVlEdUI5STZXT3FYbkFvczFqRmxNUG5NbFpqdU5kSGxpT1BjTU1oNndLaTZzZFhpVStHYTJ2RUVLY01jSVUyRgpvU2djUWdMYTk0aEpacGk3ZnNMdm1OQUxoT045UHdNMGM1dVJVejV4T0dGMUtCbWRSeEgvbUNOS2JKYjFRQm1HCkkwYitEUEdaTktXTU0xMzhIQXdoV0tkNjVoVHdYOWl4V3ZHMkh4TG1WQzg0L1BHT0tWQW9FNkpsYWFHdTlQVmkKdjlOSjVaZlZrcXdCd0hKbzZXdk9xVlA3SVFjZmg3d0drWm89Ci0tLS0tRU5EIENFUlRJRklDQVRFIFJFUVVFU1QtLS0tLQo= signerName: kubernetes.io/kube-apiserver-client usages: From 0d57c5dffafe85c955a83c9213f43f7e0ffcbf99 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Tue, 20 Jul 2021 16:28:19 -0400 Subject: [PATCH 130/175] Fix note about SA OIDC discovery doc default binding Added in v1.19: https://pr.k8s.io/88344 Signed-off-by: Monis Khan --- .../configure-pod-container/configure-service-account.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/content/en/docs/tasks/configure-pod-container/configure-service-account.md b/content/en/docs/tasks/configure-pod-container/configure-service-account.md index cf31ecb3c3..e5a0e26ea0 100644 --- a/content/en/docs/tasks/configure-pod-container/configure-service-account.md +++ b/content/en/docs/tasks/configure-pod-container/configure-service-account.md @@ -349,8 +349,11 @@ JSON Web Key Set (JWKS) at `/openid/v1/jwks`. The OpenID Provider Configuration is sometimes referred to as the _discovery document_. Clusters include a default RBAC ClusterRole called -`system:service-account-issuer-discovery`. No role bindings are provided -by default. Administrators may, for example, choose whether to bind the role to +`system:service-account-issuer-discovery`. A default RBAC ClusterRoleBinding +assigns this role to the `system:serviceaccounts` group, which all service +accounts implicitly belong to. This allows pods running on the cluster to access +the service account discovery document via their mounted service account token. +Administrators may, additionally, choose to bind the role to `system:authenticated` or `system:unauthenticated` depending on their security requirements and which external systems they intend to federate with. From 6d273bda2f22008121a37a271f0c1249a1454994 Mon Sep 17 00:00:00 2001 From: Arhell Date: Wed, 21 Jul 2021 00:52:41 +0300 Subject: [PATCH 131/175] [es] Operator: Exists missing --- content/es/examples/controllers/daemonset.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/content/es/examples/controllers/daemonset.yaml b/content/es/examples/controllers/daemonset.yaml index f6c598c9bf..e41e0a6b6f 100644 --- a/content/es/examples/controllers/daemonset.yaml +++ b/content/es/examples/controllers/daemonset.yaml @@ -16,6 +16,7 @@ spec: spec: tolerations: - key: node-role.kubernetes.io/master + operator: Exists effect: NoSchedule containers: - name: fluentd-elasticsearch From 98b355fa2b64ea3bfac39374079dcdc819096de1 Mon Sep 17 00:00:00 2001 From: Mengjiao Liu Date: Tue, 20 Jul 2021 15:22:19 +0800 Subject: [PATCH 132/175] [zh] sync glossary kube-controller-manager --- content/zh/docs/reference/glossary/kube-controller-manager.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/zh/docs/reference/glossary/kube-controller-manager.md b/content/zh/docs/reference/glossary/kube-controller-manager.md index 5d18857fde..43aa192d17 100644 --- a/content/zh/docs/reference/glossary/kube-controller-manager.md +++ b/content/zh/docs/reference/glossary/kube-controller-manager.md @@ -29,9 +29,9 @@ tags: --> -在主节点上运行 {{< glossary_tooltip text="控制器" term_id="controller" >}} 的组件。 +运行{{< glossary_tooltip text="控制器" term_id="controller" >}}进程的控制平面组件。 From 77067866e2a9bae45cb6cfa02bf3e19be2c0d6a3 Mon Sep 17 00:00:00 2001 From: JinlongHe <2969921454@qq.com> Date: Wed, 21 Jul 2021 13:38:09 +0800 Subject: [PATCH 133/175] Update guestbook.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修正 “PHP 留言板应用程序” 教程中错误的描述 --- .../docs/tutorials/stateless-application/guestbook.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/content/zh/docs/tutorials/stateless-application/guestbook.md b/content/zh/docs/tutorials/stateless-application/guestbook.md index 158ed7a727..04d252c0e4 100644 --- a/content/zh/docs/tutorials/stateless-application/guestbook.md +++ b/content/zh/docs/tutorials/stateless-application/guestbook.md @@ -89,7 +89,7 @@ The manifest file, included below, specifies a Deployment controller that runs a 1. Apply the Redis Deployment from the `redis-leader-deployment.yaml` file: --> 1. 在下载清单文件的目录中启动终端窗口。 -2. 从 `mongo-deployment.yaml` 文件中应用 MongoDB Deployment： +2. 从 `redis-leader-deployment.yaml` 文件中应用 Redis Deployment： -3. 查询 Pod 列表以验证 MongoDB Pod 是否正在运行： +3. 查询 Pod 列表以验证 Redis Pod 是否正在运行： ```shell kubectl get pods @@ -122,7 +122,7 @@ The manifest file, included below, specifies a Deployment controller that runs a -4. 运行以下命令查看 MongoDB Deployment 中的日志： +4. 运行以下命令查看 Redis Deployment 中的日志： ```shell kubectl logs -f deployment/redis-leader @@ -136,7 +136,7 @@ The manifest file, included below, specifies a Deployment controller that runs a -留言板应用程序需要往 MongoDB 中写数据。因此，需要创建 +留言板应用程序需要往 Redis 中写数据。因此，需要创建 [Service](/zh/docs/concepts/services-networking/service/) 来转发 Redis Pod 的流量。Service 定义了访问 Pod 的策略。 @@ -632,4 +632,3 @@ Deleting the Deployments and Services also deletes any running Pods. Use labels [MySQL 和 Wordpress 的持久卷](/zh/docs/tutorials/stateful-application/mysql-wordpress-persistent-volume/#visit-your-new-wordpress-blog) * 进一步阅读[连接应用程序](/zh/docs/concepts/services-networking/connect-applications-service/) * 进一步阅读[管理资源](/zh/docs/concepts/cluster-administration/manage-deployment/#using-labels-effectively) - From be30472ac0e61c240086e2ebf10d14e7597e9133 Mon Sep 17 00:00:00 2001 From: Samuel Walker Date: Wed, 21 Jul 2021 01:40:15 -0400 Subject: [PATCH 134/175] Env for Architecture Supports multiple architectures for the "Without a Package Manager" step such as arm64. Useful in Arch + ARM's scenario as kubeadm pacman repo is currently not maintained for ARM. --- .../tools/kubeadm/install-kubeadm.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/install-kubeadm.md b/content/en/docs/setup/production-environment/tools/kubeadm/install-kubeadm.md index c997156827..922687094a 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/install-kubeadm.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/install-kubeadm.md @@ -240,8 +240,9 @@ Install CNI plugins (required for most pod network): ```bash CNI_VERSION="v0.8.2" +ARCH="amd64" #or arm64 sudo mkdir -p /opt/cni/bin -curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" | sudo tar -C /opt/cni/bin -xz +curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" | sudo tar -C /opt/cni/bin -xz ``` Define the directory to download command files @@ -260,15 +261,17 @@ Install crictl (required for kubeadm / Kubelet Container Runtime Interface (CRI) ```bash CRICTL_VERSION="v1.17.0" -curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | sudo tar -C $DOWNLOAD_DIR -xz +ARCH="amd64" #or arm64 +curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | sudo tar -C $DOWNLOAD_DIR -xz ``` Install `kubeadm`, `kubelet`, `kubectl` and add a `kubelet` systemd service: ```bash RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)" +ARCH="amd64" #or arm64 cd $DOWNLOAD_DIR -sudo curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} +sudo curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/${ARCH}/{kubeadm,kubelet,kubectl} sudo chmod +x {kubeadm,kubelet,kubectl} RELEASE_VERSION="v0.4.0" From e76f7a72cbe7205bb76afc5244121adf61084314 Mon Sep 17 00:00:00 2001 From: JinlongHe Date: Wed, 21 Jul 2021 16:40:08 +0800 Subject: [PATCH 135/175] Update _index.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修改教程标题 --- content/zh/docs/tutorials/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/zh/docs/tutorials/_index.md b/content/zh/docs/tutorials/_index.md index 821855b712..c30f00019c 100644 --- a/content/zh/docs/tutorials/_index.md +++ b/content/zh/docs/tutorials/_index.md @@ -72,7 +72,7 @@ Kubernetes 文档的这一部分包含教程。每个教程展示了如何完成 * [公开外部 IP 地址访问集群中的应用程序](/zh/docs/tutorials/stateless-application/expose-external-ip-address/) -* [示例：使用 MongoDB 部署 PHP 留言板应用程序](/zh/docs/tutorials/stateless-application/guestbook/) +* [示例：使用 Redis 部署 PHP 留言板应用程序](/zh/docs/tutorials/stateless-application/guestbook/) ### kubelet 驱逐时 Pod 的选择 @@ -414,7 +414,7 @@ The kubelet uses the following parameters to determine pod eviction order: kubelet 使用以下参数来确定 Pod 驱逐顺序： 1. Pod 的资源使用是否超过其请求 -1. [Pod 优先级](/zh/docs/concepts/configuration/pod-priority-preemption/) +1. [Pod 优先级](/zh/docs/concepts/scheduling-eviction/pod-priority-preemption/) 1. Pod 相对于请求的资源使用情况 ## DaemonSet Update Strategy From dfd2edc70c710afd794f76c0cd6d8c8e58de2b9d Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Sun, 30 May 2021 02:03:13 -0400 Subject: [PATCH 150/175] zh: sync concepts/cluster-administration/addons.md Signed-off-by: Rui Chen --- .../zh/docs/concepts/cluster-administration/addons.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/content/zh/docs/concepts/cluster-administration/addons.md b/content/zh/docs/concepts/cluster-administration/addons.md index c5295d9518..adadc7954f 100644 --- a/content/zh/docs/concepts/cluster-administration/addons.md +++ b/content/zh/docs/concepts/cluster-administration/addons.md @@ -33,7 +33,7 @@ Add-ons 扩展了 Kubernetes 的功能。 * [CNI-Genie](https://github.com/Huawei-PaaS/CNI-Genie) enables Kubernetes to seamlessly connect to a choice of CNI plugins, such as Calico, Canal, Flannel, Romana, or Weave. * [Contiv](http://contiv.github.io) provides configurable networking (native L3 using BGP, overlay using vxlan, classic L2, and Cisco-SDN/ACI) for various use cases and a rich policy framework. Contiv project is fully [open sourced](http://github.com/contiv). The [installer](http://github.com/contiv/install) provides both kubeadm and non-kubeadm based installation options. * [Contrail](http://www.juniper.net/us/en/products-services/sdn/contrail/contrail-networking/), based on [Tungsten Fabric](https://tungsten.io), is an open source, multi-cloud network virtualization and policy management platform. Contrail and Tungsten Fabric are integrated with orchestration systems such as Kubernetes, OpenShift, OpenStack and Mesos, and provide isolation modes for virtual machines, containers/pods and bare metal workloads. -* [Flannel](https://github.com/coreos/flannel/blob/master/Documentation/kubernetes.md) is an overlay network provider that can be used with Kubernetes. +* [Flannel](https://github.com/flannel-io/flannel#deploying-flannel-manually) is an overlay network provider that can be used with Kubernetes. * [Knitter](https://github.com/ZTE/Knitter/) is a network solution supporting multiple networking in Kubernetes. * [Multus](https://github.com/Intel-Corp/multus-cni) is a Multi plugin for multiple network support in Kubernetes to support all CNI plugins (e.g. Calico, Cilium, Contiv, Flannel), in addition to SRIOV, DPDK, OVS-DPDK and VPP based workloads in Kubernetes. * [OVN-Kubernetes](https://github.com/ovn-org/ovn-kubernetes/) is a networking provider for Kubernetes based on [OVN (Open Virtual Network)](https://github.com/ovn-org/ovn/), a virtual networking implementation that came out of the Open vSwitch (OVS) project. OVN-Kubernetes provides an overlay based networking implementation for Kubernetes, including an OVS based implementation of load balancing and network policy. @@ -46,7 +46,7 @@ Add-ons 扩展了 Kubernetes 的功能。 ## 网络和网络策略 * [ACI](https://www.github.com/noironetworks/aci-containers) 通过 Cisco ACI 提供集成的容器网络和安全网络。 -* [Antrea](https://antrea.io/) 在第 3/4 层执行操作，为 Kubernetes +* [Antrea](https://antrea.io/) 在第 3/4 层执行操作，为 Kubernetes 提供网络连接和安全服务。Antrea 利用 Open vSwitch 作为网络的数据面。 * [Calico](https://docs.projectcalico.org/v3.11/getting-started/kubernetes/installation/calico) 是一个安全的 L3 网络和网络策略驱动。 @@ -63,7 +63,7 @@ Add-ons 扩展了 Kubernetes 的功能。 是一个开源的多云网络虚拟化和策略管理平台，Contrail 和 Tungsten Fabric 与业务流程系统 （例如 Kubernetes、OpenShift、OpenStack和Mesos）集成在一起， 为虚拟机、容器或 Pod 以及裸机工作负载提供了隔离模式。 -* [Flannel](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml) +* [Flannel](https://github.com/flannel-io/flannel#deploying-flannel-manually) 是一个可以用于 Kubernetes 的 overlay 网络提供者。 * [Knitter](https://github.com/ZTE/Knitter/) 是为 kubernetes 提供复合网络解决方案的网络组件。 * [Multus](https://github.com/Intel-Corp/multus-cni) 是一个多插件，可在 Kubernetes 中提供多种网络支持， @@ -86,7 +86,7 @@ Add-ons 扩展了 Kubernetes 的功能。 * [Romana](https://romana.io) 是一个 pod 网络的第三层解决方案，并支持[ NetworkPolicy API](/zh/docs/concepts/services-networking/network-policies/)。 Kubeadm add-on 安装细节可以在[这里](https://github.com/romana/romana/tree/master/containerize)找到。 -* [Weave Net](https://www.weave.works/docs/net/latest/kubernetes/kube-addon/) +* [Weave Net](https://www.weave.works/docs/net/latest/kubernetes/kube-addon/) 提供在网络分组两端参与工作的网络和网络策略，并且不需要额外的数据库。 @@ -191,4 +191,3 @@ kubectl delete ds fluentd-elasticsearch -n kube-system * See [Performing a rollback on a DaemonSet](/docs/tasks/manage-daemon/rollback-daemon-set/) * See [Creating a DaemonSet to adopt existing DaemonSet pods](/docs/concepts/workloads/controllers/daemonset/) - From 71fd082bad561809f54d7cdbb7fab25853a85f1f Mon Sep 17 00:00:00 2001 From: Mengjiao Liu Date: Fri, 23 Jul 2021 10:07:06 +0800 Subject: [PATCH 153/175] [zh] Fix link for debian-base --- content/zh/docs/tutorials/stateful-application/cassandra.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/zh/docs/tutorials/stateful-application/cassandra.md b/content/zh/docs/tutorials/stateful-application/cassandra.md index 461e3d1632..d3a4d7e118 100644 --- a/content/zh/docs/tutorials/stateful-application/cassandra.md +++ b/content/zh/docs/tutorials/stateful-application/cassandra.md @@ -405,7 +405,7 @@ to also be deleted. Never assume you'll be able to access data if its volume cla The Pods in this tutorial use the [`gcr.io/google-samples/cassandra:v13`](https://github.com/kubernetes/examples/blob/master/cassandra/image/Dockerfile) image from Google's [container registry](https://cloud.google.com/container-registry/docs/). -The Docker image above is based on [debian-base](https://github.com/kubernetes/kubernetes/tree/master/build/debian-base) +The Docker image above is based on [debian-base](https://github.com/kubernetes/release/tree/master/images/build/debian-base) and includes OpenJDK 8. This image includes a standard Cassandra installation from the Apache Debian repo. @@ -414,7 +414,7 @@ By using environment variables you can change values that are inserted into `cas ## Cassandra 容器环境变量 本教程中的 Pod 使用来自 Google [container registry](https://cloud.google.com/container-registry/docs/) 的 [`gcr.io/google-samples/cassandra:v13`](https://github.com/kubernetes/examples/blob/master/cassandra/image/Dockerfile) 镜像。 -上面的 Docker 镜像基于 [debian-base](https://github.com/kubernetes/kubernetes/tree/master/build/debian-base)，并且包含 OpenJDK 8。 +上面的 Docker 镜像基于 [debian-base](https://github.com/kubernetes/release/tree/master/images/build/debian-base)，并且包含 OpenJDK 8。 该映像包括来自 Apache Debian 存储库的标准 Cassandra 安装。 通过使用环境变量，您可以更改插入到 `cassandra.yaml` 中的值。 From 723cb6392994fa2935f0e0e2e2aeca1ca658f4af Mon Sep 17 00:00:00 2001 From: Mengjiao Liu Date: Fri, 23 Jul 2021 10:13:08 +0800 Subject: [PATCH 154/175] [ja] Fix link for debian-base --- content/ja/docs/tutorials/stateful-application/cassandra.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/ja/docs/tutorials/stateful-application/cassandra.md b/content/ja/docs/tutorials/stateful-application/cassandra.md index c5f8ed3986..4283a378b5 100644 --- a/content/ja/docs/tutorials/stateful-application/cassandra.md +++ b/content/ja/docs/tutorials/stateful-application/cassandra.md @@ -246,7 +246,7 @@ StatefulSetに関連するすべてのリソースを自動的に破棄するよ ## Cassandraコンテナの環境変数 -このチュートリアルのPodでは、Googleの[コンテナレジストリ](https://cloud.google.com/container-registry/docs/)の[`gcr.io/google-samples/cassandra:v13`](https://github.com/kubernetes/examples/blob/master/cassandra/image/Dockerfile)イメージを使用しました。このDockerイメージは[debian-base](https://github.com/kubernetes/kubernetes/tree/master/build/debian-base)をベースにしており、OpenJDK 8が含まれています。 +このチュートリアルのPodでは、Googleの[コンテナレジストリ](https://cloud.google.com/container-registry/docs/)の[`gcr.io/google-samples/cassandra:v13`](https://github.com/kubernetes/examples/blob/master/cassandra/image/Dockerfile)イメージを使用しました。このDockerイメージは[debian-base](https://github.com/kubernetes/release/tree/master/images/build/debian-base)をベースにしており、OpenJDK 8が含まれています。 このイメージには、Apache Debianリポジトリの標準のCassandraインストールが含まれます。 環境変数を利用すると、`cassandra.yaml`に挿入された値を変更できます。 From b734ae26ccaa6537197eb1eeefcaee5ec86b9bd9 Mon Sep 17 00:00:00 2001 From: Mengjiao Liu Date: Fri, 23 Jul 2021 10:16:54 +0800 Subject: [PATCH 155/175] [ko] Fix link for debian-base --- content/ko/docs/tutorials/stateful-application/cassandra.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/ko/docs/tutorials/stateful-application/cassandra.md b/content/ko/docs/tutorials/stateful-application/cassandra.md index 8f7c2b37ba..3ebb7ec387 100644 --- a/content/ko/docs/tutorials/stateful-application/cassandra.md +++ b/content/ko/docs/tutorials/stateful-application/cassandra.md @@ -266,7 +266,7 @@ kubectl apply -f cassandra-statefulset.yaml 이 튜토리얼의 *파드* 는 구글의 [컨테이너 레지스트리](https://cloud.google.com/container-registry/docs/)에 [`gcr.io/google-samples/cassandra:v13`](https://github.com/kubernetes/examples/blob/master/cassandra/image/Dockerfile) 이미지를 이용한다. -이 도커 이미지는 [debian-base](https://github.com/kubernetes/kubernetes/tree/master/build/debian-base)에 +이 도커 이미지는 [debian-base](https://github.com/kubernetes/release/tree/master/images/build/debian-base)에 기반하였고 OpenJDK 8을 포함한다. 이 이미지는 아파치 데비안 리포의 표준 카산드라 설치본을 포함한다. From 1779ade51b857aa1bc7cec1b2cf08c5b11ced601 Mon Sep 17 00:00:00 2001 From: XinYang Date: Fri, 23 Jul 2021 10:23:58 +0800 Subject: [PATCH 156/175] Update anchor for hairpin mode Signed-off-by: XinYang --- .../tools/kubeadm/troubleshooting-kubeadm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md b/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md index 42a2f4c59e..35551e70df 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md @@ -163,7 +163,7 @@ services](/docs/concepts/services-networking/service/#nodeport) or use `HostNetw ## Pods are not accessible via their Service IP -- Many network add-ons do not yet enable [hairpin mode](/docs/tasks/debug-application-cluster/debug-service/#a-pod-cannot-reach-itself-via-service-ip) +- Many network add-ons do not yet enable [hairpin mode](/docs/tasks/debug-application-cluster/debug-service/#a-pod-fails-to-reach-itself-via-the-service-ip) which allows pods to access themselves via their Service IP. This is an issue related to [CNI](https://github.com/containernetworking/cni/issues/476). Please contact the network add-on provider to get the latest status of their support for hairpin mode. From 68c66893ab08e929419ac58a4aa7c6cbedaa9ef6 Mon Sep 17 00:00:00 2001 From: Mengjiao Liu Date: Fri, 23 Jul 2021 10:29:27 +0800 Subject: [PATCH 157/175] [zh] Fix secret name to be consistent with examples --- .../tasks/configmap-secret/managing-secret-using-kubectl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/zh/docs/tasks/configmap-secret/managing-secret-using-kubectl.md b/content/zh/docs/tasks/configmap-secret/managing-secret-using-kubectl.md index a4be3c9d04..8e2dd7d202 100644 --- a/content/zh/docs/tasks/configmap-secret/managing-secret-using-kubectl.md +++ b/content/zh/docs/tasks/configmap-secret/managing-secret-using-kubectl.md @@ -104,7 +104,7 @@ run the following command: 可以像下面一样执行命令： ```shell -kubectl create secret generic dev-db-secret \ +kubectl create secret generic db-user-pass \ --from-literal=username=devuser \ --from-literal=password='S!B\*d$zDsb=' ``` From d095a148f2fd90827ba2fb17612d096e57de7331 Mon Sep 17 00:00:00 2001 From: XinYang Date: Thu, 22 Jul 2021 20:27:32 +0800 Subject: [PATCH 158/175] Update kubectl deprecated flag delete-local-data Signed-off-by: XinYang revert changes Signed-off-by: XinYang update more Signed-off-by: XinYang --- .../tools/kubeadm/create-cluster-kubeadm.md | 2 +- .../run-replicated-stateful-application.md | 2 +- .../en/docs/tutorials/stateful-application/zookeeper.md | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm.md b/content/en/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm.md index 0a394ad022..56deeb1985 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm.md @@ -415,7 +415,7 @@ and make sure that the node is empty, then deconfigure the node. Talking to the control-plane node with the appropriate credentials, run: ```bash -kubectl drain --delete-local-data --force --ignore-daemonsets +kubectl drain --delete-emptydir-data --force --ignore-daemonsets ``` Before removing the node, reset the state installed by `kubeadm`: diff --git a/content/en/docs/tasks/run-application/run-replicated-stateful-application.md b/content/en/docs/tasks/run-application/run-replicated-stateful-application.md index 22f929c06f..e98830b9e3 100644 --- a/content/en/docs/tasks/run-application/run-replicated-stateful-application.md +++ b/content/en/docs/tasks/run-application/run-replicated-stateful-application.md @@ -379,7 +379,7 @@ This might impact other applications on the Node, so it's best to **only do this in a test cluster**. ```shell -kubectl drain --force --delete-local-data --ignore-daemonsets +kubectl drain --force --delete-emptydir-data --ignore-daemonsets ``` Now you can watch as the Pod reschedules on a different Node: diff --git a/content/en/docs/tutorials/stateful-application/zookeeper.md b/content/en/docs/tutorials/stateful-application/zookeeper.md index 6d517ef229..2844ae6a0e 100644 --- a/content/en/docs/tutorials/stateful-application/zookeeper.md +++ b/content/en/docs/tutorials/stateful-application/zookeeper.md @@ -937,7 +937,7 @@ Use [`kubectl drain`](/docs/reference/generated/kubectl/kubectl-commands/#drain) drain the node on which the `zk-0` Pod is scheduled. ```shell -kubectl drain $(kubectl get pod zk-0 --template {{.spec.nodeName}}) --ignore-daemonsets --force --delete-local-data +kubectl drain $(kubectl get pod zk-0 --template {{.spec.nodeName}}) --ignore-daemonsets --force --delete-emptydir-data ``` ``` @@ -972,7 +972,7 @@ Keep watching the `StatefulSet`'s Pods in the first terminal and drain the node `zk-1` is scheduled. ```shell -kubectl drain $(kubectl get pod zk-1 --template {{.spec.nodeName}}) --ignore-daemonsets --force --delete-local-data "kubernetes-node-ixsl" cordoned +kubectl drain $(kubectl get pod zk-1 --template {{.spec.nodeName}}) --ignore-daemonsets --force --delete-emptydir-data "kubernetes-node-ixsl" cordoned ``` ``` @@ -1015,7 +1015,7 @@ Continue to watch the Pods of the stateful set, and drain the node on which `zk-2` is scheduled. ```shell -kubectl drain $(kubectl get pod zk-2 --template {{.spec.nodeName}}) --ignore-daemonsets --force --delete-local-data +kubectl drain $(kubectl get pod zk-2 --template {{.spec.nodeName}}) --ignore-daemonsets --force --delete-emptydir-data ``` ``` @@ -1101,7 +1101,7 @@ zk-1 1/1 Running 0 13m Attempt to drain the node on which `zk-2` is scheduled. ```shell -kubectl drain $(kubectl get pod zk-2 --template {{.spec.nodeName}}) --ignore-daemonsets --force --delete-local-data +kubectl drain $(kubectl get pod zk-2 --template {{.spec.nodeName}}) --ignore-daemonsets --force --delete-emptydir-data ``` The output: From 9eb90fddd8f5519f0c6940ae6beed1ca080e0489 Mon Sep 17 00:00:00 2001 From: Mengjiao Liu Date: Fri, 23 Jul 2021 11:09:55 +0800 Subject: [PATCH 159/175] Fix Observability format to be consistent with other items --- content/en/docs/concepts/overview/what-is-kubernetes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/concepts/overview/what-is-kubernetes.md b/content/en/docs/concepts/overview/what-is-kubernetes.md index 1ace280139..d72f1beb48 100644 --- a/content/en/docs/concepts/overview/what-is-kubernetes.md +++ b/content/en/docs/concepts/overview/what-is-kubernetes.md @@ -45,7 +45,7 @@ Containers have become popular because they provide extra benefits, such as: * Agile application creation and deployment: increased ease and efficiency of container image creation compared to VM image use. * Continuous development, integration, and deployment: provides for reliable and frequent container image build and deployment with quick and efficient rollbacks (due to image immutability). * Dev and Ops separation of concerns: create application container images at build/release time rather than deployment time, thereby decoupling applications from infrastructure. -* Observability not only surfaces OS-level information and metrics, but also application health and other signals. +* Observability: not only surfaces OS-level information and metrics, but also application health and other signals. * Environmental consistency across development, testing, and production: Runs the same on a laptop as it does in the cloud. * Cloud and OS distribution portability: Runs on Ubuntu, RHEL, CoreOS, on-premises, on major public clouds, and anywhere else. * Application-centric management: Raises the level of abstraction from running an OS on virtual hardware to running an application on an OS using logical resources. From 8bd5598f3d69d5b975d0a5b7286bddeeb1e1ec6f Mon Sep 17 00:00:00 2001 From: Qiming Teng Date: Fri, 23 Jul 2021 12:01:32 +0800 Subject: [PATCH 160/175] Fix kubeadm config reference There are some types incorrectly parsed because the related go pkg was not properly updated/referenced. --- .../config-api/kubeadm-config.v1beta2.md | 144 +++++++++--------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/content/en/docs/reference/config-api/kubeadm-config.v1beta2.md b/content/en/docs/reference/config-api/kubeadm-config.v1beta2.md index 293c7dc779..95cc3161e0 100644 --- a/content/en/docs/reference/config-api/kubeadm-config.v1beta2.md +++ b/content/en/docs/reference/config-api/kubeadm-config.v1beta2.md @@ -116,7 +116,7 @@ The ClusterConfiguration type should be used to configure cluster-wide settings, including settings for: - Networking, that holds configuration for the networking topology of the cluster; use it e.g. to customize - node subnet or services subnet. + pod subnet or services subnet. - Etcd configurations; use it e.g. to customize the local etcd or to configure the API server for using an external etcd cluster. - kube-apiserver, kube-scheduler, kube-controller-manager configurations; use it to customize control-plane @@ -164,19 +164,19 @@ bootstrapTokens: groups: - system:bootstrappers:kubeadm:default-node-token nodeRegistration: - name: "ec2-10-100-0-1" - criSocket: "/var/run/dockershim.sock" - taints: - - key: "kubeadmNode" - value: "master" - effect: "NoSchedule" - kubeletExtraArgs: - cgroup-driver: "cgroupfs" - ignorePreflightErrors: - - IsPrivilegedUser + name: "ec2-10-100-0-1" + criSocket: "/var/run/dockershim.sock" + taints: + - key: "kubeadmNode" + value: "master" + effect: "NoSchedule" + kubeletExtraArgs: + cgroup-driver: "cgroupfs" + ignorePreflightErrors: + - IsPrivilegedUser localAPIEndpoint: - advertiseAddress: "10.100.0.1" - bindPort: 6443 + advertiseAddress: "10.100.0.1" + bindPort: 6443 certificateKey: "e6a2eb8581237ab72a4f494f30285ec12a9694d750b9785706a83bfcbbbd2204" --- apiVersion: kubeadm.k8s.io/v1beta2 @@ -184,59 +184,59 @@ kind: ClusterConfiguration etcd: # one of local or external local: - imageRepository: "k8s.gcr.io" - imageTag: "3.2.24" - dataDir: "/var/lib/etcd" - extraArgs: - listen-client-urls: "http://10.100.0.1:2379" - serverCertSANs: - - "ec2-10-100-0-1.compute-1.amazonaws.com" - peerCertSANs: - - "10.100.0.1" - # external: - # endpoints: - # - "10.100.0.1:2379" - # - "10.100.0.2:2379" - # caFile: "/etcd/kubernetes/pki/etcd/etcd-ca.crt" - # certFile: "/etcd/kubernetes/pki/etcd/etcd.crt" - # keyFile: "/etcd/kubernetes/pki/etcd/etcd.key" - networking: - serviceSubnet: "10.96.0.0/12" - podSubnet: "10.100.0.1/24" - dnsDomain: "cluster.local" - kubernetesVersion: "v1.12.0" - controlPlaneEndpoint: "10.100.0.1:6443" - apiServer: - extraArgs: - authorization-mode: "Node,RBAC" - extraVolumes: - - name: "some-volume" - hostPath: "/etc/some-path" - mountPath: "/etc/some-pod-path" - readOnly: false - pathType: File - certSANs: - - "10.100.1.1" - - "ec2-10-100-0-1.compute-1.amazonaws.com" - timeoutForControlPlane: 4m0s - controllerManager: - extraArgs: - "node-cidr-mask-size": "20" - extraVolumes: - - name: "some-volume" - hostPath: "/etc/some-path" - mountPath: "/etc/some-pod-path" - readOnly: false - pathType: File - scheduler: - extraArgs: - address: "10.100.0.1" - extraVolumes: - - name: "some-volume" - hostPath: "/etc/some-path" - mountPath: "/etc/some-pod-path" - readOnly: false - pathType: File + imageRepository: "k8s.gcr.io" + imageTag: "3.2.24" + dataDir: "/var/lib/etcd" + extraArgs: + listen-client-urls: "http://10.100.0.1:2379" + serverCertSANs: + - "ec2-10-100-0-1.compute-1.amazonaws.com" + peerCertSANs: + - "10.100.0.1" + # external: + # endpoints: + # - "10.100.0.1:2379" + # - "10.100.0.2:2379" + # caFile: "/etcd/kubernetes/pki/etcd/etcd-ca.crt" + # certFile: "/etcd/kubernetes/pki/etcd/etcd.crt" + # keyFile: "/etcd/kubernetes/pki/etcd/etcd.key" + networking: + serviceSubnet: "10.96.0.0/12" + podSubnet: "10.100.0.1/24" + dnsDomain: "cluster.local" + kubernetesVersion: "v1.12.0" + controlPlaneEndpoint: "10.100.0.1:6443" + apiServer: + extraArgs: + authorization-mode: "Node,RBAC" + extraVolumes: + - name: "some-volume" + hostPath: "/etc/some-path" + mountPath: "/etc/some-pod-path" + readOnly: false + pathType: File + certSANs: + - "10.100.1.1" + - "ec2-10-100-0-1.compute-1.amazonaws.com" + timeoutForControlPlane: 4m0s + controllerManager: + extraArgs: + "node-cidr-mask-size": "20" + extraVolumes: + - name: "some-volume" + hostPath: "/etc/some-path" + mountPath: "/etc/some-pod-path" + readOnly: false + pathType: File + scheduler: + extraArgs: + address: "10.100.0.1" + extraVolumes: + - name: "some-volume" + hostPath: "/etc/some-path" + mountPath: "/etc/some-pod-path" + readOnly: false + pathType: File certificatesDir: "/etc/kubernetes/pki" imageRepository: "k8s.gcr.io" useHyperKubeImage: false @@ -663,7 +663,7 @@ APIServer holds settings necessary for API server deployments in the cluster timeoutForControlPlane [Required]
-invalid type +meta/v1.Duration `timeoutForControlPlane` controls the timeout that we use for API server to appear @@ -712,7 +712,7 @@ for, so other administrators can know its purpose. ttl [Required]
-invalid type +meta/v1.Duration `ttl` defines the time to live for this token. Defaults to "24h". @@ -721,7 +721,7 @@ for, so other administrators can know its purpose. expires [Required]
-invalid type +meta/v1.Time `expires` specifies the timestamp when this token expires. Defaults to being set @@ -1004,7 +1004,7 @@ not contain any other authentication information timeout [Required]
-invalid type +meta/v1.Duration `timeout` modifies the discovery timeout. @@ -1199,7 +1199,7 @@ HostPathMount contains elements describing volumes that are mounted from the hos pathType [Required]
-invalid type +core/v1.HostPathType `pathType` is the type of the `hostPath` volume. @@ -1451,7 +1451,7 @@ annotated to the Node API object, for later re-use. taints [Required]
-[]invalid type +[]core/v1.Taint `taints` specifies the taints the Node API object should be registered with. If From 626b1013403833374253db6e6a9c998bfb62f090 Mon Sep 17 00:00:00 2001 From: Vijay Date: Fri, 23 Jul 2021 17:09:19 +1200 Subject: [PATCH 161/175] Update pod-lifecycle.md I think there is a typo, `readiness problem` needs to be `readiness probe`. --- content/en/docs/concepts/workloads/pods/pod-lifecycle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/concepts/workloads/pods/pod-lifecycle.md b/content/en/docs/concepts/workloads/pods/pod-lifecycle.md index 166d81e424..8f9a423edf 100644 --- a/content/en/docs/concepts/workloads/pods/pod-lifecycle.md +++ b/content/en/docs/concepts/workloads/pods/pod-lifecycle.md @@ -311,7 +311,7 @@ is different from the liveness probe. If your app has a strict dependency on back-end services, you can implement both a liveness and a readiness probe. The liveness probe passes when the app itself -is healthy, but the readiness problem additionally checks that each required +is healthy, but the readiness probe additionally checks that each required back-end service is available. This helps you avoid directing traffic to Pods that can only respond with error messages. From bc05d880e3bd549d304e2d759a20738c7441d029 Mon Sep 17 00:00:00 2001 From: Mengjiao Liu Date: Fri, 23 Jul 2021 14:18:53 +0800 Subject: [PATCH 162/175] [zh] Fix Observability format to be consistent with other list items. --- content/zh/docs/concepts/overview/what-is-kubernetes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/zh/docs/concepts/overview/what-is-kubernetes.md b/content/zh/docs/concepts/overview/what-is-kubernetes.md index 761d4736c1..97110f4f51 100644 --- a/content/zh/docs/concepts/overview/what-is-kubernetes.md +++ b/content/zh/docs/concepts/overview/what-is-kubernetes.md @@ -107,7 +107,7 @@ Containers are becoming popular because they have many benefits. Some of the con * Agile application creation and deployment: increased ease and efficiency of container image creation compared to VM image use. * Continuous development, integration, and deployment: provides for reliable and frequent container image build and deployment with quick and easy rollbacks (due to image immutability). * Dev and Ops separation of concerns: create application container images at build/release time rather than deployment time, thereby decoupling applications from infrastructure. -* Observability not only surfaces OS-level information and metrics, but also application health and other signals. +* Observability: not only surfaces OS-level information and metrics, but also application health and other signals. * Environmental consistency across development, testing, and production: Runs the same on a laptop as it does in the cloud. * Cloud and OS distribution portability: Runs on Ubuntu, RHEL, CoreOS, on-prem, Google Kubernetes Engine, and anywhere else. * Application-centric management: Raises the level of abstraction from running an OS on virtual hardware to running an application on an OS using logical resources. @@ -120,7 +120,7 @@ Containers are becoming popular because they have many benefits. Some of the con 容器镜像构建和部署。 * 关注开发与运维的分离：在构建/发布时而不是在部署时创建应用程序容器镜像， 从而将应用程序与基础架构分离。 -* 可观察性不仅可以显示操作系统级别的信息和指标，还可以显示应用程序的运行状况和其他指标信号。 +* 可观察性：不仅可以显示操作系统级别的信息和指标，还可以显示应用程序的运行状况和其他指标信号。 * 跨开发、测试和生产的环境一致性：在便携式计算机上与在云中相同地运行。 * 跨云和操作系统发行版本的可移植性：可在 Ubuntu、RHEL、CoreOS、本地、 Google Kubernetes Engine 和其他任何地方运行。 From b078e4916eb971d66a9049a2eebb2e580b00303e Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 23 Jul 2021 08:34:11 -0700 Subject: [PATCH 163/175] Apply suggestions from code review Co-authored-by: Tim Bannister --- content/en/docs/reference/labels-annotations-taints.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/docs/reference/labels-annotations-taints.md b/content/en/docs/reference/labels-annotations-taints.md index bb36561997..c123c8da5b 100644 --- a/content/en/docs/reference/labels-annotations-taints.md +++ b/content/en/docs/reference/labels-annotations-taints.md @@ -200,7 +200,7 @@ Used on: Service The kube-proxy has this label for custom proxy, which delegates service control to custom proxy. -## experimental.windows.kubernetes.io/isolation-type (deprecated) +## experimental.windows.kubernetes.io/isolation-type (deprecated) {#experimental-windows-kubernetes-io-isolation-type} Example: `experimental.windows.kubernetes.io/isolation-type: "hyperv"` @@ -210,7 +210,7 @@ The annotation is used to run Windows containers with Hyper-V isolation. To use {{< note >}} You can only set this annotation on Pods that have a single container. -Starting in v1.20, this annotation is deprecated as experimental hyper-v support was removed in 1.21. +Starting from v1.20, this annotation is deprecated. Experimental Hyper-V support was removed in 1.21. {{< /note >}} ## ingressclass.kubernetes.io/is-default-class From ee312ddc56ac3914d50c955056af2c848fac3561 Mon Sep 17 00:00:00 2001 From: Maciej Filocha Date: Fri, 23 Jul 2021 22:37:46 +0200 Subject: [PATCH 164/175] Add Polish Release index page --- content/pl/releases/_index.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 content/pl/releases/_index.md diff --git a/content/pl/releases/_index.md b/content/pl/releases/_index.md new file mode 100644 index 0000000000..5df8f36264 --- /dev/null +++ b/content/pl/releases/_index.md @@ -0,0 +1,27 @@ +--- +linktitle: Historia wydań +title: Wydania +type: docs +--- + + + + +Projekt Kubernetes zapewnia wsparcie dla trzech ostatnich wydań _minor_ ({{< skew latestVersion >}}, {{< skew prevMinorVersion >}}, {{< skew oldestMinorVersion >}}). Poprawki do wydania 1.19 i nowszych będą publikowane przez około rok. Kuberetes w wersji 1.18 i wcześniejszych będzie otrzymywał poprawki przez 9 miesięcy. + +Wersje Kubernetesa oznaczane są jako **x.y.z**, +gdzie **x** jest oznaczeniem wersji głównej (_major_), **y** — podwersji (_minor_), a **z** — numer poprawki (_patch_), zgodnie z terminologią [Semantic Versioning](https://semver.org/). + +Więcej informacji można z znaleźć w dokumencie [version skew policy](/releases/version-skew-policy/). + + + +## Historia wydań + +{{< release-data >}} + +## Nadchodzące wydania + +Zajrzyj na [harmonogram](https://github.com/kubernetes/sig-release/tree/master/releases/release-{{< skew nextMinorVersion >}}) nadchodzącego wydania Kubernetesa numer **{{< skew nextMinorVersion >}}**! + +## Przydatne zasoby \ No newline at end of file From d4184d45cbe96436b7d56069c2ca5e840b1224e8 Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Sat, 24 Jul 2021 02:50:30 +0100 Subject: [PATCH 165/175] Set publication date for article --- ...ngress-nginx.md => 2021-07-26-update-with-ingress-nginx.md} | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename content/en/blog/_posts/{2021-07-15-update-with-ingress-nginx.md => 2021-07-26-update-with-ingress-nginx.md} (99%) diff --git a/content/en/blog/_posts/2021-07-15-update-with-ingress-nginx.md b/content/en/blog/_posts/2021-07-26-update-with-ingress-nginx.md similarity index 99% rename from content/en/blog/_posts/2021-07-15-update-with-ingress-nginx.md rename to content/en/blog/_posts/2021-07-26-update-with-ingress-nginx.md index c95c96a82f..761b0b0575 100644 --- a/content/en/blog/_posts/2021-07-15-update-with-ingress-nginx.md +++ b/content/en/blog/_posts/2021-07-26-update-with-ingress-nginx.md @@ -1,8 +1,7 @@ --- -draft: true layout: blog title: 'Updating NGINX-Ingress to use the stable Ingress API' -date: 2021-07-15 +date: 2021-07-26 slug: update-with-ingress-nginx --- From 8c471377843dd723c428d9c32e00f66c1ccff0e2 Mon Sep 17 00:00:00 2001 From: Arhell Date: Sat, 24 Jul 2021 13:13:37 +0300 Subject: [PATCH 166/175] [id] update link to Flannel --- content/id/docs/concepts/cluster-administration/addons.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/id/docs/concepts/cluster-administration/addons.md b/content/id/docs/concepts/cluster-administration/addons.md index ca50347492..e8a52a4910 100644 --- a/content/id/docs/concepts/cluster-administration/addons.md +++ b/content/id/docs/concepts/cluster-administration/addons.md @@ -27,7 +27,7 @@ Laman ini akan menjabarkan beberapa *add-ons* yang tersedia serta tautan instruk * [CNI-Genie](https://github.com/Huawei-PaaS/CNI-Genie) memungkinkan Kubernetes agar dapat terkoneksi dengan beragam *plugin* CNI, seperti Calico, Canal, Flannel, Romana, atau Weave dengan mulus. * [Contiv](http://contiv.github.io) menyediakan jaringan yang dapat dikonfigurasi (*native* L3 menggunakan BGP, *overlay* menggunakan vxlan, klasik L2, dan Cisco-SDN/ACI) untuk berbagai penggunaan serta *policy framework* yang kaya dan beragam. Proyek Contiv merupakan proyek [open source](http://github.com/contiv). Laman [instalasi](http://github.com/contiv/install) ini akan menjabarkan cara instalasi, baik untuk klaster dengan kubeadm maupun non-kubeadm. * [Contrail](http://www.juniper.net/us/en/products-services/sdn/contrail/contrail-networking/), yang berbasis dari [Tungsten Fabric](https://tungsten.io), merupakan sebuah proyek *open source* yang menyediakan virtualisasi jaringan *multi-cloud* serta platform manajemen *policy*. Contrail dan Tungsten Fabric terintegrasi dengan sistem orkestrasi lainnya seperti Kubernetes, OpenShift, OpenStack dan Mesos, serta menyediakan mode isolasi untuk mesin virtual (VM), kontainer/pod dan *bare metal*. -* [Flannel](https://github.com/coreos/flannel/blob/master/Documentation/kubernetes.md) merupakan penyedia jaringan *overlay* yang dapat digunakan pada Kubernetes. +* [Flannel](https://github.com/flannel-io/flannel#deploying-flannel-manually) merupakan penyedia jaringan *overlay* yang dapat digunakan pada Kubernetes. * [Knitter](https://github.com/ZTE/Knitter/) merupakan solusi jaringan yang mendukung multipel jaringan pada Kubernetes. * [Multus](https://github.com/Intel-Corp/multus-cni) merupakan sebuah multi *plugin* agar Kubernetes mendukung multipel jaringan secara bersamaan sehingga dapat menggunakan semua *plugin* CNI (contoh: Calico, Cilium, Contiv, Flannel), ditambah pula dengan SRIOV, DPDK, OVS-DPDK dan VPP pada *workload* Kubernetes. * [NSX-T](https://docs.vmware.com/en/VMware-NSX-T/2.0/nsxt_20_ncp_kubernetes.pdf) Container Plug-in (NCP) menyediakan integrasi antara VMware NSX-T dan orkestrator kontainer seperti Kubernetes, termasuk juga integrasi antara NSX-T dan platform CaaS/PaaS berbasis kontainer seperti *Pivotal Container Service* (PKS) dan OpenShift. From 403dbdcc15fad0095bd6d64331555355c7ab3ceb Mon Sep 17 00:00:00 2001 From: Arhell Date: Sun, 25 Jul 2021 00:34:29 +0300 Subject: [PATCH 167/175] [pt-br] update link to Flannel --- content/pt-br/docs/concepts/cluster-administration/addons.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/pt-br/docs/concepts/cluster-administration/addons.md b/content/pt-br/docs/concepts/cluster-administration/addons.md index 0a50c96190..e72834ec38 100644 --- a/content/pt-br/docs/concepts/cluster-administration/addons.md +++ b/content/pt-br/docs/concepts/cluster-administration/addons.md @@ -27,7 +27,7 @@ Os Add-ons de cada sessão são classificados em ordem alfabética - a ordem nã * [CNI-Genie](https://github.com/Huawei-PaaS/CNI-Genie) permite que o Kubernetes se conecte facilmente a uma variedade de plugins CNI, como Calico, Canal, Flannel, Romana ou Weave. * [Contiv](http://contiv.github.io) fornece um rede configurável (L3 nativa usando BGP, sobreposição usando vxlan, L2 clássico e Cisco-SDN / ACI) para vários casos de uso e uma estrutura rica de políticas de rede. O projeto Contiv é totalmente [open source](http://github.com/contiv). O script de [instalação](http://github.com/contiv/install) fornece opções de instalação com ou sem kubeadm. * [Contrail](http://www.juniper.net/us/en/products-services/sdn/contrail/contrail-networking/), baseado no [Tungsten Fabric](https://tungsten.io), é um projeto open source, multi-cloud com uma rede virtualizada e com uma plataforma de gerenciamento de políticas de rede. O Contrail e o Tungsten Fabric estão integrados a sistemas de orquestração, como Kubernetes, OpenShift, OpenStack e Mesos, e fornecem modos de isolamento para máquinas virtuais, containers / pods e cargas em servidores físicos. -* [Flannel](https://github.com/coreos/flannel/blob/master/Documentation/kubernetes.md) é um provedor de rede de sobreposição que pode ser usado com o Kubernetes. +* [Flannel](https://github.com/flannel-io/flannel#deploying-flannel-manually) é um provedor de rede de sobreposição que pode ser usado com o Kubernetes. * [Knitter](https://github.com/ZTE/Knitter/) é uma solução de rede que suporta múltiplas redes no Kubernetes. * [Multus](https://github.com/Intel-Corp/multus-cni) é um plugin Multi para suporte a várias redes no Kubernetes para suportar todos os plugins CNI (por exemplo, Calico, Cilium, Contiv, Flannel), além das cargas de trabalho baseadas em SRIOV, DPDK, OVS-DPDK e VPP no Kubernetes. * [NSX-T](https://docs.vmware.com/en/VMware-NSX-T/2.0/nsxt_20_ncp_kubernetes.pdf) O Plugin de contêiner (NCP) fornece integração entre o VMware NSX-T e orquestradores de contêineres como o Kubernetes, além da integração entre o NSX-T e as plataformas CaaS / PaaS baseadas em contêiner, como Pivotal Container Service (PKS) e OpenShift. From 4aa2b6df61284f10fce9f9a4af99b3673e88e483 Mon Sep 17 00:00:00 2001 From: Sachin <57769917+slayer321@users.noreply.github.com> Date: Sun, 25 Jul 2021 16:47:06 +0530 Subject: [PATCH 168/175] docs: mention container probes (#29085) * mention container probe * fix style errors * fix errors --- content/en/docs/concepts/workloads/pods/_index.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/content/en/docs/concepts/workloads/pods/_index.md b/content/en/docs/concepts/workloads/pods/_index.md index 5dd6bac9de..20dbdcb9e8 100644 --- a/content/en/docs/concepts/workloads/pods/_index.md +++ b/content/en/docs/concepts/workloads/pods/_index.md @@ -282,6 +282,17 @@ on the Kubernetes API server for each static Pod. This means that the Pods running on a node are visible on the API server, but cannot be controlled from there. +## Container probes + +A _probe_ is a diagnostic performed periodically by the kubelet on a container. To perform a diagnostic, the kubelet can invoke different actions: + +- `ExecAction` (performed with the help of the container runtime) +- `TCPSocketAction` (checked directly by the kubelet) +- `HTTPGetAction` (checked directly by the kubelet) + +You can read more about [probes](/docs/concepts/workloads/pods/pod-lifecycle/#container-probes) +in the Pod Lifecycle documentation. + ## {{% heading "whatsnext" %}} * Learn about the [lifecycle of a Pod](/docs/concepts/workloads/pods/pod-lifecycle/). From 26748e36b0b3d733a4f87fd327123ce58c0250d5 Mon Sep 17 00:00:00 2001 From: Qiming Teng Date: Sun, 25 Jul 2021 19:57:50 +0800 Subject: [PATCH 169/175] Fix service name requirement --- .../overview/working-with-objects/names.md | 15 +++++++++++++-- .../docs/concepts/services-networking/service.md | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/content/en/docs/concepts/overview/working-with-objects/names.md b/content/en/docs/concepts/overview/working-with-objects/names.md index 8e74eb5c0b..9bafb1584c 100644 --- a/content/en/docs/concepts/overview/working-with-objects/names.md +++ b/content/en/docs/concepts/overview/working-with-objects/names.md @@ -28,7 +28,7 @@ For non-unique user-provided attributes, Kubernetes provides [labels](/docs/conc In cases when objects represent a physical entity, like a Node representing a physical host, when the host is re-created under the same name without deleting and re-creating the Node, Kubernetes treats the new host as the old one, which may lead to inconsistencies. {{< /note >}} -Below are three types of commonly used name constraints for resources. +Below are four types of commonly used name constraints for resources. ### DNS Subdomain Names @@ -41,7 +41,7 @@ This means the name must: - start with an alphanumeric character - end with an alphanumeric character -### DNS Label Names +### RFC 1123 Label Names {#dns-label-names} Some resource types require their names to follow the DNS label standard as defined in [RFC 1123](https://tools.ietf.org/html/rfc1123). @@ -52,6 +52,17 @@ This means the name must: - start with an alphanumeric character - end with an alphanumeric character +### RFC 1035 Label Names + +Some resource types require their names to follow the DNS +label standard as defined in [RFC 1035](https://tools.ietf.org/html/rfc1035). +This means the name must: + +- contain at most 63 characters +- contain only lowercase alphanumeric characters or '-' +- start with an alphabetic character +- end with an alphanumeric character + ### Path Segment Names Some resource types require their names to be able to be safely encoded as a diff --git a/content/en/docs/concepts/services-networking/service.md b/content/en/docs/concepts/services-networking/service.md index 011e1d395f..9c4b90914b 100644 --- a/content/en/docs/concepts/services-networking/service.md +++ b/content/en/docs/concepts/services-networking/service.md @@ -72,7 +72,7 @@ A Service in Kubernetes is a REST object, similar to a Pod. Like all of the REST objects, you can `POST` a Service definition to the API server to create a new instance. The name of a Service object must be a valid -[DNS label name](/docs/concepts/overview/working-with-objects/names#dns-label-names). +[RFC 1035 label name](/docs/concepts/overview/working-with-objects/names#rfc-1035-label-names). For example, suppose you have a set of Pods where each listens on TCP port 9376 and contains a label `app=MyApp`: From b2084c9928e596af04421db6cfd010d9f88109a6 Mon Sep 17 00:00:00 2001 From: Vaibhav Date: Mon, 26 Jul 2021 14:10:07 +0530 Subject: [PATCH 170/175] Update the upcoming-changes-in-kubernetes-1-22/index.md --- .../2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md b/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md index bfed8d9407..4c3fbdefee 100644 --- a/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md +++ b/content/en/blog/_posts/2021-07-14-upcoming-changes-in-kubernetes-1-22/index.md @@ -154,7 +154,7 @@ removals before you upgrade to Kubernetes v1.22. To do that, add the following to the kube-apiserver command line arguments: -`--runtime-config=admissionregistration.k8s.io/v1beta1=false,apiextensions.k8s.io/v1beta1=false,apiregistration.k8s.io/v1beta1=false,authentication.k8s.io/v1beta1=false,authorization.k9s.io/v1=false,certificates.k8s.io/v1beta=false,coordination.k8s.io/v1beta1=false,extensions/v1beta1/ingresses=false,networking.k8s.io/v1beta1=false` +`--runtime-config=admissionregistration.k8s.io/v1beta1=false,apiextensions.k8s.io/v1beta1=false,apiregistration.k8s.io/v1beta1=false,authentication.k8s.io/v1beta1=false,authorization.k8s.io/v1beta1=false,certificates.k8s.io/v1beta1=false,coordination.k8s.io/v1beta1=false,extensions/v1beta1/ingresses=false,networking.k8s.io/v1beta1=false` (as a side effect, this also turns off v1beta1 of EndpointSlice - watch out for that when you're testing). From e4d7ffb6d653cb80a9b5a6cd50dfb4885aadd856 Mon Sep 17 00:00:00 2001 From: Sam Wheating Date: Mon, 26 Jul 2021 09:14:03 -0700 Subject: [PATCH 171/175] Update content/en/docs/tasks/run-application/horizontal-pod-autoscale.md Co-authored-by: Tim Bannister --- .../run-application/horizontal-pod-autoscale.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md b/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md index ed095fa0cd..bc32cf97f2 100644 --- a/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md +++ b/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md @@ -198,8 +198,17 @@ The detailed documentation of `kubectl autoscale` can be found [here](/docs/refe ## Autoscaling during rolling update -Currently in Kubernetes, a rolling update is performed using a deployment object, which manages the underlying replica sets for you. -A Horizontal Pod Autoscaler is bound to a single deployment object - it sets the size for the deployment object, and the deployment is responsible for setting sizes of underlying replica sets. +Kubernetes lets you perform a rolling update on a Deployment. In that +case, the Deployment manages the underlying ReplicaSets for you. +When you configure autoscaling for a Deployment, you bind a +HorizontalPodAutoscaler to a single Deployment. The HorizontalPodAutoscaler +manages the `replicas` field of the Deployment. The deployment controller is responsible +for setting the `replicas` of the underlying ReplicaSets so that they add up to a suitable +number during the rollout and also afterwards. + +If you perform a rolling update of a StatefulSet that has an autoscaled number of +replicas, the StatefulSet directly manages its set of Pods (there is no intermediate resource +similar to ReplicaSet). Horizontal Pod Autoscaler does not work with rolling update using direct manipulation of replication controllers, i.e. you cannot bind a Horizontal Pod Autoscaler to a replication controller and do rolling update. From 498647c4388636ccbbabb9fdbdc4f9f2aba67072 Mon Sep 17 00:00:00 2001 From: Sam Wheating Date: Mon, 26 Jul 2021 09:17:10 -0700 Subject: [PATCH 172/175] Removing paragraph about ReplicationController --- .../docs/tasks/run-application/horizontal-pod-autoscale.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md b/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md index bc32cf97f2..f1d008edce 100644 --- a/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md +++ b/content/en/docs/tasks/run-application/horizontal-pod-autoscale.md @@ -210,11 +210,6 @@ If you perform a rolling update of a StatefulSet that has an autoscaled number o replicas, the StatefulSet directly manages its set of Pods (there is no intermediate resource similar to ReplicaSet). -Horizontal Pod Autoscaler does not work with rolling update using direct manipulation of replication controllers, -i.e. you cannot bind a Horizontal Pod Autoscaler to a replication controller and do rolling update. -The reason this doesn't work is that when rolling update creates a new replication controller, -the Horizontal Pod Autoscaler will not be bound to the new replication controller. - ## Support for cooldown/delay When managing the scale of a group of replicas using the Horizontal Pod Autoscaler, From 526f0d29722b5141546a078631c3c4ea8ea5aee5 Mon Sep 17 00:00:00 2001 From: Swetha Repakula Date: Mon, 26 Jul 2021 11:24:29 -0700 Subject: [PATCH 173/175] Update EPS docs to indicate behavior is unique to 1.21 --- content/en/docs/concepts/services-networking/service.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/concepts/services-networking/service.md b/content/en/docs/concepts/services-networking/service.md index 011e1d395f..921ab7480e 100644 --- a/content/en/docs/concepts/services-networking/service.md +++ b/content/en/docs/concepts/services-networking/service.md @@ -188,7 +188,7 @@ selectors and uses DNS names instead. For more information, see the [ExternalName](#externalname) section later in this document. ### Over Capacity Endpoints -If an Endpoints resource has more than 1000 endpoints then a Kubernetes v1.21 (or later) +If an Endpoints resource has more than 1000 endpoints then a Kubernetes v1.21 cluster annotates that Endpoints with `endpoints.kubernetes.io/over-capacity: warning`. This annotation indicates that the affected Endpoints object is over capacity. From 4226eb76c6258107cacbff5bb6cd33bb30037075 Mon Sep 17 00:00:00 2001 From: Arhell Date: Tue, 27 Jul 2021 00:19:08 +0300 Subject: [PATCH 174/175] [de] update link to Flannel --- content/de/docs/concepts/cluster-administration/addons.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/de/docs/concepts/cluster-administration/addons.md b/content/de/docs/concepts/cluster-administration/addons.md index f5eedeb59b..abf15e453f 100644 --- a/content/de/docs/concepts/cluster-administration/addons.md +++ b/content/de/docs/concepts/cluster-administration/addons.md @@ -26,7 +26,7 @@ Die Add-Ons in den einzelnen Kategorien sind alphabetisch sortiert - Die Reihenf * [CNI-Genie](https://github.com/Huawei-PaaS/CNI-Genie) ermöglicht das nahtlose Verbinden von Kubernetes mit einer Reihe an CNI-Plugins wie z.B. Calico, Canal, Flannel, Romana, oder Weave. * [Contiv](http://contiv.github.io) bietet konfigurierbares Networking (Native L3 auf BGP, Overlay mit vxlan, Klassisches L2, Cisco-SDN/ACI) für verschiedene Anwendungszwecke und auch umfangreiches Policy-Framework. Das Contiv-Projekt ist vollständig [Open Source](http://github.com/contiv). Der [installer](http://github.com/contiv/install) bietet sowohl kubeadm als auch nicht-kubeadm basierte Installationen. * [Contrail](http://www.juniper.net/us/en/products-services/sdn/contrail/contrail-networking/), basierend auf [Tungsten Fabric](https://tungsten.io), ist eine Open Source, multi-Cloud Netzwerkvirtualisierungs- und Policy-Management Plattform. Contrail und Tungsten Fabric sind mit Orechstratoren wie z.B. Kubernetes, OpenShift, OpenStack und Mesos integriert und bieten Isolationsmodi für Virtuelle Maschinen, Container (bzw. Pods) und Bare Metal workloads. -* [Flannel](https://github.com/coreos/flannel/blob/master/Documentation/kubernetes.md) ist ein Overlay-Network-Provider der mit Kubernetes genutzt werden kann. +* [Flannel](https://github.com/flannel-io/flannel#deploying-flannel-manually) ist ein Overlay-Network-Provider der mit Kubernetes genutzt werden kann. * [Knitter](https://github.com/ZTE/Knitter/) ist eine Network-Lösung die Mehrfach-Network in Kubernetes ermöglicht. * [Multus](https://github.com/Intel-Corp/multus-cni) ist ein Multi-Plugin für Mehrfachnetzwerk-Unterstützung um alle CNI-Plugins (z.B. Calico, Cilium, Contiv, Flannel), zusätzlich zu SRIOV-, DPDK-, OVS-DPDK- und VPP-Basierten Workloads in Kubernetes zu unterstützen. * [NSX-T](https://docs.vmware.com/en/VMware-NSX-T/2.0/nsxt_20_ncp_kubernetes.pdf) Container Plug-in (NCP) bietet eine Integration zwischen VMware NSX-T und einem Orchestator wie z.B. Kubernetes. Außerdem bietet es eine Integration zwischen NSX-T und Containerbasierten CaaS/PaaS-Plattformen wie z.B. Pivotal Container Service (PKS) und OpenShift. From a6edadd674eb1dbff88600cb0ff2227ff2b8473a Mon Sep 17 00:00:00 2001 From: "Chen, Xu Chun (Simon)" Date: Tue, 27 Jul 2021 17:48:45 +0800 Subject: [PATCH 175/175] add nfs storageClass example (#28941) * add nfs example * --amend --- .../docs/concepts/storage/storage-classes.md | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/content/en/docs/concepts/storage/storage-classes.md b/content/en/docs/concepts/storage/storage-classes.md index 5fb8ce6791..416f2f3a15 100644 --- a/content/en/docs/concepts/storage/storage-classes.md +++ b/content/en/docs/concepts/storage/storage-classes.md @@ -76,7 +76,7 @@ for provisioning PVs. This field must be specified. | Glusterfs | ✓ | [Glusterfs](#glusterfs) | | iSCSI | - | - | | Quobyte | ✓ | [Quobyte](#quobyte) | -| NFS | - | - | +| NFS | - | [NFS](#nfs) | | RBD | ✓ | [Ceph RBD](#ceph-rbd) | | VsphereVolume | ✓ | [vSphere](#vsphere) | | PortworxVolume | ✓ | [Portworx Volume](#portworx-volume) | @@ -423,6 +423,29 @@ parameters: `gluster-dynamic-`. The dynamic endpoint and service are automatically deleted when the persistent volume claim is deleted. +### NFS + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: example-nfs +provisioner: example.com/external-nfs +parameters: + server: nfs-server.example.com + path: /share + readOnly: false +``` + +* `server`: Server is the hostname or IP address of the NFS server. +* `path`: Path that is exported by the NFS server. +* `readOnly`: A flag indicating whether the storage will be mounted as read only (default false). + +Kubernetes doesn't include an internal NFS provisioner. You need to use an external provisioner to create a StorageClass for NFS. +Here are some examples: +* [NFS Ganesha server and external provisioner](https://github.com/kubernetes-sigs/nfs-ganesha-server-and-external-provisioner) +* [NFS subdir external provisioner](https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner) + ### OpenStack Cinder ```yaml