Merge pull request #34108 from swagmdy/fix-rbac

[ja]fix typo in k8s.io/ja/docs/reference/access-authn-authz/rbac/ #31220
fix typo
2022-06-02 02:25:47 -07:00 · 2022-06-02 14:51:24 +09:00 · 2022-03-24 21:59:51 -07:00 · 2021-12-11 17:16:50 +09:00 · 2021-12-11 15:30:24 +09:00 · 2021-12-11 15:29:04 +09:00
1980 changed files with 240234 additions and 51814 deletions
@@ -0,0 +1,15 @@
+---
+name: Scheduled Netlify site build
+on:
+  schedule: # Build twice daily: shortly after midnight and noon (UTC)
+            # Offset is to be nice to the build service
+  - cron: '4 0,12 * * *'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Trigger build on Netlify
+      env:
+        TOKEN: ${{ secrets.NETLIFY_BUILD_HOOK_KEY }}
+      run: >-
+          curl -s -H "Accept: application/json" -H "Content-Type: application/json" -X POST -d "{}" "https://api.netlify.com/build_hooks/${TOKEN}"
@@ -33,4 +33,8 @@ resources/
 # Netlify Functions build output
 package-lock.json
 functions/
-node_modules/
+node_modules/
+
+# Generated files when building with make container-build
+.config/
+.npm/
@@ -1,3 +1,6 @@
 [submodule "themes/docsy"]
 	path = themes/docsy
 	url = https://github.com/google/docsy.git
+[submodule "api-ref-generator"]
+	path = api-ref-generator
+	url = https://github.com/kubernetes-sigs/reference-docs
@@ -4,7 +4,7 @@
 # change is that the Hugo version is now an overridable argument rather than a fixed
 # environment variable.

-FROM alpine:latest
+FROM golang:1.15-alpine

 LABEL maintainer="Luc Perkins <lperkins@linuxfoundation.org>"

@@ -58,7 +58,7 @@ docker-serve:
 	@echo -e "$(CCRED)**** The use of docker-serve is deprecated. Use container-serve instead. ****$(CCEND)"
 	$(MAKE) container-serve

-container-image:
+container-image: ## Build a container image for the preview of the website
 	$(CONTAINER_ENGINE) build . \
 		--network=host \
 		--tag $(CONTAINER_IMAGE) \
@@ -67,8 +67,8 @@ container-image:
 container-build: module-check
 	$(CONTAINER_RUN) --read-only --mount type=tmpfs,destination=/tmp,tmpfs-mode=01777 $(CONTAINER_IMAGE) sh -c "npm ci && hugo --minify"

-container-serve: module-check
-	$(CONTAINER_RUN) --read-only --mount type=tmpfs,destination=/tmp,tmpfs-mode=01777 -p 1313:1313 $(CONTAINER_IMAGE) hugo server --buildFuture --bind 0.0.0.0 --destination /tmp/hugo --cleanDestinationDir
+container-serve: module-check ## Boot the development server using container. Run `make container-image` before this.
+	$(CONTAINER_RUN) --cap-drop=ALL --cap-add=AUDIT_WRITE --read-only --mount type=tmpfs,destination=/tmp,tmpfs-mode=01777 -p 1313:1313 $(CONTAINER_IMAGE) hugo server --buildFuture --bind 0.0.0.0 --destination /tmp/hugo --cleanDestinationDir

 test-examples:
 	scripts/test_examples.sh install
@@ -85,3 +85,10 @@ docker-internal-linkcheck:
 container-internal-linkcheck: link-checker-image-pull
 	$(CONTAINER_RUN) $(CONTAINER_IMAGE) hugo --config config.toml,linkcheck-config.toml --buildFuture
 	$(CONTAINER_ENGINE) run --mount type=bind,source=$(CURDIR),target=/test --rm wjdp/htmltest htmltest
+
+clean-api-reference: ## Clean all directories in API reference directory, preserve _index.md
+	rm -rf content/en/docs/reference/kubernetes-api/*/
+
+api-reference: clean-api-reference ## Build the API reference pages. go needed
+	cd api-ref-generator/gen-resourcesdocs && \
+		go run cmd/main.go kwebsite --config-dir config/v1.21/ --file api/v1.21/swagger.json --output-dir ../../content/en/docs/reference/kubernetes-api --templates templates
@@ -11,7 +11,7 @@ emeritus_approvers:
 # - jaredbhatti, commented out to disable PR assignments
 # - steveperry-53, commented out to disable PR assignments
 - stewart-yu 
- zacharysarah
+# - zacharysarah, commented out to disable PR assignments

 labels:
 - sig/docs
@@ -3,16 +3,13 @@ aliases:
    - castrojo
    - kbarnard10
    - onlydole
-    - zacharysarah
    - mrbobbytables
  sig-docs-blog-reviewers: # Reviewers for blog content
    - castrojo
-    - cody-clark
    - kbarnard10
    - mrbobbytables
    - onlydole
-    - parispittman
-    - vonguard
+    - sftim
  sig-docs-de-owners: # Admins for German content
    - bene2k1
    - mkorbi
@@ -30,11 +27,12 @@ aliases:
    - kbarnard10
    - kbhawkey
    - onlydole
+    - pi-victor
+    - reylejano
    - savitharaghunathan
    - sftim
    - steveperry-53
    - tengqm
-    - zacharysarah
    - zparnold
  sig-docs-en-reviews: # PR reviews for English content
    - bradtopol
@@ -51,11 +49,9 @@ aliases:
    - zparnold
  sig-docs-es-owners: # Admins for Spanish content
    - raelga
-    - alexbrand
+    - electrocucaracha
  sig-docs-es-reviews: # PR reviews for Spanish content
    - raelga
-    - alexbrand
-    # glo-pena
    - electrocucaracha
  sig-docs-fr-owners: # Admins for French content
    - remyleone
@@ -94,14 +90,21 @@ aliases:
    - daminisatya
    - mittalyashu
  sig-docs-id-owners: # Admins for Indonesian content
-    - girikuncoro
-    - irvifa
-  sig-docs-id-reviews: # PR reviews for Indonesian content
-    - girikuncoro
-    - irvifa
-    - wahyuoi
-    - phanama
+    - ariscahyadi
    - danninov
+    - girikuncoro
+    - habibrosyad
+    - irvifa
+    - phanama
+    - wahyuoi
+  sig-docs-id-reviews: # PR reviews for Indonesian content
+    - ariscahyadi
+    - danninov
+    - girikuncoro
+    - habibrosyad
+    - irvifa
+    - phanama
+    - wahyuoi
  sig-docs-it-owners: # Admins for Italian content
    - fabriziopandini
    - Fale
@@ -138,6 +141,7 @@ aliases:
    - seokho-son
    - ysyukr
    - pjhwa
+    - yoonian
  sig-docs-leads: # Website chairs and tech leads
    - irvifa
    - jimangel
@@ -169,14 +173,20 @@ aliases:
    # zhangxiaoyu-zidif
  sig-docs-pt-owners: # Admins for Portuguese content
    - femrtnz
+    - jailton
    - jcjesus
    - devlware
    - jhonmike
+    - rikatz
+    - yagonobre
  sig-docs-pt-reviews: # PR reviews for Portugese content
    - femrtnz
+    - jailton
    - jcjesus
    - devlware
    - jhonmike
+    - rikatz
+    - yagonobre
  sig-docs-vi-owners: # Admins for Vietnamese content
    - huynguyennovem
    - ngtuna
@@ -214,3 +224,36 @@ aliases:
    - idvoretskyi
    - MaxymVlasov
    - Potapy4
+  # authoritative source: git.k8s.io/community/OWNERS_ALIASES
+  committee-steering: # provide PR approvals for announcements
+    - cblecker
+    - derekwaynecarr
+    - dims
+    - liggitt
+    - mrbobbytables
+    - nikhita
+    - parispittman
+  # authoritative source: https://git.k8s.io/sig-release/OWNERS_ALIASES
+  sig-release-leads:
+    - hasheddan # SIG Technical Lead
+    - jeremyrickard # SIG Technical Lead
+    - justaugustus # SIG Chair
+    - LappleApple # SIG Program Manager
+    - saschagrunert # SIG Chair
+  release-engineering-approvers:
+    - cpanato # Release Manager
+    - hasheddan # subproject owner / Release Manager
+    - puerco # Release Manager
+    - saschagrunert # subproject owner / Release Manager
+    - justaugustus # subproject owner / Release Manager
+    - xmudrii # Release Manager
+  release-engineering-reviewers:
+    - ameukam # Release Manager Associate
+    - jimangel # Release Manager Associate
+    - mkorbi # Release Manager Associate
+    - palnabarun # Release Manager Associate
+    - onlydole # Release Manager Associate
+    - sethmccombs # Release Manager Associate
+    - thejoycekung # Release Manager Associate
+    - verolop # Release Manager Associate
+    - wilsonehusin # Release Manager Associate
@@ -9,14 +9,14 @@ Herzlich willkommen! Dieses Repository enthält alle Assets, die zur Erstellung

 Sie können auf die Schaltfläche **Fork** im oberen rechten Bereich des Bildschirms klicken, um eine Kopie dieses Repositorys in Ihrem GitHub-Konto zu erstellen. Diese Kopie wird als *Fork* bezeichnet. Nehmen Sie die gewünschten Änderungen an Ihrem Fork vor. Wenn Sie bereit sind, diese Änderungen an uns zu senden, gehen Sie zu Ihrem Fork und erstellen Sie eine neue Pull-Anforderung, um uns darüber zu informieren.

-Sobald Ihre Pull-Anfrage erstellt wurde, übernimmt ein Rezensent von Kubernetes die Verantwortung für klares, umsetzbares Feedback. Als Eigentümer des Pull-Request **liegt es in Ihrer Verantwortung Ihren Pull-Reqest enstsprechend des Feedbacks, dass Sie vom Kubernetes-Reviewer erhalten haben abzuändern.** Beachten Sie auch, dass Sie am Ende mehr als einen Rezensenten von Kubernetes erhalten, der Ihnen Feedback gibt, oder dass Sie Rückmeldungen von einem Rezensenten von Kubernetes erhalten, der sich von demjenigen unterscheidet, der ursprünglich für das Feedback zugewiesen wurde. In einigen Fällen kann es vorkommen, dass einer Ihrer Prüfer bei Bedarf eine technische Überprüfung von einem [Kubernetes Tech-Reviewer](https://github.com/kubernetes/website/wiki/tech-reviewers) anfordert. Reviewer geben ihr Bestes, um zeitnah Feedback zu geben, die Antwortzeiten können jedoch je nach den Umständen variieren.
+Sobald Ihre Pull-Anfrage erstellt wurde, übernimmt ein Rezensent von Kubernetes die Verantwortung für klares, umsetzbares Feedback. Als Eigentümer des Pull-Request **liegt es in Ihrer Verantwortung Ihren Pull-Reqest entsprechend des Feedbacks, dass Sie vom Kubernetes-Reviewer erhalten haben abzuändern.** Beachten Sie auch, dass Sie am Ende mehr als einen Rezensenten von Kubernetes erhalten, der Ihnen Feedback gibt, oder dass Sie Rückmeldungen von einem Rezensenten von Kubernetes erhalten, der sich von demjenigen unterscheidet, der ursprünglich für das Feedback zugewiesen wurde. In einigen Fällen kann es vorkommen, dass einer Ihrer Prüfer bei Bedarf eine technische Überprüfung von einem [Kubernetes Tech-Reviewer](https://github.com/kubernetes/website/wiki/tech-reviewers) anfordert. Reviewer geben ihr Bestes, um zeitnah Feedback zu geben, die Antwortzeiten können jedoch je nach den Umständen variieren.

 Weitere Informationen zum Beitrag zur Kubernetes-Dokumentation finden Sie unter:

 * [Mitwirkung beginnen](https://kubernetes.io/docs/contribute/start/)
-* [Ihre Dokumentationsänderungen bereitstellen](http://kubernetes.io/docs/contribute/intermediate#view-your-changes-locally)
-* [Seitenvorlagen verwenden](http://kubernetes.io/docs/contribute/style/page-content-types/)
-* [Dokumentationsstil-Handbuch](http://kubernetes.io/docs/contribute/style/style-guide/)
+* [Ihre Dokumentationsänderungen bereitstellen](https://kubernetes.io/docs/contribute/intermediate#view-your-changes-locally)
+* [Seitenvorlagen verwenden](https://kubernetes.io/docs/contribute/style/page-content-types/)
+* [Dokumentationsstil-Handbuch](https://kubernetes.io/docs/contribute/style/style-guide/)
 * [Übersetzung der Kubernetes-Dokumentation](https://kubernetes.io/docs/contribute/localization/)

 ## `README.md`'s Localizing Kubernetes Documentation
@@ -37,6 +37,13 @@ Um die Kubernetes-Website lokal laufen zu lassen, empfiehlt es sich, ein speziel

 > Wenn Sie die Website lieber lokal ohne Docker ausführen möchten, finden Sie weitere Informationen unter [Website lokal mit Hugo ausführen](#Die-Site-lokal-mit-Hugo-ausführen).

+Das benötigte [Docsy Hugo theme](https://github.com/google/docsy#readme) muss als git submodule installiert werden:
+
+```
+#Füge das Docsy submodule hinzu
+git submodule update --init --recursive --depth 1
+```
+
 Wenn Sie Docker [installiert](https://www.docker.com/get-started) haben, erstellen Sie das Docker-Image `kubernetes-hugo` lokal:

 ```bash
@@ -55,9 +62,18 @@ make container-serve

 Hugo-Installationsanweisungen finden Sie in der [offiziellen Hugo-Dokumentation](https://gohugo.io/getting-started/installing/). Stellen Sie sicher, dass Sie die Hugo-Version installieren, die in der Umgebungsvariablen `HUGO_VERSION` in der Datei [`netlify.toml`](netlify.toml#L9) angegeben ist.

+Das benötigte [Docsy Hugo theme](https://github.com/google/docsy#readme) muss als git submodule installiert werden:
+
+```
+#Füge das Docsy submodule hinzu
+git submodule update --init --recursive --depth 1
+```
+
 So führen Sie die Site lokal aus, wenn Sie Hugo installiert haben:

 ```bash
+# Installieren der JavaScript Abhängigkeiten
+npm ci
 make serve
 ```

@@ -65,7 +81,7 @@ Dadurch wird der lokale Hugo-Server an Port 1313 gestartet. Öffnen Sie Ihren Br

 ## Community, Diskussion, Beteiligung und Unterstützung

-Erfahren Sie auf der [Community-Seite](http://kubernetes.io/community/) wie Sie mit der Kubernetes-Community interagieren können.
+Erfahren Sie auf der [Community-Seite](https://kubernetes.io/community/) wie Sie mit der Kubernetes-Community interagieren können.

 Sie können die Betreuer dieses Projekts unter folgender Adresse erreichen:

@@ -17,9 +17,9 @@ Los revisores harán todo lo posible para proporcionar toda la información nece
 Para obtener más información sobre cómo contribuir a la documentación de Kubernetes, puede consultar:

 * [Empezando a contribuir](https://kubernetes.io/docs/contribute/start/)
-* [Visualizando sus cambios en su entorno local](http://kubernetes.io/docs/contribute/intermediate#view-your-changes-locally)
-* [Utilizando las plantillas de las páginas](http://kubernetes.io/docs/contribute/style/page-content-types/)
-* [Guía de estilo de la documentación](http://kubernetes.io/docs/contribute/style/style-guide/)
+* [Visualizando sus cambios en su entorno local](https://kubernetes.io/docs/contribute/intermediate#view-your-changes-locally)
+* [Utilizando las plantillas de las páginas](https://kubernetes.io/docs/contribute/style/page-content-types/)
+* [Guía de estilo de la documentación](https://kubernetes.io/docs/contribute/style/style-guide/)
 * [Traduciendo la documentación de Kubernetes](https://kubernetes.io/docs/contribute/localization/)

 ## Levantando el sitio web kubernetes.io en su entorno local con Docker
@@ -30,6 +30,17 @@ El método recomendado para levantar una copia local del sitio web kubernetes.io

 > Si prefiere levantar el sitio web sin utilizar **Docker**, puede seguir las instrucciones disponibles en la sección [Levantando kubernetes.io en local con Hugo](#levantando-kubernetesio-en-local-con-hugo).

+**`Nota`: Para el procedimiento de construir una imagen de Docker e iniciar el servidor.**
+El sitio web de Kubernetes utiliza Docsy Hugo theme. Se sugiere que se instale si aún no se ha hecho, los **submódulos** y otras dependencias de herramientas de desarrollo ejecutando el siguiente comando de `git`:
+
+```bash
+# pull de los submódulos del repositorio
+git submodule update --init --recursive --depth 1
+
+```
+
+Si identifica que `git` reconoce una cantidad innumerable de cambios nuevos en el proyecto, la forma más simple de solucionarlo es cerrando y volviendo a abrir el proyecto en el editor. Los submódulos son automáticamente detectados por `git`, pero los plugins usados por los editores pueden tener dificultades para ser cargados.
+
 Una vez tenga Docker [configurado en su máquina](https://www.docker.com/get-started), puede construir la imagen de Docker `kubernetes-hugo` localmente ejecutando el siguiente comando en la raíz del repositorio:

 ```bash
@@ -73,4 +84,4 @@ La participación en la comunidad de Kubernetes está regulada por el [Código d

 Kubernetes es posible gracias a la participación de la comunidad y la documentación es vital para facilitar el acceso al proyecto.

-Agradecemos muchísimo sus contribuciones a nuestro sitio web y nuestra documentación.
+Agradecemos muchísimo sus contribuciones a nuestro sitio web y nuestra documentación.
@@ -1,60 +1,45 @@
 # Dokumentacja projektu Kubernetes

-[![Build Status](https://api.travis-ci.org/kubernetes/website.svg?branch=master)](https://travis-ci.org/kubernetes/website)
-[![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest)
-
-Witamy!
+[![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-master-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest)

 W tym repozytorium znajdziesz wszystko, czego potrzebujesz do zbudowania [strony internetowej Kubernetesa wraz z dokumentacją](https://kubernetes.io/). Bardzo nam miło, że chcesz wziąć udział w jej współtworzeniu!

-## Twój wkład w dokumentację
+ [Twój wkład w dokumentację](#twój-wkład-w-dokumentację)
+ [Informacje o wersjach językowych](#informacje-o-wersjach-językowych)

-Możesz kliknąć w przycisk **Fork** w prawym górnym rogu ekranu, aby stworzyć kopię tego repozytorium na swoim koncie GitHub. Taki rodzaj kopii (odgałęzienia) nazywa się *fork*. Zmieniaj w nim, co chcesz, a kiedy będziesz już gotowy/a przesłać te zmiany do nas, przejdź do swojej kopii i stwórz nowy *pull request*, abyśmy zostali o tym poinformowani.
+# Jak używać tego repozytorium

-Po stworzeniu *pull request*, jeden z recenzentów projektu Kubernetes podejmie się przekazania jasnych wskazówek pozwalających podjąć następne działania. Na Tobie, jako właścicielu *pull requesta*, **spoczywa odpowiedzialność za wprowadzenie poprawek zgodnie z uwagami recenzenta.** Może też się zdarzyć, że swoje uwagi zgłosi więcej niż jeden recenzent, lub że recenzję będzie robił ktoś inny, niż ten, kto został przydzielony na początku. W niektórych przypadkach, jeśli zajdzie taka potrzeba, recenzent może poprosić dodatkowo o recenzję jednego z [recenzentów technicznych](https://github.com/kubernetes/website/wiki/Tech-reviewers). Recenzenci zrobią wszystko, aby odpowiedzieć sprawnie, ale konkretny czas odpowiedzi zależy od wielu czynników.
+Możesz uruchomić serwis lokalnie poprzez Hugo (Extended version) lub ze środowiska kontenerowego. Zdecydowanie zalecamy korzystanie z kontenerów, bo dzięki temu lokalna wersja będzie spójna z tym, co jest na oficjalnej stronie.

-Więcej informacji na temat współpracy przy tworzeniu dokumentacji znajdziesz na stronach:
+## Wymagania wstępne

-* [Jak rozpocząć współpracę](https://kubernetes.io/docs/contribute/start/)
-* [Podgląd wprowadzanych zmian w dokumentacji](http://kubernetes.io/docs/contribute/intermediate#view-your-changes-locally)
-* [Szablony stron](https://kubernetes.io/docs/contribute/style/page-content-types/)
-* [Styl pisania dokumentacji](http://kubernetes.io/docs/contribute/style/style-guide/)
-* [Lokalizacja dokumentacji Kubernetes](https://kubernetes.io/docs/contribute/localization/)
+Aby móc skorzystać z tego repozytorium, musisz lokalnie zainstalować:

-## Różne wersje językowe `README.md`
+- [npm](https://www.npmjs.com/)
+- [Go](https://golang.org/)
+- [Hugo (Extended version)](https://gohugo.io/)
+- Środowisko obsługi kontenerów, np. [Docker-a](https://www.docker.com/).

-|                                        |                                        |
-|----------------------------------------|----------------------------------------|
-| [README po angielsku](README.md)       | [README po francusku](README-fr.md)    |
-| [README po koreańsku](README-ko.md)    | [README po niemiecku](README-de.md)    |
-| [README po portugalsku](README-pt.md)  | [README w hindi](README-hi.md)        |
-| [README po hiszpańsku](README-es.md)   | [README po indonezyjsku](README-id.md) |
-| [README po chińsku](README-zh.md)      | [README po japońsku](README-ja.md)     |
-| [README po wietnamsku](README-vi.md)   | [README po rosyjsku](README-ru.md)     |
-| [README po włosku](README-it.md)       | [README po ukraińsku](README-uk.md)    |
-|                                        |                                        |
+Przed rozpoczęciem zainstaluj niezbędne zależności. Sklonuj repozytorium i przejdź do odpowiedniego katalogu:

-## Jak uruchomić lokalną kopię strony przy pomocy Dockera?
-
-Zalecaną metodą uruchomienia serwisu internetowego Kubernetesa lokalnie jest użycie specjalnego obrazu [Dockera](https://docker.com), który zawiera generator stron statycznych [Hugo](https://gohugo.io).
-
-> Użytkownicy Windows będą potrzebowali dodatkowych narzędzi, które mogą zainstalować przy pomocy [Chocolatey](https://chocolatey.org).
-
-```bash
-choco install make
+```
+git clone https://github.com/kubernetes/website.git
+cd website
 ```

-> Jeśli wolisz uruchomić serwis lokalnie bez Dockera, przeczytaj [jak uruchomić serwis lokalnie przy pomocy Hugo](#jak-uruchomić-lokalną-kopię-strony-przy-pomocy-hugo) poniżej.
+Strona Kubernetesa używa [Docsy Hugo theme](https://github.com/google/docsy#readme). Nawet jeśli planujesz uruchomić serwis w środowisku kontenerowym, zalecamy pobranie podmodułów i innych zależności za pomocą polecenia:

-Jeśli [zainstalowałeś i uruchomiłeś](https://www.docker.com/get-started) już Dockera, zbuduj obraz `kubernetes-hugo` lokalnie:
+```
+# pull in the Docsy submodule
+git submodule update --init --recursive --depth 1
+```

-```bash
+## Uruchomienie serwisu w kontenerze
+
+Aby zbudować i uruchomić serwis wewnątrz środowiska kontenerowego, wykonaj następujące polecenia:
+
+```
 make container-image
-```
-
-Po zbudowaniu obrazu, możesz uruchomić serwis lokalnie:
-
-```bash
 make container-serve
 ```

@@ -62,29 +47,106 @@ Aby obejrzeć zawartość serwisu otwórz w przeglądarce adres http://localhost

 ## Jak uruchomić lokalną kopię strony przy pomocy Hugo?

-Zajrzyj do [oficjalnej dokumentacji Hugo](https://gohugo.io/getting-started/installing/) po instrukcję instalacji. Upewnij się, że instalujesz rozszerzoną wersję Hugo, określoną przez zmienną środowiskową `HUGO_VERSION` w pliku [`netlify.toml`](netlify.toml#L9).
+Upewnij się, że zainstalowałeś odpowiednią wersję Hugo "extended", określoną przez zmienną środowiskową `HUGO_VERSION` w pliku [`netlify.toml`](netlify.toml#L10).

-Aby uruchomić serwis lokalnie po instalacji Hugo, napisz:
+Aby uruchomić i przetestować serwis lokalnie, wykonaj:

 ```bash
+# install dependencies
+npm ci
 make serve
 ```

 Zostanie uruchomiony lokalny serwer Hugo na porcie 1313. Otwórz w przeglądarce adres http://localhost:1313, aby obejrzeć zawartość serwisu. Po każdej zmianie plików źródłowych, Hugo automatycznie aktualizuje stronę i odświeża jej widok w przeglądarce.

-## Społeczność, listy dyskusyjne, uczestnictwo i wsparcie
+## Budowanie dokumentacji źródłowej API

-Zajrzyj na stronę [społeczności](http://kubernetes.io/community/), aby dowiedzieć się, jak możesz zaangażować się w jej działania.
+Budowanie dokumentacji źródłowej API zostało opisane w [angielskiej wersji pliku README.md](README.md#building-the-api-reference-pages).
+
+## Rozwiązywanie problemów
+### error: failed to transform resource: TOCSS: failed to transform "scss/main.scss" (text/x-scss): this feature is not available in your current Hugo version
+
+Z przyczyn technicznych, Hugo jest rozprowadzany w dwóch wersjach. Aktualny serwis używa tylko wersji **Hugo Extended**. Na stronie z [wydaniami](https://github.com/gohugoio/hugo/releases) poszukaj archiwum z `extended` w nazwie. Dla potwierdzenia, uruchom `hugo version` i poszukaj słowa `extended`.
+
+### Błąd w środowisku macOS: "too many open files"
+
+Jeśli po uruchomieniu `make serve` na macOS widzisz następujący błąd:
+
+```
+ERROR 2020/08/01 19:09:18 Error: listen tcp 127.0.0.1:1313: socket: too many open files
+make: *** [serve] Error 1
+```
+
+sprawdź aktualny limit otwartych plików:
+
+`launchctl limit maxfiles`
+
+Uruchom następujące polecenia: (na podstawie https://gist.github.com/tombigel/d503800a282fcadbee14b537735d202c):
+
+```shell
+#!/bin/sh
+
+# These are the original gist links, linking to my gists now.
+# curl -O https://gist.githubusercontent.com/a2ikm/761c2ab02b7b3935679e55af5d81786a/raw/ab644cb92f216c019a2f032bbf25e258b01d87f9/limit.maxfiles.plist
+# curl -O https://gist.githubusercontent.com/a2ikm/761c2ab02b7b3935679e55af5d81786a/raw/ab644cb92f216c019a2f032bbf25e258b01d87f9/limit.maxproc.plist
+
+curl -O https://gist.githubusercontent.com/tombigel/d503800a282fcadbee14b537735d202c/raw/ed73cacf82906fdde59976a0c8248cce8b44f906/limit.maxfiles.plist
+curl -O https://gist.githubusercontent.com/tombigel/d503800a282fcadbee14b537735d202c/raw/ed73cacf82906fdde59976a0c8248cce8b44f906/limit.maxproc.plist
+
+sudo mv limit.maxfiles.plist /Library/LaunchDaemons
+sudo mv limit.maxproc.plist /Library/LaunchDaemons
+
+sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
+sudo chown root:wheel /Library/LaunchDaemons/limit.maxproc.plist
+
+sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist
+```
+
+Przedstawiony sposób powinien działać dla MacOS w wersji Catalina i Mojave.
+
+
+# Zaangażowanie w prace SIG Docs
+
+O społeczności SIG Docs i terminach spotkań dowiesz z [jej strony](https://github.com/kubernetes/community/tree/master/sig-docs#meetings).

 Możesz kontaktować się z gospodarzami projektu za pomocą:

-* [Komunikatora Slack](https://kubernetes.slack.com/messages/sig-docs)
-* [List dyskusyjnych](https://groups.google.com/forum/#!forum/kubernetes-sig-docs)
+- [Komunikatora Slack](https://kubernetes.slack.com/messages/sig-docs) [Tutaj możesz dostać zaproszenie do tej grupy Slack-a](https://slack.k8s.io/)
+- [List dyskusyjnych](https://groups.google.com/forum/#!forum/kubernetes-sig-docs)

-### Zasady postępowania
+# Twój wkład w dokumentację

-Udział w działaniach społeczności Kubernetes jest regulowany przez [Kodeks postępowania](code-of-conduct.md).
+Możesz kliknąć w przycisk **Fork** w prawym górnym rogu ekranu, aby stworzyć kopię tego repozytorium na swoim koncie GitHub. Taki rodzaj kopii (odgałęzienia) nazywa się *fork*. Zmieniaj w nim, co chcesz, a kiedy będziesz już gotowy/a przesłać te zmiany do nas, przejdź do swojej kopii i stwórz nowy *pull request*, abyśmy zostali o tym poinformowani.

-## Dziękujemy!
+Po stworzeniu *pull request*, jeden z recenzentów projektu Kubernetes podejmie się przekazania jasnych wskazówek pozwalających podjąć następne działania. Na Tobie, jako właścicielu *pull requesta*, **spoczywa odpowiedzialność za wprowadzenie poprawek zgodnie z uwagami recenzenta.** 
+
+Może też się zdarzyć, że swoje uwagi zgłosi więcej niż jeden recenzent, lub że recenzję będzie robił ktoś inny, niż ten, kto został przydzielony na początku.
+
+W niektórych przypadkach, jeśli zajdzie taka potrzeba, recenzent może poprosić dodatkowo o recenzję jednego z [recenzentów technicznych](https://github.com/kubernetes/website/wiki/Tech-reviewers). Recenzenci zrobią wszystko, aby odpowiedzieć sprawnie, ale konkretny czas odpowiedzi zależy od wielu czynników.
+
+Więcej informacji na temat współpracy przy tworzeniu dokumentacji znajdziesz na stronach:
+
+* [Udział w rozwijaniu dokumentacji](https://kubernetes.io/docs/contribute/)
+* [Rodzaje stron](https://kubernetes.io/docs/contribute/style/page-content-types/)
+* [Styl pisania dokumentacji](http://kubernetes.io/docs/contribute/style/style-guide/)
+* [Lokalizacja dokumentacji Kubernetes](https://kubernetes.io/docs/contribute/localization/)
+
+# Różne wersje językowe `README.md`
+
+| Język  | Język |
+|---|---|
+| [angielski](README.md)       | [francuski](README-fr.md)    |
+| [koreański](README-ko.md)    | [niemiecki](README-de.md)    |
+| [portugalski](README-pt.md)  | [hindi](README-hi.md)        |
+| [hiszpański](README-es.md)   | [indonezyjski](README-id.md) |
+| [chiński](README-zh.md)      | [japoński](README-ja.md)     |
+| [wietnamski](README-vi.md)   | [rosyjski](README-ru.md)     |
+| [włoski](README-it.md)       | [ukraiński](README-uk.md)    |
+
+# Zasady postępowania
+
+Udział w działaniach społeczności Kubernetesa jest regulowany przez [Kodeks postępowania CNCF](https://github.com/cncf/foundation/blob/master/code-of-conduct-languages/pl.md).
+
+# Dziękujemy!

 Kubernetes rozkwita dzięki zaangażowaniu społeczności — doceniamy twój wkład w tworzenie naszego serwisu i dokumentacji!
@@ -1,76 +1,193 @@
 # A documentação do Kubernetes

-[![Build Status](https://api.travis-ci.org/kubernetes/website.svg?branch=master)](https://travis-ci.org/kubernetes/website)
-[![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest)
+[![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-master-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest)

-Bem vindos! Este repositório abriga todos os recursos necessários para criar o [site e documentação do Kubernetes](https://kubernetes.io/). Estamos muito satisfeitos por você querer contribuir!
+Bem-vindos! Este repositório contém todos os recursos necessários para criar o [website e documentação do Kubernetes](https://kubernetes.io/). Estamos muito satisfeitos por você querer contribuir!

-## Contribuindo com os documentos
+# Utilizando este repositório

-Você pode clicar no botão **Fork** na área superior direita da tela para criar uma cópia desse repositório na sua conta do GitHub. Esta cópia é chamada de *fork*. Faça as alterações desejadas no seu fork e, quando estiver pronto para enviar as alterações para nós, vá até o fork e crie uma nova solicitação de pull para nos informar sobre isso.
+Você pode executar o website localmente utilizando o Hugo (versão Extended), ou você pode executa-ló em um container runtime. É altamente recomendável utilizar um container runtime, pois garante a consistência na implantação do website real.

-Depois que seu **pull request** for criado, um revisor do Kubernetes assumirá a responsabilidade de fornecer um feedback claro e objetivo. Como proprietário do pull request, **é sua responsabilidade modificar seu pull request para abordar o feedback que foi fornecido a você pelo revisor do Kubernetes.** Observe também que você pode acabar tendo mais de um revisor do Kubernetes para fornecer seu feedback ou você pode acabar obtendo feedback de um revisor do Kubernetes que é diferente daquele originalmente designado para lhe fornecer feedback. Além disso, em alguns casos, um de seus revisores pode solicitar uma revisão técnica de um [revisor de tecnologia Kubernetes](https://github.com/kubernetes/website/wiki/Tech-reviewers) quando necessário. Os revisores farão o melhor para fornecer feedback em tempo hábil, mas o tempo de resposta pode variar de acordo com as circunstâncias.
+## Pré-requisitos
+
+Para usar este repositório, você precisa instalar:
+
+- [npm](https://www.npmjs.com/)
+- [Go](https://golang.org/)
+- [Hugo (versão Extended)](https://gohugo.io/)
+- Um container runtime, por exemplo [Docker](https://www.docker.com/).
+
+Antes de você iniciar, instale as dependências, clone o repositório e navegue até o diretório:
+
+```
+git clone https://github.com/kubernetes/website.git
+cd website
+```
+
+O website do Kubernetes utiliza o [tema Docsy Hugo](https://github.com/google/docsy#readme). Mesmo se você planeje executar o website em um container, é altamente recomendado baixar os submódulos e outras dependências executando o seguinte comando:
+
+```
+# Baixar o submódulo Docsy
+git submodule update --init --recursive --depth 1
+```
+
+## Executando o website usando um container
+
+Para executar o build do website em um container, execute o comando abaixo para criar a imagem do container e executa-lá:
+
+```
+make container-image
+make container-serve
+```
+
+Abra seu navegador em http://localhost:1313 para visualizar o website. Conforme você faz alterações nos arquivos fontes, o Hugo atualiza o website e força a atualização do navegador.
+
+## Executando o website localmente utilizando o Hugo
+
+Consulte a [documentação oficial do Hugo](https://gohugo.io/getting-started/installing/) para instruções de instalação do Hugo. Certifique-se de instalar a versão do Hugo especificada pela variável de ambiente `HUGO_VERSION` no arquivo [`netlify.toml`](netlify.toml#L9).
+
+Para executar o build e testar o website localmente, execute:
+
+```bash
+# instalar dependências
+npm ci
+make serve
+```
+
+Isso iniciará localmente o Hugo na porta 1313. Abra o seu navegador em http://localhost:1313 para visualizar o website. Conforme você faz alterações nos arquivos fontes, o Hugo atualiza o website e força uma atualização no navegador.
+
+## Construindo a página de referência da API
+
+A página de referência da API localizada em `content/en/docs/reference/kubernetes-api` é construída a partir da especificação do Swagger utilizando https://github.com/kubernetes-sigs/reference-docs/tree/master/gen-resourcesdocs.
+
+Siga os passos abaixo para atualizar a página de referência para uma nova versão do Kubernetes:
+
+OBS: modifique o "v1.20" no exemplo a seguir pela versão a ser atualizada
+
+1. Obter o submódulo `kubernetes-resources-reference`:
+
+```
+git submodule update --init --recursive --depth 1
+```
+
+2. Criar a nova versão da API no submódulo e adicionar à especificação do Swagger:
+
+```
+mkdir api-ref-generator/gen-resourcesdocs/api/v1.20
+curl 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/api/openapi-spec/swagger.json' > api-ref-generator/gen-resourcesdocs/api/v1.20/swagger.json
+```
+
+3. Copiar o sumário e os campos de configuração para a nova versão a partir da versão anterior:
+
+```
+mkdir api-ref-generator/gen-resourcesdocs/api/v1.20
+cp api-ref-generator/gen-resourcesdocs/api/v1.19/* api-ref-generator/gen-resourcesdocs/api/v1.20/
+```
+
+4. Ajustar os arquivos `toc.yaml` e `fields.yaml` para refletir as mudanças entre as duas versões.
+
+5. Em seguida, gerar as páginas:
+
+```
+make api-reference
+```
+
+Você pode validar o resultado localmente gerando e disponibilizando o site a partir da imagem do container:
+
+```
+make container-image
+make container-serve
+```
+
+Abra o seu navegador em http://localhost:1313/docs/reference/kubernetes-api/ para visualizar a página de referência da API.
+
+6. Quando todas as mudanças forem refletidas nos arquivos de configuração `toc.yaml` e `fields.yaml`, crie um pull request com a nova página de referência de API.
+
+## Troubleshooting
+### error: failed to transform resource: TOCSS: failed to transform "scss/main.scss" (text/x-scss): this feature is not available in your current Hugo version
+
+Por motivos técnicos, o Hugo é disponibilizado em dois conjuntos de binários. O website atual funciona apenas na versão **Hugo Extended**. Na [página de releases](https://github.com/gohugoio/hugo/releases) procure por arquivos com `extended` no nome. Para confirmar, execute `hugo version` e procure pela palavra `extended`.
+
+### Troubleshooting macOS for too many open files
+
+Se você executar o comando `make serve` no macOS e retornar o seguinte erro:
+
+```
+ERROR 2020/08/01 19:09:18 Error: listen tcp 127.0.0.1:1313: socket: too many open files
+make: *** [serve] Error 1
+```
+
+Verifique o limite atual para arquivos abertos:
+
+`launchctl limit maxfiles`
+
+Em seguida, execute os seguintes comandos (adaptado de https://gist.github.com/tombigel/d503800a282fcadbee14b537735d202c):
+
+```shell
+#!/bin/sh
+
+# Esse são os links do gist original, vinculados ao meu gists agora.
+# curl -O https://gist.githubusercontent.com/a2ikm/761c2ab02b7b3935679e55af5d81786a/raw/ab644cb92f216c019a2f032bbf25e258b01d87f9/limit.maxfiles.plist
+# curl -O https://gist.githubusercontent.com/a2ikm/761c2ab02b7b3935679e55af5d81786a/raw/ab644cb92f216c019a2f032bbf25e258b01d87f9/limit.maxproc.plist
+
+curl -O https://gist.githubusercontent.com/tombigel/d503800a282fcadbee14b537735d202c/raw/ed73cacf82906fdde59976a0c8248cce8b44f906/limit.maxfiles.plist
+curl -O https://gist.githubusercontent.com/tombigel/d503800a282fcadbee14b537735d202c/raw/ed73cacf82906fdde59976a0c8248cce8b44f906/limit.maxproc.plist
+
+sudo mv limit.maxfiles.plist /Library/LaunchDaemons
+sudo mv limit.maxproc.plist /Library/LaunchDaemons
+
+sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
+sudo chown root:wheel /Library/LaunchDaemons/limit.maxproc.plist
+
+sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist
+```
+
+Esta solução funciona tanto para o MacOS Catalina quanto para o MacOS Mojave.
+
+### Erro de "Out of Memory"
+
+Se você executar o comando `make container-serve` e retornar o seguinte erro:
+```
+make: *** [container-serve] Error 137
+```
+
+Verifique a quantidade de memória disponível para o agente de execução de contêiner. No caso do Docker Desktop para macOS, abra o menu "Preferences..." -> "Resources..." e tente disponibilizar mais memória.
+
+# Comunidade, discussão, contribuição e apoio
+
+Saiba mais sobre a comunidade Kubernetes SIG Docs e reuniões na [página da comunidade](http://kubernetes.io/community/).
+
+Você também pode entrar em contato com os mantenedores deste projeto em:
+
+- [Slack](https://kubernetes.slack.com/messages/sig-docs) ([Obter o convide para o este slack](https://slack.k8s.io/))
+- [Mailing List](https://groups.google.com/forum/#!forum/kubernetes-sig-docs)
+
+# Contribuindo com os documentos
+
+Você pode clicar no botão **Fork** na área superior direita da tela para criar uma cópia desse repositório na sua conta do GitHub. Esta cópia é chamada de *fork*. Faça as alterações desejadas no seu fork e, quando estiver pronto para enviar as alterações para nós, vá até o fork e crie um novo **pull request** para nos informar sobre isso.
+
+Depois que seu **pull request** for criado, um revisor do Kubernetes assumirá a responsabilidade de fornecer um feedback claro e objetivo. Como proprietário do pull request, **é sua responsabilidade modificar seu pull request para atender ao feedback que foi fornecido a você pelo revisor do Kubernetes.**
+
+Observe também que você pode acabar tendo mais de um revisor do Kubernetes para fornecer seu feedback ou você pode acabar obtendo feedback de um outro revisor do Kubernetes diferente daquele originalmente designado para lhe fornecer o feedback.
+
+Além disso, em alguns casos, um de seus revisores pode solicitar uma revisão técnica de um [revisor técnico do Kubernetes](https://github.com/kubernetes/website/wiki/Tech-reviewers) quando necessário. Os revisores farão o melhor para fornecer feedbacks em tempo hábil, mas o tempo de resposta pode variar de acordo com as circunstâncias.

 Para mais informações sobre como contribuir com a documentação do Kubernetes, consulte:

-* [Comece a contribuir](https://kubernetes.io/docs/contribute/start/)
-* [Preparando suas alterações na documentação](http://kubernetes.io/docs/contribute/intermediate#view-your-changes-locally)
-* [Usando Modelos de Página](http://kubernetes.io/docs/contribute/style/page-templates/)
+* [Contribua com a documentação do Kubernetes](https://kubernetes.io/docs/contribute/)
+* [Tipos de conteúdo de página](https://kubernetes.io/docs/contribute/style/page-content-types/)
 * [Guia de Estilo da Documentação](http://kubernetes.io/docs/contribute/style/style-guide/)
 * [Localizando documentação do Kubernetes](https://kubernetes.io/docs/contribute/localization/)

-Você pode contactar os mantenedores da localização em Português em:
+Você pode contatar os mantenedores da localização em Português em:

 * Felipe ([GitHub - @femrtnz](https://github.com/femrtnz))
 * [Slack channel](https://kubernetes.slack.com/messages/kubernetes-docs-pt)

-## Executando o site localmente usando o Docker
-
-A maneira recomendada de executar o site do Kubernetes localmente é executar uma imagem especializada do [Docker](https://docker.com) que inclui o gerador de site estático [Hugo](https://gohugo.io).
-
-> Se você está rodando no Windows, você precisará de mais algumas ferramentas que você pode instalar com o [Chocolatey](https://chocolatey.org). `choco install make`
-
-> Se você preferir executar o site localmente sem o Docker, consulte [Executando o site localmente usando o Hugo](#executando-o-site-localmente-usando-o-hugo) abaixo.
-
-Se você tiver o Docker [em funcionamento](https://www.docker.com/get-started), crie a imagem do Docker do `kubernetes-hugo` localmente:
-
-```bash
-make container-image
-```
-
-Depois que a imagem foi criada, você pode executar o site localmente:
-
-```bash
-make container-serve
-```
-
-Abra seu navegador para http://localhost:1313 para visualizar o site. Conforme você faz alterações nos arquivos de origem, Hugo atualiza o site e força a atualização do navegador.
-
-## Executando o site localmente usando o Hugo
-
-Veja a [documentação oficial do Hugo](https://gohugo.io/getting-started/installing/) para instruções de instalação do Hugo. Certifique-se de instalar a versão do Hugo especificada pela variável de ambiente `HUGO_VERSION` no arquivo [`netlify.toml`](netlify.toml#L9).
-
-Para executar o site localmente quando você tiver o Hugo instalado:
-
-```bash
-make serve
-```
-
-Isso iniciará o servidor Hugo local na porta 1313. Abra o navegador para http://localhost:1313 para visualizar o site. Conforme você faz alterações nos arquivos de origem, Hugo atualiza o site e força a atualização do navegador.
-
-## Comunidade, discussão, contribuição e apoio
-
-Aprenda a se envolver com a comunidade do Kubernetes na [página da comunidade](http://kubernetes.io/community/).
-
-Você pode falar com os mantenedores deste projeto:
-
- [Slack](https://kubernetes.slack.com/messages/sig-docs)
- [Mailing List](https://groups.google.com/forum/#!forum/kubernetes-sig-docs)
-
-### Código de conduta
+# Código de conduta

 A participação na comunidade Kubernetes é regida pelo [Código de Conduta da Kubernetes](code-of-conduct.md).

-## Obrigado!
+# Obrigado!

-O Kubernetes conta com a participação da comunidade e nós realmente agradecemos suas contribuições para o nosso site e nossa documentação!
+O Kubernetes prospera com a participação da comunidade e nós realmente agradecemos suas contribuições para o nosso website e nossa documentação!
@@ -2,38 +2,117 @@

 [![Netlify Status](https://api.netlify.com/api/v1/badges/be93b718-a6df-402a-b4a4-855ba186c97d/deploy-status)](https://app.netlify.com/sites/kubernetes-io-master-staging/deploys) [![GitHub release](https://img.shields.io/github/release/kubernetes/website.svg)](https://github.com/kubernetes/website/releases/latest)

-Добро пожаловать! Данный репозиторий содержит все необходимые файлы для сборки [сайта Kubernetes и документации](https://kubernetes.io/). Мы благодарим вас за старания!
+Данный репозиторий содержит все необходимые файлы для сборки [сайта Kubernetes и документации](https://kubernetes.io/). Мы благодарим вас за желание внести свой вклад!

-## Запуск сайта с помощью Hugo
+# Использование этого репозитория

-Обратитесь к [официальной документации Hugo](https://gohugo.io/getting-started/installing/), чтобы установить Hugo. Убедитесь, что вы установили правильную версию Hugo, которая устанавливается в переменной окружения `HUGO_VERSION` в файле [`netlify.toml`](netlify.toml#L10).
+Запустить сайт локально можно с помощью Hugo (Extended version) или же в исполняемой среде для контейнеров. Мы настоятельно рекомендуем воспользоваться контейнерной средой, поскольку она обеспечивает консистивность развёртывания с оригинальным сайтом.

-После установки Hugo, чтобы запустить сайт, выполните в консоли:
+## Предварительные требования

-```bash
+Чтобы работать с этим репозиторием, понадобятся следующие компоненты, установленные локально:
+
+- [npm](https://www.npmjs.com/)
+- [Go](https://golang.org/)
+- [Hugo (Extended version)](https://gohugo.io/)
+- Исполняемая среда для контейнеров вроде [Docker](https://www.docker.com/)
+
+Перед тем, как начать, установите зависимости. Склонируйте репозиторий и перейдите в его директорию:
+
+```
 git clone https://github.com/kubernetes/website.git
 cd website
-hugo server --buildFuture
 ```

-Эта команда запустит сервер Hugo на порту 1313. Откройте браузер и перейдите по ссылке http://localhost:1313, чтобы открыть сайт. Если вы отредактируете исходные файлы сайта, Hugo автоматически применит изменения и обновит страницу в браузере.
+Сайт Kubernetes использует [тему для Hugo под названием Docsy](https://github.com/google/docsy). Даже если вы планируете запускать сайт в контейнере, мы настоятельно рекомендуем загрузить соответствующий подмодуль и другие зависимости для разработки, выполнив следующую команду:

-## Сообщество, обсуждение, вклад и поддержка
+```
+# загружаем Git-подмодуль Docsy
+git submodule update --init --recursive --depth 1
+```

-Узнайте, как поучаствовать в жизни сообщества Kubernetes на [странице сообщества](http://kubernetes.io/community/).
+## Запуск сайта в контейнере

-Вы можете связаться с сопровождающими этого проекта по следующим ссылкам:
+Чтобы собрать сайт в контейнере, выполните следующие команды — они собирают образ контейнера и запускают его:

- [Канал в Slack](https://kubernetes.slack.com/messages/sig-docs)
- [Рассылка](https://groups.google.com/forum/#!forum/kubernetes-sig-docs)
+```
+make container-image
+make container-serve
+```

-## Вклад в документацию
+Откройте браузер и перейдите по ссылке http://localhost:1313, чтобы увидеть сайт. Если вы отредактируете исходные файлы сайта, Hugo автоматически обновит сам сайт и выполнит обновление страницы в браузере.
+ 
+## Запуск сайта с помощью Hugo

-Нажмите на кнопку **Fork** в правом верхнем углу, чтобы создать копию этого репозитория в ваш GitHub-аккаунт. Эта копия называется *форк-репозиторием*. Делайте любые изменения в вашем форк-репозитории, и когда вы будете готовы опубликовать изменения, откройте форк-репозиторий и создайте новый пулреквест, чтобы уведомить нас.
+Убедитесь, что вы установили расширенную версию Hugo (extended version): она определена в переменной окружения `HUGO_VERSION` в файле [`netlify.toml`](netlify.toml#L10).

-После того, как вы отправите пулреквест, ревьювер Kubernetes даст по нему обратную связь. Вы, как автор пулреквеста, **должны обновить свой пулреквест после его рассмотрения ревьювером Kubernetes.**
+Чтобы собрать и протестировать сайт локально, выполните:

-Вполне возможно, что более одного ревьювера Kubernetes оставят свои комментарии или даже может быть так, что новый комментарий ревьювера Kubernetes будет отличаться от первоначального назначенного ревьювера. Кроме того, в некоторых случаях один из ревьюверов может запросить технический обзор у [технического ревьювера Kubernetes](https://github.com/kubernetes/website/wiki/Tech-reviewers), если это будет необходимо. Ревьюверы сделают все возможное, чтобы как можно оперативно оставить свои предложения и пожелания, но время ответа может варьироваться в зависимости от обстоятельств.
+```bash
+# install dependencies
+npm ci
+make serve
+```
+
+Эти команды запустят локальный сервер Hugo на порту 1313. Откройте браузер и перейдите по ссылке http://localhost:1313, чтобы увидеть сайт. Если вы отредактируете исходные файлы сайта, Hugo автоматически обновит сам сайт и выполнит обновление страницы в браузере.
+
+## Решение проблем
+### error: failed to transform resource: TOCSS: failed to transform "scss/main.scss" (text/x-scss): this feature is not available in your current Hugo version
+
+По техническим причинам Hugo поставляется с двумя наборами бинарников. Текущий сайт Kubernetes работает только в версии **Hugo Extended**. На [странице релизов](https://github.com/gohugoio/hugo/releases) ищите архивы со словом `extended` в названии. Чтобы убедиться в корректности, запустите команду `hugo version` и найдите в выводе слово `extended`.
+
+### Решение проблемы на macOS с "too many open files"
+
+Если вы запускаете `make serve` на macOS и получаете следующую ошибку:
+
+```
+ERROR 2020/08/01 19:09:18 Error: listen tcp 127.0.0.1:1313: socket: too many open files
+make: *** [serve] Error 1
+```
+
+Попробуйте проверить текущий лимит для открытых файлов:
+
+`launchctl limit maxfiles`
+
+Затем выполните следующие команды (они взяты и адаптированы из https://gist.github.com/tombigel/d503800a282fcadbee14b537735d202c):
+
+```shell
+#!/bin/sh
+
+# Ссылки на оригинальные gist-файлы закомментированы в пользу моих адаптированных.
+# curl -O https://gist.githubusercontent.com/a2ikm/761c2ab02b7b3935679e55af5d81786a/raw/ab644cb92f216c019a2f032bbf25e258b01d87f9/limit.maxfiles.plist
+# curl -O https://gist.githubusercontent.com/a2ikm/761c2ab02b7b3935679e55af5d81786a/raw/ab644cb92f216c019a2f032bbf25e258b01d87f9/limit.maxproc.plist
+
+curl -O https://gist.githubusercontent.com/tombigel/d503800a282fcadbee14b537735d202c/raw/ed73cacf82906fdde59976a0c8248cce8b44f906/limit.maxfiles.plist
+curl -O https://gist.githubusercontent.com/tombigel/d503800a282fcadbee14b537735d202c/raw/ed73cacf82906fdde59976a0c8248cce8b44f906/limit.maxproc.plist
+
+sudo mv limit.maxfiles.plist /Library/LaunchDaemons
+sudo mv limit.maxproc.plist /Library/LaunchDaemons
+
+sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
+sudo chown root:wheel /Library/LaunchDaemons/limit.maxproc.plist
+
+sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist
+```
+
+Данное решение работает для macOS Catalina и Mojave.
+
+# Участие в SIG Docs
+
+Узнайте о Kubernetes-сообществе SIG Docs и его встречах на [странице сообщества](https://github.com/kubernetes/community/tree/master/sig-docs#meetings).
+
+Вы можете связаться с сопровождающими этот проект по следующим ссылкам:
+
+- [Канал в Slack](https://kubernetes.slack.com/messages/sig-docs) ([получите приглашение в этот Slack](https://slack.k8s.io/))
+- [Почтовая рассылка](https://groups.google.com/forum/#!forum/kubernetes-sig-docs)
+
+# Вклад в документацию
+
+Нажмите на кнопку **Fork** в правом верхнем углу, чтобы создать копию этого репозитория для вашего GitHub-аккаунта. Эта копия называется *форк-репозиторием*. Делайте любые изменения в своем форк-репозитории и, когда будете готовы опубликовать изменения, зайдите в свой форк-репозиторий и создайте новый pull-запрос (PR), чтобы уведомить нас.
+
+После того, как вы отправите pull-запрос, ревьювер из проекта Kubernetes даст по нему обратную связь. Вы, как автор pull-запроса, **должны обновить свой PR после его рассмотрения ревьювером Kubernetes.**
+
+Вполне возможно, что более одного ревьювера Kubernetes оставят свои комментарии. Может быть даже так, что вы будете получать обратную связь уже не от того ревьювера, что был первоначально вам назначен. Кроме того, в некоторых случаях один из ревьюверов может запросить техническую рецензию от [технического ревьювера Kubernetes](https://github.com/kubernetes/website/wiki/Tech-reviewers), если это потребуется. Ревьюверы сделают все возможное, чтобы как можно оперативнее оставить свои предложения и пожелания, но время ответа может варьироваться в зависимости от обстоятельств.

 Узнать подробнее о том, как поучаствовать в документации Kubernetes, вы можете по ссылкам ниже:

@@ -42,21 +121,22 @@ hugo server --buildFuture
 * [Руководство по оформлению документации](https://kubernetes.io/docs/contribute/style/style-guide/)
 * [Руководство по локализации Kubernetes](https://kubernetes.io/docs/contribute/localization/)

-## Файл `README.md` на других языках
+# Файл `README.md` на других языках
+
 |           другие языки        |       другие языки            |
 |-------------------------------|-------------------------------|
-| [Английский](README.md)       | [Французский](README-fr.md)   |
-| [Корейский](README-ko.md)     | [Немецкий](README-de.md)      |
-| [Португальский](README-pt.md) | [Хинди](README-hi.md)         |
-| [Испанский](README-es.md)     | [Индонезийский](README-id.md) |
-| [Китайский](README-zh.md)     | [Японский](README-ja.md)      |
-| [Вьетнамский](README-vi.md)   | [Итальянский](README-it.md)   |
-| [Польский]( README-pl.md)     | [Украинский](README-uk.md)    |
+| [Английский](README.md)       | [Немецкий](README-de.md)      |
+| [Вьетнамский](README-vi.md)   | [Польский]( README-pl.md)     |
+| [Индонезийский](README-id.md) | [Португальский](README-pt.md) |
+| [Испанский](README-es.md)     | [Украинский](README-uk.md)    |
+| [Итальянский](README-it.md)   | [Французский](README-fr.md)   |
+| [Китайский](README-zh.md)     | [Хинди](README-hi.md)         |
+| [Корейский](README-ko.md)     | [Японский](README-ja.md)      |

-### Кодекс поведения
+# Кодекс поведения

-Участие в сообществе Kubernetes регулируется [кодексом поведения CNCF](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
+Участие в сообществе Kubernetes регулируется [кодексом поведения CNCF](https://github.com/cncf/foundation/blob/master/code-of-conduct-languages/ru.md).

-## Спасибо!
+# Спасибо!

 Kubernetes процветает благодаря сообществу и мы ценим ваш вклад в сайт и документацию!
@@ -18,7 +18,8 @@
 ```bash
 git clone https://github.com/kubernetes/website.git
 cd website
-hugo server --buildFuture
+git submodule update --init --recursive --depth 1
+make serve
 ```

 <!-- This will start the local Hugo server on port 1313. Open up your browser to http://localhost:1313 to view the website. As you make changes to the source files, Hugo updates the website and forces a browser refresh. -->
@@ -82,4 +83,4 @@ hugo server --buildFuture
 ## Дякуємо!

 <!-- Kubernetes thrives on community participation, and we appreciate your contributions to our website and our documentation! -->
-Долучення до спільноти - запорука успішного розвитку Kubernetes. Ми цінуємо ваш внесок у наш сайт і документацію!
+Долучення до спільноти - запорука успішного розвитку Kubernetes. Ми цінуємо ваш внесок у наш сайт і документацію!
@@ -174,7 +174,7 @@ Learn more about SIG Docs Kubernetes community and meetings on the [community pa

 You can also reach the maintainers of this project at:

- [Slack](https://kubernetes.slack.com/messages/sig-docs)
+- [Slack](https://kubernetes.slack.com/messages/sig-docs) [Get an invite for this Slack](https://slack.k8s.io/)
 - [Mailing List](https://groups.google.com/forum/#!forum/kubernetes-sig-docs)
 -->
 # 参与 SIG Docs 工作
@@ -184,7 +184,7 @@ You can also reach the maintainers of this project at:

 你也可以通过以下渠道联系本项目的维护人员：

- [Slack](https://kubernetes.slack.com/messages/sig-docs)
+- [Slack](https://kubernetes.slack.com/messages/sig-docs) [加入Slack](https://slack.k8s.io/)
 - [邮件列表](https://groups.google.com/forum/#!forum/kubernetes-sig-docs)

 <!--
@@ -228,8 +228,8 @@ For more information about contributing to the Kubernetes documentation, see:
 有关为 Kubernetes 文档做出贡献的更多信息，请参阅：

 * [贡献 Kubernetes 文档](https://kubernetes.io/docs/contribute/)
-* [页面内容类型](http://kubernetes.io/docs/contribute/style/page-content-types/)
-* [文档风格指南](http://kubernetes.io/docs/contribute/style/style-guide/)
+* [页面内容类型](https://kubernetes.io/docs/contribute/style/page-content-types/)
+* [文档风格指南](https://kubernetes.io/docs/contribute/style/style-guide/)
 * [本地化 Kubernetes 文档](https://kubernetes.io/docs/contribute/localization/)

 # 中文本地化
@@ -4,6 +4,9 @@

 This repository contains the assets required to build the [Kubernetes website and documentation](https://kubernetes.io/). We're glad that you want to contribute!

+ [Contributing to the docs](#contributing-to-the-docs)
+ [Localization ReadMes](#localization-readmemds)
+
 # Using this repository

 You can run the website locally using Hugo (Extended version), or you can run it in a container runtime. We strongly recommend using the container runtime, as it gives deployment consistency with the live website.
@@ -40,6 +43,8 @@ make container-image
 make container-serve
 ```

+If you see errors, it probably means that the hugo container did not have enough computing resources available. To solve it, increase the amount of allowed CPU and memory usage for Docker on your machine ([MacOSX](https://docs.docker.com/docker-for-mac/#resources) and [Windows](https://docs.docker.com/docker-for-windows/#resources)).
+
 Open up your browser to http://localhost:1313 to view the website. As you make changes to the source files, Hugo updates the website and forces a browser refresh.

 ## Running the website locally using Hugo
@@ -56,6 +61,51 @@ make serve

 This will start the local Hugo server on port 1313. Open up your browser to http://localhost:1313 to view the website. As you make changes to the source files, Hugo updates the website and forces a browser refresh.

+## Building the API reference pages
+
+The API reference pages located in `content/en/docs/reference/kubernetes-api` are built from the Swagger specification, using https://github.com/kubernetes-sigs/reference-docs/tree/master/gen-resourcesdocs.
+
+To update the reference pages for a new Kubernetes release (replace v1.20 in the following examples with the release to update to):
+
+1. Pull the `kubernetes-resources-reference` submodule:
+
+```
+git submodule update --init --recursive --depth 1
+```
+
+2. Create a new API revision into the submodule, and add the Swagger specification:
+
+```
+mkdir api-ref-generator/gen-resourcesdocs/api/v1.20
+curl 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/api/openapi-spec/swagger.json' > api-ref-generator/gen-resourcesdocs/api/v1.20/swagger.json
+```
+
+3. Copy the table of contents and fields configuration for the new release from a previous one:
+
+```
+mkdir api-ref-generator/gen-resourcesdocs/api/v1.20
+cp api-ref-generator/gen-resourcesdocs/api/v1.19/* api-ref-generator/gen-resourcesdocs/api/v1.20/
+```
+
+4. Adapt the files `toc.yaml` and `fields.yaml` to reflect the changes between the two releases
+
+5. Next, build the pages:
+
+```
+make api-reference
+```
+
+You can test the results locally by making and serving the site from a container image:
+
+```
+make container-image
+make container-serve
+```
+
+In a web browser, go to http://localhost:1313/docs/reference/kubernetes-api/ to view the API reference.
+
+6. When all changes of the new contract are reflected into the configuration files `toc.yaml` and `fields.yaml`, create a Pull Request with the newly generated API reference pages.
+
 ## Troubleshooting
 ### error: failed to transform resource: TOCSS: failed to transform "scss/main.scss" (text/x-scss): this feature is not available in your current Hugo version

@@ -76,7 +126,7 @@ Try checking the current limit for open files:

 Then run the following commands (adapted from https://gist.github.com/tombigel/d503800a282fcadbee14b537735d202c):

-```
+```shell
 #!/bin/sh

 # These are the original gist links, linking to my gists now.
@@ -810,6 +810,13 @@ section#cncf {
  }
 }

+.td-search {
+  header > .header-filler {
+    height: $hero-padding-top;
+    background-color: black;
+  }
+}
+
 // Docs specific

 #editPageButton {
@@ -862,3 +869,22 @@ body.td-documentation {
      display: none;
    }
 }
+
+// nav-tabs and tab-content
+.nav-tabs {
+  border-bottom: none !important;
+}
+
+.td-content .tab-content .highlight {
+  margin: 0;
+}
+
+.tab-pane {
+  border-radius: 0.25rem;
+  padding: 0 16px 16px;
+
+  border: 1px solid #dee2e6;
+  &:first-of-type.active {
+    border-top-left-radius: 0;
+  }
+}
@@ -444,7 +444,7 @@ body.cid-community > #deprecation-warning > .deprecation-warning > * {
 .td-sidebar__inner {
  form.td-sidebar__search {

-    button.td-sidebar__toggle {
+    .td-sidebar__toggle {
      &:hover {
        color: #000000;
      }
@@ -482,10 +482,6 @@ main.content {

 .td-blog {

-  .td-sidebar-nav {
-    max-height: calc(100vh - 8rem);
-  }
-
  .widget-link {
    margin-bottom: 1rem;

@@ -578,3 +574,71 @@ body.td-documentation {
  color: black;
  text-decoration: none !important;
 }
+
+@media print {
+  /* Do not print announcements */
+  #announcement, section#announcement, #fp-announcement, section#fp-announcement {
+    display: none;
+  }
+}
+
+#announcement, #fp-announcement {
+  > * {
+    color: inherit;
+    background: inherit;
+  }
+
+  a {
+    color: inherit;
+    border-bottom: 1px solid #fff;
+  }
+
+  a:hover {
+    color: inherit;
+    border-bottom: none;
+  }
+}
+
+#announcement {
+  padding-top: 105px;
+  padding-bottom: 25px;
+}
+
+.header-hero {
+  padding-top: 40px;
+}
+
+/* Extra announcement height only for landscape viewports */
+@media (min-aspect-ratio: 8/9) {
+  #fp-announcement {
+    min-height: 25vh;
+  }
+}
+
+#fp-announcement aside {
+  padding-top: 115px;
+  padding-bottom: 25px;
+}
+
+.announcement {
+  .content {
+    margin-bottom: 0px;
+  }
+
+
+  > p {
+    .gridPage #announcement .content p,
+    .announcement > h4,
+    .announcement > h3 {
+      color: #ffffff;
+    }
+  }
+}
+
+.td-content {
+  table code {
+    background-color: inherit !important;
+    color: inherit !important;
+    font-size: inherit !important;
+  }
+}
@@ -77,7 +77,6 @@ $feature-box-div-width: 45%;
      position: relative;
      clear: both;
      display: table;
-      height: 160px;

      .content {
        display: table-cell;
@@ -124,6 +123,7 @@ $feature-box-div-width: 45%;
          position: relative;
          display: block;
          float: none;
+          text-align: center;
          max-width: 100%;
          transform: none;
        }
@@ -13,7 +13,7 @@ disableBrowserError = true

 disableKinds = ["taxonomy", "taxonomyTerm"]

-ignoreFiles = [ "^OWNERS$", "README[-]+[a-z]*\\.md", "^node_modules$", "content/en/docs/doc-contributor-tools" ]
+ignoreFiles = [ "(?:^|/)OWNERS$", "README[-]+[a-z]*\\.md", "^node_modules$", "content/en/docs/doc-contributor-tools" ]

 timeout = 3000

@@ -91,7 +91,7 @@ blog = "/:section/:year/:month/:day/:slug/"
 [outputs]
 home = [ "HTML", "RSS", "HEADERS" ]
 page = [ "HTML"]
-section = [ "HTML"]
+section = [ "HTML", "print" ]

 # Add a "text/netlify" media type for auto-generating the _headers file
 [mediaTypes]
@@ -138,10 +138,10 @@ time_format_default = "January 02, 2006 at 3:04 PM PST"
 description = "Production-Grade Container Orchestration"
 showedit = true

-latest = "v1.19"
+latest = "v1.21"

-fullversion = "v1.19.0"
-version = "v1.19"
+fullversion = "v1.21.0"
+version = "v1.21"
 githubbranch = "master"
 docsbranch = "master"
 deprecated = false
@@ -154,11 +154,6 @@ githubWebsiteRaw = "raw.githubusercontent.com/kubernetes/website"
 # GitHub repository link for editing a page and opening issues.
 github_repo = "https://github.com/kubernetes/website"

-# param for displaying an announcement block on every page.
-# See /i18n/en.toml for message text and title.
-announcement = true
-announcement_bg = "#000000" #choose a dark color – text is white
-
 #Searching
 k8s_search = true

@@ -183,40 +178,40 @@ js = [
 ]

 [[params.versions]]
-fullversion = "v1.19.0"
-version = "v1.19"
-githubbranch = "v1.19.0"
+fullversion = "v1.21.0"
+version = "v1.21"
+githubbranch = "v1.21.0"
 docsbranch = "master"
 url = "https://kubernetes.io"

 [[params.versions]]
-fullversion = "v1.18.8"
+fullversion = "v1.20.5"
+version = "v1.20"
+githubbranch = "v1.20.5"
+docsbranch = "release-1.20"
+url = "https://v1-20.docs.kubernetes.io"
+
+[[params.versions]]
+fullversion = "v1.19.9"
+version = "v1.19"
+githubbranch = "v1.19.9"
+docsbranch = "release-1.19"
+url = "https://v1-19.docs.kubernetes.io"
+
+[[params.versions]]
+fullversion = "v1.18.17"
 version = "v1.18"
-githubbranch = "v1.18.8"
+githubbranch = "v1.18.17"
 docsbranch = "release-1.18"
 url = "https://v1-18.docs.kubernetes.io"

 [[params.versions]]
-fullversion = "v1.17.11"
+fullversion = "v1.17.17"
 version = "v1.17"
-githubbranch = "v1.17.11"
+githubbranch = "v1.17.17"
 docsbranch = "release-1.17"
 url = "https://v1-17.docs.kubernetes.io"

-[[params.versions]]
-fullversion = "v1.16.14"
-version = "v1.16"
-githubbranch = "v1.16.14"
-docsbranch = "release-1.16"
-url = "https://v1-16.docs.kubernetes.io"
-
-[[params.versions]]
-fullversion = "v1.15.12"
-version = "v1.15"
-githubbranch = "v1.15.12"
-docsbranch = "release-1.15"
-url = "https://v1-15.docs.kubernetes.io"
-

 # User interface configuration
 [params.ui]
@@ -404,15 +399,15 @@ time_format_blog = "02.01.2006"
 # A list of language codes to look for untranslated content, ordered from left to right.
 language_alternatives = ["en"]

-[languages.pt]
+[languages.pt-br]
 title = "Kubernetes"
 description = "Orquestração de contêineres em nível de produção"
 languageName ="Português"
 weight = 9
-contentDir = "content/pt"
+contentDir = "content/pt-br"
 languagedirection = "ltr"

-[languages.pt.params]
+[languages.pt-br.params]
 time_format_blog = "02.01.2006"
 # A list of language codes to look for untranslated content, ordered from left to right.
 language_alternatives = ["en"]
@@ -8,7 +8,7 @@ cid: community
    <main>
        <div class="content">
            <h3>Die Gewissheit, dass Kubernetes überall und für alle gut funktioniert.</h3>
-            <p>Verbinden Sie sich mit der Kubernetes-Community in unserem <a href="http://slack.k8s.io/">Slack Kanal</a>, <a href="https://discuss.kubernetes.io/">Diskussionsforum</a>, oder beteiligen Sie sich an der <a href="https://groups.google.com/forum/#!forum/kubernetes-dev"> Kubernetes-dev-Google-Gruppe</a>. Eine wöchentliches Community-Meeting findet per Videokonferenz statt, um den Stand der Dinge zu diskutieren, folgen Sie
+            <p>Verbinden Sie sich mit der Kubernetes-Community in unserem <a href="http://slack.k8s.io/">Slack Kanal</a>, <a href="https://discuss.kubernetes.io/">Diskussionsforum</a>, oder beteiligen Sie sich an der <a href="https://groups.google.com/g/kubernetes-dev"> Kubernetes-dev-Google-Gruppe</a>. Eine wöchentliches Community-Meeting findet per Videokonferenz statt, um den Stand der Dinge zu diskutieren, folgen Sie
 		<a href="https://github.com/kubernetes/community/blob/master/events/community-meeting.md">diesen Anweisungen</a> für Informationen wie Sie teilnehmen können.</p>
            <p>Sie können Kubernetes auch auf der ganzen Welt über unsere
                <a href="https://www.meetup.com/topics/kubernetes/">Kubernetes Meetup Community</a> und der
@@ -23,7 +23,7 @@ Dieser Verhaltenskodex gilt sowohl innerhalb von Projekträumen als auch in öff

 Fälle von missbräuchlichem, belästigendem oder anderweitig unzumutbarem Verhalten in Kubernetes können gemeldet werden, indem Sie sich an das [Kubernetes Komitee für Verhaltenskodex](https://git.k8s.io/community/committee-code-of-conduct) wenden unter <conduct@kubernetes.io>. Für andere Projekte wenden Sie sich bitte an einen CNCF-Projektbetreuer oder an unseren Mediator, Mishi Choudhary <mishi@linux.com>.

-Dieser Verhaltenskodex wurde aus dem Contributor Covenant übernommen (http://contributor-covenant.org), Version 1.2.0, verfügbar unter http://contributor-covenant.org/version/1/2/0/
+Dieser Verhaltenskodex wurde aus dem Contributor Covenant übernommen (https://contributor-covenant.org), Version 1.2.0, verfügbar unter https://contributor-covenant.org/version/1/2/0/

 ### CNCF Verhaltenskodex für Veranstaltungen

@@ -147,7 +147,8 @@ Die zweite ist, die interne Node-Liste des Node Controllers mit der Liste der ve
 Wenn ein Node in einer Cloud-Umgebung ausgeführt wird und sich in einem schlechten Zustand befindet, fragt der Node Controller den Cloud-Anbieter, ob die virtuelle Maschine für diesen Node noch verfügbar ist. Wenn nicht, löscht der Node Controller den Node aus seiner Node-Liste.

 Der dritte ist die Überwachung des Zustands der Nodes. Der Node Controller ist dafür verantwortlich,
-die NodeReady-Bedingung von NodeStatus auf ConditionUnknown zu aktualisieren, wenn ein wenn ein Node unerreichbar wird (der Node Controller empfängt aus irgendeinem Grund keine Herzschläge mehr, z.B. weil der Node heruntergefahren ist) und später alle Pods aus dem Node zu entfernen (und diese ordnungsgemäss zu beenden), wenn der Node weiterhin unzugänglich ist. (Die Standard-Timeouts sind 40s, um ConditionUnknown zu melden und 5 Minuten, um mit der Evakuierung der Pods zu beginnen).
+die NodeReady-Bedingung von NodeStatus auf ConditionUnknown zu aktualisieren, wenn ein Node unerreichbar wird (der Node Controller empfängt aus irgendeinem Grund keine Herzschläge mehr, z.B. weil der Node heruntergefahren ist) und später alle Pods aus dem Node zu entfernen (und diese ordnungsgemäss zu beenden), wenn der Node weiterhin unzugänglich ist. (Die Standard-Timeouts sind 40s, um ConditionUnknown zu melden und 5 Minuten, um mit der Evakuierung der Pods zu beginnen).
+
 Der Node Controller überprüft den Zustand jedes Nodes alle `--node-monitor-period` Sekunden.


@@ -0,0 +1,369 @@
+---
+title: Pods
+content_type: concept
+weight: 10
+no_list: true
+card:
+ name: concepts
+ weight: 60
+---
+
+<!-- overview -->
+
+_Pods_ sind die kleinsten einsetzbaren Einheiten, die in Kubernetes
+erstellt und verwaltet werden können.
+
+Ein _Pod_ (übersetzt Gruppe/Schote, wie z. B. eine Gruppe von Walen oder eine 
+Erbsenschote) ist eine Gruppe von einem oder mehreren 
+{{< glossary_tooltip text="Containern" term_id="container" >}} mit gemeinsam 
+genutzten Speicher- und Netzwerkressourcen und einer Spezifikation für die 
+Ausführung der Container. Die Ressourcen eines Pods befinden sich immer auf dem 
+gleichen (virtuellen) Server, werden gemeinsam geplant und in einem
+gemeinsamen Kontext ausgeführt. Ein Pod modelliert einen anwendungsspezifischen 
+"logischen Server": Er enthält eine oder mehrere containerisierte Anwendungen, 
+die relativ stark voneinander abhängen. 
+In Nicht-Cloud-Kontexten sind Anwendungen, die auf
+demselben physischen oder virtuellen Server ausgeführt werden, vergleichbar zu 
+Cloud-Anwendungen, die auf demselben logischen Server ausgeführt werden.
+
+Ein Pod kann neben Anwendungs-Containern auch sogenannte 
+[Initialisierungs-Container](/docs/concepts/workloads/pods/init-containers/)
+enthalten, die beim Starten des Pods ausgeführt werden.
+Es können auch 
+kurzlebige/[ephemere Container](/docs/concepts/workloads/pods/ephemeral-containers/)
+zum Debuggen gestartet werden, wenn dies der Cluster anbietet.
+
+<!-- body -->
+
+## Was ist ein Pod?
+
+{{< note >}}
+Obwohl Kubernetes abgesehen von [Docker](https://www.docker.com/) auch andere
+{{<glossary_tooltip text="Container-Laufzeitumgebungen" 
+term_id="container-runtime">}} unterstützt, ist Docker am bekanntesten und
+ es ist hilfreich, Pods mit der Terminologie von Docker zu beschreiben.
+{{< /note >}}
+
+Der gemeinsame Kontext eines Pods besteht aus einer Reihe von Linux-Namespaces, 
+Cgroups und möglicherweise anderen Aspekten der Isolation, also die gleichen
+Dinge, die einen Dockercontainer isolieren. Innerhalb des Kontexts eines Pods 
+können die einzelnen Anwendungen weitere Unterisolierungen haben. 
+
+Im Sinne von Docker-Konzepten ähnelt ein Pod einer Gruppe von Docker-Containern, 
+die gemeinsame Namespaces und Dateisystem-Volumes nutzen.
+
+## Pods verwenden
+
+Normalerweise müssen keine Pods erzeugt werden, auch keine Singleton-Pods. 
+Stattdessen werden sie mit Workload-Ressourcen wie {{<glossary_tooltip 
+text="Deployment" term_id="deployment">}} oder {{<glossary_tooltip 
+text="Job" term_id="job">}} erzeugt. Für Pods, die von einem Systemzustand
+abhängen, ist die Nutzung von {{<glossary_tooltip text="StatefulSet" 
+term_id="statefulset">}}-Ressourcen zu erwägen.
+
+Pods in einem Kubernetes-Cluster werden hauptsächlich auf zwei Arten verwendet:
+
+* **Pods, die einen einzelnen Container ausführen**. Das 
+"Ein-Container-per-Pod"-Modell ist der häufigste Kubernetes-Anwendungsfall. In 
+diesem Fall kannst du dir einen einen Pod als einen Behälter vorstellen, der einen 
+einzelnen Container enthält; Kubernetes verwaltet die Pods anstatt die 
+Container direkt zu verwalten.
+* **Pods, in denen mehrere Container ausgeführt werden, die zusammenarbeiten
+müssen**. Wenn eine Softwareanwendung aus co-lokaliserten Containern besteht, 
+die sich gemeinsame Ressourcen teilen und stark voneinander abhängen, kann ein 
+Pod die Container verkapseln. 
+Diese Container bilden eine einzelne zusammenhängende 
+Serviceeinheit, z. B. ein Container, der Daten in einem gemeinsam genutzten 
+Volume öffentlich verfügbar macht, während ein separater _Sidecar_-Container 
+die Daten aktualisiert. Der Pod fasst die Container, die Speicherressourcen
+und eine kurzlebiges Netzwerk-Identität als eine Einheit zusammen.
+
+{{< note >}}
+Das Gruppieren mehrerer gemeinsam lokalisierter und gemeinsam verwalteter
+Container in einem einzigen Pod ist ein relativ fortgeschrittener 
+Anwendungsfall. Du solltest diese Architektur nur in bestimmten  Fällen 
+verwenden, wenn deine Container stark voneinander abhängen.
+{{< /note >}}
+
+Jeder Pod sollte eine einzelne Instanz einer gegebenen Anwendung ausführen. Wenn 
+du deine Anwendung horizontal skalieren willst (um mehr Instanzen auszuführen
+und  dadurch mehr Gesamtressourcen bereitstellen), solltest du mehrere Pods 
+verwenden, einen für jede Instanz. 
+In Kubernetes wird dies typischerweise als Replikation bezeichnet.
+Replizierte Pods werden normalerweise als eine Gruppe durch eine 
+Workload-Ressource und deren 
+{{<glossary_tooltip text="Controller" term_id="controller">}} erstellt 
+und verwaltet.
+
+Der Abschnitt [Pods und Controller](#pods-und-controller) beschreibt, wie 
+Kubernetes Workload-Ressourcen und deren Controller verwendet, um Anwendungen 
+zu skalieren und zu heilen.
+
+### Wie Pods mehrere Container verwalten
+
+Pods unterstützen mehrere kooperierende Prozesse (als Container), die eine 
+zusammenhängende Serviceeinheit bilden. Kubernetes plant und stellt automatisch 
+sicher, dass sich die Container in einem Pod  auf demselben physischen oder 
+virtuellen Server im Cluster befinden. Die Container können Ressourcen und
+Abhängigkeiten gemeinsam nutzen, miteinander kommunizieren und
+ferner koordinieren wann und wie sie beendet werden.
+
+Zum Beispiel könntest du einen Container haben, der als Webserver für Dateien in
+einem gemeinsamen Volume arbeitet. Und ein separater "Sidecar" -Container 
+aktualisiert die Daten von einer externen Datenquelle, siehe folgenden 
+Abbildung:
+
+{{< figure src="/images/docs/pod.svg" alt="Pod-Beispieldiagramm" width="50%" >}}
+
+Einige Pods haben sowohl {{<glossary_tooltip text="Initialisierungs-Container" 
+term_id="init-container">}} als auch {{<glossary_tooltip 
+text="Anwendungs-Container" term_id="app-container">}}. 
+Initialisierungs-Container werden gestartet und beendet bevor die 
+Anwendungs-Container gestartet werden.
+
+Pods stellen standardmäßig zwei Arten von gemeinsam Ressourcen für die 
+enthaltenen Container bereit:
+[Netzwerk](#pod-netzwerk) und [Speicher](#datenspeicherung-in-pods).
+
+
+## Mit Pods arbeiten
+
+Du wirst selten einzelne Pods direkt in Kubernetes erstellen, selbst 
+Singleton-Pods. Das liegt daran, dass Pods als relativ kurzlebige 
+Einweg-Einheiten konzipiert sind. Wann Ein Pod erstellt wird (entweder direkt 
+von Ihnen oder indirekt von einem
+{{<glossary_tooltip text="Controller" term_id="controller">}}), wird die 
+Ausführung auf einem {{<glossary_tooltip term_id="node">}} in Ihrem Cluster 
+geplant. Der Pod bleibt auf diesem (virtuellen) Server, bis entweder der Pod die
+Ausführung beendet hat, das Pod-Objekt gelöscht wird, der Pod aufgrund 
+mangelnder Ressourcen *evakuiert* wird oder oder der Node ausfällt. 
+
+{{< note >}}
+Das Neustarten eines Containers in einem Pod sollte nicht mit dem Neustarten 
+eines Pods verwechselt werden. Ein Pod ist kein Prozess, sondern eine Umgebung 
+zur Ausführung von Containern. Ein Pod bleibt bestehen bis er gelöscht wird.
+{{< /note >}}
+
+Stelle beim Erstellen des Manifests für ein Pod-Objekt sicher, dass der 
+angegebene Name ein gültiger
+[DNS-Subdomain-Name](/docs/concepts/overview/working-with-objects/names#dns-subdomain-names)
+ist.
+
+### Pods und Controller
+
+Mit Workload-Ressourcen kannst du mehrere Pods erstellen und verwalten. Ein 
+Controller für die Ressource kümmert sich um Replikation, Roll-Out sowie 
+automatische Wiederherstellung im Fall von versagenden Pods. Wenn beispielsweise ein Node 
+ausfällt, bemerkt ein Controller, dass die Pods auf dem Node nicht mehr laufen 
+und plant die Ausführung eines Ersatzpods auf einem funktionierenden Node.
+Hier sind einige Beispiele für Workload-Ressourcen, die einen oder mehrere Pods 
+verwalten:
+
+* {{< glossary_tooltip text="Deployment" term_id="deployment" >}}
+* {{< glossary_tooltip text="StatefulSet" term_id="statefulset" >}}
+* {{< glossary_tooltip text="DaemonSet" term_id="daemonset" >}}
+
+### Pod Vorlagen
+
+Controller für 
+{{<glossary_tooltip text="Workload" term_id="workload">}}-Ressourcen 
+erstellen Pods von einer _Pod Vorlage_ und verwalten diese Pods für dich.
+
+Pod Vorlagen sind Spezifikationen zum Erstellen von Pods und sind in 
+Workload-Ressourcen enthalten wie z. B.
+[Deployments](/docs/concepts/workloads/controllers/deployment/),
+[Jobs](/docs/concepts/workloads/controllers/job/), and
+[DaemonSets](/docs/concepts/workloads/controllers/daemonset/).
+
+Jeder Controller für eine Workload-Ressource verwendet die Pod Vorlage innerhalb 
+des Workload-Objektes, um Pods zu erzeugen. Die Pod Vorlage ist Teil des 
+gewünschten Zustands der Workload-Ressource, mit der du deine Anwendung 
+ausgeführt hast.
+
+Das folgende Beispiel ist ein Manifest für einen einfachen Job mit einer 
+`Vorlage`, die einen Container startet. Der Container in diesem Pod druckt 
+eine Nachricht und pausiert dann.
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: hello
+spec:
+ template:
+ # Dies is the Pod Vorlage
+ spec:
+ containers:
+ - name: hello
+ image: busybox
+ command: ['sh', '-c', 'echo "Hello, Kubernetes!" && sleep 3600']
+ restartPolicy: OnFailure
+ # Die Pod Vorlage endet hier
+```
+Das Ändern der Pod Vorlage oder der Wechsel zu einer neuen Pod Vorlage hat keine 
+direkten Auswirkungen auf bereits existierende Pods. Wenn du die Pod Vorlage für
+eine Workload-Ressource änderst, dann muss diese Ressource die Ersatz-Pods
+erstellen, welche die aktualisierte Vorlage verwenden. 
+
+Beispielsweise stellt der StatefulSet-Controller sicher, dass für jedes
+StatefulSet-Objekt die ausgeführten Pods mit der aktueller Pod Vorlage 
+übereinstimmen. Wenn du das StatefulSet bearbeitest und die Vorlage änderst, 
+beginnt das StatefulSet mit der Erstellung neuer Pods basierend auf der 
+aktualisierten Vorlage. Schließlich werden alle alten Pods durch neue Pods 
+ersetzt, und das Update ist abgeschlossen.
+
+Jede Workload-Ressource implementiert eigenen Regeln für die Umsetzung von 
+Änderungen der Pod Vorlage. Wenn du mehr über StatefulSet erfahren möchtest, 
+dann lese die Seite
+[Update-Strategien](/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) 
+im Tutorial StatefulSet Basics.
+
+
+Auf Nodes beobachtet oder verwaltet das 
+{{< glossary_tooltip term_id="kubelet" text="Kubelet" >}}
+nicht direkt die Details zu Pod Vorlagen und Updates. Diese Details sind 
+abstrahiert. Die Abstraktion und Trennung von Aufgaben vereinfacht die 
+Systemsemantik und ermöglicht so das Verhalten des Clusters zu ändern ohne 
+vorhandenen Code zu ändern.
+
+## Pod Update und Austausch
+
+Wie im vorherigen Abschnitt erwähnt, erstellt der Controller neue Pods basierend
+auf der aktualisierten Vorlage, wenn die Pod Vorlage für eine Workload-Ressource
+geändert wird anstatt die vorhandenen Pods zu aktualisieren oder zu patchen.
+
+Kubernetes hindert dich nicht daran, Pods direkt zu verwalten. Es ist möglich, 
+einige Felder eines laufenden Pods zu aktualisieren. Allerdings haben 
+Pod-Aktualisierungsvorgänge wie zum Beispiel
+[`patch`](/docs/reference/generated/kubernetes-api/{{< param "version" >}}/#patch-pod-v1-core), 
+und
+[`replace`](/docs/reference/generated/kubernetes-api/{{< param "version" >}}/#replace-pod-v1-core)
+einige Einschränkungen:
+
+- Die meisten Metadaten zu einem Pod können nicht verändert werden. Zum Beispiel kannst
+  du nicht die Felder `namespace`, `name`, `uid`, oder `creationTimestamp` 
+  ändern. Das `generation`-Feld muss eindeutig sein. Es werden nur Aktualisierungen 
+  akzeptiert, die den Wert des Feldes inkrementieren.
+- Wenn das Feld `metadata.deletionTimestamp` gesetzt ist, kann kein neuer 
+  Eintrag zur Liste `metadata.finalizers` hinzugefügt werden.
+- Pod-Updates dürfen keine Felder ändern, die Ausnahmen sind 
+  `spec.containers[*].image`,
+ `spec.initContainers[*].image`,` spec.activeDeadlineSeconds` oder
+ `spec.tolerations`. Für `spec.tolerations` kannnst du nur neue Einträge 
+  hinzufügen.
+- Für `spec.activeDeadlineSeconds` sind nur zwei Änderungen erlaubt:
+
+  1. ungesetztes Feld in eine positive Zahl
+  1. positive Zahl in eine kleinere positive Zahl, die nicht negativ ist
+  
+## Gemeinsame Nutzung von Ressourcen und Kommunikation 
+
+Pods ermöglichen den Datenaustausch und die Kommunikation zwischen den 
+Containern, die im Pod enthalten sind.
+
+### Datenspeicherung in Pods
+
+Ein Pod kann eine Reihe von gemeinsam genutzten Speicher-
+{{<glossary_tooltip text="Volumes" term_id="volume">}} spezifizieren. Alle 
+Container im Pod können auf die gemeinsamen Volumes zugreifen und dadurch Daten 
+austauschen. Volumes ermöglichen auch, dass Daten ohne Verlust gespeichert 
+werden, falls einer der Container neu gestartet werden muss. 
+Im Kapitel [Datenspeicherung](/docs/concepts/storage/) findest du weitere 
+Informationen, wie Kubernetes gemeinsam genutzten Speicher implementiert und 
+Pods zur Verfügung stellt.
+
+### Pod-Netzwerk
+
+Jedem Pod wird für jede Adressenfamilie eine eindeutige IP-Adresse zugewiesen. 
+Jeder Container in einem Pod nutzt den gemeinsamen Netzwerk-Namespace, 
+einschließlich der IP-Adresse und der Ports. In einem Pod (und **nur** dann) 
+können die Container, die zum Pod gehören, über `localhost` miteinander 
+kommunizieren. Wenn Container in einem Pod mit Entitäten *außerhalb des Pods* 
+kommunizieren, müssen sie koordinieren, wie die gemeinsam genutzten 
+Netzwerkressourcen (z. B. Ports) verwenden werden. Innerhalb eines Pods teilen 
+sich Container eine IP-Adresse und eine Reihe von Ports und können sich 
+gegenseitig über `localhost` finden. Die Container in einem Pod können auch die 
+üblichen Kommunikationsverfahren zwischen Prozessen nutzen, wie z. B. 
+SystemV-Semaphoren oder "POSIX Shared Memory". Container in verschiedenen Pods 
+haben unterschiedliche IP-Adressen und können nicht per IPC ohne 
+[spezielle Konfiguration](/docs/concepts/policy/pod-security-policy/) 
+kommunizieren. Container, die mit einem Container in einem anderen Pod 
+interagieren möchten, müssen IP Netzwerke verwenden.
+
+Für die Container innerhalb eines Pods stimmt der "hostname" mit dem 
+konfigurierten `Namen` des Pods überein. Mehr dazu im Kapitel 
+[Netzwerke](/docs/concepts/cluster-administration/networking/).
+
+## Privilegierter Modus für Container
+
+Jeder Container in einem Pod kann den privilegierten Modus aktivieren, indem 
+das Flag `privileged` im 
+[Sicherheitskontext](/docs/tasks/configure-pod-container/security-context/)
+der Container-Spezifikation verwendet wird.
+Dies ist nützlich für Container, die Verwaltungsfunktionen des Betriebssystems 
+verwenden möchten, z. B. das Manipulieren des Netzwerk-Stacks oder den Zugriff 
+auf Hardware. Prozesse innerhalb eines privilegierten Containers erhalten fast 
+die gleichen Rechte wie sie Prozessen außerhalb eines Containers zur Verfügung 
+stehen.
+
+{{< note >}}
+Ihre 
+{{<glossary_tooltip text="Container-Umgebung" term_id="container-runtime">}} 
+muss das Konzept eines privilegierten Containers unterstützen, damit diese 
+Einstellung relevant ist.
+{{< /note >}}
+
+
+## Statische Pods
+
+_Statische Pods_ werden direkt vom Kubelet-Daemon auf einem bestimmten Node 
+verwaltet ohne dass sie vom 
+{{<glossary_tooltip text="API Server" term_id="kube-apiserver">}} überwacht 
+werden.
+
+Die meisten Pods werden von der Kontrollebene verwaltet (z. B.
+{{< glossary_tooltip text="Deployment" term_id="deployment" >}}). Aber für 
+statische Pods überwacht das Kubelet jeden statischen Pod direkt (und startet 
+ihn neu, wenn er ausfällt).
+
+Statische Pods sind immer an ein {{<glossary_tooltip term_id="kubelet">}} auf 
+einem bestimmten Node gebunden. Der Hauptanwendungsfall für statische Pods 
+besteht darin, eine selbst gehostete Steuerebene auszuführen. Mit anderen 
+Worten: Das Kubelet dient zur Überwachung der einzelnen 
+[Komponenten der Kontrollebene](/docs/concepts/overview/components/#control-plane-components).
+
+Das Kubelet versucht automatisch auf dem Kubernetes API-Server für jeden 
+statischen Pod einen spiegelbildlichen Pod 
+(im Englischen: {{<glossary_tooltip text="mirror pod" term_id="mirror-pod">}}) 
+zu erstellen.
+Das bedeutet, dass die auf einem Node ausgeführten Pods auf dem API-Server 
+sichtbar sind jedoch von dort nicht gesteuert werden können.
+
+## {{% heading "whatsnext" %}}
+
+* Verstehe den 
+  [Lebenszyklus eines Pods](/docs/concepts/workloads/pods/pod-lifecycle/).
+* Erfahre mehr über [RuntimeClass](/docs/concepts/containers/runtime-class/) 
+  und wie du damit verschiedene Pods mit unterschiedlichen 
+  Container-Laufzeitumgebungen konfigurieren kannst.
+* Mehr zum Thema 
+  [Restriktionen für die Verteilung von Pods](/docs/concepts/workloads/pods/pod-topology-spread-constraints/).
+* Lese
+  [Pod-Disruption-Budget](/docs/concepts/workloads/pods/disruptions/) 
+  und wie du es verwenden kannst, um die Verfügbarkeit von Anwendungen bei 
+  Störungen zu verwalten. Die 
+  [Pod](/docs/reference/generated/kubernetes-api/{{< param "version" >}}/#pod-v1-core)
+  -Objektdefinition beschreibt das Objekt im Detail.
+* [The Distributed System Toolkit: Patterns for Composite Containers](https://kubernetes.io/blog/2015/06/the-distributed-system-toolkit-patterns) 
+  erläutert allgemeine Layouts für Pods mit mehr als einem Container.
+
+Um den Hintergrund zu verstehen, warum Kubernetes eine gemeinsame Pod-API in 
+andere Ressourcen, wie z. B. 
+{{< glossary_tooltip text="StatefulSets" term_id="statefulset" >}} 
+oder {{< glossary_tooltip text="Deployments" term_id="deployment" >}} einbindet, 
+kannst du Artikel zu früheren Technologien lesen, unter anderem: 
+ * [Aurora](https://aurora.apache.org/documentation/latest/reference/configuration/#job-schema)
+ * [Borg](https://research.google.com/pubs/pub43438.html)
+ * [Marathon](https://mesosphere.github.io/marathon/docs/rest-api.html)
+ * [Omega](https://research.google/pubs/pub41684/)
+ * [Tupperware](https://engineering.fb.com/data-center-engineering/tupperware/).
@@ -9,7 +9,7 @@ content_type: concept

 Diese Sektion umfasst verschiedene Optionen zum Einrichten und Betrieb von Kubernetes.

-Verschiedene Kubernetes Lösungen haben verschiedene Anforderungen:  Einfache Wartung, Sicherheit, Kontrolle, verfügbare Resourcen und erforderliches Fachwissen zum Betrieb und zur Verwaltung dess folgende Diagramm zeigt die möglichen Abstraktionen eines Kubernetes-Clusters und ob eine Abstraktion selbst verwaltet oder von einem Anbieter verwaltet wird.
+Verschiedene Kubernetes Lösungen haben verschiedene Anforderungen:  Einfache Wartung, Sicherheit, Kontrolle, verfügbare Resourcen und erforderliches Fachwissen zum Betrieb und zur Verwaltung. Das folgende Diagramm zeigt die möglichen Abstraktionen eines Kubernetes-Clusters und ob eine Abstraktion selbst verwaltet oder von einem Anbieter verwaltet wird.

 Sie können einen Kubernetes-Cluster auf einer lokalen Maschine, Cloud, On-Prem Datacenter bereitstellen; oder wählen Sie einen verwalteten Kubernetes-Cluster. Sie können auch eine individuelle Lösung über eine grosse Auswahl an Cloud Anbietern oder Bare-Metal-Umgebungen nutzen.

@@ -26,7 +26,7 @@ On the other hand, CNI is more philosophically aligned with Kubernetes. It's far

 Additionally, it's trivial to wrap a CNI plugin and produce a more customized CNI plugin — it can be done with a simple shell script. CNM is much more complex in this regard. This makes CNI an attractive option for rapid development and iteration. Early prototypes have proven that it's possible to eject almost 100% of the currently hard-coded network logic in kubelet into a plugin.  

-We investigated [writing a "bridge" CNM driver](https://groups.google.com/forum/#!topic/kubernetes-sig-network/5MWRPxsURUw) for Docker that ran CNI drivers. This turned out to be very complicated. First, the CNM and CNI models are very different, so none of the "methods" lined up. We still have the global vs. local and key-value issues discussed above. Assuming this driver would declare itself local, we have to get info about logical networks from Kubernetes.  
+We investigated [writing a "bridge" CNM driver](https://groups.google.com/g/kubernetes-sig-network/c/5MWRPxsURUw) for Docker that ran CNI drivers. This turned out to be very complicated. First, the CNM and CNI models are very different, so none of the "methods" lined up. We still have the global vs. local and key-value issues discussed above. Assuming this driver would declare itself local, we have to get info about logical networks from Kubernetes.  

 Unfortunately, Docker drivers are hard to map to other control planes like Kubernetes. Specifically, drivers are not told the name of the network to which a container is being attached — just an ID that Docker allocates internally. This makes it hard for a driver to map back to any concept of network that exists in another system.  

@@ -34,6 +34,6 @@ This and other issues have been brought up to Docker developers by network vendo

 For all of these reasons we have chosen to invest in CNI as the Kubernetes plugin model. There will be some unfortunate side-effects of this. Most of them are relatively minor (for example, `docker inspect` will not show an IP address), but some are significant. In particular, containers started by `docker run` might not be able to communicate with containers started by Kubernetes, and network integrators will have to provide CNI drivers if they want to fully integrate with Kubernetes. On the other hand, Kubernetes will get simpler and more flexible, and a lot of the ugliness of early bootstrapping (such as configuring Docker to use our bridge) will go away.  

-As we proceed down this path, we’ll certainly keep our eyes and ears open for better ways to integrate and simplify. If you have thoughts on how we can do that, we really would like to hear them — find us on [slack](http://slack.k8s.io/) or on our [network SIG mailing-list](https://groups.google.com/forum/#!forum/kubernetes-sig-network).  
+As we proceed down this path, we’ll certainly keep our eyes and ears open for better ways to integrate and simplify. If you have thoughts on how we can do that, we really would like to hear them — find us on [slack](http://slack.k8s.io/) or on our [network SIG mailing-list](https://groups.google.com/g/kubernetes-sig-network).  

 Tim Hockin, Software Engineer, Google
@@ -20,21 +20,14 @@ For example, if we want to require scheduling on a node that is in the us-centra


 ```
-affinity:
-
-  nodeAffinity:
-
-    requiredDuringSchedulingIgnoredDuringExecution:
-
-      nodeSelectorTerms:
-
-        - matchExpressions:
-
-          - key: "failure-domain.beta.kubernetes.io/zone"
-
-            operator: In
-
-            values: ["us-central1-a"]
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+            - key: "failure-domain.beta.kubernetes.io/zone"
+              operator: In
+              values: ["us-central1-a"]
 ```


@@ -44,21 +37,14 @@ Preferred rules mean that if nodes match the rules, they will be chosen first, a


 ```
-affinity:
-
-  nodeAffinity:
-
-    preferredDuringSchedulingIgnoredDuringExecution:
-
-      nodeSelectorTerms:
-
-        - matchExpressions:
-
-          - key: "failure-domain.beta.kubernetes.io/zone"
-
-            operator: In
-
-            values: ["us-central1-a"]
+  affinity:
+    nodeAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+            - key: "failure-domain.beta.kubernetes.io/zone"
+              operator: In
+              values: ["us-central1-a"]
 ```


@@ -67,21 +53,14 @@ Node anti-affinity can be achieved by using negative operators. So for instance


 ```
-affinity:
-
-  nodeAffinity:
-
-    requiredDuringSchedulingIgnoredDuringExecution:
-
-      nodeSelectorTerms:
-
-        - matchExpressions:
-
-          - key: "failure-domain.beta.kubernetes.io/zone"
-
-            operator: NotIn
-
-            values: ["us-central1-a"]
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+            - key: "failure-domain.beta.kubernetes.io/zone"
+              operator: NotIn
+              values: ["us-central1-a"]
 ```


@@ -99,7 +78,7 @@ The kubectl command allows you to set taints on nodes, for example:

 ```
 kubectl taint nodes node1 key=value:NoSchedule
- ```
+```


 creates a taint that marks the node as unschedulable by any pods that do not have a toleration for taint with key key, value value, and effect NoSchedule. (The other taint effects are PreferNoSchedule, which is the preferred version of NoSchedule, and NoExecute, which means any pods that are running on the node when the taint is applied will be evicted unless they tolerate the taint.) The toleration you would add to a PodSpec to have the corresponding pod tolerate this taint would look like this  
@@ -107,15 +86,11 @@ creates a taint that marks the node as unschedulable by any pods that do not hav


 ```
-tolerations:
-
- key: "key"
-
-  operator: "Equal"
-
-  value: "value"
-
-  effect: "NoSchedule"
+  tolerations:
+  - key: "key"
+    operator: "Equal"
+    value: "value"
+    effect: "NoSchedule"
 ```


@@ -138,21 +113,13 @@ Let’s look at an example. Say you have front-ends in service S1, and they comm

 ```
 affinity:
-
    podAffinity:
-
      requiredDuringSchedulingIgnoredDuringExecution:
-
      - labelSelector:
-
          matchExpressions:
-
          - key: service
-
            operator: In
-
            values: [“S1”]
-
        topologyKey: failure-domain.beta.kubernetes.io/zone
 ```

@@ -172,25 +139,15 @@ Here we have a Pod where we specify the schedulerName field:

 ```
 apiVersion: v1
-
 kind: Pod
-
 metadata:
-
  name: nginx
-
  labels:
-
    app: nginx
-
 spec:
-
  schedulerName: my-scheduler
-
  containers:
-
  - name: nginx
-
    image: nginx:1.10
 ```

@@ -56,13 +56,13 @@ Cri-containerd uses containerd to manage the full container lifecycle and all co

 Let’s use an example to demonstrate how cri-containerd works for the case when Kubelet creates a single-container pod:  

-1. 1.Kubelet calls cri-containerd, via the CRI runtime service API, to create a pod;
-2. 2.cri-containerd uses containerd to create and start a special [pause container](https://www.ianlewis.org/en/almighty-pause-container) (the _sandbox container_) and put that container inside the pod’s cgroups and namespace (steps omitted for brevity);
-3. 3.cri-containerd configures the pod’s network namespace using CNI;
-4. 4.Kubelet subsequently calls cri-containerd, via the CRI image service API, to pull the application container image;
-5. 5.cri-containerd further uses containerd to pull the image if the image is not present on the node;
-6. 6.Kubelet then calls cri-containerd, via the CRI runtime service API, to create and start the application container inside the pod using the pulled container image;
-7. 7.cri-containerd finally calls containerd to create the application container, put it inside the pod’s cgroups and namespace, then to start the pod’s new application container.
+1. Kubelet calls cri-containerd, via the CRI runtime service API, to create a pod;
+2. cri-containerd uses containerd to create and start a special [pause container](https://www.ianlewis.org/en/almighty-pause-container) (the _sandbox container_) and put that container inside the pod’s cgroups and namespace (steps omitted for brevity);
+3. cri-containerd configures the pod’s network namespace using CNI;
+4. Kubelet subsequently calls cri-containerd, via the CRI image service API, to pull the application container image;
+5. cri-containerd further uses containerd to pull the image if the image is not present on the node;
+6. Kubelet then calls cri-containerd, via the CRI runtime service API, to create and start the application container inside the pod using the pulled container image;
+7. cri-containerd finally calls containerd to create the application container, put it inside the pod’s cgroups and namespace, then to start the pod’s new application container.
 After these steps, a pod and its corresponding application container is created and running.


@@ -176,7 +176,7 @@ Cluster-distributed stateful services (e.g., Cassandra) can benefit from splitti

 [Logs](/docs/concepts/cluster-administration/logging/) and [metrics](/docs/tasks/debug-application-cluster/resource-usage-monitoring/) (if collected and persistently retained) are valuable to diagnose outages, but given the variety of technologies available it will not be addressed in this blog. If Internet connectivity is available, it may be desirable to retain logs and metrics externally at a central location.

-Your production deployment should utilize an automated installation, configuration and update tool (e.g., [Ansible](https://github.com/kubernetes-incubator/kubespray), [BOSH](https://github.com/cloudfoundry-incubator/kubo-deployment), [Chef](https://github.com/chef-cookbooks/kubernetes), [Juju](/docs/getting-started-guides/ubuntu/installation/), [kubeadm](/docs/reference/setup-tools/kubeadm/kubeadm/), [Puppet](https://forge.puppet.com/puppetlabs/kubernetes), etc.). A manual process will have repeatability issues, be labor intensive, error prone, and difficult to scale. [Certified distributions](https://www.cncf.io/certification/software-conformance/#logos) are likely to include a facility for retaining configuration settings across updates, but if you implement your own install and config toolchain, then retention, backup and recovery of the configuration artifacts is essential. Consider keeping your deployment components and settings under a version control system such as Git.
+Your production deployment should utilize an automated installation, configuration and update tool (e.g., [Ansible](https://github.com/kubernetes-incubator/kubespray), [BOSH](https://github.com/cloudfoundry-incubator/kubo-deployment), [Chef](https://github.com/chef-cookbooks/kubernetes), [Juju](/docs/getting-started-guides/ubuntu/installation/), [kubeadm](/docs/reference/setup-tools/kubeadm/), [Puppet](https://forge.puppet.com/puppetlabs/kubernetes), etc.). A manual process will have repeatability issues, be labor intensive, error prone, and difficult to scale. [Certified distributions](https://www.cncf.io/certification/software-conformance/#logos) are likely to include a facility for retaining configuration settings across updates, but if you implement your own install and config toolchain, then retention, backup and recovery of the configuration artifacts is essential. Consider keeping your deployment components and settings under a version control system such as Git.

 ## Outage recovery

@@ -17,7 +17,7 @@ Let’s dive into the key features of this release:

 ## Simplified Kubernetes Cluster Management with kubeadm in GA

-Most people who have gotten hands-on with Kubernetes have at some point been hands-on with kubeadm. It's an essential tool for managing the cluster lifecycle, from creation to configuration to upgrade; and now kubeadm is officially GA. [kubeadm](/docs/reference/setup-tools/kubeadm/kubeadm/) handles the bootstrapping of production clusters on existing hardware and configuring the core Kubernetes components in a best-practice-manner to providing a secure yet easy joining flow for new nodes and supporting easy upgrades. What’s notable about this GA release are the now graduated advanced features, specifically around pluggability and configurability. The scope of kubeadm is to be a toolbox for both admins and automated, higher-level system and this release is a significant step in that direction.
+Most people who have gotten hands-on with Kubernetes have at some point been hands-on with kubeadm. It's an essential tool for managing the cluster lifecycle, from creation to configuration to upgrade; and now kubeadm is officially GA. [kubeadm](/docs/reference/setup-tools/kubeadm/) handles the bootstrapping of production clusters on existing hardware and configuring the core Kubernetes components in a best-practice-manner to providing a secure yet easy joining flow for new nodes and supporting easy upgrades. What’s notable about this GA release are the now graduated advanced features, specifically around pluggability and configurability. The scope of kubeadm is to be a toolbox for both admins and automated, higher-level system and this release is a significant step in that direction.

 ## Container Storage Interface (CSI) Goes GA

@@ -66,6 +66,7 @@ Vagrant.configure("2") do |config|
            end
        end
    end
+end
 ```

 ### Step 2: Create an Ansible playbook for Kubernetes master.
@@ -120,7 +120,7 @@ Run the following command to have Docker run the application in a container and
 ```
 docker run -p 5001:5000 hello-python
 ```
-Now navigate to http://localhost:5001, and you should see the “Hello form Python!” message. 
+Now navigate to http://localhost:5001, and you should see the “Hello from Python!” message. 

 ### More info
 * [Get started with Docker](https://docs.docker.com/get-started/)
@@ -201,7 +201,7 @@ kubectl get pods
 ```
 <img src="/images/blog/get-started-with-kubernetes-using-python/kubectl-get-pods.png" alt="Pod listing" />

-Now navigate to http://localhost:6000, and you should see the “Hello form Python!” message. 
+Now navigate to http://localhost:6000, and you should see the “Hello from Python!” message. 

 That’s it! The application is now running in Kubernetes!

@@ -58,7 +58,7 @@ Take maven project as example, adding the following dependencies into your depen

 Then we can make use of the provided builder libraries to write your own controller.
 For example, the following one is a simple controller prints out node information
-on watch notification, see complete example [here](https://github.com/kubernetes-client/java/blob/master/examples/src/main/java/io/kubernetes/client/examples/ControllerExample.java):
+on watch notification, see complete example [here](https://github.com/kubernetes-client/java/blob/master/examples/examples-release-13/src/main/java/io/kubernetes/client/examples/ControllerExample.java):

 ```java
 ...
@@ -31,9 +31,9 @@ Standard labels are used by Kubernetes components to support some features. For

 The labels are reaching general availability in this release. Kubernetes components have been updated to populate the GA and beta labels and to react to both. However, if you are using the beta labels in your pod specs for features such as node affinity, or in your custom controllers, we recommend that you start migrating them to the new GA labels. You can find the documentation for the new labels here:

- [node.kubernetes.io/instance-type](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#nodekubernetesioinstance-type)
- [topology.kubernetes.io/region](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#topologykubernetesioregion)
- [topology.kubernetes.io/zone](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#topologykubernetesiozone)
+- [node.kubernetes.io/instance-type](/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type)
+- [topology.kubernetes.io/region](/docs/reference/labels-annotations-taints/#topologykubernetesioregion)
+- [topology.kubernetes.io/zone](/docs/reference/labels-annotations-taints/#topologykubernetesiozone)

 ## Volume Snapshot Moves to Beta

@@ -44,7 +44,7 @@ As mentioned above, with the promotion of Volume Snapshot to beta, the feature i

 In order to use the Kubernetes Volume Snapshot feature, you must ensure the following components have been deployed on your Kubernetes cluster:

- [Kubernetes Volume Snapshot CRDs](https://github.com/kubernetes-csi/external-snapshotter/tree/master/config/crd)
+- [Kubernetes Volume Snapshot CRDs](https://github.com/kubernetes-csi/external-snapshotter/tree/53469c21962339229dd150cbba50c34359acec73/config/crd)
 - [Volume snapshot controller](https://github.com/kubernetes-csi/external-snapshotter/tree/master/pkg/common-controller)
 - CSI Driver supporting Kubernetes volume snapshot beta

@@ -180,7 +180,7 @@ If your cluster does not come pre-installed with the correct components, you may
 #### Install Snapshot Beta CRDs

 - `kubectl create -f config/crd`
- [https://github.com/kubernetes-csi/external-snapshotter/tree/master/config/crd](https://github.com/kubernetes-csi/external-snapshotter/tree/master/config/crd)
+- [https://github.com/kubernetes-csi/external-snapshotter/tree/53469c21962339229dd150cbba50c34359acec73/config/crd](https://github.com/kubernetes-csi/external-snapshotter/tree/53469c21962339229dd150cbba50c34359acec73/config/crd)
 - Do this once per cluster


@@ -325,7 +325,7 @@ Now that we have a way to communicate helpful information to users in context,
 we're already considering other ways we can use this to improve people's experience with Kubernetes.
 A couple areas we're looking at next are warning about [known problematic values](http://issue.k8s.io/64841#issuecomment-395141013)
 we cannot reject outright for compatibility reasons, and warning about use of deprecated fields or field values
-(like selectors using beta os/arch node labels, [deprecated in v1.14](/docs/reference/kubernetes-api/labels-annotations-taints/#beta-kubernetes-io-arch-deprecated)).
+(like selectors using beta os/arch node labels, [deprecated in v1.14](/docs/reference/labels-annotations-taints/#beta-kubernetes-io-arch-deprecated)).
 I'm excited to see progress in this area, continuing to make it easier to use Kubernetes.

 ---
@@ -0,0 +1,96 @@
+---
+layout: blog
+title: "A Custom Kubernetes Scheduler to Orchestrate Highly Available Applications"
+date: 2020-12-21
+slug: writing-crl-scheduler
+---
+
+**Author**: Chris Seto (Cockroach Labs)
+
+As long as you're willing to follow the rules, deploying on Kubernetes and air travel can be quite pleasant. More often than not, things will "just work". However, if one is interested in travelling with an alligator that must remain alive or scaling a database that must remain available, the situation is likely to become a bit more complicated. It may even be easier to build one's own plane or database for that matter. Travelling with reptiles aside, scaling a highly available stateful system is no trivial task.
+
+Scaling any system has two main components:
+1. Adding or removing infrastructure that the system will run on, and
+2. Ensuring that the system knows how to handle additional instances of itself being added and removed.
+
+Most stateless systems, web servers for example, are created without the need to be aware of peers. Stateful systems, which includes databases like CockroachDB, have to coordinate with their peer instances and shuffle around data. As luck would have it, CockroachDB handles data redistribution and replication. The tricky part is being able to tolerate failures during these operations by ensuring that data and instances are distributed across many failure domains (availability zones).
+
+One of Kubernetes' responsibilities is to place "resources" (e.g, a disk or container) into the cluster and satisfy the constraints they request. For example: "I must be in availability zone _A_" (see [Running in multiple zones](/docs/setup/best-practices/multiple-zones/#nodes-are-labeled)), or "I can't be placed onto the same node as this other Pod" (see [Affinity and anti-affinity](/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity)).
+
+As an addition to those constraints, Kubernetes offers [Statefulsets](/docs/concepts/workloads/controllers/statefulset/) that provide identity to Pods as well as persistent storage that "follows" these identified pods. Identity in a StatefulSet is handled by an increasing integer at the end of a pod's name. It's important to note that this integer must always be contiguous: in a StatefulSet, if pods 1 and 3 exist then pod 2 must also exist.
+
+Under the hood, CockroachCloud deploys each region of CockroachDB as a StatefulSet in its own Kubernetes cluster - see [Orchestrate CockroachDB in a Single Kubernetes Cluster](https://www.cockroachlabs.com/docs/stable/orchestrate-cockroachdb-with-kubernetes.html).
+In this article, I'll be looking at an individual region, one StatefulSet and one Kubernetes cluster which is distributed across at least three availability zones.
+
+A three-node CockroachCloud cluster would look something like this:
+
+![3-node, multi-zone cockroachdb cluster](image01.png)
+
+When adding additional resources to the cluster we also distribute them across zones. For the speediest user experience, we add all Kubernetes nodes at the same time and then scale up the StatefulSet.
+
+![illustration of phases: adding Kubernetes nodes to the multi-zone cockroachdb cluster](image02.png)
+
+Note that anti-affinities are satisfied no matter the order in which pods are assigned to Kubernetes nodes. In the example, pods 0, 1 and 2 were assigned to zones A, B, and C respectively, but pods 3 and 4 were assigned in a different order, to zones B and A respectively. The anti-affinity is still satisfied because the pods are still placed in different zones.
+
+To remove resources from a cluster, we perform these operations in reverse order.
+
+We first scale down the StatefulSet and then remove from the cluster any nodes lacking a CockroachDB pod.
+
+![illustration of phases: scaling down pods in a multi-zone cockroachdb cluster in Kubernetes](image03.png)
+
+Now, remember that pods in a StatefulSet of size _n_ must have ids in the range `[0,n)`. When scaling down a StatefulSet by _m_, Kubernetes removes _m_ pods, starting from the highest ordinals and moving towards the lowest, [the reverse in which they were added](/docs/concepts/workloads/controllers/statefulset/#deployment-and-scaling-guarantees).
+Consider the cluster topology below:
+
+![illustration: cockroachdb cluster: 6 nodes distributed across 3 availability zones](image04.png)
+
+As ordinals 5 through 3 are removed from this cluster, the statefulset continues to have a presence across all 3 availability zones.
+
+![illustration: removing 3 nodes from a 6-node, 3-zone cockroachdb cluster](image05.png)
+
+However, Kubernetes' scheduler doesn't _guarantee_ the placement above as we expected at first.
+
+Our combined knowledge of the following is what lead to this misconception.
+* Kubernetes' ability to [automatically spread Pods across zone](/docs/setup/best-practices/multiple-zones/#pods-are-spread-across-zones)
+* The behavior that a StatefulSet with _n_ replicas, when Pods are being deployed, they are created sequentially, in order from `{0..n-1}`. See [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#deployment-and-scaling-guarantees) for more details.
+
+Consider the following topology:
+
+![illustration: 6-node cockroachdb cluster distributed across 3 availability zones](image06.png)
+
+These pods were created in order and they are spread across all availability zones in the cluster. When ordinals 5 through 3 are terminated, this cluster will lose its presence in zone C!
+
+![illustration: terminating 3 nodes in 6-node cluster spread across 3 availability zones, where 2/2 nodes in the same availability zone are terminated, knocking out that AZ](image07.png)
+
+Worse yet, our automation, at the time, would remove Nodes A-2, B-2, and C-2. Leaving CRDB-1 in an unscheduled state as persistent volumes are only available in the zone they are initially created in.
+
+To correct the latter issue, we now employ a "hunt and peck" approach to removing machines from a cluster. Rather than blindly removing Kubernetes nodes from the cluster, only nodes without a CockroachDB pod would be removed. The much more daunting task was to wrangle the Kubernetes scheduler.
+
+## A session of brainstorming left us with 3 options:
+
+### 1. Upgrade to kubernetes 1.18 and make use of Pod Topology Spread Constraints
+
+While this seems like it could have been the perfect solution, at the time of writing Kubernetes 1.18 was unavailable on the two most common managed Kubernetes services in public cloud, EKS and GKE.
+Furthermore, [pod topology spread constraints](/docs/concepts/workloads/pods/pod-topology-spread-constraints/) were still a [beta feature in 1.18](https://v1-18.docs.kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) which meant that it [wasn't guaranteed to be available in managed clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters#kubernetes_feature_choices) even when v1.18 became available.
+The entire endeavour was concerningly reminiscent of checking [caniuse.com](https://caniuse.com/) when Internet Explorer 8 was still around.
+
+### 2. Deploy a statefulset _per zone_.
+
+Rather than having one StatefulSet distributed across all availability zones, a single StatefulSet with node affinities per zone would allow manual control over our zonal topology.
+Our team had considered this as an option in the past which made it particularly appealing.
+Ultimately, we decided to forego this option as it would have required a massive overhaul to our codebase and performing the migration on existing customer clusters would have been an equally large undertaking.
+
+### 3. Write a custom Kubernetes scheduler.
+
+Thanks to an example from [Kelsey Hightower](https://github.com/kelseyhightower/scheduler) and a blog post from [Banzai Cloud](https://banzaicloud.com/blog/k8s-custom-scheduler/), we decided to dive in head first and write our own [custom Kubernetes scheduler](/docs/tasks/extend-kubernetes/configure-multiple-schedulers/).
+Once our proof-of-concept was deployed and running, we quickly discovered that the Kubernetes' scheduler is also responsible for mapping persistent volumes to the Pods that it schedules.
+The output of [`kubectl get events`](/docs/tasks/extend-kubernetes/configure-multiple-schedulers/#verifying-that-the-pods-were-scheduled-using-the-desired-schedulers) had led us to believe there was another system at play.
+In our journey to find the component responsible for storage claim mapping, we discovered the [kube-scheduler plugin system](/docs/concepts/scheduling-eviction/scheduling-framework/). Our next POC was a `Filter` plugin that determined the appropriate availability zone by pod ordinal, and it worked flawlessly!
+
+Our [custom scheduler plugin](https://github.com/cockroachlabs/crl-scheduler) is open source and runs in all of our CockroachCloud clusters.
+Having control over how our StatefulSet pods are being scheduled has let us scale out with confidence.
+We may look into retiring our plugin once pod topology spread constraints are available in GKE and EKS, but the maintenance overhead has been surprisingly low.
+Better still: the plugin's implementation is orthogonal to our business logic. Deploying it, or retiring it for that matter, is as simple as changing the `schedulerName` field in our StatefulSet definitions.
+
+---
+
+_[Chris Seto](https://twitter.com/_ostriches) is a software engineer at Cockroach Labs and works on their Kubernetes automation for [CockroachCloud](https://cockroachlabs.cloud), CockroachDB._
@@ -47,6 +47,15 @@ and other ecosystem groups to ensure a smooth transition and will evaluate thing
 as the situation evolves.


+### Can I still use dockershim after it is removed from Kubernetes?
+
+Update:
+Mirantis and Docker have [committed][mirantis] to maintaining the dockershim after
+it is removed from Kubernetes.
+
+[mirantis]: https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/
+
+
 ### Will my existing Docker images still work?

 Yes, the images produced from `docker build` will work with all CRI implementations.
@@ -55,7 +64,7 @@ All your existing images will still work exactly the same.

 ### What about private images?

-Also yes. All CRI runtimes support the same pull secrets configuration used in
+Yes. All CRI runtimes support the same pull secrets configuration used in
 Kubernetes, either via the PodSpec or ServiceAccount.


@@ -82,7 +91,7 @@ usability of other container runtimes. As an example, OpenShift 4.x has been
 using the [CRI-O] runtime in production since June 2019.

 For other examples and references you can look at the adopters of containerd and
-cri-o, two container runtimes under the Cloud Native Computing Foundation ([CNCF]).
+CRI-O, two container runtimes under the Cloud Native Computing Foundation ([CNCF]).
 - [containerd](https://github.com/containerd/containerd/blob/master/ADOPTERS.md)
 - [CRI-O](https://github.com/cri-o/cri-o/blob/master/ADOPTERS.md)

@@ -110,11 +119,11 @@ provide an end-to-end standard for managing containers.

 That’s a complex question and it depends on a lot of factors. If Docker is
 working for you, moving to containerd should be a relatively easy swap and
-has have strictly better performance and less overhead. However we encourage you
+will have strictly better performance and less overhead. However, we encourage you
 to explore all the options from the [CNCF landscape] in case another would be an
 even better fit for your environment.

-[CNCF landscape]: https://landscape.cncf.io/category=container-runtime&format=card-mode&grouping=category
+[CNCF landscape]: https://landscape.cncf.io/card-mode?category=container-runtime&grouping=category


 ### What should I look out for when changing CRI implementations?
@@ -129,7 +138,7 @@ common things to consider when migrating are:
 - Kubectl plugins that require docker CLI or the control socket
 - Kubernetes tools that require direct access to Docker (e.g. kube-imagepuller)
 - Configuration of functionality like `registry-mirrors` and insecure registries 
- Other support scripts or daemons that expect docker to be available and are run
+- Other support scripts or daemons that expect Docker to be available and are run
  outside of Kubernetes (e.g. monitoring or security agents)
 - GPUs or special hardware and how they integrate with your runtime and Kubernetes

@@ -140,14 +149,15 @@ runtime where possible.

 Another thing to look out for is anything expecting to run for system maintenance
 or nested inside a container when building images will no longer work. For the
-former, you can use the [`crictl`][cr] tool as a drop-in replacement and for the
-latter you can use newer container build options like [img], [buildah], or
-[kaniko] that don’t require Docker.
+former, you can use the [`crictl`][cr] tool as a drop-in replacement (see [mapping from docker cli to crictl](https://kubernetes.io/docs/tasks/debug-application-cluster/crictl/#mapping-from-docker-cli-to-crictl)) and for the
+latter you can use newer container build options like [img], [buildah],
+[kaniko], or [buildkit-cli-for-kubectl] that don’t require Docker.

 [cr]: https://github.com/kubernetes-sigs/cri-tools
 [img]: https://github.com/genuinetools/img
 [buildah]: https://github.com/containers/buildah
 [kaniko]: https://github.com/GoogleContainerTools/kaniko
+[buildkit-cli-for-kubectl]: https://github.com/vmware-tanzu/buildkit-cli-for-kubectl

 For containerd, you can start with their [documentation] to see what configuration
 options are available as you migrate things over.
@@ -177,4 +187,3 @@ discussion of the changes.

 Always and whenever you want!  🤗🤗

-
@@ -13,8 +13,8 @@ as a container runtime after v1.20.

 **You do not need to panic. It’s not as dramatic as it sounds.**

-tl;dr Docker as an underlying runtime is being deprecated in favor of runtimes
-that use the [Container Runtime Interface(CRI)](https://kubernetes.io/blog/2016/12/container-runtime-interface-cri-in-kubernetes/)
+TL;DR Docker as an underlying runtime is being deprecated in favor of runtimes
+that use the [Container Runtime Interface (CRI)](https://kubernetes.io/blog/2016/12/container-runtime-interface-cri-in-kubernetes/)
 created for Kubernetes. Docker-produced images will continue to work in your
 cluster with all runtimes, as they always have.

@@ -48,7 +48,7 @@ is a popular choice for that runtime (other common options include containerd
 and CRI-O), but Docker was not designed to be embedded inside Kubernetes, and
 that causes a problem. 

-You see, the thing we call “Docker” isn’t actually one thing -- it’s an entire
+You see, the thing we call “Docker” isn’t actually one thing&mdash;it’s an entire
 tech stack, and one part of it is a thing called “containerd,” which is a
 high-level container runtime by itself. Docker is cool and useful because it has
 a lot of UX enhancements that make it really easy for humans to interact with
@@ -66,11 +66,11 @@ does Kubernetes need the Dockershim?

 Docker isn’t compliant with CRI, the [Container Runtime Interface](https://kubernetes.io/blog/2016/12/container-runtime-interface-cri-in-kubernetes/).
 If it were, we wouldn’t need the shim, and this wouldn’t be a thing. But it’s
-not the end of the world, and you don’t need to panic -- you just need to change
+not the end of the world, and you don’t need to panic&mdash;you just need to change
 your container runtime from Docker to another supported container runtime.

 One thing to note: If you are relying on the underlying docker socket
-(/var/run/docker.sock) as part of a workflow within your cluster today, moving
+(`/var/run/docker.sock`) as part of a workflow within your cluster today, moving
 to a different runtime will break your ability to use it. This pattern is often
 called Docker in Docker. There are lots of options out there for this specific
 use case including things like
@@ -82,10 +82,10 @@ use case including things like
 
 This change addresses a different environment than most folks use to interact
 with Docker. The Docker installation you’re using in development is unrelated to
-the Docker runtime inside your Kubernetes cluster. It’s confusing, I know. As a
-developer, Docker is still useful to you in all the ways it was before this
+the Docker runtime inside your Kubernetes cluster. It’s confusing, we understand.
+As a developer, Docker is still useful to you in all the ways it was before this
 change was announced. The image that Docker produces isn’t really a
-Docker-specific image -- it’s an OCI ([Open Container Initiative](https://opencontainers.org/)) image. 
+Docker-specific image&mdash;it’s an OCI ([Open Container Initiative](https://opencontainers.org/)) image. 
 Any OCI-compliant image, regardless of the tool you use to build it, will look
 the same to Kubernetes. Both [containerd](https://containerd.io/) and
 [CRI-O](https://cri-o.io/) know how to pull those images and run them. This is
@@ -95,10 +95,10 @@ So, this change is coming. It’s going to cause issues for some, but it isn’t
 catastrophic, and generally it’s a good thing. Depending on how you interact
 with Kubernetes, this could mean nothing to you, or it could mean a bit of work.
 In the long run, it’s going to make things easier. If this is still confusing
-for you, that’s okay -- there’s a lot going on here, Kubernetes has a lot of
+for you, that’s okay&mdash;there’s a lot going on here; Kubernetes has a lot of
 moving parts, and nobody is an expert in 100% of it. We encourage any and all
 questions regardless of experience level or complexity! Our goal is to make sure
-everyone is educated as much as possible on the upcoming changes. `<3` We hope
-this has answered most of your questions and soothed some anxieties!
+everyone is educated as much as possible on the upcoming changes. We hope
+this has answered most of your questions and soothed some anxieties! ❤️

 Looking for more answers? Check out our accompanying [Dockershim Deprecation FAQ](/blog/2020/12/02/dockershim-faq/).
@@ -0,0 +1,144 @@
+---
+layout: blog
+title: 'Kubernetes 1.20: The Raddest Release'
+date: 2020-12-08
+slug: kubernetes-1-20-release-announcement
+---
+
+**Authors:** [Kubernetes 1.20 Release Team](https://github.com/kubernetes/sig-release/blob/master/releases/release-1.20/release_team.md)
+
+We’re pleased to announce the release of Kubernetes 1.20, our third and final release of 2020! This release consists of 42 enhancements: 11 enhancements have graduated to stable, 15 enhancements are moving to beta, and 16 enhancements are entering alpha.
+
+The 1.20 release cycle returned to its normal cadence of 11 weeks following the previous extended release cycle. This is one of the most feature dense releases in a while: the Kubernetes innovation cycle is still trending upward. This release has more alpha than stable enhancements, showing that there is still much to explore in the cloud native ecosystem.
+
+## Major Themes
+
+### Volume Snapshot Operations Goes Stable
+
+This feature provides a standard way to trigger volume snapshot operations and allows users to incorporate snapshot operations in a portable manner on any Kubernetes environment and supported storage providers.
+
+Additionally, these Kubernetes snapshot primitives act as basic building blocks that unlock the ability to develop advanced, enterprise-grade, storage administration features for Kubernetes, including application or cluster level backup solutions.
+
+Note that snapshot support requires Kubernetes distributors to bundle the Snapshot controller, Snapshot CRDs, and validation webhook. A CSI driver supporting the snapshot functionality must also be deployed on the cluster.
+
+### Kubectl Debug Graduates to Beta
+
+The `kubectl alpha debug` features graduates to beta in 1.20, becoming `kubectl debug`. The feature provides support for common debugging workflows directly from kubectl. Troubleshooting scenarios supported in this release of kubectl include:
+
+* Troubleshoot workloads that crash on startup by creating a copy of the pod that uses a different container image or command.
+* Troubleshoot distroless containers by adding a new container with debugging tools, either in a new copy of the pod or using an ephemeral container. (Ephemeral containers are an alpha feature that are not enabled by default.)
+* Troubleshoot on a node by creating a container running in the host namespaces and with access to the host’s filesystem.
+
+Note that as a new built-in command, `kubectl debug` takes priority over any kubectl plugin named “debug”. You must rename the affected plugin.
+
+Invocations using `kubectl alpha debug` are now deprecated and will be removed in a subsequent release. Update your scripts to use `kubectl debug`. For more information about `kubectl debug`, see [Debugging Running Pods](https://kubernetes.io/docs/tasks/debug-application-cluster/debug-running-pod/).
+
+### Beta: API Priority and Fairness
+
+Introduced in 1.18, Kubernetes 1.20 now enables API Priority and Fairness (APF) by default. This allows `kube-apiserver` to categorize incoming requests by priority levels.
+
+### Alpha with updates: IPV4/IPV6
+
+The IPv4/IPv6 dual stack has been reimplemented to support dual stack services based on user and community feedback. This allows both IPv4 and IPv6 service cluster IP addresses to be assigned to a single service, and also enables a service to be transitioned from single to dual IP stack and vice versa.
+
+### GA: Process PID Limiting for Stability
+
+Process IDs (pids) are a fundamental resource on Linux hosts. It is trivial to hit the task limit without hitting any other resource limits and cause instability to a host machine.
+
+Administrators require mechanisms to ensure that user pods cannot induce pid exhaustion that prevents host daemons (runtime, kubelet, etc) from running. In addition, it is important to ensure that pids are limited among pods in order to ensure they have limited impact to other workloads on the node.
+After being enabled-by-default for a year, SIG Node graduates PID Limits to GA on both `SupportNodePidsLimit` (node-to-pod PID isolation) and `SupportPodPidsLimit` (ability to limit PIDs per pod).
+
+### Alpha: Graceful node shutdown
+
+Users and cluster administrators expect that pods will adhere to expected pod lifecycle including pod termination. Currently, when a node shuts down, pods do not follow the expected pod termination lifecycle and are not terminated gracefully which can cause issues for some workloads.
+The `GracefulNodeShutdown` feature is now in Alpha. `GracefulNodeShutdown` makes the kubelet aware of node system shutdowns, enabling graceful termination of pods during a system shutdown.
+
+## Major Changes
+
+### Dockershim Deprecation
+
+Dockershim, the container runtime interface (CRI) shim for Docker is being deprecated. Support for Docker is deprecated and will be removed in a future release. Docker-produced images will continue to work in your cluster with all CRI compliant runtimes as Docker images follow the Open Container Initiative (OCI) image specification.
+The Kubernetes community has written a [detailed blog post about deprecation](https://blog.k8s.io/2020/12/02/dont-panic-kubernetes-and-docker/) with [a dedicated FAQ page for it](https://blog.k8s.io/2020/12/02/dockershim-faq/).
+
+### Exec Probe Timeout Handling
+
+A longstanding bug regarding exec probe timeouts that may impact existing pod definitions has been fixed. Prior to this fix, the field `timeoutSeconds` was not respected for exec probes. Instead, probes would run indefinitely, even past their configured deadline, until a result was returned. With this change, the default value of `1 second` will be applied if a value is not specified and existing pod definitions may no longer be sufficient if a probe takes longer than one second. A feature gate, called `ExecProbeTimeout`, has been added with this fix that enables cluster operators to revert to the previous behavior, but this will be locked and removed in subsequent releases. In order to revert to the previous behavior, cluster operators should set this feature gate to `false`.
+
+Please review the updated documentation regarding [configuring probes](/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes) for more details.
+
+## Other Updates
+
+### Graduated to Stable
+
+* [RuntimeClass](https://github.com/kubernetes/enhancements/issues/585)
+* [Built-in API Types Defaults](https://github.com/kubernetes/enhancements/issues/1929)
+* [Add Pod-Startup Liveness-Probe Holdoff](https://github.com/kubernetes/enhancements/issues/950)
+* [Support CRI-ContainerD On Windows](https://github.com/kubernetes/enhancements/issues/1001)
+* [SCTP Support for Services](https://github.com/kubernetes/enhancements/issues/614)
+* [Adding AppProtocol To Services And Endpoints](https://github.com/kubernetes/enhancements/issues/1507)
+
+### Notable Feature Updates
+
+* [CronJobs](https://github.com/kubernetes/enhancements/issues/19)
+
+# Release notes
+
+You can check out the full details of the 1.20 release in the [release notes](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.20.md).
+
+# Availability of release
+
+Kubernetes 1.20 is available for [download on GitHub](https://github.com/kubernetes/kubernetes/releases/tag/v1.20.0). There are some great resources out there for getting started with Kubernetes. You can check out some [interactive tutorials](https://kubernetes.io/docs/tutorials/) on the main Kubernetes site, or run a local cluster on your machine using Docker containers with [kind](https://kind.sigs.k8s.io). If you’d like to try building a cluster from scratch, check out the [Kubernetes the Hard Way](https://github.com/kelseyhightower/kubernetes-the-hard-way) tutorial by Kelsey Hightower.
+
+# Release Team
+
+This release was made possible by a very dedicated group of individuals, who came together as a team in the midst of a lot of things happening out in the world. A huge thank you to the release lead Jeremy Rickard, and to everyone else on the release team for supporting each other, and working so hard to deliver the 1.20 release for the community.
+
+# Release Logo
+
+![Kubernetes 1.20 Release Logo](/images/blog/2020-12-08-kubernetes-1.20-release-announcement/laser.png)
+
+[raddest](https://www.dictionary.com/browse/rad): *adjective*, Slang. excellent; wonderful; cool:
+
+> The Kubernetes 1.20 Release has been the raddest release yet.
+
+2020 has been a challenging year for many of us, but Kubernetes contributors have delivered a record-breaking number of enhancements in this release. That is a great accomplishment, so the release lead wanted to end the year with a little bit of levity and pay homage to [Kubernetes 1.14 - Caturnetes](https://github.com/kubernetes/sig-release/tree/master/releases/release-1.14) with a "rad" cat named Humphrey.
+
+Humphrey is the release lead's cat and has a permanent [`blep`](https://www.inverse.com/article/42316-why-do-cats-blep-science-explains). *Rad* was pretty common slang in the 1990s in the United States, and so were laser backgrounds. Humphrey in a 1990s style school picture felt like a fun way to end the year. Hopefully, Humphrey and his *blep* bring you a little joy at the end of 2020!
+
+The release logo was created by [Henry Hsu - @robotdancebattle](https://www.instagram.com/robotdancebattle/).
+
+# User Highlights
+
+- Apple is operating multi-thousand node Kubernetes clusters in data centers all over the world. Watch [Alena Prokharchyk's KubeCon NA Keynote](https://youtu.be/Tx8qXC-U3KM) to learn more about their cloud native journey.
+
+# Project Velocity
+
+The [CNCF K8s DevStats project](https://k8s.devstats.cncf.io/) aggregates a number of interesting data points related to the velocity of Kubernetes and various sub-projects. This includes everything from individual contributions to the number of companies that are contributing, and is a neat illustration of the depth and breadth of effort that goes into evolving this ecosystem.
+
+In the v1.20 release cycle, which ran for 11 weeks (September 25 to December 9), we saw contributions from [967 companies](https://k8s.devstats.cncf.io/d/9/companies-table?orgId=1&var-period_name=v1.19.0%20-%20now&var-metric=contributions) and [1335 individuals](https://k8s.devstats.cncf.io/d/66/developer-activity-counts-by-companies?orgId=1&var-period_name=v1.19.0%20-%20now&var-metric=contributions&var-repogroup_name=Kubernetes&var-country_name=All&var-companies=All) ([44 of whom](https://k8s.devstats.cncf.io/d/52/new-contributors?orgId=1&from=1601006400000&to=1607576399000&var-repogroup_name=Kubernetes) made their first Kubernetes contribution) from [26 countries](https://k8s.devstats.cncf.io/d/50/countries-stats?orgId=1&from=1601006400000&to=1607576399000&var-period_name=Quarter&var-countries=All&var-repogroup_name=Kubernetes&var-metric=rcommitters&var-cum=countries).
+
+# Ecosystem Updates
+
+- KubeCon North America just wrapped up three weeks ago, the second such event to be virtual! All talks are [now available to all on-demand](https://www.youtube.com/playlist?list=PLj6h78yzYM2Pn8RxfLh2qrXBDftr6Qjut) for anyone still needing to catch up!
+- In June, the Kubernetes community formed a new working group as a direct response to the Black Lives Matter protests occurring across America. WG Naming's goal is to remove harmful and unclear language in the Kubernetes project as completely as possible and to do so in a way that is portable to other CNCF projects. A great introductory talk on this important work and how it is conducted was given [at KubeCon 2020 North America](https://sched.co/eukp), and the initial impact of this labor [can actually be seen in the v1.20 release](https://github.com/kubernetes/enhancements/issues/2067).
+- Previously announced this summer, [The Certified Kubernetes Security Specialist (CKS) Certification](https://www.cncf.io/announcements/2020/11/17/kubernetes-security-specialist-certification-now-available/) was released during Kubecon NA for immediate scheduling!  Following the model of CKA and CKAD, the CKS is a performance-based exam, focused on security-themed competencies and domains.  This exam is targeted at current CKA holders, particularly those who want to round out their baseline knowledge in securing cloud workloads (which is all of us, right?).
+
+# Event Updates
+
+KubeCon + CloudNativeCon Europe 2021 will take place May 4 - 7, 2021! Registration will open on January 11. You can find more information about the conference [here](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/). Remember that [the CFP](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/program/cfp/) closes on Sunday, December 13, 11:59pm PST!
+
+# Upcoming release webinar
+
+Stay tuned for the upcoming release webinar happening this January.
+
+# Get Involved
+
+If you’re interested in contributing to the Kubernetes community, Special Interest Groups (SIGs) are a great starting point. Many of them may align with your interests! If there are things you’d like to share with the community, you can join the weekly community meeting, or use any of the following channels:
+
+* Find out more about contributing to Kubernetes at the new [Kubernetes Contributor website](https://www.kubernetes.dev/)
+* Follow us on Twitter [@Kubernetesio](https://twitter.com/kubernetesio) for latest updates
+* Join the community discussion on [Discuss](https://discuss.kubernetes.io/)
+* Join the community on [Slack](http://slack.k8s.io/)
+* Share your Kubernetes [story](https://docs.google.com/a/linuxfoundation.org/forms/d/e/1FAIpQLScuI7Ye3VQHQTwBASrgkjQDSS5TP0g3AXfFhwSM9YpHgxRKFA/viewform)
+* Read more about what’s happening with Kubernetes on the [blog](https://kubernetes.io/blog/)
+* Learn more about the [Kubernetes Release Team](https://github.com/kubernetes/sig-release/tree/master/release-team)
@@ -0,0 +1,224 @@
+---
+layout: blog
+title: 'Kubernetes 1.20: Kubernetes Volume Snapshot Moves to GA'
+date: 2020-12-10
+slug: kubernetes-1.20-volume-snapshot-moves-to-ga
+---
+
+**Authors**: Xing Yang, VMware & Xiangqian Yu, Google
+
+The Kubernetes Volume Snapshot feature is now GA in Kubernetes v1.20. It was introduced as [alpha](https://kubernetes.io/blog/2018/10/09/introducing-volume-snapshot-alpha-for-kubernetes/) in Kubernetes v1.12, followed by a [second alpha](https://kubernetes.io/blog/2019/01/17/update-on-volume-snapshot-alpha-for-kubernetes/) with breaking changes in Kubernetes v1.13, and promotion to [beta](https://kubernetes.io/blog/2019/12/09/kubernetes-1-17-feature-cis-volume-snapshot-beta/) in Kubernetes 1.17. This blog post summarizes the changes releasing the feature from beta to GA.
+
+## What is a volume snapshot?
+
+Many storage systems (like Google Cloud Persistent Disks, Amazon Elastic Block Storage, and many on-premise storage systems) provide the ability to create a “snapshot” of a persistent volume. A snapshot represents a point-in-time copy of a volume. A snapshot can be used either to rehydrate a new volume (pre-populated with the snapshot data) or to restore an existing volume to a previous state (represented by the snapshot).
+
+## Why add volume snapshots to Kubernetes?
+
+Kubernetes aims to create an abstraction layer between distributed applications and underlying clusters so that applications can be agnostic to the specifics of the cluster they run on and application deployment requires no “cluster-specific” knowledge.
+
+The Kubernetes Storage SIG identified snapshot operations as critical functionality for many stateful workloads. For example, a database administrator may want to snapshot a database’s volumes before starting a database operation.
+
+By providing a standard way to trigger volume snapshot operations in Kubernetes, this feature allows Kubernetes users to incorporate snapshot operations in a portable manner on any Kubernetes environment regardless of the underlying storage.
+
+Additionally, these Kubernetes snapshot primitives act as basic building blocks that unlock the ability to develop advanced enterprise-grade storage administration features for Kubernetes,  including application or cluster level backup solutions.
+
+## What’s new since beta?
+
+With the promotion of Volume Snapshot to GA, the feature is enabled by default on standard Kubernetes deployments and cannot be turned off.
+
+Many enhancements have been made to improve the quality of this feature and to make it production-grade.
+
+- The Volume Snapshot APIs and client library were moved to a separate Go module.
+
+- A snapshot validation webhook has been added to perform necessary validation on volume snapshot objects. More details can be found in the [Volume Snapshot Validation Webhook Kubernetes Enhancement Proposal](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1900-volume-snapshot-validation-webhook).
+
+- Along with the validation webhook, the volume snapshot controller will start labeling invalid snapshot objects that already existed. This allows users to identify, remove any invalid objects, and correct their workflows. Once the API is switched to the v1 type, those invalid objects will not be deletable from the system.
+
+- To provide better insights into how the snapshot feature is performing, an initial set of operation metrics has been added to the volume snapshot controller.
+
+- There are more end-to-end tests, running on GCP, that validate the feature in a real Kubernetes cluster. Stress tests (based on Google Persistent Disk and `hostPath` CSI Drivers) have been introduced to test the robustness of the system.
+
+Other than introducing tightening validation, there is no difference between the v1beta1 and v1 Kubernetes volume snapshot API. In this release (with Kubernetes 1.20), both v1 and v1beta1 are served while the stored API version is still v1beta1. Future releases will switch the stored version to v1 and gradually remove v1beta1 support.
+
+## Which CSI drivers support volume snapshots?
+
+Snapshots are only supported for CSI drivers, not for in-tree or FlexVolume drivers. Ensure the deployed CSI driver on your cluster has implemented the snapshot interfaces. For more information, see [Container Storage Interface (CSI) for Kubernetes GA](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/).
+
+Currently more than [50 CSI drivers](https://kubernetes-csi.github.io/docs/drivers.html) support the Volume Snapshot feature. The [GCE Persistent Disk CSI Driver](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) has gone through the tests for upgrading from volume snapshots beta to GA. GA level support for other CSI drivers should be available soon.
+
+## Who builds products using volume snapshots?
+
+As of the publishing of this blog, the following participants from the [Kubernetes Data Protection Working Group](https://github.com/kubernetes/community/tree/master/wg-data-protection) are building products or have already built products using Kubernetes volume snapshots.
+
+- [Dell-EMC: PowerProtect](https://www.delltechnologies.com/en-us/data-protection/powerprotect-data-manager.htm)
+- [Druva](https://www.druva.com/)
+- [Kasten K10](https://www.kasten.io/)
+- [NetApp: Project Astra](https://cloud.netapp.com/project-astra)
+- [Portworx (PX-Backup)](https://portworx.com/products/px-backup/)
+- [Pure Storage (Pure Service Orchestrator)](https://github.com/purestorage/pso-csi)
+- [Red Hat OpenShift Container Storage](https://www.redhat.com/en/technologies/cloud-computing/openshift-container-storage)
+- [Robin Cloud Native Storage](https://robin.io/storage/)
+- [TrilioVault for Kubernetes](https://docs.trilio.io/kubernetes/)
+- [Velero plugin for CSI](https://github.com/vmware-tanzu/velero-plugin-for-csi)
+
+## How to deploy volume snapshots?
+
+Volume Snapshot feature contains the following components:
+
+- [Kubernetes Volume Snapshot CRDs](https://github.com/kubernetes-csi/external-snapshotter/tree/master/client/config/crd)
+- [Volume snapshot controller](https://github.com/kubernetes-csi/external-snapshotter/tree/master/pkg/common-controller)
+- [Snapshot validation webhook](https://github.com/kubernetes-csi/external-snapshotter/tree/master/pkg/validation-webhook)
+- CSI Driver along with [CSI Snapshotter sidecar](https://github.com/kubernetes-csi/external-snapshotter/tree/master/pkg/sidecar-controller)
+
+It is strongly recommended that Kubernetes distributors bundle and deploy the volume snapshot controller, CRDs, and validation webhook as part of their Kubernetes cluster management process (independent of any CSI Driver).
+
+{{< warning >}}
+
+The snapshot validation webhook serves as a critical component to transition smoothly from using v1beta1 to v1 API. Not installing the snapshot validation webhook makes prevention of invalid volume snapshot objects from creation/updating impossible, which in turn will block deletion of invalid volume snapshot objects in coming upgrades.
+
+{{< /warning >}}
+
+If your cluster does not come pre-installed with the correct components, you may manually install them. See the [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter#readme) README for details.
+
+## How to use volume snapshots?
+
+Assuming all the required components (including CSI driver) have been already deployed and running on your cluster, you can create volume snapshots using the `VolumeSnapshot` API object, or use an existing `VolumeSnapshot` to restore a PVC by specifying the VolumeSnapshot data source on it. For more details, see the [volume snapshot documentation](/docs/concepts/storage/volume-snapshots/).
+
+{{< note >}} The Kubernetes Snapshot API does not provide any application consistency guarantees. You have to prepare your application (pause application, freeze filesystem etc.) before taking the snapshot for data consistency either manually or using higher level APIs/controllers. {{< /note >}}
+
+### Dynamically provision a volume snapshot
+
+To dynamically provision a volume snapshot, create a `VolumeSnapshotClass` API object first.
+
+```yaml
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: test-snapclass
+driver: testdriver.csi.k8s.io
+deletionPolicy: Delete
+parameters:
+  csi.storage.k8s.io/snapshotter-secret-name: mysecret
+  csi.storage.k8s.io/snapshotter-secret-namespace: mysecretnamespace
+```  
+
+Then create a `VolumeSnapshot` API object from a PVC by specifying the volume snapshot class.
+
+```yaml
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  name: test-snapshot
+  namespace: ns1
+spec:
+  volumeSnapshotClassName: test-snapclass
+  source:
+    persistentVolumeClaimName: test-pvc
+```
+
+### Importing an existing volume snapshot with Kubernetes
+
+To import a pre-existing volume snapshot into Kubernetes, manually create a `VolumeSnapshotContent` object first.
+
+```yaml
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotContent
+metadata:
+  name: test-content
+spec:
+  deletionPolicy: Delete
+  driver: testdriver.csi.k8s.io
+  source:
+    snapshotHandle: 7bdd0de3-xxx
+  volumeSnapshotRef:
+    name: test-snapshot
+    namespace: default
+```
+
+Then create a `VolumeSnapshot` object pointing to the `VolumeSnapshotContent` object.
+
+```yaml
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  name: test-snapshot
+spec:
+  source:
+        volumeSnapshotContentName: test-content
+```
+
+### Rehydrate volume from snapshot
+
+A bound and ready `VolumeSnapshot` object can be used to rehydrate a new volume with data pre-populated from snapshotted data as shown here:
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-restore
+  namespace: demo-namespace
+spec:
+  storageClassName: test-storageclass
+  dataSource:
+    name: test-snapshot
+    kind: VolumeSnapshot
+    apiGroup: snapshot.storage.k8s.io
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+```
+
+## How to add support for snapshots in a CSI driver?
+
+See the [CSI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md) and the [Kubernetes-CSI Driver Developer Guide](https://kubernetes-csi.github.io/docs/snapshot-restore-feature.html) for more details on how to implement the snapshot feature in a CSI driver.
+
+## What are the limitations?
+
+The GA implementation of volume snapshots for Kubernetes has the following limitations:
+
+- Does not support reverting an existing PVC to an earlier state represented by a snapshot (only supports provisioning a new volume from a snapshot).
+
+### How to learn more?
+
+The code repository for snapshot APIs and controller is here: https://github.com/kubernetes-csi/external-snapshotter
+
+Check out additional documentation on the snapshot feature here: http://k8s.io/docs/concepts/storage/volume-snapshots and https://kubernetes-csi.github.io/docs/
+
+## How to get involved?
+
+This project, like all of Kubernetes, is the result of hard work by many contributors from diverse backgrounds working together.
+
+We offer a huge thank you to the contributors who stepped up these last few quarters to help the project reach GA. We want to thank Saad Ali, Michelle Au, Tim Hockin, and Jordan Liggitt for their insightful reviews and thorough consideration with the design, thank Andi Li for his work on adding the support of the snapshot validation webhook, thank Grant Griffiths on implementing metrics support in the snapshot controller and handling password rotation in the validation webhook, thank Chris Henzie, Raunak Shah, and Manohar Reddy for writing critical e2e tests to meet the scalability and stability requirements for graduation, thank Kartik Sharma for moving snapshot APIs and client lib to a separate go module, and thank Raunak Shah and Prafull Ladha for their help with upgrade testing from beta to GA.
+
+There are many more people who have helped to move the snapshot feature from beta to GA. We want to thank everyone who has contributed to this effort:
+- [Andi Li](https://github.com/AndiLi99)
+- [Ben Swartzlander](https://github.com/bswartz)
+- [Chris Henzie](https://github.com/chrishenzie)
+- [Christian Huffman](https://github.com/huffmanca)
+- [Grant Griffiths](https://github.com/ggriffiths)
+- [Humble Devassy Chirammal](https://github.com/humblec)
+- [Jan Šafránek](https://github.com/jsafrane)
+- [Jiawei Wang](https://github.com/Jiawei0227)
+- [Jing Xu](https://github.com/jingxu97)
+- [Jordan Liggitt](https://github.com/liggitt)
+- [Kartik Sharma](https://github.com/Kartik494)
+- [Madhu Rajanna](https://github.com/Madhu-1)
+- [Manohar Reddy](https://github.com/boddumanohar)
+- [Michelle Au](https://github.com/msau42)
+- [Patrick Ohly](https://github.com/pohly)
+- [Prafull Ladha](https://github.com/prafull01)
+- [Prateek Pandey](https://github.com/prateekpandey14)
+- [Raunak Shah](https://github.com/RaunakShah)
+- [Saad Ali](https://github.com/saad-ali)
+- [Saikat Roychowdhury](https://github.com/saikat-royc)
+- [Tim Hockin](https://github.com/thockin)
+- [Xiangqian Yu](https://github.com/yuxiangqian)
+- [Xing Yang](https://github.com/xing-yang)
+- [Zhu Can](https://github.com/zhucan)
+
+For those interested in getting involved with the design and development of CSI or any part of the Kubernetes Storage system, join the [Kubernetes Storage Special Interest Group](https://github.com/kubernetes/community/tree/master/sig-storage)  (SIG). We’re rapidly growing and always welcome new contributors.
+
+We also hold regular [Data Protection Working Group meetings](https://docs.google.com/document/d/15tLCV3csvjHbKb16DVk-mfUmFry_Rlwo-2uG6KNGsfw/edit#). New attendees are welcome to join in discussions.
@@ -0,0 +1,53 @@
+---
+layout: blog
+title: 'Kubernetes 1.20: Pod Impersonation and Short-lived Volumes in CSI Drivers'
+date: 2020-12-18
+slug: kubernetes-1.20-pod-impersonation-short-lived-volumes-in-csi
+---
+
+**Author**: Shihang Zhang (Google)
+
+Typically when a [CSI](https://github.com/container-storage-interface/spec/blob/baa71a34651e5ee6cb983b39c03097d7aa384278/spec.md) driver mounts credentials such as secrets and certificates, it has to authenticate against storage providers to access the credentials. However, the access to those credentials are controlled on the basis of the pods' identities rather than the CSI driver's identity. CSI drivers, therefore, need some way to retrieve pod's service account token. 
+
+Currently there are two suboptimal approaches to achieve this, either by granting CSI drivers the permission to use TokenRequest API or by reading tokens directly from the host filesystem. 
+
+Both of them exhibit the following drawbacks:
+
+- Violating the principle of least privilege
+- Every CSI driver needs to re-implement the logic of getting the pod’s service account token
+
+The second approach is more problematic due to:
+
+- The audience of the token defaults to the kube-apiserver
+- The token is not guaranteed to be available (e.g. `AutomountServiceAccountToken=false`)
+- The approach does not work for CSI drivers that run as a different (non-root) user from the pods. See [file permission section for service account token](https://github.com/kubernetes/enhancements/blob/f40c24a5da09390bd521be535b38a4dbab09380c/keps/sig-storage/20180515-svcacct-token-volumes.md#file-permission)
+- The token might be legacy Kubernetes service account token which doesn’t expire if `BoundServiceAccountTokenVolume=false`
+
+Kubernetes 1.20 introduces an alpha feature, `CSIServiceAccountToken`, to improve the security posture. The new feature allows CSI drivers to receive pods' [bound service account tokens](https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/1205-bound-service-account-tokens/README.md).
+
+This feature also provides a knob to re-publish volumes so that short-lived volumes can be refreshed.
+
+## Pod Impersonation
+
+### Using GCP APIs
+
+Using [Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity), a Kubernetes service account can authenticate as a Google service account when accessing Google Cloud APIs. If a CSI driver needs to access GCP APIs on behalf of the pods that it is mounting volumes for, it can use the pod's service account token to [exchange for GCP tokens](https://cloud.google.com/iam/docs/reference/sts/rest). The pod's service account token is plumbed through the volume context in `NodePublishVolume` RPC calls when the feature `CSIServiceAccountToken` is enabled. For example: accessing [Google Secret Manager](https://cloud.google.com/secret-manager/) via a [secret store CSI driver](https://github.com/GoogleCloudPlatform/secrets-store-csi-driver-provider-gcp).
+
+### Using Vault
+
+If users configure [Kubernetes as an auth method](https://www.vaultproject.io/docs/auth/kubernetes), Vault uses the `TokenReview` API to validate the Kubernetes service account token. For CSI drivers using Vault as resources provider, they need to present the pod's service account to Vault. For example, [secrets store CSI driver](https://github.com/hashicorp/secrets-store-csi-driver-provider-vault) and [cert manager CSI driver](https://github.com/jetstack/cert-manager-csi).
+
+## Short-lived Volumes
+
+To keep short-lived volumes such as certificates effective, CSI drivers can specify `RequiresRepublish=true` in their`CSIDriver` object to have the kubelet periodically call `NodePublishVolume` on mounted volumes. These republishes allow CSI drivers to ensure that the volume content is up-to-date.
+
+## Next steps
+
+This feature is alpha and projected to move to beta in 1.21. See more in the following KEP and CSI documentation:
+
+- [KEP-1855: Service Account Token for CSI Driver](https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/1855-csi-driver-service-account-token/README.md)
+- [Token Requests](https://kubernetes-csi.github.io/docs/token-requests.html)
+
+Your feedback is always welcome!
+- SIG-Auth [meets regularly](https://github.com/kubernetes/community/tree/master/sig-auth#meetings) and can be reached via [Slack and the mailing list](https://github.com/kubernetes/community/tree/master/sig-auth#contact)
+- SIG-Storage [meets regularly](https://github.com/kubernetes/community/tree/master/sig-storage#meetings) and can be reached via [Slack and the mailing list](https://github.com/kubernetes/community/tree/master/sig-storage#contact).
@@ -0,0 +1,59 @@
+--- 
+layout: blog 
+title: 'Kubernetes 1.20: Granular Control of Volume Permission Changes'
+date: 2020-12-14 
+slug: kubernetes-release-1.20-fsGroupChangePolicy-fsGroupPolicy
+---
+
+**Authors**: Hemant Kumar, Red Hat & Christian Huffman, Red Hat
+
+Kubernetes 1.20 brings two important beta features, allowing Kubernetes admins and users alike to have more adequate control over how volume permissions are applied when a volume is mounted inside a Pod.
+
+### Allow users to skip recursive permission changes on mount
+Traditionally if your pod is running as a non-root user ([which you should](https://twitter.com/thockin/status/1333892204490735617)), you must specify a `fsGroup` inside the pod’s security context so that the volume can be readable and writable by the Pod. This requirement is covered in more detail in [here](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/).
+
+But one side-effect of setting `fsGroup` is that, each time a volume is mounted, Kubernetes must recursively `chown()` and `chmod()` all the files and directories inside the volume - with a few exceptions noted below. This happens even if group ownership of the volume already matches the requested `fsGroup`, and can be pretty expensive for larger volumes with lots of small files, which causes pod startup to take a long time. This scenario has been a [known problem](https://github.com/kubernetes/kubernetes/issues/69699) for a while, and in Kubernetes 1.20 we are providing knobs to opt-out of recursive permission changes if the volume already has the correct permissions.
+
+When configuring a pod’s security context, set `fsGroupChangePolicy` to "OnRootMismatch" so if the root of the volume already has the correct permissions, the recursive permission change can be skipped. Kubernetes ensures that permissions of the top-level directory are changed last the first time it applies permissions.
+
+```yaml
+securityContext:
+  runAsUser: 1000
+  runAsGroup: 3000
+  fsGroup: 2000
+  fsGroupChangePolicy: "OnRootMismatch"
+```
+You can learn more about this in [Configure volume permission and ownership change policy for Pods](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#configure-volume-permission-and-ownership-change-policy-for-pods).
+
+### Allow CSI Drivers to declare support for fsGroup based permissions
+
+Although the previous section implied that Kubernetes _always_ recursively changes permissions of a volume if a Pod has a `fsGroup`, this is not strictly true. For certain multi-writer volume types, such as NFS or Gluster, the cluster doesn’t perform recursive permission changes even if the pod has a `fsGroup`. Other volume types may not even support `chown()`/`chmod()`, which rely on Unix-style permission control primitives. 
+
+So how do we know when to apply recursive permission changes and when we shouldn't? For in-tree storage drivers, this was relatively simple. For [CSI](https://kubernetes-csi.github.io/docs/introduction.html#introduction) drivers that could span a multitude of platforms and storage types, this problem can be a bigger challenge.
+
+Previously, whenever a CSI volume was mounted to a Pod, Kubernetes would attempt to automatically determine if the permissions and ownership should be modified. These methods were imprecise and could cause issues as we already mentioned, depending on the storage type.
+
+The CSIDriver custom resource now has a `.spec.fsGroupPolicy` field, allowing storage drivers to explicitly opt in or out of these recursive modifications. By having the CSI driver specify a policy for the backing volumes, Kubernetes can avoid needless modification attempts. This optimization helps to reduce volume mount time and also cuts own reporting errors about modifications that would never succeed.
+
+#### CSIDriver FSGroupPolicy API
+
+Three FSGroupPolicy values are available as of Kubernetes 1.20, with more planned for future releases.
+
+- **ReadWriteOnceWithFSType** - This is the default policy, applied if no `fsGroupPolicy` is defined; this preserves the behavior from previous Kubernetes releases. Each volume is examined at mount time to determine if permissions should be recursively applied.
+- **File** - Always attempt to apply permission modifications, regardless of the filesystem type or PersistentVolumeClaim’s access mode.
+- **None** - Never apply permission modifications.
+
+#### How do I use it?
+The only configuration needed is defining `fsGroupPolicy` inside of the `.spec` for a CSIDriver. Once that element is defined, any subsequently mounted volumes will automatically use the defined policy. There’s no additional deployment required!
+
+#### What’s next?
+
+Depending on feedback and adoption, the Kubernetes team plans to push these implementations to GA in either 1.21 or 1.22. 
+
+### How can I learn more?
+This feature is explained in more detail in Kubernetes project documentation: [CSI Driver fsGroup Support](https://kubernetes-csi.github.io/docs/support-fsgroup.html) and [Configure volume permission and ownership change policy for Pods ](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#configure-volume-permission-and-ownership-change-policy-for-pods).
+
+### How do I get involved?
+The [Kubernetes Slack channel #csi](https://kubernetes.slack.com/messages/csi) and any of the [standard SIG Storage communication channels](https://github.com/kubernetes/community/blob/master/sig-storage/README.md#contact) are great mediums to reach out to the SIG Storage and the CSI team.
+
+Those interested in getting involved with the design and development of CSI or any part of the Kubernetes Storage system, join the [Kubernetes Storage Special Interest Group (SIG)](https://github.com/kubernetes/community/tree/master/sig-storage). We’re rapidly growing and always welcome new contributors.
@@ -0,0 +1,134 @@
+---
+layout: blog
+title: 'Third Party Device Metrics Reaches GA'
+date: 2020-12-16
+slug: third-party-device-metrics-reaches-ga
+---
+
+ **Authors:** Renaud Gaubert (NVIDIA), David Ashpole (Google), and Pramod Ramarao (NVIDIA)
+
+ With Kubernetes 1.20, infrastructure teams who manage large scale Kubernetes clusters, are seeing the graduation of two exciting and long awaited features:
+ * The Pod Resources API (introduced in 1.13) is finally graduating to GA. This allows Kubernetes plugins to obtain information about the node’s resource usage and assignment; for example: which pod/container consumes which device.
+ * The `DisableAcceleratorMetrics` feature (introduced in 1.19) is graduating to beta and will be enabled by default. This removes device metrics reported by the kubelet in favor of the new plugin architecture.
+
+Many of the features related to fundamental device support (device discovery, plugin, and monitoring) are reaching a strong level of stability.
+Kubernetes users should see these features as stepping stones to enable more complex use cases (networking, scheduling, storage, etc.)!
+
+One such example is Non Uniform Memory Access (NUMA) placement where, when selecting a device, an application typically wants to ensure that data transfer between CPU Memory and Device Memory is as fast as possible. In some cases, incorrect NUMA placement can nullify the benefit of offloading compute to an external device.
+
+If these are topics of interest to you, consider joining the [Kubernetes Node Special Insterest Group](https://github.com/kubernetes/community/tree/master/sig-node) (SIG) for all topics related to the Kubernetes node, the COD (container orchestrated device) workgroup for topics related to runtimes, or the resource management forum for topics related to resource management!
+
+## The Pod Resources API - Why does it need to exist?
+
+Kubernetes is a vendor neutral platform. If we want it to support device monitoring, adding vendor-specific code in the Kubernetes code base is not an ideal solution. Ultimately, devices are a domain where deep expertise is needed and the best people to add and maintain code in that area are the device vendors themselves.
+
+The Pod Resources API was built as a solution to this issue. Each vendor can build and maintain their own out-of-tree monitoring plugin. This monitoring plugin, often deployed as a separate pod within a cluster, can then associate the metrics a device emits with the associated pod that's using it. 
+
+For example, use the NVIDIA GPU dcgm-exporter to scrape metrics in Prometheus format:
+
+```
+$ curl -sL http://127.0.01:8080/metrics
+
+
+# HELP DCGM_FI_DEV_SM_CLOCK SM clock frequency (in MHz).
+# TYPE DCGM_FI_DEV_SM_CLOCK gauge
+# HELP DCGM_FI_DEV_MEM_CLOCK Memory clock frequency (in MHz).
+# TYPE DCGM_FI_DEV_MEM_CLOCK gauge
+# HELP DCGM_FI_DEV_MEMORY_TEMP Memory temperature (in C).
+# TYPE DCGM_FI_DEV_MEMORY_TEMP gauge
+...
+DCGM_FI_DEV_SM_CLOCK{gpu="0", UUID="GPU-604ac76c-d9cf-fef3-62e9-d92044ab6e52",container="foo",namespace="bar",pod="baz"} 139
+DCGM_FI_DEV_MEM_CLOCK{gpu="0", UUID="GPU-604ac76c-d9cf-fef3-62e9-d92044ab6e52",container="foo",namespace="bar",pod="baz"} 405
+DCGM_FI_DEV_MEMORY_TEMP{gpu="0", UUID="GPU-604ac76c-d9cf-fef3-62e9-d92044ab6e52",container="foo",namespace="bar",pod="baz"} 9223372036854775794
+```
+
+Each agent is expected to adhere to the node monitoring guidelines. In other words, plugins are expected to generate metrics in Prometheus format, and new metrics should not have any dependency on the Kubernetes base directly.
+
+This allows consumers of the metrics to use a compatible monitoring pipeline to collect and analyze metrics from a variety of agents, even if they are maintained by different vendors.
+
+![Device metrics flowchart](/images/blog/2020-12-16-third-party-device-metrics-hits-ga/metrics-chart.png)
+
+## Disabling the NVIDIA GPU metrics - Warning {#nvidia-gpu-metrics-deprecated}
+
+With the graduation of the plugin monitoring system, Kubernetes is deprecating the NVIDIA GPU metrics that are being reported by the kubelet.
+
+With the [DisableAcceleratorMetrics](/docs/concepts/cluster-administration/system-metrics/#disable-accelerator-metrics) feature being enabled by default in Kubernetes 1.20, NVIDIA GPUs are no longer special citizens  in Kubernetes. This is a good thing in the spirit of being vendor-neutral, and enables the most suited people to maintain their plugin on their own release schedule!
+
+Users will now need to either install the [NVIDIA GDGM exporter](https://github.com/NVIDIA/gpu-monitoring-tools) or use [bindings](https://github.com/nvidia/go-nvml) to gather more accurate and complete metrics about NVIDIA GPUs. This deprecation means that you can no longer rely on metrics that were reported by kubelet, such as `container_accelerator_duty_cycle` or `container_accelerator_memory_used_bytes` which were used to gather NVIDIA GPU memory utilization.
+
+This means that users who used to rely on the NVIDIA GPU metrics reported by the kubelet, will need to update their reference and deploy the NVIDIA plugin. Namely the different metrics reported by Kubernetes map to the following metrics:
+
+| Kubernetes Metrics                         | NVIDIA dcgm-exporter metric                 |
+| ------------------------------------------ | ------------------------------------------- |
+| `container_accelerator_duty_cycle`         | `DCGM_FI_DEV_GPU_UTIL`                      |
+| `container_accelerator_memory_used_bytes`  | `DCGM_FI_DEV_FB_USED`                       |
+| `container_accelerator_memory_total_bytes` | `DCGM_FI_DEV_FB_FREE + DCGM_FI_DEV_FB_USED` |
+
+You might also be interested in other metrics such as `DCGM_FI_DEV_GPU_TEMP` (the GPU temperature) or DCGM_FI_DEV_POWER_USAGE (the power usage). The [default set](https://github.com/NVIDIA/gpu-monitoring-tools/blob/d5c9bb55b4d1529ca07068b7f81e690921ce2b59/etc/dcgm-exporter/default-counters.csv) is available in Nvidia's [Data Center GPU Manager documentation](https://docs.nvidia.com/datacenter/dcgm/latest/dcgm-api/group__dcgmFieldIdentifiers.html).
+
+Note that for this release you can still set the `DisableAcceleratorMetrics` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) to _false_, effectively re-enabling the ability for the kubelet to report NVIDIA GPU metrics.
+
+Paired with the graduation of the Pod Resources API, these tools can be used to generate GPU telemetry [that can be used in visualization dashboards](https://grafana.com/grafana/dashboards/12239), below is an example: 
+
+![Grafana visualization of device metrics](/images/blog/2020-12-16-third-party-device-metrics-hits-ga/grafana.png)
+
+## The Pod Resources API - What can I go on to do with this?
+
+As soon as this interface was introduced, many vendors started using it for widely different use cases! To list a few examples:
+
+The [kuryr-kubernetes](https://github.com/openstack/kuryr-kubernetes) CNI plugin in tandem with [intel-sriov-device-plugin](https://github.com/intel/sriov-network-device-plugin). This allowed the CNI plugin to know which allocation of SR-IOV Virtual Functions (VFs) the kubelet made and use that information to correctly setup the container network namespace and use a device with the appropriate NUMA node. We also expect this interface to be used to track the allocated and available resources with information about the NUMA topology of the worker node.
+
+Another use-case is GPU telemetry, where GPU metrics can be associated with the containers and pods that the GPU is assigned to. One such example is the NVIDIA `dcgm-exporter`, but others can be easily built in the same paradigm.
+
+The Pod Resources API is a simple gRPC service which informs clients of the pods the kubelet knows. The information concerns the devices assignment the kubelet made and the assignment of CPUs. This information is obtained from the internal state of the kubelet's Device Manager and CPU Manager respectively.
+
+You can see below a sample example of the API and how a go client could use that information in a few lines:
+
+```
+service PodResourcesLister {
+    rpc List(ListPodResourcesRequest) returns (ListPodResourcesResponse) {}
+    rpc GetAllocatableResources(AllocatableResourcesRequest) returns (AllocatableResourcesResponse) {}
+
+    // Kubernetes 1.21
+    rpc Watch(WatchPodResourcesRequest) returns (stream WatchPodResourcesResponse) {}
+}
+```
+
+```go
+func main() {
+	ctx, cancel := context.WithTimeout(context.Background(), connectionTimeout)
+	defer cancel()
+
+	socket := "/var/lib/kubelet/pod-resources/kubelet.sock"
+	conn, err := grpc.DialContext(ctx, socket, grpc.WithInsecure(), grpc.WithBlock(),
+		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
+			return net.DialTimeout("unix", addr, timeout)
+		}),
+	)
+
+	if err != nil {
+		panic(err)
+	}
+
+    client := podresourcesapi.NewPodResourcesListerClient(conn)
+    resp, err := client.List(ctx, &podresourcesapi.ListPodResourcesRequest{})
+	if err != nil {
+		panic(err)
+	}
+	net.Printf("%+v\n", resp)
+}
+```
+
+Finally, note that you can watch the number of requests made to the Pod Resources endpoint by watching the new kubelet metric called `pod_resources_endpoint_requests_total` on the kubelet's `/metrics` endpoint.
+
+## Is device monitoring suitable for production? Can I extend it? Can I contribute?
+
+Yes! This feature released in 1.13, almost 2 years ago, has seen broad adoption, is already used by different cloud managed services, and with its graduation to G.A in Kubernetes 1.20 is production ready!
+
+If you are a device vendor, you can start using it today! If you just want to monitor the devices in your cluster, go get the latest version of your monitoring plugin!
+
+If you feel passionate about that area, join the kubernetes community, help improve the API or contribute the device monitoring plugins!
+
+## Acknowledgements
+
+We thank the members of the community who have contributed to this feature or given feedback including members of WG-Resource-Management, SIG-Node and the Resource management forum!
@@ -0,0 +1,63 @@
+---
+layout: blog
+title: "The Evolution of Kubernetes Dashboard"
+date: 2021-03-09
+slug: the-evolution-of-kubernetes-dashboard
+---
+
+Authors: Marcin Maciaszczyk, Kubermatic & Sebastian Florek, Kubermatic
+
+In October 2020, the Kubernetes Dashboard officially turned five. As main project maintainers, we can barely believe that so much time has passed since our very first commits to the project. However, looking back with a bit of nostalgia, we realize that quite a lot has happened since then. Now it’s due time to celebrate “our baby” with a short recap.
+
+## How It All Began
+
+The initial idea behind the Kubernetes Dashboard project was to provide a web interface for Kubernetes. We wanted to reflect the kubectl functionality through an intuitive web UI. The main benefit from using the UI is to be able to quickly see things that do not work as expected (monitoring and troubleshooting). Also, the Kubernetes Dashboard is a great starting point for users that are new to the Kubernetes ecosystem.
+
+The very [first commit](https://github.com/kubernetes/dashboard/commit/5861187fa807ac1cc2d9b2ac786afeced065076c) to the Kubernetes Dashboard was made by Filip Grządkowski from Google on 16th October 2015 – just a few months from the initial commit to the Kubernetes repository. Our initial commits go back to November 2015 ([Sebastian committed on 16 November 2015](https://github.com/kubernetes/dashboard/commit/09e65b6bb08c49b926253de3621a73da05e400fd); [Marcin committed on 23 November 2015](https://github.com/kubernetes/dashboard/commit/1da4b1c25ef040818072c734f71333f9b4733f55)). Since that time, we’ve become regular contributors to the project. For the next two years, we worked closely with the Googlers, eventually becoming main project maintainers ourselves.
+
+{{< figure src="first-ui.png" caption="The First Version of the User Interface" >}}
+
+{{< figure src="along-the-way-ui.png" caption="Prototype of the New User Interface" >}}
+
+{{< figure src="current-ui.png" caption="The Current User Interface" >}}
+
+As you can see, the initial look and feel of the project were completely different from the current one. We have changed the design multiple times. The same has happened with the code itself.
+
+## Growing Up - The Big Migration
+
+At [the beginning of 2018](https://github.com/kubernetes/dashboard/pull/2727), we reached a point where AngularJS was getting closer to the end of its life, while the new Angular versions were published quite often. A lot of the libraries and the modules that we were using were following the trend. That forced us to spend a lot of the time rewriting the frontend part of the project to make it work with newer technologies.
+
+The migration came with many benefits like being able to refactor a lot of the code, introduce design patterns, reduce code complexity, and benefit from the new modules. However, you can imagine that the scale of the migration was huge. Luckily, there were a number of contributions from the community helping us with the resource support, new Kubernetes version support, i18n, and much more. After many long days and nights, we finally released the [first beta version](https://github.com/kubernetes/dashboard/releases/tag/v2.0.0-beta1) in July 2019, followed by the [2.0 release](https://github.com/kubernetes/dashboard/releases/tag/v2.0.0) in April 2020 — our baby had grown up.
+
+## Where Are We Standing in 2021?
+
+Due to limited resources, unfortunately, we were not able to offer extensive support for many different Kubernetes versions. So, we’ve decided to always try and support the latest Kubernetes version available at the time of the Kubernetes Dashboard release. The latest release, [Dashboard v2.2.0](https://github.com/kubernetes/dashboard/releases/tag/v2.2.0) provides support for Kubernetes v1.20.
+
+On top of that, we put in a great deal of effort into [improving resource support](https://github.com/kubernetes/dashboard/issues/5232). Meanwhile, we do offer support for most of the Kubernetes resources. Also, the Kubernetes Dashboard supports multiple languages: English, German, French, Japanese, Korean, Chinese (Traditional, Simplified, Traditional Hong Kong). Persian and Russian localizations are currently in progress. Moreover, we are working on the support for 3rd party themes and the design of the app in general. As you can see, quite a lot of things are going on.
+
+Luckily, we do have regular contributors with domain knowledge who are taking care of the project, updating the Helm charts, translations, Go modules, and more. But as always, there could be many more hands on deck. So if you are thinking about contributing to Kubernetes, keep us in mind ;)
+
+## What’s Next
+
+The Kubernetes Dashboard has been growing and prospering for more than 5 years now. It provides the community with an intuitive Web UI, thereby decreasing the complexity of Kubernetes and increasing its accessibility to new community members. We are proud of what the project has achieved so far, but this is by far not the end. These are our priorities for the future:
+
+*   Keep providing support for the new Kubernetes versions
+*   Keep improving the support for the existing resources
+*   Keep working on auth system improvements
+*   [Rewrite the API to use gRPC and shared informers](https://github.com/kubernetes/dashboard/pull/5449): This will allow us to improve the performance of the application but, most importantly, to support live updates coming from the Kubernetes project. It is one of the most requested features from the community.
+*   Split the application into two containers, one with the UI and the second with the API running inside.
+
+## The Kubernetes Dashboard in Numbers
+
+* Initial commit made on October 16, 2015
+* Over 100 million pulls from Dockerhub since the v2 release
+* 8 supported languages and the next 2 in progress
+* Over 3360 closed PRs
+* Over 2260 closed issues
+* 100% coverage of the supported core Kubernetes resources
+* Over 9000 stars on GitHub
+* Over 237 000 lines of code
+
+## Join Us
+
+As mentioned earlier, we are currently looking for more people to help us further develop and grow the project. We are open to contributions in multiple areas, i.e., [issues with help wanted label](https://github.com/kubernetes/dashboard/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22). Please feel free to reach out via GitHub or the #sig-ui channel in the [Kubernetes Slack](https://slack.k8s.io/).
@@ -0,0 +1,74 @@
+---
+layout: blog
+title: "PodSecurityPolicy Deprecation: Past, Present, and Future"
+date: 2021-04-06
+slug: podsecuritypolicy-deprecation-past-present-and-future
+---
+
+**Author:** Tabitha Sable (Kubernetes SIG Security)
+
+PodSecurityPolicy (PSP) is being deprecated in Kubernetes 1.21, to be released later this week. This starts the countdown to its removal, but doesn’t change anything else. PodSecurityPolicy will continue to be fully functional for several more releases before being removed completely. In the meantime, we are developing a replacement for PSP that covers key use cases more easily and sustainably.
+
+What are Pod Security Policies? Why did we need them? Why are they going away, and what’s next? How does this affect you? These key questions come to mind as we prepare to say goodbye to PSP, so let’s walk through them together. We’ll start with an overview of how features get removed from Kubernetes.
+
+## What does deprecation mean in Kubernetes?
+
+Whenever a Kubernetes feature is set to go away, our [deprecation policy](/docs/reference/using-api/deprecation-policy/) is our guide. First the feature is marked as deprecated, then after enough time has passed, it can finally be removed.
+
+Kubernetes 1.21 starts the deprecation process for PodSecurityPolicy. As with all feature deprecations, PodSecurityPolicy will continue to be fully functional for several more releases. The current plan is to remove PSP from Kubernetes in the 1.25 release.
+
+Until then, PSP is still PSP. There will be at least a year during which the newest Kubernetes releases will still support PSP, and nearly two years until PSP will pass fully out of all supported Kubernetes versions.
+
+## What is PodSecurityPolicy?
+
+[PodSecurityPolicy](/docs/concepts/policy/pod-security-policy/) is a built-in [admission controller](/blog/2019/03/21/a-guide-to-kubernetes-admission-controllers/) that allows a cluster administrator to control security-sensitive aspects of the Pod specification. 
+
+First, one or more PodSecurityPolicy resources are created in a cluster to define the requirements Pods must meet. Then, RBAC rules are created to control which PodSecurityPolicy applies to a given pod. If a pod meets the requirements of its PSP, it will be admitted to the cluster as usual. In some cases, PSP can also modify Pod fields, effectively creating new defaults for those fields. If a Pod does not meet the PSP requirements, it is rejected, and cannot run. 
+
+One more important thing to know about PodSecurityPolicy: it’s not the same as [PodSecurityContext](/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context).
+
+A part of the Pod specification, PodSecurityContext (and its per-container counterpart `SecurityContext`) is the collection of fields that specify many of the security-relevant settings for a Pod. The security context dictates to the kubelet and container runtime how the Pod should actually be run. In contrast, the PodSecurityPolicy only constrains (or defaults) the values that may be set on the security context.
+
+The deprecation of PSP does not affect PodSecurityContext in any way.
+
+## Why did we need PodSecurityPolicy?
+
+In Kubernetes, we define resources such as Deployments, StatefulSets, and Services that represent the building blocks of software applications. The various controllers inside a Kubernetes cluster react to these resources, creating further Kubernetes resources or configuring some software or hardware to accomplish our goals.
+
+In most Kubernetes clusters, RBAC (Role-Based Access Control) [rules](/docs/reference/access-authn-authz/rbac/#role-and-clusterrole) control access to these resources. `list`, `get`, `create`, `edit`, and `delete` are the sorts of API operations that RBAC cares about, but _RBAC does not consider what settings are being put into the resources it controls_. For example, a Pod can be almost anything from a simple webserver to a privileged command prompt offering full access to the underlying server node and all the data. It’s all the same to RBAC: a Pod is a Pod is a Pod.
+
+To control what sorts of settings are allowed in the resources defined in your cluster, you need Admission Control in addition to RBAC. Since Kubernetes 1.3, PodSecurityPolicy has been the built-in way to do that for security-related Pod fields. Using PodSecurityPolicy, you can prevent “create Pod” from automatically meaning “root on every cluster node,” without needing to deploy additional external admission controllers.
+
+## Why is PodSecurityPolicy going away?
+
+In the years since PodSecurityPolicy was first introduced, we have realized that PSP has some serious usability problems that can’t be addressed without making breaking changes.
+
+The way PSPs are applied to Pods has proven confusing to nearly everyone that has attempted to use them. It is easy to accidentally grant broader permissions than intended, and difficult to inspect which PSP(s) apply in a given situation. The “changing Pod defaults” feature can be handy, but is only supported for certain Pod settings and it’s not obvious when they will or will not apply to your Pod. Without a “dry run” or audit mode, it’s impractical to retrofit PSP to existing clusters safely, and it’s impossible for PSP to ever be enabled by default.
+
+For more information about these and other PSP difficulties, check out SIG Auth’s KubeCon NA 2019 Maintainer Track session video: {{< youtube "SFtHRmPuhEw?start=953" youtube-quote-sm >}}
+
+Today, you’re not limited only to deploying PSP or writing your own custom admission controller. Several external admission controllers are available that incorporate lessons learned from PSP to provide a better user experience. [K-Rail](https://github.com/cruise-automation/k-rail), [Kyverno](https://github.com/kyverno/kyverno/), and [OPA/Gatekeeper](https://github.com/open-policy-agent/gatekeeper/) are all well-known, and each has its fans.
+
+Although there are other good options available now, we believe there is still value in having a built-in admission controller available as a choice for users. With this in mind, we turn toward building what’s next, inspired by the lessons learned from PSP.
+
+## What’s next?
+
+Kubernetes SIG Security, SIG Auth, and a diverse collection of other community members have been working together for months to ensure that what’s coming next is going to be awesome. We have developed a Kubernetes Enhancement Proposal ([KEP 2579](https://github.com/kubernetes/enhancements/issues/2579)) and a prototype for a new feature, currently being called by the temporary name "PSP Replacement Policy." We are targeting an Alpha release in Kubernetes 1.22.
+
+PSP Replacement Policy starts with the realization that since there is a robust ecosystem of external admission controllers already available, PSP’s replacement doesn’t need to be all things to all people. Simplicity of deployment and adoption is the key advantage a built-in admission controller has compared to an external webhook, so we have focused on how to best utilize that advantage.
+
+PSP Replacement Policy is designed to be as simple as practically possible while providing enough flexibility to really be useful in production at scale. It has soft rollout features to enable retrofitting it to existing clusters, and is configurable enough that it can eventually be active by default. It can be deactivated partially or entirely, to coexist with external admission controllers for advanced use cases.
+
+## What does this mean for you?
+
+What this all means for you depends on your current PSP situation. If you’re already using PSP, there’s plenty of time to plan your next move. Please review the PSP Replacement Policy KEP and think about how well it will suit your use case.
+
+If you’re making extensive use of the flexibility of PSP with numerous PSPs and complex binding rules, you will likely find the simplicity of PSP Replacement Policy too limiting. Use the next year to evaluate the other admission controller choices in the ecosystem. There are resources available to ease this transition, such as the [Gatekeeper Policy Library](https://github.com/open-policy-agent/gatekeeper-library).
+
+If your use of PSP is relatively simple, with a few policies and straightforward binding to service accounts in each namespace, you will likely find PSP Replacement Policy to be a good match for your needs. Evaluate your PSPs compared to the Kubernetes [Pod Security Standards](/docs/concepts/security/pod-security-standards/) to get a feel for where you’ll be able to use the Restricted, Baseline, and Privileged policies. Please follow along with or contribute to the KEP and subsequent development, and try out the Alpha release of PSP Replacement Policy when it becomes available.
+
+If you’re just beginning your PSP journey, you will save time and effort by keeping it simple. You can approximate the functionality of PSP Replacement Policy today by using the Pod Security Standards’ PSPs. If you set the cluster default by binding a Baseline or Restricted policy to the `system:serviceaccounts` group, and then make a more-permissive policy available as needed in certain Namespaces [using ServiceAccount bindings](/docs/concepts/policy/pod-security-policy/#run-another-pod), you will avoid many of the PSP pitfalls and have an easy migration to PSP Replacement Policy. If your needs are much more complex than this, your effort is probably better spent adopting one of the more fully-featured external admission controllers mentioned above.
+
+We’re dedicated to making Kubernetes the best container orchestration tool we can, and sometimes that means we need to remove longstanding features to make space for better things to come. When that happens, the Kubernetes deprecation policy ensures you have plenty of time to plan your next move. In the case of PodSecurityPolicy, several options are available to suit a range of needs and use cases. Start planning ahead now for PSP’s eventual removal, and please consider contributing to its replacement! Happy securing!
+
+**Acknowledgment:** It takes a wonderful group to make wonderful software. Thanks are due to everyone who has contributed to the PSP replacement effort, especially (in alphabetical order) Tim Allclair, Ian Coldwater, and Jordan Liggitt. It’s been a joy to work with y’all on this.
@@ -0,0 +1,105 @@
+---
+layout: blog
+title: 'Kubernetes 1.21: CronJob Reaches GA'
+date: 2021-04-09
+slug: kubernetes-release-1.21-cronjob-ga
+---
+
+ **Authors:** Alay Patel (Red Hat), and Maciej Szulik (Red Hat)
+
+In Kubernetes v1.21, the 
+[CronJob](/docs/concepts/workloads/controllers/cron-jobs/) resource
+reached general availability (GA). We've also substantially improved the 
+performance of CronJobs since Kubernetes v1.19, by implementing a new
+controller.
+
+In Kubernetes v1.20 we launched a revised v2 controller for CronJobs, 
+initially as an alpha feature. Kubernetes 1.21 uses the newer controller by
+default, and the CronJob resource itself is now GA (group version: `batch/v1`).
+
+In this article, we'll take you through the driving forces behind this new
+development, give you a brief description of controller design for core 
+Kubernetes, and we'll outline what you will gain from this improved controller.
+
+The driving force behind promoting the API was Kubernetes' policy choice to
+[ensure APIs move beyond beta](/blog/2020/08/21/moving-forward-from-beta/).
+That policy aims to prevent APIs from being stuck in a “permanent beta” state.
+Over the years the old CronJob controller implementation had received healthy
+feedback from the community, with reports of several widely recognized
+[issues](https://github.com/kubernetes/kubernetes/issues/82659).
+
+If the beta API for CronJob was to be supported as GA, the existing controller
+code would need substantial rework. Instead, the SIG Apps community decided
+to introduce a new controller and gradually replace the old one.
+
+## How do controllers work?
+
+Kubernetes [controllers](/docs/concepts/architecture/controller/) are control
+loops that watch the state of resource(s) in your cluster, then make or
+request changes where needed. Each controller tries to move part of the
+current cluster state closer to the desired state.
+
+The v1 CronJob controller works by performing a periodic poll and sweep of all
+the CronJob objects in your cluster, in order to act on them. It is a single
+worker implementation that gets all CronJobs  every 10 seconds, iterates over
+each one of them, and syncs them to their desired state. This was the default 
+way of doing things almost 5 years ago when the controller was initially
+written. In hindsight, we can certainly say that such an approach can
+overload the API server at scale.
+
+These days, every core controller in kubernetes must follow the guidelines 
+described in [Writing Controllers](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/controllers.md#readme).
+Among many details, that document prescribes using
+[shared informers](https://www.cncf.io/blog/2019/10/15/extend-kubernetes-via-a-shared-informer/)
+to “receive notifications of adds, updates, and deletes for a particular
+resource”. Upon any such events, the related object(s) is placed in a queue. 
+Workers pull items from the queue and process them one at a time. This
+approach ensures consistency and scalability.
+ 
+The picture below shows the flow of information from kubernetes API server, 
+through shared informers and queue, to the main part of a controller - a 
+reconciliation loop which is responsible for performing the core functionality.
+
+![Controller flowchart](controller-flowchart.svg)
+
+The CronJob controller V2 uses a queue that implements the DelayingInterface to 
+handle the scheduling aspect. This queue allows processing an element after a 
+specific time interval. Every time there is a change in a CronJob or its related
+Jobs, the key that represents the CronJob is pushed to the queue. The main 
+handler pops the key, processes the CronJob, and after completion 
+pushes the key back into the queue for the next scheduled time interval. This is
+immediately a more performant implementation, as it no longer requires a linear 
+scan of all the CronJobs. On top of that, this controller can be scaled by 
+increasing the number of workers processing the CronJobs in parallel.
+
+## Performance impact of the new controller {#performance-impact}
+
+In order to test the performance difference of the two controllers a VM instance
+with 128 GiB RAM and 64 vCPUs was used to set up a single node Kubernetes cluster.
+Initially, a sample workload was created with 20 CronJob instances with a schedule
+to run every minute, and 2100 CronJobs running every 20 hours. Additionally, 
+over the next few minutes we added 1000 CronJobs with a schedule to run every 
+20 hours, until we reached a total of 5120 CronJobs. 
+
+![Visualization of performance](performance-impact-graph.svg)
+
+We observed that for every 1000 CronJobs added, the old controller used 
+around 90 to 120 seconds more wall-clock time to schedule 20 Jobs every cycle.
+That is, at 5120 CronJobs, the old controller took approximately 9 minutes
+to create 20 Jobs. Hence, during each cycle, about 8 schedules were missed.
+The new controller, implemented with architectural change explained above, 
+created 20 Jobs without any delay, even when we created an additional batch
+of 1000 CronJobs reaching a total of 6120. 
+
+As a closing remark, the new controller exposes a histogram metric 
+`cronjob_controller_cronjob_job_creation_skew_duration_seconds` which helps 
+monitor the time difference between when a CronJob is meant to run and when 
+the actual Job is created.
+
+Hopefully the above description is a sufficient argument to follow the 
+guidelines and standards set in the Kubernetes project, even for your own 
+controllers. As mentioned before, the new controller is on by default starting
+from Kubernetes v1.21; if you want to check it out in the previous release (1.20),
+you can enable the `CronJobControllerV2`
+[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) 
+for the kube-controller-manager: `--feature-gate="CronJobControllerV2=true"`.
@@ -0,0 +1,128 @@
+---
+layout: blog
+title: 'Kubernetes 1.21: Power to the Community'
+date: 2021-04-08
+slug: kubernetes-1-21-release-announcement
+---
+
+**Authors:** [Kubernetes 1.21 Release Team](https://github.com/kubernetes/sig-release/blob/master/releases/release-1.21/release-team.md)
+
+We’re pleased to announce the release of Kubernetes 1.21, our first release of 2021! This release consists of 51 enhancements: 13 enhancements have graduated to stable, 16 enhancements are moving to beta, 20 enhancements are entering alpha, and 2 features have been deprecated.
+
+This release cycle, we saw a major shift in ownership of processes around the release team. We moved from a synchronous mode of communication, where we periodically asked the community for inputs, to a mode where the community opts-in to contribute features and/or blogs to the release. These changes have resulted in an increase in collaboration and teamwork across the community. The result of all that is reflected in Kubernetes 1.21 having the most number of features in the recent times.
+
+## Major Themes
+
+### CronJobs Graduate to Stable!
+[CronJobs](/docs/concepts/workloads/controllers/cron-jobs/) (previously ScheduledJobs) has been a beta feature since Kubernetes 1.8! With 1.21 we get to finally see this widely used API graduate to stable.
+
+CronJobs are meant for performing regular scheduled actions such as backups, report generation, and so on. Each of those tasks should be configured to recur indefinitely (for example: once a day / week / month); you can define the point in time within that interval when the job should start.
+
+### Immutable Secrets and ConfigMaps
+[Immutable Secrets](/docs/concepts/configuration/secret/#secret-immutable) and [ConfigMaps](/docs/concepts/configuration/configmap/#configmap-immutable) add a new field to those resource types that will reject changes to those objects if set. Secrets and ConfigMaps by default are mutable which is beneficial for pods that are able to consume changes. Mutating Secrets and ConfigMaps can also cause problems if a bad configuration is pushed for pods that use them.
+
+By marking Secrets and ConfigMaps as immutable you can be sure your application configuration won't change. If you want to make changes you'll need to create a new, uniquly named Secret or ConfigMap and deploy a new pod to consume that resource. Immutable resources also have scaling benefits because controllers do not need to poll the API server to watch for changes.
+
+This feature has graduated to stable in Kubernetes 1.21.
+
+### IPv4/IPv6 dual-stack support
+IP addresses are a consumable resource that cluster operators and administrators need to make sure are not exhausted. In particular, public IPv4 addresses are now scarce. Having dual-stack support enables native IPv6 routing to pods and services, whilst still allowing your cluster to talk IPv4 where needed. Dual-stack cluster networking also improves a possible scaling limitation for workloads.
+
+Dual-stack support in Kubernetes means that pods, services, and nodes can get IPv4 addresses and IPv6 addresses. In Kubernetes 1.21 [dual-stack networking](/docs/concepts/services-networking/dual-stack/) has graduated from alpha to beta, and is now enabled by default.
+
+### Graceful Node Shutdown
+[Graceful Node shutdown](/docs/concepts/architecture/nodes/#graceful-node-shutdown) also graduated to beta with this release (and will now be available to a much larger group of users)! This is a hugely beneficial feature that allows the kubelet to be aware of node shutdown, and gracefully terminate pods that are scheduled to that node.
+
+Currently, when a node shuts down, pods do not follow the expected termination lifecycle and are not shut down gracefully. This can introduce problems with a lot of different workloads. Going forward, the kubelet will be able to detect imminent system shutdown through systemd, then inform running pods so they can terminate as gracefully as possible.
+
+### PersistentVolume Health Monitor
+Persistent Volumes (PV) are commonly used in applications to get local, file-based storage. They can be used in many different ways and help users migrate applications without needing to re-write storage backends.
+
+Kubernetes 1.21 has a new alpha feature which allows PVs to be monitored for health of the volume and marked accordingly if the volume becomes unhealthy. Workloads will be able to react to the health state to protect data from being written or read from a volume that is unhealthy.
+
+### Reducing Kubernetes Build Maintenance
+Previously Kubernetes has maintained multiple build systems. This has often been a source of friction and complexity for new and current contributors.
+
+Over the last release cycle, a lot of work has been put in to simplify the build process, and standardize on the native Golang build tools. This should empower broader community maintenance, and lower the barrier to entry for new contributors.
+
+## Major Changes
+
+### PodSecurityPolicy Deprecation
+In Kubernetes 1.21, PodSecurityPolicy is deprecated. As with all Kubernetes feature deprecations, PodSecurityPolicy will continue to be available and fully-functional for several more releases. PodSecurityPolicy, previously in the beta stage, is planned for removal in Kubernetes 1.25.
+
+What's next? We're developing a new built-in mechanism to help limit Pod privileges, with a working title of “PSP Replacement Policy.” Our plan is for this new mechanism to cover the key PodSecurityPolicy use cases, with greatly improved ergonomics and maintainability. To learn more, read [PodSecurityPolicy Deprecation: Past, Present, and Future](/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future).
+
+### TopologyKeys Deprecation
+The Service field `topologyKeys` is now deprecated; all the component features that used this field were previously alpha, and are now also deprecated.
+We've replaced `topologyKeys` with a way to implement topology-aware routing, called topology-aware hints.  Topology-aware hints are an alpha feature in Kubernetes 1.21. You can read more details about the replacement feature in [Topology Aware Hints](/docs/concepts/services-networking/service-topology/); the related [KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/2433-topology-aware-hints/README.md) explains the context for why we switched.
+
+## Other Updates
+
+### Graduated to Stable
+
+* [EndpointSlice](https://github.com/kubernetes/enhancements/issues/752)
+* [Add sysctl support](https://github.com/kubernetes/enhancements/issues/34)
+* [PodDisruptionBudgets](https://github.com/kubernetes/enhancements/issues/85)
+
+### Notable Feature Updates
+
+* [External client-go credential providers](https://github.com/kubernetes/enhancements/issues/541) - beta in 1.21
+* [Structured logging](https://github.com/kubernetes/enhancements/issues/1602) - graduating to beta in 1.22
+* [TTL after finish cleanup for Jobs and Pods](https://github.com/kubernetes/enhancements/issues/592) - graduated to beta
+
+# Release notes
+
+You can check out the full details of the 1.21 release in the [release notes](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.21.md).
+
+# Availability of release
+
+Kubernetes 1.21 is available for [download on GitHub](https://github.com/kubernetes/kubernetes/releases/tag/v1.21.0). There are some great resources out there for getting started with Kubernetes. You can check out some [interactive tutorials](https://kubernetes.io/docs/tutorials/) on the main Kubernetes site, or run a local cluster on your machine using Docker containers with [kind](https://kind.sigs.k8s.io). If you’d like to try building a cluster from scratch, check out the [Kubernetes the Hard Way](https://github.com/kelseyhightower/kubernetes-the-hard-way) tutorial by Kelsey Hightower.
+
+# Release Team
+
+This release was made possible by a very dedicated group of individuals, who came together as a team in the midst of a lot of things happening out in the world. A huge thank you to the release lead Nabarun Pal, and to everyone else on the release team for supporting each other, and working so hard to deliver the 1.21 release for the community.
+
+# Release Logo
+
+![Kubernetes 1.21 Release Logo](/images/blog/2021-04-08-kubernetes-release-1.21/globe_250px.png)
+
+The Kubernetes 1.21 Release Logo portrays the global nature of the Release Team, with release team members residing in timezones from UTC+8 all the way to UTC-8. The diversity of the release team brought in a lot of challenges, but the team tackled them all by adopting more asynchronous communication practices. The heptagonal globe in the release logo signifies the sheer determination of the community to overcome the challenges as they come. It celebrates the amazing teamwork of the release team over the last 3 months to bring in a fun packed Kubernetes release!
+
+The logo is designed by [Aravind Sekar](https://www.behance.net/noblebatman), an independent designer based out of India. Aravind helps open source communities like PyCon India in their design efforts.
+
+# User Highlights
+
+- CNCF welcomes 47 new organizations across the globe as members to advance Cloud Native technology further at the start of 2021! These [new members](https://www.cncf.io/announcements/2021/02/24/cloud-native-computing-foundation-welcomes-47-new-members-at-the-start-of-2021/) will join CNCF at the upcoming 2021 KubeCon + CloudNativeCon events, including [KubeCon + CloudNativeCom EU – Virtual](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/) from May 4 – 7, 2021, and [KubeCon + CloudNativeCon NA in Los Angeles](https://events.linuxfoundation.org/kubecon-cloudnativecon-north-america/) from October 12 – 15, 2021.
+
+# Project Velocity
+
+The [CNCF K8s DevStats project](https://k8s.devstats.cncf.io/) aggregates a number of interesting data points related to the velocity of Kubernetes and various sub-projects. This includes everything from individual contributions to the number of companies that are contributing, and is a neat illustration of the depth and breadth of effort that goes into evolving this ecosystem.
+
+In the v1.21 release cycle, which ran for 12 weeks (January 11 to April 8), we saw contributions from [999 companies](https://k8s.devstats.cncf.io/d/9/companies-table?orgId=1&var-period_name=v1.20.0%20-%20now&var-metric=contributions) and [1279 individuals](https://k8s.devstats.cncf.io/d/66/developer-activity-counts-by-companies?orgId=1&var-period_name=v1.20.0%20-%20now&var-metric=contributions&var-repogroup_name=Kubernetes&var-country_name=All&var-companies=All).
+
+# Ecosystem Updates
+
+- In the wake of rising racism & attacks on global Asian communities, read the statement from CNCF General Priyanka Sharma on the [CNCF blog](https://www.cncf.io/blog/2021/03/18/statement-from-cncf-general-manager-priyanka-sharma-on-the-unacceptable-attacks-against-aapi-and-asian-communities/) reinstating the community's commitment towards inclusive values & diversity-powered resilience.
+- We now have a process in place for migration of the default branch from master → main. Learn more about the guidelines [here](k8s.dev/rename)
+- CNCF and the Linux Foundation have announced the availability of their new training course, [LFS260 – Kubernetes Security Essentials](https://training.linuxfoundation.org/training/kubernetes-security-essentials-lfs260/). In addition to providing skills and knowledge on a broad range of best practices for securing container-based applications and Kubernetes platforms, the course is also a great way to prepare for the recently launched [Certified Kubernetes Security Specialist](https://training.linuxfoundation.org/certification/certified-kubernetes-security-specialist/) certification exam.
+
+# Event Updates
+
+- KubeCon + CloudNativeCon Europe 2021 will take place May 4 - 7, 2021! You can find more information about the conference [here](https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/).
+- [Kubernetes Community Days](https://kubernetescommunitydays.org/) are being relaunched! Q2 2021 will start with Africa and Bengaluru.
+
+# Upcoming release webinar
+
+Join the members of the Kubernetes 1.21 release team on May 13th, 2021 to learn about the major features in this release including IPv4/IPv6 dual-stack support, PersistentVolume Health Monitor, Immutable Secrets and ConfigMaps, and many more. Register here: [https://community.cncf.io/events/details/cncf-cncf-online-programs-presents-cncf-live-webinar-kubernetes-121-release/](https://community.cncf.io/events/details/cncf-cncf-online-programs-presents-cncf-live-webinar-kubernetes-121-release/)
+
+# Get Involved
+
+If you’re interested in contributing to the Kubernetes community, Special Interest Groups (SIGs) are a great starting point. Many of them may align with your interests! If there are things you’d like to share with the community, you can join the weekly community meeting, or use any of the following channels:
+
+* Find out more about contributing to Kubernetes at the [Kubernetes Contributor website](https://www.kubernetes.dev/)
+* Follow us on Twitter [@Kubernetesio](https://twitter.com/kubernetesio) for latest updates
+* Join the community discussion on [Discuss](https://discuss.kubernetes.io/)
+* Join the community on [Slack](http://slack.k8s.io/)
+* Share your Kubernetes [story](https://github.com/cncf/foundation/blob/master/case-study-guidelines.md)
+* Read more about what’s happening with Kubernetes on the [blog](https://kubernetes.io/blog/)
+* Learn more about the [Kubernetes Release Team](https://github.com/kubernetes/sig-release/tree/master/release-team)
@@ -0,0 +1,110 @@
+---
+title: "Introducing Suspended Jobs"
+date: 2021-04-12
+slug: introducing-suspended-jobs
+layout: blog
+---
+
+**Author:** Adhityaa Chandrasekar (Google)
+
+[Jobs](/docs/concepts/workloads/controllers/job/) are a crucial part of
+Kubernetes' API. While other kinds of workloads such as [Deployments](/docs/concepts/workloads/controllers/deployment/),
+[ReplicaSets](/docs/concepts/workloads/controllers/replicaset/),
+[StatefulSets](/docs/concepts/workloads/controllers/statefulset/), and
+[DaemonSets](/docs/concepts/workloads/controllers/daemonset/)
+solve use-cases that require Pods to run forever, Jobs are useful when Pods need
+to run to completion. Commonly used in parallel batch processing, Jobs can be
+used in a variety of applications ranging from video rendering and database
+maintenance to sending bulk emails and scientific computing.
+
+While the amount of parallelism and the conditions for Job completion are
+configurable, the Kubernetes API lacked the ability to suspend and resume Jobs.
+This is often desired when cluster resources are limited and a higher priority
+Job needs to execute in the place of another Job. Deleting the lower priority
+Job is a poor workaround as Pod completion history and other metrics associated
+with the Job will be lost.
+
+With the recent Kubernetes 1.21 release, you will be able to suspend a Job by
+updating its spec. The feature is currently in **alpha** and requires you to
+enable the `SuspendJob` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/)
+on the [API server](/docs/reference/command-line-tools-reference/kube-apiserver/)
+and the [controller manager](/docs/reference/command-line-tools-reference/kube-controller-manager/)
+in order to use it.
+
+## API changes
+
+We introduced a new boolean field `suspend` into the `.spec` of Jobs. Let's say
+I create the following Job:
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: my-job
+spec:
+  suspend: true
+  parallelism: 2
+  completions: 10
+  template:
+    spec:
+      containers:
+      - name: my-container
+        image: busybox
+        command: ["sleep", "5"]
+      restartPolicy: Never
+```
+
+Jobs are not suspended by default, so I'm explicitly setting the `suspend` field
+to _true_ in the `.spec` of the above Job manifest. In the above example, the
+Job controller will refrain from creating Pods until I'm ready to start the Job,
+which I can do by updating `suspend` to false.
+
+As another example, consider a Job that was created with the `suspend` field
+omitted. The Job controller will happily create Pods to work towards Job
+completion. However, before the Job completes, if I explicitly set the field to
+true with a Job update, the Job controller will terminate all active Pods that
+are running and will wait indefinitely for the flag to be flipped back to false.
+Typically, Pod termination is done by sending a SIGTERM signal to all container
+processes in the Pod; the [graceful termination period](/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination)
+defined in the Pod spec will be honoured. Pods terminated this way will not be
+counted as failures by the Job controller.
+
+It is important to understand that succeeded and failed Pods from the past will
+continue to exist after you suspend a Job. That is, that they will count towards
+Job completion once you resume it. You can verify this by looking at Job's
+status before and after suspension.
+
+Read the [documentation](/docs/concepts/workloads/controllers/job#suspending-a-job)
+for a full overview of this new feature.
+
+## Where is this useful?
+
+Let's say I'm the operator of a large cluster. I have many users submitting Jobs
+to the cluster, but not all Jobs are created equal — some Jobs are more
+important than others. Cluster resources aren't infinite either, so all users
+must share resources. If all Jobs were created in the suspended state and placed
+in a pending queue, I can achieve priority-based Job scheduling by resuming Jobs
+in the right order.
+
+As another motivational use-case, consider a cloud provider where compute
+resources are cheaper at night than in the morning. If I have a long-running Job
+that takes multiple days to complete, being able to suspend the Job in the
+morning and then resume it in the evening every day can reduce costs.
+
+Since this field is a part of the Job spec, [CronJobs](/docs/concepts/workloads/controllers/cron-jobs/)
+automatically get this feature for free too.
+
+## References and next steps
+
+If you're interested in a deeper dive into the rationale behind this feature and
+the decisions we have taken, consider reading the [enhancement proposal](https://github.com/kubernetes/enhancements/tree/master/keps/sig-apps/2232-suspend-jobs).
+There's more detail on suspending and resuming jobs in the documentation for [Job](/docs/concepts/workloads/controllers/job#suspending-a-job).
+
+As previously mentioned, this feature is currently in alpha and is available
+only if you explicitly opt-in through the `SuspendJob` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/).
+If this is a feature you're interested in, please consider testing suspended
+Jobs in your cluster and providing feedback. You can discuss this enhancement [on GitHub](https://github.com/kubernetes/enhancements/issues/2232).
+The SIG Apps community also [meets regularly](https://github.com/kubernetes/community/tree/master/sig-apps#meetings)
+and can be reached through [Slack or the mailing list](https://github.com/kubernetes/community/tree/master/sig-apps#contact).
+Barring any unexpected changes to the API, we intend to graduate the feature to
+beta in Kubernetes 1.22, so that the feature becomes available by default.
@@ -0,0 +1,45 @@
+---
+layout: blog
+title: "kube-state-metrics goes v2.0"
+date: 2021-04-13
+slug: kube-state-metrics-v-2-0
+---
+
+**Authors:** Lili Cosic (Red Hat), Frederic Branczyk (Polar Signals), Manuel Rüger (Sony Interactive Entertainment), Tariq Ibrahim (Salesforce)
+
+## What?
+
+[kube-state-metrics](https://github.com/kubernetes/kube-state-metrics), a project under the Kubernetes organization, generates Prometheus format metrics based on the current state of the Kubernetes native resources. It does this by listening to the Kubernetes API and gathering information about resources and objects, e.g. Deployments, Pods, Services, and StatefulSets. A full list of resources is available in the [documentation](https://github.com/kubernetes/kube-state-metrics/tree/master/docs) of kube-state-metrics.
+
+## Why?
+
+There are numerous useful metrics and insights provided by `kube-state-metrics` right out of the box! These metrics can be used to serve as an insight into your cluster: Either through metrics alone, in the form of dashboards, or through an alerting pipeline. To provide a few examples:
+
+* `kube_pod_container_status_restarts_total` can be used to alert on a crashing pod.
+* `kube_deployment_status_replicas` which together with `kube_deployment_status_replicas_available` can be used to alert on whether a deployment is rolled out successfully or stuck.
+* `kube_pod_container_resource_requests` and `kube_pod_container_resource_limits` can be used in capacity planning dashboards.
+
+And there are many more metrics available! To learn more about the other metrics and their details, please check out the [documentation](https://github.com/kubernetes/kube-state-metrics/tree/master/docs#readme).
+
+## What is new in v2.0?
+
+So now that we know what kube-state-metrics is, we are excited to announce the next release: kube-state-metrics v2.0! This release was long-awaited and started with an alpha release in September 2020. To ease maintenance we removed tech debt and also adjusted some confusing wording around user-facing flags and APIs. We also removed some metrics that caused unnecessarily high cardinality in Prometheus! For the 2.0 release, we took the time to set up scale and performance testing. This allows us to better understand if we hit any issues in large clusters and also to document resource request recommendations for your clusters. In this release (and v1.9.8) container builds providing support for multiple architectures were introduced allowing you to run kube-state-metrics on ARM, ARM64, PPC64 and S390x as well!
+
+So without further ado, here is the list of more noteworthy user-facing breaking changes. A full list of changes, features and bug fixes is available in the changelog at the end of this post.
+
+* Flag `--namespace` was renamed to `--namespaces`. If you are using the former, please make sure to update the flag before deploying the latest release.
+* Flag `--collectors` was renamed to `--resources`.
+* Flags `--metric-blacklist` and `--metric-whitelist` were renamed to `--metric-denylist` and `--metric-allowlist`.
+* Flag `--metric-labels-allowlist` allows you to specify a list of Kubernetes labels that get turned into the dimensions of the `kube_<resource-name>_labels` metrics. By default, the metric contains only name and namespace labels.
+* All metrics with a prefix of `kube_hpa_*` were renamed to `kube_horizontalpodautoscaler_*`.
+* Metric labels that relate to Kubernetes were converted to snake_case.
+* If you are importing kube-state-metrics as a library, we have updated our go module path to `k8s.io/kube-state-metrics/v2`
+* All deprecated stable metrics were removed as per the [notice in the v1.9 release](https://github.com/kubernetes/kube-state-metrics/tree/release-1.9/docs#metrics-deprecation).
+* `quay.io/coreos/kube-state-metrics` images will no longer be updated. `k8s.gcr.io/kube-state-metrics/kube-state-metrics` is the new canonical location.
+* The helm chart that is part of the kubernetes/kube-state-metrics repository is deprecated. https://github.com/prometheus-community/helm-charts will be its new location. 
+
+For the full list of v2.0 release changes includes features, bug fixes and other breaking changes see the full [CHANGELOG](https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md).
+
+## Found a problem?
+
+Thanks to all our users for testing so far and thank you to all our contributors for your issue reports as well as code and documentation changes! If you find any problems, we the [maintainers](https://github.com/kubernetes/kube-state-metrics/blob/master/OWNERS) are more than happy to look into them, so please report them by opening a [GitHub issue](https://github.com/kubernetes/kube-state-metrics/issues/new/choose).
@@ -0,0 +1,216 @@
+---
+layout: blog
+title: "Local Storage: Storage Capacity Tracking, Distributed Provisioning and Generic Ephemeral Volumes hit Beta"
+date: 2021-04-14
+slug: local-storage-features-go-beta
+---
+
+ **Authors:** Patrick Ohly (Intel)
+
+The ["generic ephemeral
+volumes"](/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes)
+and ["storage capacity
+tracking"](/docs/concepts/storage/storage-capacity/)
+features in Kubernetes are getting promoted to beta in Kubernetes
+1.21. Together with the [distributed provisioning
+support](https://github.com/kubernetes-csi/external-provisioner#deployment-on-each-node)
+in the CSI external-provisioner, development and deployment of
+Container Storage Interface (CSI) drivers which manage storage locally
+on a node become a lot easier.
+
+This blog post explains how such drivers worked before and how these
+features can be used to make drivers simpler.
+
+## Problems we are solving
+
+There are drivers for local storage, like
+[TopoLVM](https://github.com/cybozu-go/topolvm) for traditional disks
+and [PMEM-CSI](https://intel.github.io/pmem-csi/latest/README.html)
+for [persistent memory](https://pmem.io/). They work and are ready for
+usage today also on older Kubernetes releases, but making that possible
+was not trivial.
+
+### Central component required
+
+The first problem is volume provisioning: it is handled through the
+Kubernetes control plane. Some component must react to
+[PersistentVolumeClaims](/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims)
+(PVCs)
+and create volumes. Usually, that is handled by a central deployment
+of the [CSI
+external-provisioner](https://kubernetes-csi.github.io/docs/external-provisioner.html)
+and a CSI driver component that then connects to the storage
+backplane. But for local storage, there is no such backplane.
+
+TopoLVM solved this by having its different components communicate
+with each other through the Kubernetes API server by creating and
+reacting to custom resources. So although TopoLVM is based on CSI, a
+standard that is independent of a particular container orchestrator,
+TopoLVM only works on Kubernetes.
+
+PMEM-CSI created its own storage backplane with communication through
+gRPC calls. Securing that communication depends on TLS certificates,
+which made driver deployment more complicated.
+
+### Informing Pod scheduler about capacity
+
+The next problem is scheduling. When volumes get created independently
+of pods ("immediate binding"), the CSI driver must pick a node without
+knowing anything about the pod(s) that are going to use it. Topology
+information then forces those pods to run on the node where the volume
+was created. If other resources like RAM or CPU are exhausted there,
+the pod cannot start. This can be avoided by configuring in the
+StorageClass that volume creation is meant to wait for the first pod
+that uses a volume (`volumeBinding: WaitForFirstConsumer`). In that
+mode, the Kubernetes scheduler tentatively picks a node based on other
+constraints and then the external-provisioner is asked to create a
+volume such that it is usable there. If local storage is exhausted,
+the provisioner [can
+ask](https://github.com/kubernetes-csi/external-provisioner/blob/master/doc/design.md)
+for another scheduling round. But without information about available
+capacity, the scheduler might always pick the same unsuitable node.
+
+Both TopoLVM and PMEM-CSI solved this with scheduler extenders. This
+works, but it is hard to configure when deploying the driver because
+communication between kube-scheduler and the driver is very dependent
+on how the cluster was set up.
+
+### Rescheduling
+
+A common use case for local storage is scratch space. A better fit for
+that use case than persistent volumes are ephemeral volumes that get
+created for a pod and destroyed together with it. The initial API for
+supporting ephemeral volumes with CSI drivers (hence called ["*CSI*
+ephemeral
+volumes"](/docs/concepts/storage/ephemeral-volumes/#csi-ephemeral-volumes))
+was [designed for light-weight
+volumes](https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190122-csi-inline-volumes.md)
+where volume creation is unlikely to fail. Volume creation happens
+after pods have been permanently scheduled onto a node, in contrast to
+the traditional provisioning where volume creation is tried before
+scheduling a pod onto a node. CSI drivers must be modified to support
+"CSI ephemeral volumes", which was done for TopoLVM and PMEM-CSI. But
+due to the design of the feature in Kubernetes, pods can get stuck
+permanently if storage capacity runs out on a node. The scheduler
+extenders try to avoid that, but cannot be 100% reliable.
+
+## Enhancements in Kubernetes 1.21
+
+### Distributed provisioning
+
+Starting with [external-provisioner
+v2.1.0](https://github.com/kubernetes-csi/external-provisioner/releases/tag/v2.1.0),
+released for Kubernetes 1.20, provisioning can be handled by
+external-provisioner instances that get [deployed together with the
+CSI driver on each
+node](https://github.com/kubernetes-csi/external-provisioner#deployment-on-each-node)
+and then cooperate to provision volumes ("distributed
+provisioning"). There is no need any more to have a central component
+and thus no need for communication between nodes, at least not for
+provisioning.
+
+### Storage capacity tracking
+
+A scheduler extender still needs some way to find out about capacity
+on each node. When PMEM-CSI switched to distributed provisioning in
+v0.9.0, this was done by querying the metrics data exposed by the
+local driver containers. But it is better also for users to eliminate
+the need for a scheduler extender completely because the driver
+deployment becomes simpler. [Storage capacity
+tracking](/docs/concepts/storage/storage-capacity/), [introduced in
+1.19](/blog/2020/09/01/ephemeral-volumes-with-storage-capacity-tracking/)
+and promoted to beta in Kubernetes 1.21, achieves that. It works by
+publishing information about capacity in `CSIStorageCapacity`
+objects. The scheduler itself then uses that information to filter out
+unsuitable nodes. Because information might be not quite up-to-date,
+pods may still get assigned to nodes with insufficient storage, it's
+just less likely and the next scheduling attempt for a pod should work
+better once the information got refreshed.
+
+### Generic ephemeral volumes
+
+So CSI drivers still need the ability to recover from a bad scheduling
+decision, something that turned out to be impossible to implement for
+"CSI ephemeral volumes". ["*Generic* ephemeral
+volumes"](/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes),
+another feature that got promoted to beta in 1.21, don't have that
+limitation. This feature adds a controller that will create and manage
+PVCs with the lifetime of the Pod and therefore the normal recovery
+mechanism also works for them. Existing storage drivers will be able
+to process these PVCs without any new logic to handle this new
+scenario.
+
+## Known limitations
+
+Both generic ephemeral volumes and storage capacity tracking increase
+the load on the API server. Whether that is a problem depends a lot on
+the kind of workload, in particular how many pods have volumes and how
+often those need to be created and destroyed.
+
+No attempt was made to model how scheduling decisions affect storage
+capacity. That's because the effect can vary considerably depending on
+how the storage system handles storage. The effect is that multiple
+pods with unbound volumes might get assigned to the same node even
+though there is only sufficient capacity for one pod. Scheduling
+should recover, but it would be more efficient if the scheduler knew
+more about storage.
+
+Because storage capacity gets published by a running CSI driver and
+the cluster autoscaler needs information about a node that hasn't been
+created yet, it will currently not scale up a cluster for pods that
+need volumes. There is an [idea how to provide that
+information](https://github.com/kubernetes/autoscaler/pull/3887), but
+more work is needed in that area.
+
+Distributed snapshotting and resizing are not currently supported. It
+should be doable to adapt the respective sidecar and there are
+tracking issues for external-snapshotter and external-resizer open
+already, they just need some volunteer.
+
+The recovery from a bad scheduling decising can fail for pods with
+multiple volumes, in particular when those volumes are local to nodes:
+if one volume can be created and then storage is insufficient for
+another volume, the first volume continues to exist and forces the
+scheduler to put the pod onto the node of that volume. There is an
+idea how do deal with this, [rolling back the provision of the
+volume](https://github.com/kubernetes/enhancements/pull/1703), but
+this is only in the very early stages of brainstorming and not even a
+merged KEP yet. For now it is better to avoid creating pods with more
+than one persistent volume.
+
+## Enabling the new features and next steps
+
+With the feature entering beta in the 1.21 release, no additional actions are needed to enable it. Generic
+ephemeral volumes also work without changes in CSI drivers. For more
+information, see the
+[documentation](/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes)
+and the [previous blog
+post](/blog/2020/09/01/ephemeral-volumes-with-storage-capacity-tracking/)
+about it. The API has not changed at all between alpha and beta.
+
+For the other two features, the external-provisioner documentation
+explains how CSI driver developers must change how their driver gets
+deployed to support [storage capacity
+tracking](https://github.com/kubernetes-csi/external-provisioner#capacity-support)
+and [distributed
+provisioning](https://github.com/kubernetes-csi/external-provisioner#deployment-on-each-node).
+These two features are independent, therefore it is okay to enable
+only one of them.
+
+[SIG
+Storage](https://github.com/kubernetes/community/tree/master/sig-storage)
+would like to hear from you if you are using these new features. We
+can be reached through
+[email](https://groups.google.com/forum/#!forum/kubernetes-sig-storage),
+[Slack](https://slack.k8s.io/) (channel [`#sig-storage`](https://kubernetes.slack.com/messages/sig-storage)) and in the
+[regular SIG
+meeting](https://github.com/kubernetes/community/tree/master/sig-storage#meeting).
+A description of your workload would be very useful to validate design
+decisions, set up performance tests and eventually promote these
+features to GA.
+
+## Acknowledgements
+
+Thanks a lot to the members of the community who have contributed to these
+features or given feedback including members of SIG Scheduling, SIG Auth,
+and of course SIG Storage!
@@ -0,0 +1,80 @@
+---
+layout: blog
+title: 'Three Tenancy Models For Kubernetes'
+date: 2021-04-15
+slug: three-tenancy-models-for-kubernetes
+---
+
+**Authors:** Ryan Bezdicek (Medtronic), Jim Bugwadia (Nirmata), Tasha Drew (VMware),  Fei Guo (Alibaba), Adrian Ludwin (Google)
+
+Kubernetes clusters are typically used by several teams in an organization. In other cases, Kubernetes may be used to deliver applications to end users requiring segmentation and isolation of resources across users from different organizations. Secure sharing of Kubernetes control plane and worker node resources allows maximizing productivity and saving costs in both cases.
+
+The Kubernetes Multi-Tenancy Working Group is chartered with defining tenancy models for Kubernetes and making it easier to operationalize tenancy related use cases. This blog post, from the working group members, describes three common tenancy models and introduces related working group projects.
+
+We will also be presenting on this content and discussing different use cases at our Kubecon EU 2021 panel session, [Multi-tenancy vs. Multi-cluster: When Should you Use What?](https://sched.co/iE66).
+
+## Namespaces as a Service
+
+With the *namespaces-as-a-service* model, tenants share a cluster and tenant workloads are restricted to a set of Namespaces assigned to the tenant. The cluster control plane resources like the API server and scheduler, and worker node resources like CPU, memory, etc. are available for use across all tenants. 
+
+To isolate tenant workloads, each namespace must also contain:
+* **[role bindings](/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding):** for controlling access to the namespace
+* **[network policies](/docs/concepts/services-networking/network-policies/):** to prevent network traffic across tenants
+* **[resource quotas](/docs/concepts/policy/resource-quotas/):** to limit usage and ensure fairness across tenants
+
+With this model, tenants share cluster-wide resources like ClusterRoles and CustomResourceDefinitions (CRDs) and hence cannot create or update these cluster-wide resources.
+
+The [Hierarchical Namespace Controller (HNC)](/blog/2020/08/14/introducing-hierarchical-namespaces/) project makes it easier to manage namespace based tenancy by allowing users to create additional namespaces under a namespace, and propagating resources within the namespace hierarchy. This allows self-service namespaces for tenants, without requiring cluster-wide permissions.
+
+The [Multi-Tenancy Benchmarks (MTB)](https://github.com/kubernetes-sigs/multi-tenancy/tree/master/benchmarks) project provides benchmarks and a command-line tool that performs several configuration and runtime checks to report if tenant namespaces are properly isolated and the necessary security controls are implemented.
+
+## Clusters as a Service
+
+With the *clusters-as-a-service* usage model, each tenant gets their own cluster.  This model allows tenants to have different versions of cluster-wide resources such as CRDs, and provides full isolation of the Kubernetes control plane.
+
+The tenant clusters may be provisioned using projects like [Cluster API (CAPI)](https://cluster-api.sigs.k8s.io/) where a management cluster is used to provision multiple workload clusters. A workload cluster is assigned to a tenant and tenants have full control over cluster resources. Note that in most enterprises a central platform team may be responsible for managing required add-on services such as security and monitoring services, and for providing cluster lifecycle management services such as patching and upgrades. A tenant administrator may be restricted from modifying the centrally managed services and other critical cluster information. 
+
+## Control planes as a Service
+
+In a variation of the *clusters-as-a-service* model, the tenant cluster may be a **virtual cluster** where each tenant gets their own dedicated Kubernetes control plane but share worker node resources. As with other forms of virtualization, users of a virtual cluster see no significant differences between a virtual cluster and other Kubernetes clusters. This is sometimes referred to as `Control Planes as a Service` (CPaaS).
+
+A virtual cluster of this type shares worker node resources and workload state independent control plane components, like the scheduler. Other workload aware control-plane components, like the API server, are created on a per-tenant basis to allow overlaps, and additional components are used to synchronize and manage state across the per-tenant control plane and the underlying shared cluster resources. With this model users can manage their own cluster-wide resources. 
+
+The [Virtual Cluster](https://github.com/kubernetes-sigs/multi-tenancy/tree/master/incubator/virtualcluster) project implements this model, where a `supercluster` is shared by multiple `virtual clusters`. The [Cluster API Nested](https://github.com/kubernetes-sigs/cluster-api-provider-nested) project is extending this work to conform to the CAPI model, allowing use of familiar API resources to provision and manage virtual clusters.
+
+## Security considerations
+
+Cloud native security involves different system layers and lifecycle phases as described in the [Cloud Native Security Whitepaper](/blog/2020/11/18/cloud-native-security-for-your-clusters) from CNCF SIG Security.  Without proper security measures implemented across all layers and phases, Kubernetes tenant isolation can be compromised and a security breach with one tenant can threaten other tenants.
+
+It is important for any new user to Kubernetes to realize that the default installation of a new upstream Kubernetes cluster is not secure, and you are going to need to invest in hardening it in order to avoid security issues.
+
+At a minimum, the following security measures are required:
+* image scanning: container image vulnerabilities can be exploited to execute commands and access additional resources. 
+* [RBAC](/docs/reference/access-authn-authz/rbac/): for *namespaces-as-a-service* user roles and permissions must be properly configured at a per-namespace level; for other models tenants may need to be restricted from accessing centrally managed add-on services and other cluster-wide resources.
+* [network policies](/docs/concepts/services-networking/network-policies/): for *namespaces-as-a-service* default network policies that deny all ingress and egress traffic are recommended to prevent cross-tenant network traffic and may also be used as a best practice for other tenancy models.
+* [Kubernetes Pod Security Standards](/docs/concepts/security/pod-security-standards/): to enforce Pod hardening best practices the `Restricted` policy is recommended as the default for tenant workloads with exclusions configured only as needed.
+* [CIS Benchmarks for Kubernetes](https://www.cisecurity.org/benchmark/kubernetes/): the CIS Benchmarks for Kubernetes guidelines should be used to properly configure Kubernetes control-plane and worker node components.
+
+Additional recommendations include using:
+* policy engines: for configuration security best practices, such as only allowing trusted registries.  
+* runtime scanners: to detect and report runtime security events.
+* VM-based container sandboxing: for stronger data plane isolation.
+
+While proper security is required independently of tenancy models, not having essential security controls like [pod security](/docs/concepts/security/pod-security-standards/) in a shared cluster provides attackers with means to compromise tenancy models and possibly access sensitive information across tenants increasing the overall risk profile.
+
+## Summary
+
+A 2020 CNCF survey showed that production Kubernetes usage has increased by over 300% since 2016. As an increasing number of Kubernetes workloads move to production, organizations are looking for ways to share Kubernetes resources across teams for agility and cost savings. 
+
+The **namespaces as a service** tenancy model allows sharing clusters and hence enables resource efficiencies. However, it requires proper security configurations and has limitations as all tenants share the same cluster-wide resources. 
+
+The **clusters as a service** tenancy model addresses these limitations, but with higher management and resource overhead. 
+
+The **control planes as a service** model provides a way to share resources of a single Kubernetes cluster and also let tenants manage their own cluster-wide resources. Sharing worker node resources increases resource effeciencies, but also exposes cross tenant security and isolation concerns that exist for shared clusters.
+
+ In many cases, organizations will use multiple tenancy models to address different use cases and as different product and development teams will have varying needs. Following security and management best practices, such as applying [Pod Security Standards](/docs/concepts/security/pod-security-standards/) and not using the `default` namespace, makes it easer to switch from one model to another.
+
+The [Kubernetes Multi-Tenancy Working Group](https://github.com/kubernetes-sigs/multi-tenancy) has created several projects like [Hierarchical Namespaces Controller](https://github.com/kubernetes-sigs/multi-tenancy/tree/master/incubator/hnc), [Virtual Cluster](https://github.com/kubernetes-sigs/multi-tenancy/tree/master/incubator/virtualcluster) / [CAPI Nested](https://github.com/kubernetes-sigs/cluster-api-provider-nested), and [Multi-Tenancy Benchmarks](https://github.com/kubernetes-sigs/multi-tenancy/tree/master/benchmarks) to make it easier to provision and manage multi-tenancy models.
+
+If you are interested in multi-tenancy topics, or would like to share your use cases, please join us in an upcoming [community meeting](https://github.com/kubernetes/community/blob/master/wg-multitenancy/README.md) or reach out on the *wg-multitenancy channel* on the [Kubernetes slack](https://slack.k8s.io/).
+
@@ -0,0 +1,95 @@
+---
+layout: blog
+title: "Volume Health Monitoring Alpha Update"
+date: 2021-04-16
+slug: volume-health-monitoring-alpha-update
+---
+
+**Author:** Xing Yang (VMware)
+
+The CSI Volume Health Monitoring feature, originally introduced in 1.19 has undergone a large update for the 1.21 release.
+
+## Why add Volume Health Monitoring to Kubernetes?
+
+Without Volume Health Monitoring, Kubernetes has no knowledge of the state of the underlying volumes of a storage system after a PVC is provisioned and used by a Pod. Many things could happen to the underlying storage system after a volume is provisioned in Kubernetes. For example, the volume could be deleted by accident outside of Kubernetes, the disk that the volume resides on could fail, it could be out of capacity, the disk may be degraded which affects its performance, and so on. Even when the volume is mounted on a pod and used by an application, there could be problems later on such as read/write I/O errors, file system corruption, accidental unmounting of the volume outside of Kubernetes, etc. It is very hard to debug and detect root causes when something happened like this.
+
+Volume health monitoring can be very beneficial to Kubernetes users. It can communicate with the CSI driver to retrieve errors detected by the underlying storage system. PVC events can be reported up to the user to take action. For example, if the volume is out of capacity, they could request a volume expansion to get more space.
+
+## What is Volume Health Monitoring?
+
+CSI Volume Health Monitoring allows CSI Drivers to detect abnormal volume conditions from the underlying storage systems and report them as events on PVCs or Pods.
+
+The Kubernetes components that monitor the volumes and report events with volume health information include the following:
+
+* Kubelet, in addition to gathering the existing volume stats will watch the volume health of the PVCs on that node. If a PVC has an abnormal health condition, an event will be reported on the pod object using the PVC. If multiple pods are using the same PVC, events will be reported on all pods using that PVC.
+* An [External Volume Health Monitor Controller](https://github.com/kubernetes-csi/external-health-monitor) watches volume health of the PVCs and reports events on the PVCs.
+
+Note that the node side volume health monitoring logic was an external agent when this feature was first introduced in the Kubernetes 1.19 release. In Kubernetes 1.21, the node side volume health monitoring logic was moved from the external agent into the Kubelet, to avoid making duplicate CSI function calls. With this change in 1.21, a new alpha [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) `CSIVolumeHealth` was introduced for the volume health monitoring logic in Kubelet.
+
+Currently the Volume Health Monitoring feature is informational only as it only reports abnormal volume health events on PVCs or Pods. Users will need to check these events and manually fix the problems. This feature serves as a stepping stone towards programmatic detection and resolution of volume health issues by Kubernetes in the future.
+
+## How do I use Volume Health on my Kubernetes Cluster?
+
+To use the Volume Health feature, first make sure the CSI driver you are using supports this feature. Refer to this [CSI drivers doc](https://kubernetes-csi.github.io/docs/drivers.html) to find out which CSI drivers support this feature.
+
+To enable Volume Health Monitoring from the node side, the alpha feature gate `CSIVolumeHealth` needs to be enabled.
+
+If a CSI driver supports the Volume Health Monitoring feature from the controller side, events regarding abnormal volume conditions will be recorded on PVCs.
+
+If a CSI driver supports the Volume Health Monitoring feature from the controller side, user can also get events regarding node failures if the `enable-node-watcher` flag is set to true when deploying the External Health Monitor Controller. When a node failure event is detected, an event will be reported on the PVC to indicate that pods using this PVC are on a failed node.
+
+If a CSI driver supports the Volume Health Monitoring feature from the node side, events regarding abnormal volume conditions will be recorded on pods using the PVCs.
+
+## As a storage vendor, how do I add support for volume health to my CSI driver?
+
+Volume Health Monitoring includes two parts:
+* An External Volume Health Monitoring Controller monitors volume health from the controller side.
+* Kubelet monitors volume health from the node side.
+
+For details, see the [CSI spec](https://github.com/container-storage-interface/spec/blob/master/spec.md) and the [Kubernetes-CSI Driver Developer Guide](https://kubernetes-csi.github.io/docs/volume-health-monitor.html).
+
+There is a sample implementation for volume health in [CSI host path driver](https://github.com/kubernetes-csi/csi-driver-host-path).
+
+### Controller Side Volume Health Monitoring
+
+To learn how to deploy the External Volume Health Monitoring controller, see [CSI external-health-monitor-controller](https://kubernetes-csi.github.io/docs/external-health-monitor-controller.html) in the CSI documentation.
+
+The External Health Monitor Controller calls either `ListVolumes` or `ControllerGetVolume` CSI RPC and reports VolumeConditionAbnormal events with messages on PVCs if abnormal volume conditions are detected. Only CSI drivers with `LIST_VOLUMES` and `VOLUME_CONDITION` controller capability or `GET_VOLUME` and `VOLUME_CONDITION` controller capability support Volume Health Monitoring in the external controller.
+
+To implement the volume health feature from the controller side, a CSI driver **must** add support for the new controller capabilities.
+
+If a CSI driver supports `LIST_VOLUMES` and `VOLUME_CONDITION` controller capabilities, it **must** implement controller RPC `ListVolumes` and report the volume condition in the response.
+
+If a CSI driver supports `GET_VOLUME` and `VOLUME_CONDITION` controller capability, it **must** implement controller PRC `ControllerGetVolume` and report the volume condition in the response.
+
+If a CSI driver supports `LIST_VOLUMES`, `GET_VOLUME`, and `VOLUME_CONDITION` controller capabilities, only `ListVolumes` CSI RPC will be invoked by the External Health Monitor Controller.
+
+### Node Side Volume Health Monitoring
+
+Kubelet calls `NodeGetVolumeStats` CSI RPC and reports VolumeConditionAbnormal events with messages on Pods if abnormal volume conditions are detected. Only CSI drivers with `VOLUME_CONDITION` node capability support Volume Health Monitoring in Kubelet.
+
+To implement the volume health feature from the node side, a CSI driver **must** add support for the new node capabilities.
+
+If a CSI driver supports `VOLUME_CONDITION` node capability, it **must** report the volume condition in node RPC `NodeGetVoumeStats`.
+
+## What’s next?
+
+Depending on feedback and adoption, the Kubernetes team plans to push the CSI volume health implementation to beta in either 1.22 or 1.23.
+
+We are also exploring how to use volume health information for programmatic detection and automatic reconcile in Kubernetes.
+
+## How can I learn more?
+
+To learn the design details for Volume Health Monitoring, read the [Volume Health Monitor](https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1432-volume-health-monitor) enhancement proposal.
+
+The Volume Health Monitor controller source code is at [https://github.com/kubernetes-csi/external-health-monitor](https://github.com/kubernetes-csi/external-health-monitor).
+
+There are also more details about volume health checks in the [Container Storage Interface Documentation](https://kubernetes-csi.github.io/docs/).
+
+## How do I get involved?
+
+The [Kubernetes Slack channel #csi](https://kubernetes.slack.com/messages/csi) and any of the [standard SIG Storage communication channels](https://github.com/kubernetes/community/blob/master/sig-storage/README.md#contact) are great mediums to reach out to the SIG Storage and the CSI team.
+
+We offer a huge thank you to the contributors who helped release this feature in 1.21. We want to thank Yuquan Ren ([NickrenREN](https://github.com/nickrenren)) who implemented the initial volume health monitor controller and agent in the external health monitor repo, thank Ran Xu ([fengzixu](https://github.com/fengzixu)) who moved the volume health monitoring logic from the external agent to Kubelet in 1.21, and we offer special thanks to the following people for their insightful reviews: David Ashpole ([dashpole](https://github.com/dashpole)), Michelle Au ([msau42](https://github.com/msau42)), David Eads ([deads2k](https://github.com/deads2k)), Elana Hashman ([ehashman](https://github.com/ehashman)), Seth Jennings ([sjenning](https://github.com/sjenning)), and Jiawei Wang ([Jiawei0227](https://github.com/Jiawei0227)).
+
+Those interested in getting involved with the design and development of CSI or any part of the Kubernetes Storage system, join the [Kubernetes Storage Special Interest Group](https://github.com/kubernetes/community/tree/master/sig-storage) (SIG). We’re rapidly growing and always welcome new contributors.
@@ -0,0 +1,95 @@
+---
+title: "Introducing Indexed Jobs"
+date: 2021-04-19
+slug: introducing-indexed-jobs
+---
+
+**Author:** Aldo Culquicondor (Google)
+
+Once you have containerized a non-parallel [Job](/docs/concepts/workloads/controllers/job/),
+it is quite easy to get it up and running on Kubernetes without modifications to
+the binary. In most cases, when running parallel distributed Jobs, you had
+to set a separate system to partition the work among the workers. For
+example, you could set up a task queue to [assign one work item to each
+Pod](/docs/tasks/job/coarse-parallel-processing-work-queue/) or [multiple items
+to each Pod until the queue is emptied](/docs/tasks/job/fine-parallel-processing-work-queue/).
+
+The Kubernetes 1.21 release introduces a new field to control Job _completion mode_,
+a configuration option that allows you to control how Pod completions affect the
+overall progress of a Job, with two possible options (for now):
+
+- `NonIndexed` (default): the Job is considered complete when there has been
+  a number of successfully completed Pods equal to the specified number in
+  `.spec.completions`. In other words, each Pod completion is homologous to
+  each other. Any Job you might have created before the introduction of
+  completion modes is implicitly NonIndexed.
+- `Indexed`: the Job is considered complete when there is one successfully
+  completed Pod associated with each index from 0 to `.spec.completions-1`. The
+  index is exposed to each Pod in the `batch.kubernetes.io/job-completion-index`
+  annotation and the `JOB_COMPLETION_INDEX` environment variable.
+
+You can start using Jobs with Indexed completion mode, or Indexed Jobs, for
+short, to easily start parallel Jobs. Then, each worker Pod can have a statically
+assigned partition of the data based on the index. This saves you from having to
+set up a queuing system or even having to modify your binary!
+
+## Creating an Indexed Job
+
+To create an Indexed Job, you just have to add `completionMode: Indexed` to the
+Job spec and make use of the `JOB_COMPLETION_INDEX` environment variable.
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: 'sample-job'
+spec:
+  completions: 3
+  parallelism: 3
+  completionMode: Indexed
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - command:
+        - 'bash'
+        - '-c'
+        - 'echo "My partition: ${JOB_COMPLETION_INDEX}"'
+        image: 'docker.io/library/bash'
+        name: 'sample-load'
+```
+
+Note that completion mode is an alpha feature in the 1.21 release. To be able to
+use it in your cluster, make sure to enable the `IndexedJob` [feature
+gate](/docs/reference/command-line-tools-reference/feature-gates/) on the
+[API server](docs/reference/command-line-tools-reference/kube-apiserver/) and
+the [controller manager](/docs/reference/command-line-tools-reference/kube-controller-manager/).
+
+When you run the example, you will see that each of the three created Pods gets a
+different completion index. For the user's convenience, the control plane sets the
+`JOB_COMPLETION_INDEX` environment variable, but you can choose to [set your
+own](/docs/tasks/inject-data-application/environment-variable-expose-pod-information/)
+or [expose the index as a file](/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/).
+
+See [Indexed Job for parallel processing with static work
+assignment](/docs/tasks/job/indexed-parallel-processing-static/) for a
+step-by-step guide, and a few more examples.
+
+## Future plans
+
+SIG Apps envisions that there might be more completion modes that enable more
+use cases for the Job API. We welcome you to open issues in
+[kubernetes/kubernetes](https://github.com/kubernetes/kubernetes) with your
+suggestions.
+
+In particular, we are considering an `IndexedAndUnique` mode where the indexes
+are not just available as annotation, but they are part of the Pod names,
+similar to {{< glossary_tooltip text="StatefulSet" term_id="statefulset" >}}.
+This should facilitate inter-Pod communication for tightly coupled Pods.
+You can join the discussion in the [open issue](https://github.com/kubernetes/kubernetes/issues/99497).
+
+## Wrap-up
+
+Indexed Jobs allows you to statically partition work among the workers of your
+parallel Jobs. SIG Apps hopes that this feature facilitates the migration of
+more batch workloads to Kubernetes.
@@ -0,0 +1,479 @@
+---
+layout: blog
+title: "Defining Network Policy Conformance for Container Network Interface (CNI) providers"
+date: 2021-04-20
+slug: defining-networkpolicy-conformance-cni-providers
+---
+
+Authors: Matt Fenwick (Synopsys), Jay Vyas (VMWare), Ricardo Katz, Amim Knabben (Loadsmart), Douglas Schilling Landgraf (Red Hat), Christopher Tomkins (Tigera)
+
+Special thanks to Tim Hockin and Bowie Du (Google), Dan Winship and Antonio Ojea (Red Hat),
+Casey Davenport and Shaun Crampton (Tigera), and Abhishek Raut and Antonin Bas (VMware) for
+being supportive of this work, and working with us to resolve issues in different Container Network Interfaces (CNIs) over time.
+
+A brief conversation around "node local" Network Policies in April of 2020 inspired the creation of a NetworkPolicy subproject from SIG Network. It became clear that as a community,
+we need a rock-solid story around how to do pod network security on Kubernetes, and this story needed a community around it, so as to grow the cultural adoption of enterprise security patterns in K8s.
+
+In this post we'll discuss:
+
+- Why we created a subproject for [Network Policies](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
+- How we changed the Kubernetes e2e framework to `visualize` NetworkPolicy implementation of your CNI provider
+- The initial results of our comprehensive NetworkPolicy conformance validator, _Cyclonus_, built around these principles
+- Improvements subproject contributors have made to the NetworkPolicy user experience
+
+## Why we created a subproject for NetworkPolicies
+
+In April of 2020 it was becoming clear that many CNIs were emerging, and many vendors
+implement these CNIs in subtly different ways.  Users were beginning to express a little bit
+of confusion around how to implement policies for different scenarios, and asking for new features.
+It was clear that we needed to begin unifying the way we think about Network Policies
+in Kubernetes, to avoid API fragmentation and unnecessary complexity.
+
+For example:
+- In order to be flexible to the user’s environment, Calico as a CNI provider can be run using IPIP or VXLAN mode, or without encapsulation overhead.  CNIs such as Antrea
+  and Cilium offer similar configuration options as well.
+- Some CNI plugins offer iptables for NetworkPolicies amongst other options, whereas other CNIs use a completely
+  different technology stack (for example, the Antrea project uses Open vSwitch rules).
+- Some CNI plugins only implement a subset of the Kubernetes NetworkPolicy API, and some a superset. For example, certain plugins don't support the
+  ability to target a named port; others don't work with certain IP address types, and there are diverging semantics for similar policy types.
+- Some CNI plugins combine with OTHER CNI plugins in order to implement NetworkPolicies (canal), some CNI's might mix implementations (multus), and some clouds do routing separately from NetworkPolicy implementation.
+
+Although this complexity is to some extent necessary to support different environments, end-users find that they need to follow a multistep process to implement Network Policies to secure their applications:
+- Confirm that their network plugin supports NetworkPolicies (some don't, such as Flannel)
+- Confirm that their cluster's network plugin supports the specific NetworkPolicy features that they are interested in (again, the named port or port range examples come to mind here)
+- Confirm that their application's Network Policy definitions are doing the right thing
+- Find out the nuances of a vendor's implementation of policy, and check whether or not that implementation has a CNI neutral implementation (which is sometimes adequate for users)
+
+The NetworkPolicy project in upstream Kubernetes aims at providing a community where
+people can learn about, and contribute to, the Kubernetes NetworkPolicy API and the surrounding ecosystem.
+
+## The First step: A validation framework for NetworkPolicies that was intuitive to use and understand
+
+The Kubernetes end to end suite has always had NetworkPolicy tests, but these weren't
+run in CI, and the way they were implemented didn't provide holistic, easily consumable
+information about how a policy was working in a cluster.
+This is because the original tests didn't provide any kind of visual summary of connectivity
+across a cluster.   We thus initially set out to make it easy to confirm CNI support for NetworkPolicies by
+making the end to end tests (which are often used by administrators or users to diagnose cluster conformance) easy to interpret.
+
+To solve the problem of confirming that CNIs support the basic features most users care about
+for a policy, we built a new NetworkPolicy validation tool into the Kubernetes e2e
+framework which allows for visual inspection of policies and their effect on a standard set of pods in a cluster.
+For example, take the following test output.  We found a bug in
+[OVN Kubernetes](https://github.com/ovn-org/ovn-kubernetes/issues/1782). This bug has now been resolved. With this tool the bug was really
+easy to characterize, wherein certain policies caused a state-modification that,
+later on, caused traffic to incorrectly be blocked (even after all Network Policies were deleted from the cluster).
+
+This is the network policy for the test in question:
+```yaml
+metadata:
+  creationTimestamp: null
+  name: allow-ingress-port-80
+spec:
+  ingress:
+  - ports:
+    - port: serve-80-tcp
+  podSelector: {}
+```
+
+These are the expected connectivity results.  The test setup is 9 pods (3 namespaces: x, y, and z;
+and 3 pods in each namespace: a, b, and c); each pod runs a server on the same port and protocol
+that can be reached through HTTP calls in the absence of network policies.  Connectivity is verified
+by using the [agnhost](https://github.com/kubernetes/kubernetes/tree/master/test/images/agnhost) network utility to issue HTTP calls on a port and protocol that other pods are
+expected to be serving.  A test scenario first
+runs a connectivity check to ensure that each pod can reach each other pod, for 81 (= 9 x 9) data
+points.  This is the "control".  Then perturbations are applied, depending on the test scenario:
+policies are created, updated, and deleted; labels are added and removed from pods and namespaces,
+and so on.  After each change, the connectivity matrix is recollected and compared to the expected
+connectivity.
+
+These results give a visual indication of connectivity in a simple matrix.  Going down the leftmost column is the "source"
+pod, or the pod issuing the request; going across the topmost row is the "destination" pod, or the pod
+receiving the request.  A `.` means that the connection was allowed; an `X` means the connection was
+blocked. For example:
+
+```
+Nov  4 16:58:43.449: INFO: expected:
+
+-   x/a x/b x/c y/a y/b y/c z/a z/b z/c
+x/a .   .   .   .   .   .   .   .   .
+x/b .   .   .   .   .   .   .   .   .
+x/c .   .   .   .   .   .   .   .   .
+y/a .   .   .   .   .   .   .   .   .
+y/b .   .   .   .   .   .   .   .   .
+y/c .   .   .   .   .   .   .   .   .
+z/a .   .   .   .   .   .   .   .   .
+z/b .   .   .   .   .   .   .   .   .
+z/c .   .   .   .   .   .   .   .   .
+```
+
+Below are the observed connectivity results in the case of the OVN Kubernetes bug.  Notice how the top three rows indicate that
+all requests from namespace x regardless of pod and destination were blocked.  Since these
+experimental results do not match the expected results, a failure will be reported.  Note
+how the specific pattern of failure provides clear insight into the nature of the problem --
+since all requests from a specific namespace fail, we have a clear clue to start our
+investigation.
+
+```
+Nov  4 16:58:43.449: INFO: observed:
+
+-   x/a x/b x/c y/a y/b y/c z/a z/b z/c
+x/a X   X   X   X   X   X   X   X   X
+x/b X   X   X   X   X   X   X   X   X
+x/c X   X   X   X   X   X   X   X   X
+y/a .   .   .   .   .   .   .   .   .
+y/b .   .   .   .   .   .   .   .   .
+y/c .   .   .   .   .   .   .   .   .
+z/a .   .   .   .   .   .   .   .   .
+z/b .   .   .   .   .   .   .   .   .
+z/c .   .   .   .   .   .   .   .   .
+```
+
+This was one of our earliest wins in the Network Policy group, as we were able to
+identify and work with the OVN Kubernetes group to fix a bug in egress policy processing.
+
+However, even though this tool has made it easy to validate roughly 30 common scenarios,
+it doesn't validate *all* Network Policy scenarios - because there are an enormous number of possible
+permutations that one might create (technically, we might say this number is
+infinite given that there's an infinite number of possible namespace/pod/port/protocol variations one can create).
+
+Once these tests were in play, we worked with the Upstream SIG Network and SIG Testing communities
+(thanks to Antonio Ojea and Ben Elder) to put a testgrid Network Policy job in place.  This job
+continuously runs the entire suite of Network Policy tests against
+[GCE with Calico as a Network Policy provider](https://testgrid.k8s.io/sig-network-gce#presubmit-network-policies,%20google-gce).
+
+Part of our role as a subproject is to help make sure that, when these tests break, we can help triage them effectively.
+
+## Cyclonus: The next step towards Network Policy conformance {#cyclonus}
+
+Around the time that we were finishing the validation work, it became clear from the community that,
+in general, we needed to solve the overall problem of testing ALL possible Network Policy implementations.
+For example, a KEP was recently written which introduced the concept of micro versioning to
+Network Policies to accommodate [describing this at the API level](https://github.com/kubernetes/enhancements/pull/2137/files), by Dan Winship.
+
+In response to this increasingly obvious need to comprehensively evaluate Network
+Policy implementations from all vendors, Matt Fenwick decided to evolve our approach to Network Policy validation again by creating Cyclonus.
+
+Cyclonus is a comprehensive Network Policy fuzzing tool which verifies a CNI provider
+against hundreds of different Network Policy scenarios, by defining similar truth table/policy
+combinations as demonstrated in the end to end tests, while also providing a hierarchical
+representation of policy "categories".  We've found some interesting nuances and issues
+in almost every CNI we've tested so far, and have even contributed some fixes back.
+
+To perform a Cyclonus validation run, you create a Job manifest similar to:
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cyclonus
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+        - command:
+            - ./cyclonus
+            - generate
+            - --perturbation-wait-seconds=15
+            - --server-protocol=tcp,udp
+          name: cyclonus
+          imagePullPolicy: IfNotPresent
+          image: mfenwick100/cyclonus:latest
+      serviceAccount: cyclonus
+```
+
+Cyclonus outputs a report of all the test cases it will run:
+```
+test cases to run by tag:
+- target: 6
+- peer-ipblock: 4
+- udp: 16
+- delete-pod: 1
+- conflict: 16
+- multi-port/protocol: 14
+- ingress: 51
+- all-pods: 14
+- egress: 51
+- all-namespaces: 10
+- sctp: 10
+- port: 56
+- miscellaneous: 22
+- direction: 100
+- multi-peer: 0
+- any-port-protocol: 2
+- set-namespace-labels: 1
+- upstream-e2e: 0
+- allow-all: 6
+- namespaces-by-label: 6
+- deny-all: 10
+- pathological: 6
+- action: 6
+- rule: 30
+- policy-namespace: 4
+- example: 0
+- tcp: 16
+- target-namespace: 3
+- named-port: 24
+- update-policy: 1
+- any-peer: 2
+- target-pod-selector: 3
+- IP-block-with-except: 2
+- pods-by-label: 6
+- numbered-port: 28
+- protocol: 42
+- peer-pods: 20
+- create-policy: 2
+- policy-stack: 0
+- any-port: 14
+- delete-namespace: 1
+- delete-policy: 1
+- create-pod: 1
+- IP-block-no-except: 2
+- create-namespace: 1
+- set-pod-labels: 1
+testing 112 cases
+```
+
+Note that Cyclonus tags its tests based on the type of policy being created, because
+the policies themselves are auto-generated, and thus have no meaningful names to be recognized by.
+
+For each test, Cyclonus outputs a truth table, which is again similar to that of the
+E2E tests, along with the policy being validated:
+
+```
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  creationTimestamp: null
+  name: base
+  namespace: x
+spec:
+  egress:
+  - ports:
+    - port: 81
+    to:
+    - namespaceSelector:
+        matchExpressions:
+        - key: ns
+          operator: In
+          values:
+          - "y"
+          - z
+      podSelector:
+        matchExpressions:
+        - key: pod
+          operator: In
+          values:
+          - a
+          - b
+  - ports:
+    - port: 53
+      protocol: UDP
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchExpressions:
+        - key: ns
+          operator: In
+          values:
+          - x
+          - "y"
+      podSelector:
+        matchExpressions:
+        - key: pod
+          operator: In
+          values:
+          - b
+          - c
+    ports:
+    - port: 80
+      protocol: TCP
+  podSelector:
+    matchLabels:
+      pod: a
+  policyTypes:
+  - Ingress
+  - Egress
+
+0 wrong, 0 ignored, 81 correct
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+| TCP/80 | X/A | X/B | X/C | Y/A | Y/B | Y/C | Z/A | Z/B | Z/C |
+| TCP/81 |     |     |     |     |     |     |     |     |     |
+| UDP/80 |     |     |     |     |     |     |     |     |     |
+| UDP/81 |     |     |     |     |     |     |     |     |     |
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+| x/a    | X   | X   | X   | X   | X   | X   | X   | X   | X   |
+|        | X   | X   | X   | .   | .   | X   | .   | .   | X   |
+|        | X   | X   | X   | X   | X   | X   | X   | X   | X   |
+|        | X   | X   | X   | X   | X   | X   | X   | X   | X   |
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+| x/b    | .   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+| x/c    | .   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+| y/a    | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+| y/b    | .   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+| y/c    | .   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+| z/a    | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+| z/b    | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+| z/c    | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+|        | X   | .   | .   | .   | .   | .   | .   | .   | .   |
+--------+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+```
+
+Both Cyclonus and the e2e tests use the same strategy to validate a Network Policy - probing pods over TCP or UDP, with
+SCTP support available as well for CNIs that support it (such as Calico).
+
+As examples of how we use Cyclonus to help make CNI implementations better from a Network Policy perspective, you can see the following issues:
+
+- [Antrea: NetworkPolicy: unable to allow ingress by CIDR](https://github.com/vmware-tanzu/antrea/issues/1764)
+- [Calico: default missing protocol to TCP; don't let single port overwrite all ports](https://github.com/projectcalico/libcalico-go/pull/1373)
+- [Cilium: Egress Network Policy allows traffic that should be denied](https://github.com/cilium/cilium/issues/14678)
+
+The good news is that Antrea and Calico have already merged fixes for all the issues found and other CNI providers are working on it,
+with the support of SIG Network and the Network Policy subproject.
+
+Are you interested in verifying NetworkPolicy functionality on your cluster?
+(if you care about security or offer multi-tenant SaaS, you should be)
+If so, you can run the upstream end to end tests, or Cyclonus, or both.
+- If you're just getting started with NetworkPolicies and want to simply
+  verify the "common" NetworkPolicy cases that most CNIs should be
+  implementing correctly, in a way that is quick to diagnose, then you're
+  better off running the e2e tests only.
+- If you are deeply curious about your CNI provider's NetworkPolicy
+  implementation, and want to verify it: use Cyclonus.
+- If you want to test *hundreds* of policies, and evaluate your CNI plugin
+  for comprehensive functionality, for deep discovery of potential security
+  holes: use Cyclonus, and also consider running end-to-end cluster tests.
+- If you're thinking of getting involved with the upstream NetworkPolicy efforts:
+  use Cyclonus, and read at least an outline of which e2e tests are relevant.
+
+## Where to start with NetworkPolicy testing?
+
+- Cyclonus is easy to run on your cluster, check out the [instructions on github](https://github.com/mattfenwick/cyclonus#run-as-a-kubernetes-job),
+  and determine whether *your* specific CNI configuration is fully conformant to the hundreds of different
+  Kubernetes Network Policy API constructs.
+- Alternatively, you can use a tool like [sonobuoy](https://github.com/vmware-tanzu/sonobuoy)
+  to run the existing E2E tests in Kubernetes, with the `--ginkgo.focus=NetworkPolicy` flag.
+  Make sure that you use the K8s conformance image for K8s 1.21 or above (for example, by using the `--kube-conformance-image-version v1.21.0` flag),
+  as older images will not have the *new* Network Policy tests in them.
+
+## Improvements to the NetworkPolicy API and user experience
+
+In addition to cleaning up the validation story for CNI plugins that implement NetworkPolicies,
+subproject contributors have also spent some time improving the Kubernetes NetworkPolicy API for a few commonly requested features.
+After months of deliberation, we eventually settled on a few core areas for improvement:
+
+- Port Range policies: We now allow you to specify a *range* of ports for a policy.
+  This allows users interested in scenarios like FTP or virtualization to enable advanced policies.
+  The port range option for network policies will be available to use in Kubernetes 1.21.
+  Read more in [targeting a range of ports](/docs/concepts/services-networking/network-policies/#targeting-a-range-of-ports).
+- Namespace as name policies: Allowing users in Kubernetes >= 1.21 to target namespaces using names,
+  when building Network Policy objects.  This was done in collaboration with Jordan Liggitt and Tim Hockin on the API Machinery side.
+  This change allowed us to improve the Network Policy user experience without actually
+  changing the API! For more details, you can read
+  [Automatic labelling](/docs/concepts/overview/working-with-objects/namespaces/#automatic-labelling) in the page about Namespaces.
+  The TL,DR; is that for Kubernetes 1.21 and later, **all namespaces** have the following label added by default:
+
+    ```
+    kubernetes.io/metadata.name: <name-of-namespace>
+    ```
+
+This means you can write a namespace policy against this namespace, even if you can't edit its labels.
+For example, this policy, will 'just work', without needing to run a command such as `kubectl edit namespace`.
+In fact, it will even work if you can't edit or view this namespace's data at all, because of the magic of API server defaulting.
+
+```yaml
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: test-network-policy
+  namespace: default
+spec:
+  podSelector:
+    matchLabels:
+      role: db
+  policyTypes:
+  - Ingress
+  # Allow inbound traffic to Pods labelled role=db, in the namespace 'default'
+  # provided that the source is a Pod in the namespace 'my-namespace'
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: my-namespace
+```
+
+## Results
+
+In our tests, we found that:
+
+- Antrea and Calico are at a point where they support all of cyclonus's scenarios, modulo a few very minor tweaks which we've made.
+- Cilium also conformed to the majority of the policies, outside known features that aren't fully supported (for example, related to the way Cilium deals with pod CIDR policies).
+
+If you are a CNI provider and interested in helping us to do a better job curating large tests of network policies, please reach out!  We are continuing to curate the Network Policy conformance results from Cyclonus [here](https://raw.githubusercontent.com/K8sbykeshed/cyclonus-artifacts/), but
+we are not capable of maintaining all of the subtleties in NetworkPolicy testing data on our own.  For now, we use github actions and Kind to test in CI.
+
+## The Future
+
+We're also working on some improvements for the future of Network Policies, including:
+
+- Fully qualified Domain policies: The Google Cloud team created a prototype (which
+  we are really excited about) of [FQDN policies](https://github.com/GoogleCloudPlatform/gke-fqdnnetworkpolicies-golang).
+  This tool uses the Network Policy API to enforce policies against L7 URLs, by finding
+  their IPs and blocking them proactively when requests are made.
+- Cluster Administrative policies: We're working hard at enabling *administrative* or
+  *cluster scoped* Network Policies for the future. These are being presented iteratively to the NetworkPolicy subproject.
+  You can read about them here in [Cluster Scoped Network Policy](https://docs.google.com/presentation/d/1Jk86jtS3TcGAugVSM_I4Yds5ukXFJ4F1ZCvxN5v2BaY/).
+
+The Network Policy subproject meets on mondays at 4PM EST. For details, check out the
+[SIG Network community repo](https://github.com/kubernetes/community/tree/master/sig-network).  We'd love
+to hang out with you, hack on stuff, and help you adopt K8s Network Policies for your cluster wherever possible.
+
+### A quick note on User Feedback
+
+We've gotten a lot of ideas and feedback from users on Network Policies.  A lot of people have interesting ideas about Network Policies,
+but we've found that as a subproject, very few people were deeply interested in implementing these ideas to the full extent.
+
+Almost every change to the NetworkPolicy API includes weeks or months of discussion to cover different cases, and ensure no CVEs are being introduced.  Thus, long term ownership
+is the biggest impediment in improving the NetworkPolicy user experience for us, over time.
+
+- We've documented a lot of the history of the Network Policy dialogue [here](https://github.com/jayunit100/network-policy-subproject/blob/master/history.md).
+- We've also taken a poll of users, for what they'd like to see in the Network Policy API [here](https://github.com/jayunit100/network-policy-subproject/blob/master/p0_user_stories.md).
+
+We encourage anyone to provide us with feedback, but our most pressing issues right now
+involve finding *long term owners to help us drive changes*.  
+
+This doesn't require a lot of technical knowledge, but rather, just a long term commitment to helping us stay organized, do paperwork,
+and iterate through the many stages of the K8s feature process.  If you want to help us and get involved, please reach out on the SIG Network mailing list, or in the SIG Network room in the k8s.io slack channel!
+
+Anyone can put an oar in the water and help make NetworkPolices better!
@@ -0,0 +1,100 @@
+---
+layout: blog
+title: 'Annotating Kubernetes Services for Humans'
+date: 2021-04-20
+slug: annotating-k8s-for-humans
+---
+
+**Author:** Richard Li, Ambassador Labs
+
+Have you ever been asked to troubleshoot a failing Kubernetes service and struggled to find basic information about the service such as the source repository and owner?
+
+One of the problems as Kubernetes applications grow is the proliferation of services. As the number of services grows, developers start to specialize working with specific services. When it comes to troubleshooting, however, developers need to be able to find the source, understand the service and dependencies, and chat with the owning team for any service.
+
+## Human service discovery
+
+Troubleshooting always begins with information gathering. While much attention has been paid to centralizing machine data (e.g., logs, metrics), much less attention has been given to the human aspect of service discovery. Who owns a particular service? What Slack channel does the team work on? Where is the source for the service? What issues are currently known and being tracked?
+
+## Kubernetes annotations
+
+Kubernetes annotations are designed to solve exactly this problem. Oft-overlooked, Kubernetes annotations are designed to add metadata to Kubernetes objects. The Kubernetes documentation says annotations can “attach arbitrary non-identifying metadata to objects.” This means that annotations should be used for attaching metadata that is external to Kubernetes (i.e., metadata that Kubernetes won’t use to identify objects. As such, annotations can contain any type of data. This is a contrast to labels, which are designed for uses internal to Kubernetes. As such, label structure and values are [constrained](/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) so they can be efficiently used by Kubernetes.
+
+
+## Kubernetes annotations in action
+
+Here is an example. Imagine you have a Kubernetes service for quoting, called the quote service. You can do the following:
+
+```
+kubectl annotate service quote a8r.io/owner=”@sally”
+```
+
+In this example, we've just added an annotation called `a8r.io/owner` with the value of @sally. Now, we can use `kubectl describe` to get the information.
+
+```
+Name:              quote
+Namespace:         default
+Labels:            <none>
+Annotations:       a8r.io/owner: @sally
+Selector:          app=quote
+Type:              ClusterIP
+IP:                10.109.142.131
+Port:              http  80/TCP
+TargetPort:        8080/TCP
+Endpoints:         <none>
+Session Affinity:  None
+Events:            <none>
+```
+
+If you’re practicing GitOps (and you should be!) you’ll want to code these values directly into your Kubernetes manifest, e.g.,
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: quote
+  annotations:
+    a8r.io/owner: “@sally”
+spec:
+  ports:
+  - name: http
+    port: 80
+    targetPort: 8080
+  selector:
+    app: quote
+```
+
+## A Convention for Annotations
+
+Adopting a common convention for annotations ensures consistency and understandability. Typically, you’ll want to attach the annotation to the service object, as services are the high-level resource that maps most clearly to a team’s responsibility. Namespacing your annotations is also very important. Here is one set of conventions, documented at [a8r.io](https://a8r.io), and reproduced below:
+
+{{< table caption="Annotation convention for human-readable services">}}
+| Annotation                                 | Description                                 |
+| ------------------------------------------ | ------------------------------------------- |
+| `a8r.io/description`                       | Unstructured text description of the service for humans.      |
+| `a8r.io/owner`                             | SSO username (GitHub), email address (linked to GitHub account), or unstructured owner description. |
+| `a8r.io/chat`                              | Slack channel, or link to external chat system. |
+| `a8r.io/bugs`                              | Link to external bug tracker. |
+| `a8r.io/logs`                              | Link to external log viewer. |
+| `a8r.io/documentation`                     | Link to external project documentation. |
+| `a8r.io/repository`                        | Link to external VCS repository. |
+| `a8r.io/support`                           | Link to external support center. |
+| `a8r.io/runbook`                           | Link to external project runbook. |
+| `a8r.io/incidents`                         | Link to external incident dashboard. |
+| `a8r.io/uptime`                            | Link to external uptime dashboard. |
+| `a8r.io/performance`                       | Link to external performance dashboard. |
+| `a8r.io/dependencies`                      | Unstructured text describing the service dependencies for humans. |
+
+
+## Visualizing annotations: Service Catalogs
+
+As the number of microservices and annotations proliferate, running `kubectl describe` can get tedious. Moreover, using `kubectl describe` requires every developer to have some direct access to the Kubernetes cluster. Over the past few years, service catalogs have gained greater visibility in the Kubernetes ecosystem. Popularized by tools such as [Shopify's ServicesDB](https://shopify.engineering/scaling-mobile-development-by-treating-apps-as-services) and [Spotify's System Z](https://dzone.com/articles/modeling-microservices-at-spotify-with-petter-mari), service catalogs are internally-facing developer portals that present critical information about microservices.
+
+Note that these service catalogs should not be confused with the [Kubernetes Service Catalog project](https://svc-cat.io/). Built on the Open Service Broker API, the Kubernetes Service Catalog enables Kubernetes operators to plug in different services (e.g., databases) to their cluster.
+
+## Annotate your services now and thank yourself later
+
+Much like implementing observability within microservice systems, you often don’t realize that you need human service discovery until it’s too late. Don't wait until something is on fire in production to start wishing you had implemented better metrics and also documented how to get in touch with the part of your organization that looks after it.
+
+There's enormous benefits to building an effective “version 0” service: a [_dancing skeleton_](https://containerjournal.com/topics/container-management/dancing-skeleton-apis-and-microservices/) application with a thin slice of complete functionality that can be deployed to production with a minimal yet effective continuous delivery pipeline.
+
+Adding service annotations should be an essential part of your “version 0” for all of your services. Add them now, and you’ll thank yourself later.
@@ -0,0 +1,80 @@
+---
+layout: blog
+title: 'Graceful Node Shutdown Goes Beta'
+date: 2021-04-21
+slug: graceful-node-shutdown-beta
+---
+
+**Authors:** David Porter (Google), Mrunal Patel (Red Hat), and Tim Bannister (The Scale Factory)
+
+Graceful node shutdown, beta in 1.21, enables kubelet to gracefully evict pods during a node shutdown.
+
+Kubernetes is a distributed system and as such we need to be prepared for inevitable failures — nodes will fail, containers might crash or be restarted, and - ideally - your workloads will be able to withstand these catastrophic events.
+
+One of the common classes of issues are workload failures on node shutdown or restart. The best practice prior to bringing your node down is to [safely drain and cordon your node](/docs/tasks/administer-cluster/safely-drain-node/). This will ensure that all pods running on this node can safely be evicted. An eviction will ensure your pods can follow the expected [pod termination lifecycle](/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination) meaning receiving a SIGTERM in your container and/or running `preStopHooks`.
+
+Prior to Kubernetes 1.20 (when graceful node shutdown was introduced as an alpha feature), safe node draining was not easy: it required users to manually take action and drain the node beforehand. If someone or something shut down your node without draining it first, most likely your pods would not be safely evicted from your node and shutdown abruptly. Other services talking to those pods might see errors due to the pods exiting abruptly. Some examples of this situation may be caused by a reboot due to security patches or preemption of short lived cloud compute instances.
+
+Kubernetes 1.21 brings graceful node shutdown to beta. Graceful node shutdown gives you more control over some of those unexpected shutdown situations. With graceful node shutdown, the kubelet is aware of underlying system shutdown events and can propagate these events to pods, ensuring containers can shut down as gracefully as possible. This gives the containers a chance to checkpoint their state or release back any resources they are holding.
+
+Note, that for the best availability, even with graceful node shutdown, you should still design your deployments to be resilient to node failures.
+
+## How does it work?
+On Linux, your system can shut down in many different situations. For example:
+* A user or script running `shutdown -h now` or `systemctl poweroff` or `systemctl reboot`.
+* Physically pressing a power button on the machine.
+* Stopping a VM instance on a cloud provider, e.g. `gcloud compute instances stop` on GCP.
+* A Preemptible VM or Spot Instance that your cloud provider can terminate unexpectedly, but with a brief warning.
+
+
+Many of these situations can be unexpected and there is no guarantee that a cluster administrator drained the node prior to these events. With the graceful node shutdown feature, kubelet uses a systemd mechanism called ["Inhibitor Locks"](https://www.freedesktop.org/wiki/Software/systemd/inhibit) to allow draining in most cases. Using Inhibitor Locks, kubelet instructs systemd to postpone system shutdown for a specified duration, giving a chance for the node to drain and evict pods on the system.
+
+Kubelet makes use of this mechanism to ensure your pods will be terminated cleanly. When the kubelet starts, it acquires a systemd delay-type inhibitor lock. When the system is about to shut down, the kubelet can delay that shutdown for a configurable, short duration utilizing the delay-type inhibitor lock it acquired earlier. This gives your pods extra time to  terminate. As a result, even during unexpected shutdowns, your application will receive a SIGTERM, [preStop hooks](/docs/concepts/containers/container-lifecycle-hooks/#container-hooks) will execute, and kubelet will properly update `Ready` node condition and respective pod statuses to the api-server.
+
+For example, on a node with graceful node shutdown enabled, you can see that the inhibitor lock is taken by the kubelet:
+
+```
+kubelet-node ~ # systemd-inhibit --list
+    Who: kubelet (UID 0/root, PID 1515/kubelet)
+    What: shutdown
+    Why: Kubelet needs time to handle node shutdown
+    Mode: delay
+
+1 inhibitors listed.
+```
+
+One important consideration we took when designing this feature is that not all pods are created equal. For example, some of the pods running on a node such as a logging related daemonset should stay running as long as possible to capture important logs during the shutdown itself. As a result, pods are split into two categories: "regular" and "critical". [Critical pods](/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/#marking-pod-as-critical) are those that have `priorityClassName` set to `system-cluster-critical` or `system-node-critical`; all other pods are considered regular.
+
+In our example, the logging DaemonSet would run as a critical pod. During the graceful node shutdown, regular pods are terminated first, followed by critical pods. As an example, this would allow a critical pod associated with a logging daemonset to continue functioning, and collecting logs during the termination of regular pods.
+
+We will evaluate during the beta phase if we need more flexibility for different pod priority classes and add support if needed, please let us know if you have some scenarios in mind.
+
+
+## How do I use it?
+
+Graceful node shutdown is controlled with the `GracefulNodeShutdown` [feature gate](/docs/reference/command-line-tools-reference/feature-gates) and is enabled by default in Kubernetes 1.21.
+
+You can configure the graceful node shutdown behavior using two kubelet configuration options: `ShutdownGracePeriod` and `ShutdownGracePeriodCriticalPods`. To configure these options, you edit the kubelet configuration file that is passed to kubelet via the `--config` flag; for more details, refer to [Set kubelet parameters via a configuration file](/docs/tasks/administer-cluster/kubelet-config-file/).
+
+
+During a shutdown, kubelet terminates pods in two phases. You can configure how long each of these phases lasts.
+1. Terminate regular pods running on the node.
+2. Terminate critical pods running on the node.
+
+The settings that control the duration of shutdown are:
+* `ShutdownGracePeriod`
+    * Specifies the total duration that the node should delay the shutdown by. This is the total grace period for pod termination for both regular and critical pods.
+* `ShutdownGracePeriodCriticalPods`
+    * Specifies the duration used to terminate critical pods during a node shutdown. This should be less than `ShutdownGracePeriod`.
+
+For example, if `ShutdownGracePeriod=30s`, and `ShutdownGracePeriodCriticalPods=10s`, kubelet will delay the node shutdown by 30 seconds. During this time, the first 20 seconds (30-10) would be reserved for gracefully terminating normal pods, and the last 10 seconds would be reserved for terminating critical pods.
+
+Note that by default, both configuration options described above, `ShutdownGracePeriod` and `ShutdownGracePeriodCriticalPods` are set to zero, so you will need to configure them as appropriate for your environment to activate graceful node shutdown functionality.
+
+## How can I learn more?
+* Read the [documentation](/docs/concepts/architecture/nodes/#graceful-node-shutdown)
+* Read the enhancement proposal, [KEP 2000](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2000-graceful-node-shutdown)
+* View the [code](https://github.com/kubernetes/kubernetes/tree/release-1.21/pkg/kubelet/nodeshutdown)
+
+## How do I get involved?
+Your feedback is always welcome! SIG Node meets regularly and can be reached via [Slack](https://slack.k8s.io) (channel `#sig-node`), or the SIG's [mailing list](https://github.com/kubernetes/community/tree/master/sig-node#contact)
@@ -0,0 +1,197 @@
+
+---
+layout: blog
+title: 'Evolving Kubernetes networking with the Gateway API'
+date: 2021-04-22
+slug: evolving-kubernetes-networking-with-the-gateway-api
+---
+
+**Authors:** Mark Church (Google), Harry Bagdi (Kong), Daneyon Hanson (Red Hat), Nick Young (VMware), Manuel Zapf (Traefik Labs)
+
+The Ingress resource is one of the many Kubernetes success stories. It created a [diverse ecosystem of Ingress controllers](/docs/concepts/services-networking/ingress-controllers/) which were used across hundreds of thousands of clusters in a standardized and consistent way. This standardization helped users adopt Kubernetes. However, five years after the creation of Ingress, there are signs of fragmentation into different but [strikingly similar CRDs](https://dave.cheney.net/paste/ingress-is-dead-long-live-ingressroute.pdf) and [overloaded annotations](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/). The same portability that made Ingress pervasive also limited its future.
+
+It was at Kubecon 2019 San Diego when a passionate group of contributors gathered to discuss the [evolution of Ingress](https://static.sched.com/hosted_files/kccncna19/a5/Kubecon%20San%20Diego%202019%20-%20Evolving%20the%20Kubernetes%20Ingress%20APIs%20to%20GA%20and%20Beyond%20%5BPUBLIC%5D.pdf). The discussion overflowed to the hotel lobby across the street and what came out of it would later be known as the [Gateway API](https://gateway-api.sigs.k8s.io). This discussion was based on a few key assumptions:
+
+1. The API standards underlying route matching, traffic management, and service exposure are commoditized and provide little value to their implementers and users as custom APIs
+2. It’s possible to represent L4/L7 routing and traffic management through common core API resources
+3. It’s possible to provide extensibility for more complex capabilities in a way that does not sacrifice the user experience of the core API
+
+
+## Introducing the Gateway API
+
+This led to design principles that allow the Gateway API to improve upon Ingress:
+
+- **Expressiveness** - In addition to HTTP host/path matching and TLS, Gateway API can express capabilities like HTTP header manipulation, traffic weighting & mirroring, TCP/UDP routing, and other capabilities that were only possible in Ingress through custom annotations.
+- **Role-oriented design** - The API resource model reflects the separation of responsibilities that is common in routing and Kubernetes service networking.
+- **Extensibility** - The resources allow arbitrary configuration attachment at various layers within the API. This makes granular customization possible at the most appropriate places.
+- **Flexible conformance** - The Gateway API defines varying conformance levels - core (mandatory support), extended (portable if supported), and custom (no portability guarantee), known together as [flexible conformance](https://gateway-api.sigs.k8s.io/concepts/guidelines/#conformance). This promotes a highly portable core API (like Ingress) that still gives flexibility for Gateway controller implementers.  
+
+### What does the Gateway API look like?
+
+The Gateway API introduces a few new resource types:
+
+- **[GatewayClasses](https://gateway-api.sigs.k8s.io/references/spec/#networking.x-k8s.io/v1alpha1.GatewayClass)** are cluster-scoped resources that act as templates to explicitly define behavior for Gateways derived from them. This is similar in concept to StorageClasses, but for networking data-planes.
+- **[Gateways](https://gateway-api.sigs.k8s.io/references/spec/#networking.x-k8s.io/v1alpha1.Gateway)** are the deployed instances of GatewayClasses. They are the logical representation of the data-plane which performs routing, which may be in-cluster proxies, hardware LBs, or cloud LBs.
+- **Routes** are not a single resource, but represent many different protocol-specific Route resources. The [HTTPRoute](https://gateway-api.sigs.k8s.io/references/spec/#networking.x-k8s.io/v1alpha1.HTTPRoute) has matching, filtering, and routing rules that get applied to Gateways that can process HTTP and HTTPS traffic. Similarly, there are [TCPRoutes](https://gateway-api.sigs.k8s.io/references/spec/#networking.x-k8s.io/v1alpha1.TCPRoute), [UDPRoutes](https://gateway-api.sigs.k8s.io/references/spec/#networking.x-k8s.io/v1alpha1.UDPRoute), and [TLSRoutes](https://gateway-api.sigs.k8s.io/references/spec/#networking.x-k8s.io/v1alpha1.TLSRoute) which also have protocol-specific semantics. This model also allows the Gateway API to incrementally expand its protocol support in the future.
+
+![The resources of the Gateway API](gateway-api-resources.png)
+
+### Gateway Controller Implementations
+
+The good news is that although Gateway is in [Alpha](https://github.com/kubernetes-sigs/gateway-api/releases), there are already several [Gateway controller implementations](https://gateway-api.sigs.k8s.io/references/implementations/) that you can run. Since it’s a standardized spec, the following example could be run on any of them and should function the exact same way. Check out [getting started](https://gateway-api.sigs.k8s.io/guides/getting-started/) to see how to install and use one of these Gateway controllers.
+
+## Getting Hands-on with the Gateway API
+
+In the following example, we’ll demonstrate the relationships between the different API Resources and walk you through a common use case:
+
+*   Team foo has their app deployed in the foo Namespace. They need to control the routing logic for the different pages of their app.
+*   Team bar is running in the bar Namespace. They want to be able to do blue-green rollouts of their application to reduce risk.
+*   The platform team is responsible for managing the load balancer and network security of all the apps in the Kubernetes cluster.
+
+The following foo-route does path matching to various Services in the foo Namespace and also has a default route to a 404 server. This exposes foo-auth and foo-home Services via `foo.example.com/login` and `foo.example.com/home` respectively.:
+
+
+```yaml
+kind: HTTPRoute
+apiVersion: networking.x-k8s.io/v1alpha1
+metadata:
+  name: foo-route
+  namespace: foo
+  labels:
+    gateway: external-https-prod
+spec:
+  hostnames:
+  - "foo.example.com"
+  rules:
+  - matches:
+    - path:
+        type: Prefix
+        value: /login
+    forwardTo:
+    - serviceName: foo-auth
+      port: 8080
+  - matches:
+    - path:
+        type: Prefix
+        value: /home
+    forwardTo:
+    - serviceName: foo-home
+      port: 8080
+  - matches:
+    - path:
+        type: Prefix
+        value: /
+    forwardTo:
+    - serviceName: foo-404
+      port: 8080
+```
+
+
+The bar team, operating in the bar Namespace of the same Kubernetes cluster, also wishes to expose their application to the internet, but they also want to control their own canary and blue-green rollouts. The following HTTPRoute is configured for the following behavior:
+
+*   For traffic to `bar.example.com`:
+    *   Send 90% of the traffic to bar-v1
+    *   Send 10% of the traffic to bar-v2
+*   For traffic to `bar.example.com` with the HTTP header `env: canary`:
+    
+    *   Send all the traffic to bar-v2
+    
+![The routing rules configured for the bar-v1 and bar-v2 Services](httproute.png)
+
+
+
+```yaml
+kind: HTTPRoute
+apiVersion: networking.x-k8s.io/v1alpha1
+metadata:
+  name: bar-route
+  namespace: bar
+  labels:
+    gateway: external-https-prod
+spec:
+  hostnames:
+  - "bar.example.com"
+  rules:
+  - forwardTo:
+    - serviceName: bar-v1
+      port: 8080
+      weight: 90
+    - serviceName: bar-v2
+      port: 8080
+      weight: 10
+  - matches:
+    - headers:
+        values:
+          env: canary
+    forwardTo:
+    - serviceName: bar-v2
+      port: 8080
+```
+
+
+
+### Route and Gateway Binding
+
+So we have two HTTPRoutes matching and routing traffic to different Services. You might be wondering, where are these Services accessible? Through which networks or IPs are they exposed? 
+
+How Routes are exposed to clients is governed by [Route binding](https://gateway-api.sigs.k8s.io/concepts/api-overview/#route-binding), which describes how Routes and Gateways create a bidirectional relationship between each other. When Routes are bound to a Gateway it means their collective routing rules are configured on the underlying load balancers or proxies and the Routes are accessible through the Gateway. Thus, a Gateway is a logical representation of a networking data plane that can be configured through Routes.
+
+
+![How Routes bind with Gateways](route-binding.png )
+
+### Administrative Delegation
+
+The split between Gateway and Route resources allows the cluster administrator to delegate some of the routing configuration to individual teams while still retaining centralized control. The following Gateway resource exposes HTTPS on port 443 and terminates all traffic on the port with a certificate controlled by the cluster administrator.
+
+
+```yaml
+kind: Gateway
+apiVersion: networking.x-k8s.io/v1alpha1
+metadata:
+  name: prod-web
+spec:
+  gatewayClassName: acme-lb
+  listeners:  
+  - protocol: HTTPS
+    port: 443
+    routes:
+      kind: HTTPRoute
+      selector:
+        matchLabels:
+          gateway: external-https-prod
+      namespaces:
+        from: All
+    tls:
+      certificateRef:
+        name: admin-controlled-cert
+```
+
+
+The following HTTPRoute shows how the Route can ensure it matches the Gateway's selector via it’s `kind` (HTTPRoute) and resource labels (`gateway=external-https-prod`). 
+
+
+```yaml
+# Matches the required kind selector on the Gateway
+kind: HTTPRoute
+apiVersion: networking.x-k8s.io/v1alpha1
+metadata:
+  name: foo-route
+  namespace: foo-ns
+  labels:
+
+    # Matches the required label selector on the Gateway
+    gateway: external-https-prod
+...
+```
+
+### Role Oriented Design
+
+When you put it all together, you have a single load balancing infrastructure that can be safely shared by multiple teams. The Gateway API not only a more expressive API for advanced routing, but is also a role-oriented API, designed for multi-tenant infrastructure. Its extensibility ensures that it will evolve for future use-cases while preserving portability. Ultimately these characteristics will allow Gateway API to adapt to different organizational models and implementations well into the future.
+
+### Try it out and get involved
+
+There are many resources to check out to learn more. 
+
+*   Check out the [user guides](https://gateway-api.sigs.k8s.io/guides/getting-started/) to see what use-cases can be addressed. 
+*   Try out one of the [existing Gateway controllers ](https://gateway-api.sigs.k8s.io/references/implementations/)
+*   Or [get involved](https://gateway-api.sigs.k8s.io/contributing/community/) and help design and influence the future of Kubernetes service networking!
@@ -0,0 +1,80 @@
+---
+layout: blog
+title: 'Kubernetes 1.21: Metrics Stability hits GA'
+date: 2021-04-23
+slug: kubernetes-release-1.21-metrics-stability-ga
+---
+
+**Authors**: Han Kang (Google), Elana Hashman (Red Hat)
+
+Kubernetes 1.21 marks the graduation of the metrics stability framework and along with it, the first officially supported stable metrics. Not only do stable metrics come with supportability guarantees, the metrics stability framework brings escape hatches that you can use if you encounter problematic metrics.
+
+See the list of [stable Kubernetes metrics here](https://github.com/kubernetes/kubernetes/blob/master/test/instrumentation/testdata/stable-metrics-list.yaml)
+
+### What are stable metrics and why do we need them?
+A stable metric is one which, from a consumption point of view, can be reliably consumed across a number of Kubernetes versions without risk of ingestion failure.
+
+Metrics stability is an ongoing community concern. Cluster monitoring infrastructure often assumes the stability of some control plane metrics, so we have introduced a mechanism for versioning metrics as a proper API, with stability guarantees around a formal metrics deprecation process.
+
+### What are the stability levels for metrics?
+
+Metrics can currently have one of two stability levels: alpha or stable.
+
+_Alpha metrics_ have no stability guarantees; as such they can be modified or deleted at any time. At this time, all Kubernetes metrics implicitly fall into this category.
+
+_Stable metrics_ can be guaranteed to not change, except that the metric may become marked deprecated for a future Kubernetes version. By not change, we mean three things:
+
+1. the metric itself will not be deleted or renamed
+2. the type of metric will not be modified
+3. no labels can be added or removed from this metric
+
+From an ingestion point of view, it is backwards-compatible to add or remove possible values for labels which already do exist, but not labels themselves. Therefore, adding or removing values from an existing label is permitted. Stable metrics can also be marked as deprecated for a future Kubernetes version, since this is tracked in a metadata field and does not actually change the metric itself.
+
+Removing or adding labels from stable metrics is not permitted. In order to add or remove a label from an existing stable metric, one would have to introduce a new metric and deprecate the stable one; otherwise this would violate compatibility agreements.
+
+
+#### How are metrics deprecated?
+
+While deprecation policies only affect stability guarantees for stable metrics (and not alpha ones), deprecation information may be optionally provided on alpha metrics to help component owners inform users of future intent and assist with transition plans.
+
+A stable metric undergoing the deprecation process signals that the metric will eventually be deleted. The metrics deprecation lifecycle looks roughly like this (with each stage representing a Kubernetes release):
+
+![Stable metric → Deprecated metric → Hidden metric → Deletion](lifecycle-metric.png)
+
+_Deprecated metrics_ have the same stability guarantees of their stable counterparts. If a stable metric is deprecated, then a deprecated stable metric is guaranteed to not change. When deprecating a stable metric, a future Kubernetes release is specified as the point from which the metric will be considered deprecated.
+
+Deprecated metrics will have their description text prefixed with a deprecation notice string “(Deprecated from x.y)” and a warning log will be emitted during metric registration, in the spirit of the official Kubernetes deprecation policy.
+
+Like their stable metric counterparts, deprecated metrics will be automatically registered to the metrics endpoint. On a subsequent release (when the metric's deprecatedVersion is equal to _current\_kubernetes\_version - 4_)), a deprecated metric will become a _hidden_ metric. _Hidden metrics_ are not automatically registered, and hence are hidden by default from end users. These hidden metrics can be explicitly re-enabled for one release after they reach the hidden state, to provide a migration path for cluster operators.
+
+
+#### As an owner of a Kubernetes component, how do I add stable metrics?
+
+During metric instantiation, stability can be specified by setting the metadata field, StabilityLevel, to “Stable”. When a StabilityLevel is not explicitly set, metrics default to “Alpha” stability. Note that metrics which have fields determined at runtime cannot be marked as Stable. Stable metrics will be detected during static analysis during the pre-commit phase, and must be reviewed by sig-instrumentation.
+
+```golang
+var metricDefinition = kubemetrics.CounterOpts{
+    Name: "some_metric",
+    Help: "some description",
+    StabilityLevel: kubemetrics.STABLE,
+}
+```
+For more examples of setting metrics stability and deprecation, see the [Metrics Stability KEP](http://bit.ly/metrics-stability).
+
+
+### How do I get involved?
+
+This project, like all of Kubernetes, is the result of hard work by many contributors from diverse backgrounds working together.
+We offer a huge thank you to all the contributors in Kubernetes community who helped review the design and implementation of the project, including but not limited to the following:
+
+- Han Kang (logicalhan)
+- Frederic Branczyk (brancz)
+- Marek Siarkowicz (serathius)
+- Elana Hashman (ehashman)
+- Solly Ross (DirectXMan12)
+- Stefan Schimanski (sttts)
+- David Ashpole (dashpole)
+- Yuchen Zhou (yoyinzyc)
+- Yu Yi (erain)
+
+If you’re interested in getting involved with the design and development of instrumentation or any part of the Kubernetes metrics system, join the [Kubernetes Instrumentation Special Interest Group (SIG)](https://github.com/kubernetes/community/tree/master/sig-instrumentation). We’re rapidly growing and always welcome new contributors.
@@ -0,0 +1,268 @@
+---
+layout: blog
+title: 'Using Finalizers to Control Deletion'
+date: 2021-05-14
+slug: using-finalizers-to-control-deletion
+---
+
+**Authors:** Aaron Alpar (Kasten)
+
+Deleting objects in Kubernetes can be challenging. You may think you’ve deleted something, only to find it still persists. While issuing a `kubectl delete` command and hoping for the best might work for day-to-day operations, understanding how Kubernetes `delete` commands operate will help you understand why some objects linger after deletion. 
+
+In this post, I’ll look at:
+
+- What properties of a resource govern deletion
+- How finalizers and owner references impact object deletion
+- How the propagation policy can be used to change the order of deletions
+- How deletion works, with examples
+
+For simplicity, all examples will use ConfigMaps and basic shell commands to demonstrate the process. We’ll explore how the commands work and discuss repercussions and results from using them in practice.
+
+## The basic `delete`
+
+Kubernetes has several different commands you can use that allow you to create, read, update, and delete objects. For the purpose of this blog post, we’ll focus on four `kubectl` commands: `create`, `get`, `patch`, and `delete`. 
+
+Here are examples of the basic `kubectl delete` command:
+
+```
+kubectl create configmap mymap
+configmap/mymap created
+```
+
+```
+kubectl get configmap/mymap
+NAME    DATA   AGE
+mymap   0      12s
+```
+
+```
+kubectl delete configmap/mymap
+configmap "mymap" deleted
+```
+
+```
+kubectl get configmap/mymap
+Error from server (NotFound): configmaps "mymap" not found
+```
+
+Shell commands preceded by `$` are followed by their output. You can see that we begin with a `kubectl create configmap mymap`, which will create the empty configmap `mymap`. Next, we need to `get` the configmap to prove it exists. We can then delete that configmap. Attempting to `get` it again produces an HTTP 404 error, which means the configmap is not found.
+
+The state diagram for the basic `delete` command is very simple:
+
+
+{{<figure width="495" src="/images/blog/2021-05-14-using-finalizers-to-control-deletion/state-diagram-delete.png" caption="State diagram for delete">}}
+
+Although this operation is straightforward, other factors may interfere with the deletion, including finalizers and owner references.
+
+## Understanding Finalizers
+
+When it comes to understanding resource deletion in Kubernetes, knowledge of how finalizers work is helpful and can help you understand why some objects don’t get deleted. 
+
+Finalizers are keys on resources that signal pre-delete operations. They control the garbage collection on resources, and are designed to alert controllers what cleanup operations to perform prior to removing a resource. However, they don’t necessarily name code that should be executed; finalizers on resources are basically just lists of keys much like annotations. Like annotations, they can be manipulated.
+
+Some common finalizers you’ve likely encountered are:
+
+- `kubernetes.io/pv-protection`
+- `kubernetes.io/pvc-protection`
+
+The finalizers above are used on volumes to prevent accidental deletion. Similarly, some finalizers can be used to prevent deletion of any resource but are not managed by any controller.
+
+Below with a custom configmap, which has no properties but contains a finalizer:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mymap
+  finalizers:
+  - kubernetes
+EOF
+```
+
+The configmap resource controller doesn't understand what to do with the `kubernetes` finalizer key. I term these “dead” finalizers for configmaps as it is normally used on namespaces. Here’s what happen upon attempting to delete the configmap:
+
+```
+kubectl delete configmap/mymap &
+configmap "mymap" deleted
+jobs
+[1]+  Running kubectl delete configmap/mymap
+```
+
+Kubernetes will report back that the object has been deleted, however, it hasn’t been deleted in a traditional sense. Rather, it’s in the process of deletion. When we attempt to `get` that object again, we discover the object has been modified to include the deletion timestamp. 
+
+```
+kubectl get configmap/mymap -o yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  creationTimestamp: "2020-10-22T21:30:18Z"
+  deletionGracePeriodSeconds: 0
+  deletionTimestamp: "2020-10-22T21:30:34Z"
+  finalizers:
+  - kubernetes
+  name: mymap
+  namespace: default
+  resourceVersion: "311456"
+  selfLink: /api/v1/namespaces/default/configmaps/mymap
+  uid: 93a37fed-23e3-45e8-b6ee-b2521db81638
+```
+
+In short, what’s happened is that the object was updated, not deleted. That’s because Kubernetes saw that the object contained finalizers and put it into a read-only state. The deletion timestamp signals that the object can only be read, with the exception of removing the finalizer key updates. In other words, the deletion will not be complete until we edit the object and remove the finalizer.
+
+Here's a demonstration of using the `patch` command to remove finalizers. If we want to delete an object, we can simply patch it on the command line to remove the finalizers. In this way, the deletion that was running in the background will complete and the object will be deleted. When we attempt to `get` that configmap, it will be gone. 
+
+```
+kubectl patch configmap/mymap \
+    --type json \
+    --patch='[ { "op": "remove", "path": "/metadata/finalizers" } ]'
+configmap/mymap patched
+[1]+  Done  kubectl delete configmap/mymap
+
+kubectl get configmap/mymap -o yaml
+Error from server (NotFound): configmaps "mymap" not found
+```
+
+Here's a state diagram for finalization: 
+
+{{<figure width="617" src="/images/blog/2021-05-14-using-finalizers-to-control-deletion/state-diagram-finalize.png" caption="State diagram for finalize">}}
+
+So, if you attempt to delete an object that has a finalizer on it, it will remain in finalization until the controller has removed the finalizer keys or the finalizers are removed using Kubectl. Once that finalizer list is empty, the object can actually be reclaimed by Kubernetes and put into a queue to be deleted from the registry.
+
+## Owner References
+
+Owner references describe how groups of objects are related. They are properties on resources that specify the relationship to one another, so entire trees of resources can be deleted.
+
+Finalizer rules are processed when there are owner references. An owner reference consists of a name and a UID. Owner references link resources within the same namespace, and it also needs a UID for that reference to work. Pods typically have owner references to the owning replica set. So, when deployments or stateful sets are deleted, then the child replica sets and pods are deleted in the process. 
+
+Here are some examples of owner references and how they work. In the first example, we create a parent object first, then the child. The result is a very simple configmap that contains an owner reference to its parent:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mymap-parent
+EOF
+CM_UID=$(kubectl get configmap mymap-parent -o jsonpath="{.metadata.uid}")
+
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mymap-child
+  ownerReferences:
+  - apiVersion: v1
+    kind: ConfigMap
+    name: mymap-parent
+    uid: $CM_UID
+EOF
+```
+
+Deleting the child object when an owner reference is involved does not delete the parent:
+
+```
+kubectl get configmap
+NAME           DATA   AGE
+mymap-child    0      12m4s
+mymap-parent   0      12m4s
+
+kubectl delete configmap/mymap-child
+configmap "mymap-child" deleted
+
+kubectl get configmap
+NAME           DATA   AGE
+mymap-parent   0      12m10s
+```
+
+In this example, we re-created the parent-child configmaps from above. Now, when deleting from the parent (instead of the child) with an owner reference from the child to the parent, when we `get` the configmaps, none are in the namespace:
+
+```
+kubectl get configmap
+NAME           DATA   AGE
+mymap-child    0      10m2s
+mymap-parent   0      10m2s
+
+kubectl delete configmap/mymap-parent
+configmap "mymap-parent" deleted
+
+kubectl get configmap
+No resources found in default namespace.
+```
+
+To sum things up, when there's an override owner reference from a child to a parent, deleting the parent deletes the children automatically. This is called `cascade`. The default for cascade is `true`, however, you can use the --cascade=false option for `kubectl delete` to delete an object and orphan its children. 
+
+In the following example, there is a parent and a child. Notice the owner references are still included. If I delete the parent using --cascade=false, the parent is deleted but the child still exists:
+
+```
+kubectl get configmap
+NAME           DATA   AGE
+mymap-child    0      13m8s
+mymap-parent   0      13m8s
+
+kubectl delete --cascade=false configmap/mymap-parent
+configmap "mymap-parent" deleted
+
+kubectl get configmap
+NAME          DATA   AGE
+mymap-child   0      13m21s
+```
+
+The --cascade option links to the propagation policy in the API, which allows you to change the order in which objects are deleted within a tree. In the following example uses API access to craft a custom delete API call with the background propagation policy:
+
+```
+kubectl proxy --port=8080 &
+Starting to serve on 127.0.0.1:8080
+
+curl -X DELETE \
+  localhost:8080/api/v1/namespaces/default/configmaps/mymap-parent \
+  -d '{ "kind":"DeleteOptions", "apiVersion":"v1", "propagationPolicy":"Background" }' \
+  -H "Content-Type: application/json"
+{
+  "kind": "Status",
+  "apiVersion": "v1",
+  "metadata": {},
+  "status": "Success",
+  "details": { ... }
+}
+```
+
+Note that the propagation policy cannot be specified on the command line using kubectl. You have to specify it using a custom API call. Simply create a proxy, so you have access to the API server from the client, and execute a `curl` command with just a URL to execute that `delete` command. 
+
+There are three different options for the propagation policy:
+
+- `Foreground`: Children are deleted before the parent (post-order)
+- `Background`: Parent is deleted before the children (pre-order)
+- `Orphan`: Owner references are ignored
+
+Keep in mind that when you delete an object and owner references have been specified, finalizers will be honored in the process. This can result in trees of objects persisting, and you end up with a partial deletion. At that point, you have to look at any existing owner references on your objects, as well as any finalizers, to understand what’s happening. 
+
+## Forcing a Deletion of a Namespace
+
+There's one situation that may require forcing finalization for a namespace. If you've deleted a namespace and you've cleaned out all of the objects under it, but the namespace still exists, deletion can be forced by updating the namespace subresource, `finalize`. This informs the namespace controller that it needs to remove the finalizer from the namespace and perform any cleanup:
+
+```
+cat <<EOF | curl -X PUT \
+  localhost:8080/api/v1/namespaces/test/finalize \
+  -H "Content-Type: application/json" \
+  --data-binary @-
+{
+  "kind": "Namespace",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "test"
+  },
+  "spec": {
+    "finalizers": null
+  }
+}
+EOF
+```
+
+This should be done with caution as it may delete the namespace only and leave orphan objects within the, now non-exiting, namespace - a confusing state for Kubernetes. If this happens, the namespace can be re-created manually and sometimes the orphaned objects will re-appear under the just-created namespace which will allow manual cleanup and recovery.
+
+## Key Takeaways
+
+As these examples demonstrate, finalizers can get in the way of deleting resources in Kubernetes, especially when there are parent-child relationships between objects. Often, there is a reason for adding a finalizer into the code, so you should always investigate before manually deleting it. Owner references allow you to specify and remove trees of resources, although finalizers will be honored in the process. Finally, the propagation policy can be used to specify the order of deletion via a custom API call, giving you control over how objects are deleted. Now that you know a little more about how deletions work in Kubernetes, we recommend you try it out on your own, using a test cluster. 
+
+{{< youtube class="youtube-quote-sm" id="F7-ZxWwf4sY" title="Clean Up Your Room! What Does It Mean to Delete Something in K8s">}}
@@ -1 +1 @@
-<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:#fbfbfb;}.cls-2{fill:#64c3a5;}.cls-2,.cls-3,.cls-4,.cls-5{fill-rule:evenodd;}.cls-3{fill:#e9d661;}.cls-4{fill:#0582bd;}.cls-5{fill:#808285;}</style></defs><title>kubernetes.io-logos</title><rect class="cls-1" x="-3.21396" y="-3.67916" width="223.25536" height="134.51136"/><path class="cls-2" d="M109.66922,82.43741A19.57065,19.57065,0,0,1,90.35516,66.01248a19.57588,19.57588,0,0,1,17.35089-16.32394A19.571,19.571,0,0,0,96.842,72.59509a13.28048,13.28048,0,0,0,12.82726,9.84232"/><path class="cls-3" d="M107.70538,49.68854q.96877-.09621,1.96384-.09714a19.59034,19.59034,0,0,1,5.971.93243c-1.19879,3.42362-2.9112,5.60261-4.58257,5.4263a13.51316,13.51316,0,0,0-1.38838-.07249,13.2911,13.2911,0,0,0-12.94535,16.2528,19.572,19.572,0,0,1,10.98151-22.4419"/><path class="cls-4" d="M118.5549,59.2884c-1.24844-1.1244-.77543-3.85447.96614-7.03615a19.56137,19.56137,0,1,1-29.16566,13.762,19.57091,19.57091,0,0,0,19.31384,16.42316,13.27982,13.27982,0,0,0,8.88568-23.149"/><path class="cls-5" d="M148.72465,56.68991a6.07242,6.07242,0,0,0-9.10641,5.25828v24.793H133.331v-24.793a12.36017,12.36017,0,0,1,18.53664-10.70293l-3.143,5.44465Zm24.605-2.97568a12.35685,12.35685,0,0,1,21.57057,8.234v24.793h-6.28659v-24.793a6.07039,6.07039,0,1,0-12.14078,0v24.793h-6.28662v-24.793a6.07006,6.07006,0,1,0-12.14012,0v24.793h-6.28747v-24.793a12.35715,12.35715,0,0,1,21.571-8.234m-79.275-11.71068a6.07292,6.07292,0,0,0-9.10642,5.25869v3.15422h6.60732l-2.79511,6.27307H84.94817V86.74119H78.66091v-39.479A12.3589,12.3589,0,0,1,97.19756,36.55965L94.625,42.333l-.57036-.32949ZM20.49076,54.8028a13.15543,13.15543,0,0,1,10.7037-5.2114c7.03714,0,12.74106,5.1005,12.74106,11.39221l-.01237,16.34832c0,6.29168-5.704,11.39208-12.74112,11.39208s-12.741-5.1004-12.741-11.39208c0-5.26947,3.663-9.84144,9.772-11.47815l9.43621-2.52868.01242-2.34149c0-2.82005-2.895-5.106-6.46627-5.106a7.12669,7.12669,0,0,0-5.22007,2.0919l-.66586.73592L20.49076,54.8028ZM37.64921,69.84037l-.00073,7.49156c-.00018,2.81872-2.89507,5.10548-6.46645,5.10548s-6.46627-2.28565-6.46627-5.10548c0-2.41256,2.2001-4.61169,5.09086-5.38735l7.84259-2.10421Zm29.58343-32.952v14.2779a13.83819,13.83819,0,0,0-6.46716-1.57488c-7.03708,0-12.74094,5.1005-12.74094,11.39221V77.33193c0,6.29168,5.70386,11.39208,12.74094,11.39208s12.74117-5.1004,12.74117-11.39208V36.88838Zm0,24.09523V77.33193c0,2.81983-2.89487,5.10548-6.46628,5.10548s-6.46627-2.28565-6.46627-5.10548V60.98361c0-2.82005,2.89487-5.106,6.46627-5.106s6.46628,2.28592,6.46628,5.106"/></svg>
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:transparent;}.cls-2{fill:#64c3a5;}.cls-2,.cls-3,.cls-4,.cls-5{fill-rule:evenodd;}.cls-3{fill:#e9d661;}.cls-4{fill:#0582bd;}.cls-5{fill:#808285;}</style></defs><title>kubernetes.io-logos</title><rect class="cls-1" x="-3.21396" y="-3.67916" width="223.25536" height="134.51136"/><path class="cls-2" d="M109.66922,82.43741A19.57065,19.57065,0,0,1,90.35516,66.01248a19.57588,19.57588,0,0,1,17.35089-16.32394A19.571,19.571,0,0,0,96.842,72.59509a13.28048,13.28048,0,0,0,12.82726,9.84232"/><path class="cls-3" d="M107.70538,49.68854q.96877-.09621,1.96384-.09714a19.59034,19.59034,0,0,1,5.971.93243c-1.19879,3.42362-2.9112,5.60261-4.58257,5.4263a13.51316,13.51316,0,0,0-1.38838-.07249,13.2911,13.2911,0,0,0-12.94535,16.2528,19.572,19.572,0,0,1,10.98151-22.4419"/><path class="cls-4" d="M118.5549,59.2884c-1.24844-1.1244-.77543-3.85447.96614-7.03615a19.56137,19.56137,0,1,1-29.16566,13.762,19.57091,19.57091,0,0,0,19.31384,16.42316,13.27982,13.27982,0,0,0,8.88568-23.149"/><path class="cls-5" d="M148.72465,56.68991a6.07242,6.07242,0,0,0-9.10641,5.25828v24.793H133.331v-24.793a12.36017,12.36017,0,0,1,18.53664-10.70293l-3.143,5.44465Zm24.605-2.97568a12.35685,12.35685,0,0,1,21.57057,8.234v24.793h-6.28659v-24.793a6.07039,6.07039,0,1,0-12.14078,0v24.793h-6.28662v-24.793a6.07006,6.07006,0,1,0-12.14012,0v24.793h-6.28747v-24.793a12.35715,12.35715,0,0,1,21.571-8.234m-79.275-11.71068a6.07292,6.07292,0,0,0-9.10642,5.25869v3.15422h6.60732l-2.79511,6.27307H84.94817V86.74119H78.66091v-39.479A12.3589,12.3589,0,0,1,97.19756,36.55965L94.625,42.333l-.57036-.32949ZM20.49076,54.8028a13.15543,13.15543,0,0,1,10.7037-5.2114c7.03714,0,12.74106,5.1005,12.74106,11.39221l-.01237,16.34832c0,6.29168-5.704,11.39208-12.74112,11.39208s-12.741-5.1004-12.741-11.39208c0-5.26947,3.663-9.84144,9.772-11.47815l9.43621-2.52868.01242-2.34149c0-2.82005-2.895-5.106-6.46627-5.106a7.12669,7.12669,0,0,0-5.22007,2.0919l-.66586.73592L20.49076,54.8028ZM37.64921,69.84037l-.00073,7.49156c-.00018,2.81872-2.89507,5.10548-6.46645,5.10548s-6.46627-2.28565-6.46627-5.10548c0-2.41256,2.2001-4.61169,5.09086-5.38735l7.84259-2.10421Zm29.58343-32.952v14.2779a13.83819,13.83819,0,0,0-6.46716-1.57488c-7.03708,0-12.74094,5.1005-12.74094,11.39221V77.33193c0,6.29168,5.70386,11.39208,12.74094,11.39208s12.74117-5.1004,12.74117-11.39208V36.88838Zm0,24.09523V77.33193c0,2.81983-2.89487,5.10548-6.46628,5.10548s-6.46627-2.28565-6.46627-5.10548V60.98361c0-2.82005,2.89487-5.106,6.46627-5.106s6.46628,2.28592,6.46628,5.106"/></svg>
@@ -1 +1 @@
-<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:#fbfbfb;}.cls-2{fill:#040606;}</style></defs><title>kubernetes.io-54664</title><rect class="cls-1" x="-4.55738" y="-4.48481" width="223.25536" height="134.51136"/><path class="cls-2" d="M169.05853,71.307c.23549,6.07483,5.42615,10.38072,14.09569,10.38072,7.0783,0,12.91766-3.06693,12.91766-9.85009,0-4.71846-2.65436-7.49024-8.78885-8.66954l-4.77685-.94382c-3.06756-.59028-5.19066-1.17992-5.19066-3.0079,0-2.00568,2.06471-2.89047,4.65943-2.89047,3.77525,0,5.3081,1.887,5.42678,4.1288H194.951c-.41258-5.89838-5.1304-9.8501-12.73994-9.8501-7.845,0-12.50382,4.30588-12.50382,9.90912,0,6.84157,5.54483,7.96247,10.3217,8.84662l3.95171.70834c2.83082.53062,4.06977,1.35638,4.06977,3.0079,0,1.47444-1.41541,2.94887-4.77748,2.94887-4.89553,0-6.488-2.53631-6.54705-4.71845Zm-27.4259-5.13164a8.58413,8.58413,0,0,1,8.557-8.61113q.02706-.00009.05409,0a8.58415,8.58415,0,0,1,8.61115,8.557q.00009.02706,0,.0541a8.58413,8.58413,0,0,1-8.55641,8.61177q-.02738.00009-.05473,0a8.58413,8.58413,0,0,1-8.61113-8.557q-.00009-.02738,0-.05473M158.79589,81.098h7.49149V50.95811h-7.49149v2.41825a15.58033,15.58033,0,1,0-9.01133,28.37034q.05285.00018.10567,0a15.47693,15.47693,0,0,0,8.90566-2.77179ZM106.59844,66.17532a8.58413,8.58413,0,0,1,8.557-8.61113q.02706-.00009.05409,0a8.58414,8.58414,0,0,1,8.61114,8.557q.00009.02706,0,.05409a8.58413,8.58413,0,0,1-8.55641,8.61177q-.02738.00009-.05473,0a8.58413,8.58413,0,0,1-8.61113-8.557q-.00009-.02738,0-.05473m17.16387-25.47987V53.37636a15.58048,15.58048,0,1,0-9.01195,28.37034q.05251.00018.105,0a15.48233,15.48233,0,0,0,8.90691-2.77179V81.098h7.49023V40.69545ZM96.21772,50.95811H88.72811V81.098h7.49023Zm0-10.26266H88.72811v7.49023h7.49023ZM60.41805,66.17532a8.58414,8.58414,0,0,1,8.557-8.61113q.02673-.00009.05346,0a8.58414,8.58414,0,0,1,8.61114,8.557q.00009.02706,0,.05409a8.58414,8.58414,0,0,1-8.55642,8.61177q-.02736.00009-.05472,0a8.58412,8.58412,0,0,1-8.61113-8.557q-.00009-.02738,0-.05473M77.58067,40.69545V53.37636A15.58033,15.58033,0,1,0,68.56935,81.7467q.05283.00018.10567,0a15.4769,15.4769,0,0,0,8.90565-2.77179V81.098h7.4915V40.69545ZM25.38259,66.176a8.58414,8.58414,0,0,1,8.557-8.61114q.02736-.00009.05472,0a8.58414,8.58414,0,0,1,8.61114,8.557q.00009.027,0,.05409a8.58414,8.58414,0,0,1-8.55642,8.61176q-.02736.00009-.05472,0a8.58413,8.58413,0,0,1-8.61177-8.55641q-.00009-.02768,0-.05535m17.16388,14.9227H50.038V50.95872H42.54647V53.377a15.58048,15.58048,0,1,0-9.01069,28.37035q.0522.00016.10441,0a15.48032,15.48032,0,0,0,8.90628-2.77179Z"/></svg>
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:transparent;}.cls-2{fill:#040606;}</style></defs><title>kubernetes.io-54664</title><rect class="cls-1" x="-4.55738" y="-4.48481" width="223.25536" height="134.51136"/><path class="cls-2" d="M169.05853,71.307c.23549,6.07483,5.42615,10.38072,14.09569,10.38072,7.0783,0,12.91766-3.06693,12.91766-9.85009,0-4.71846-2.65436-7.49024-8.78885-8.66954l-4.77685-.94382c-3.06756-.59028-5.19066-1.17992-5.19066-3.0079,0-2.00568,2.06471-2.89047,4.65943-2.89047,3.77525,0,5.3081,1.887,5.42678,4.1288H194.951c-.41258-5.89838-5.1304-9.8501-12.73994-9.8501-7.845,0-12.50382,4.30588-12.50382,9.90912,0,6.84157,5.54483,7.96247,10.3217,8.84662l3.95171.70834c2.83082.53062,4.06977,1.35638,4.06977,3.0079,0,1.47444-1.41541,2.94887-4.77748,2.94887-4.89553,0-6.488-2.53631-6.54705-4.71845Zm-27.4259-5.13164a8.58413,8.58413,0,0,1,8.557-8.61113q.02706-.00009.05409,0a8.58415,8.58415,0,0,1,8.61115,8.557q.00009.02706,0,.0541a8.58413,8.58413,0,0,1-8.55641,8.61177q-.02738.00009-.05473,0a8.58413,8.58413,0,0,1-8.61113-8.557q-.00009-.02738,0-.05473M158.79589,81.098h7.49149V50.95811h-7.49149v2.41825a15.58033,15.58033,0,1,0-9.01133,28.37034q.05285.00018.10567,0a15.47693,15.47693,0,0,0,8.90566-2.77179ZM106.59844,66.17532a8.58413,8.58413,0,0,1,8.557-8.61113q.02706-.00009.05409,0a8.58414,8.58414,0,0,1,8.61114,8.557q.00009.02706,0,.05409a8.58413,8.58413,0,0,1-8.55641,8.61177q-.02738.00009-.05473,0a8.58413,8.58413,0,0,1-8.61113-8.557q-.00009-.02738,0-.05473m17.16387-25.47987V53.37636a15.58048,15.58048,0,1,0-9.01195,28.37034q.05251.00018.105,0a15.48233,15.48233,0,0,0,8.90691-2.77179V81.098h7.49023V40.69545ZM96.21772,50.95811H88.72811V81.098h7.49023Zm0-10.26266H88.72811v7.49023h7.49023ZM60.41805,66.17532a8.58414,8.58414,0,0,1,8.557-8.61113q.02673-.00009.05346,0a8.58414,8.58414,0,0,1,8.61114,8.557q.00009.02706,0,.05409a8.58414,8.58414,0,0,1-8.55642,8.61177q-.02736.00009-.05472,0a8.58412,8.58412,0,0,1-8.61113-8.557q-.00009-.02738,0-.05473M77.58067,40.69545V53.37636A15.58033,15.58033,0,1,0,68.56935,81.7467q.05283.00018.10567,0a15.4769,15.4769,0,0,0,8.90565-2.77179V81.098h7.4915V40.69545ZM25.38259,66.176a8.58414,8.58414,0,0,1,8.557-8.61114q.02736-.00009.05472,0a8.58414,8.58414,0,0,1,8.61114,8.557q.00009.027,0,.05409a8.58414,8.58414,0,0,1-8.55642,8.61176q-.02736.00009-.05472,0a8.58413,8.58413,0,0,1-8.61177-8.55641q-.00009-.02768,0-.05535m17.16388,14.9227H50.038V50.95872H42.54647V53.377a15.58048,15.58048,0,1,0-9.01069,28.37035q.0522.00016.10441,0a15.48032,15.48032,0,0,0,8.90628-2.77179Z"/></svg>
@@ -1 +1 @@
-<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:#fbfbfb;}.cls-2{fill:#57565a;}</style></defs><title>kubernetes.io-logos</title><rect class="cls-1" x="-3.37342" y="-3.34411" width="223.25536" height="134.51136"/><path class="cls-2" d="M81.14242,56.12567,77.99668,64.0731h6.29149l-3.14575-7.94743Zm7.0534,17.78238-2.58263-6.49093h-8.9411L74.089,73.90805h-3.775l9.17231-23.114h3.31256l9.17318,23.114Z"/><path class="cls-2" d="M102.9876,62.3189a2.97367,2.97367,0,0,0-2.74772-1.35853A2.91125,2.91125,0,0,0,97.4913,62.3189a3.7299,3.7299,0,0,0-.43087,1.48937v4.23808A3.73648,3.73648,0,0,0,97.4913,69.537a3.40988,3.40988,0,0,0,5.4963,0,7.0611,7.0611,0,0,0,.66165-3.60929,7.3468,7.3468,0,0,0-.66165-3.60884Zm2.94694,8.8744a6.31949,6.31949,0,0,1-5.69466,3.07919,6.89537,6.89537,0,0,1-3.17945-.72845v7.64965H93.68261v-23.379h3.37782v.49738a6.8926,6.8926,0,0,1,3.17945-.72842,6.31935,6.31935,0,0,1,5.69466,3.079,10.006,10.006,0,0,1,1.09252,5.29783,10.16391,10.16391,0,0,1-1.09252,5.23285Z"/><path class="cls-2" d="M118.79367,62.3189a2.97667,2.97667,0,0,0-2.74944-1.35853,2.9108,2.9108,0,0,0-2.74859,1.35853,3.75853,3.75853,0,0,0-.43,1.48937v4.23808a3.76517,3.76517,0,0,0,.43,1.49068,3.41159,3.41159,0,0,0,5.498,0,7.07027,7.07027,0,0,0,.66078-3.60929,7.35645,7.35645,0,0,0-.66078-3.60884Zm2.94608,8.8744a6.32,6.32,0,0,1-5.69552,3.07919,6.88941,6.88941,0,0,1-3.1786-.72845v7.64965h-3.3778v-23.379h3.3778v.49738a6.88664,6.88664,0,0,1,3.1786-.72842,6.31981,6.31981,0,0,1,5.69552,3.079,10.00267,10.00267,0,0,1,1.093,5.29783,10.16449,10.16449,0,0,1-1.093,5.23285Z"/><path class="cls-2" d="M132.96442,54.20565h-3.80825v16.259h3.80825c6.58968,0,6.35717-2.25263,6.35717-8.113,0-5.8944.23251-8.14595-6.35717-8.14595Zm9.60232,13.179c-.828,5.53-4.6687,6.52343-9.60232,6.52343h-7.31875v-23.114h7.31875c4.93362,0,8.7743.96057,9.60232,6.52326a36.3302,36.3302,0,0,1,.26577,5.03426,36.58042,36.58042,0,0,1-.26577,5.033Z"/><polygon class="cls-2" points="146.109 73.908 146.109 57.55 149.52 57.55 149.52 73.908 146.109 73.908 146.109 73.908"/><polygon class="cls-2" points="146.109 54.565 146.109 50.638 149.52 50.638 149.52 54.565 146.109 54.565 146.109 54.565"/><path class="cls-2" d="M161.67111,61.45749a3.97853,3.97853,0,0,0-2.185-.66277c-1.35871,0-3.08048.79467-3.08048,2.352V73.90805h-3.30952V57.71553h3.30952v.49609a7.36373,7.36373,0,0,1,3.08048-.662,8.37517,8.37517,0,0,1,3.676.86135l-1.491,3.04647Z"/><path class="cls-2" d="M173.35993,62.25243a3.00221,3.00221,0,0,0-2.71661-1.39145,3.30382,3.30382,0,0,0-3.609,2.98125h6.72194a4.32873,4.32873,0,0,0-.3963-1.5898ZM166.934,67.11957a5.0516,5.0516,0,0,0,.66295,2.352,3.97682,3.97682,0,0,0,3.345,1.48937,4.29333,4.29333,0,0,0,3.44307-1.55606l2.71705,2.01943a7.84563,7.84563,0,0,1-6.12728,2.78251,7.20688,7.20688,0,0,1-6.22535-2.94833,9.72791,9.72791,0,0,1-1.2589-5.397,9.71813,9.71813,0,0,1,1.2589-5.39724,6.77573,6.77573,0,0,1,5.92674-2.88061,6.353,6.353,0,0,1,5.5313,2.91333c1.19191,1.78819,1.02553,4.53689.99181,6.62261Z"/><path class="cls-2" d="M186.53109,74.17309c-4.70242,0-7.48386-3.54363-7.48386-8.411,0-4.90111,2.78144-8.44475,7.48386-8.44475,2.31728,0,3.84152.62923,6.2258,3.17962l-2.48411,2.18509c-1.78787-1.92-2.51607-2.18509-3.74169-2.18509a3.67433,3.67433,0,0,0-3.14532,1.49064,6.08445,6.08445,0,0,0-.86174,3.77449,6.04264,6.04264,0,0,0,.86174,3.74221,3.59542,3.59542,0,0,0,3.14532,1.48939c1.22562,0,1.95382-.26505,3.74169-2.15137l2.48411,2.18534c-2.38428,2.51638-3.90852,3.14543-6.2258,3.14543Z"/><path class="cls-2" d="M194.70206,57.54967V53.72892l3.24471-.55054v4.37129h2.88082v3.04669h-2.88082V69.6035a2.65978,2.65978,0,0,0,.03327.7282.3886.3886,0,0,0,.3963.23233h2.0878v3.344h-2.0878a3.50427,3.50427,0,0,1-3.07832-1.655,4.8599,4.8599,0,0,1-.596-2.64952V57.54967Z"/><polygon class="cls-2" points="16.87 77.995 30.665 71.993 37.34 52.213 30.585 41.486 16.87 77.995 16.87 77.995"/><polygon class="cls-2" points="30.277 38.06 47.2 64.828 57.151 60.51 41.284 38.06 30.277 38.06 30.277 38.06"/><polygon class="cls-2" points="64.356 59.132 14.131 80.907 24.836 88.213 58.498 73.908 64.356 59.132 64.356 59.132"/></svg>
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:transparent;}.cls-2{fill:#57565a;}</style></defs><title>kubernetes.io-logos</title><rect class="cls-1" x="-3.37342" y="-3.34411" width="223.25536" height="134.51136"/><path class="cls-2" d="M81.14242,56.12567,77.99668,64.0731h6.29149l-3.14575-7.94743Zm7.0534,17.78238-2.58263-6.49093h-8.9411L74.089,73.90805h-3.775l9.17231-23.114h3.31256l9.17318,23.114Z"/><path class="cls-2" d="M102.9876,62.3189a2.97367,2.97367,0,0,0-2.74772-1.35853A2.91125,2.91125,0,0,0,97.4913,62.3189a3.7299,3.7299,0,0,0-.43087,1.48937v4.23808A3.73648,3.73648,0,0,0,97.4913,69.537a3.40988,3.40988,0,0,0,5.4963,0,7.0611,7.0611,0,0,0,.66165-3.60929,7.3468,7.3468,0,0,0-.66165-3.60884Zm2.94694,8.8744a6.31949,6.31949,0,0,1-5.69466,3.07919,6.89537,6.89537,0,0,1-3.17945-.72845v7.64965H93.68261v-23.379h3.37782v.49738a6.8926,6.8926,0,0,1,3.17945-.72842,6.31935,6.31935,0,0,1,5.69466,3.079,10.006,10.006,0,0,1,1.09252,5.29783,10.16391,10.16391,0,0,1-1.09252,5.23285Z"/><path class="cls-2" d="M118.79367,62.3189a2.97667,2.97667,0,0,0-2.74944-1.35853,2.9108,2.9108,0,0,0-2.74859,1.35853,3.75853,3.75853,0,0,0-.43,1.48937v4.23808a3.76517,3.76517,0,0,0,.43,1.49068,3.41159,3.41159,0,0,0,5.498,0,7.07027,7.07027,0,0,0,.66078-3.60929,7.35645,7.35645,0,0,0-.66078-3.60884Zm2.94608,8.8744a6.32,6.32,0,0,1-5.69552,3.07919,6.88941,6.88941,0,0,1-3.1786-.72845v7.64965h-3.3778v-23.379h3.3778v.49738a6.88664,6.88664,0,0,1,3.1786-.72842,6.31981,6.31981,0,0,1,5.69552,3.079,10.00267,10.00267,0,0,1,1.093,5.29783,10.16449,10.16449,0,0,1-1.093,5.23285Z"/><path class="cls-2" d="M132.96442,54.20565h-3.80825v16.259h3.80825c6.58968,0,6.35717-2.25263,6.35717-8.113,0-5.8944.23251-8.14595-6.35717-8.14595Zm9.60232,13.179c-.828,5.53-4.6687,6.52343-9.60232,6.52343h-7.31875v-23.114h7.31875c4.93362,0,8.7743.96057,9.60232,6.52326a36.3302,36.3302,0,0,1,.26577,5.03426,36.58042,36.58042,0,0,1-.26577,5.033Z"/><polygon class="cls-2" points="146.109 73.908 146.109 57.55 149.52 57.55 149.52 73.908 146.109 73.908 146.109 73.908"/><polygon class="cls-2" points="146.109 54.565 146.109 50.638 149.52 50.638 149.52 54.565 146.109 54.565 146.109 54.565"/><path class="cls-2" d="M161.67111,61.45749a3.97853,3.97853,0,0,0-2.185-.66277c-1.35871,0-3.08048.79467-3.08048,2.352V73.90805h-3.30952V57.71553h3.30952v.49609a7.36373,7.36373,0,0,1,3.08048-.662,8.37517,8.37517,0,0,1,3.676.86135l-1.491,3.04647Z"/><path class="cls-2" d="M173.35993,62.25243a3.00221,3.00221,0,0,0-2.71661-1.39145,3.30382,3.30382,0,0,0-3.609,2.98125h6.72194a4.32873,4.32873,0,0,0-.3963-1.5898ZM166.934,67.11957a5.0516,5.0516,0,0,0,.66295,2.352,3.97682,3.97682,0,0,0,3.345,1.48937,4.29333,4.29333,0,0,0,3.44307-1.55606l2.71705,2.01943a7.84563,7.84563,0,0,1-6.12728,2.78251,7.20688,7.20688,0,0,1-6.22535-2.94833,9.72791,9.72791,0,0,1-1.2589-5.397,9.71813,9.71813,0,0,1,1.2589-5.39724,6.77573,6.77573,0,0,1,5.92674-2.88061,6.353,6.353,0,0,1,5.5313,2.91333c1.19191,1.78819,1.02553,4.53689.99181,6.62261Z"/><path class="cls-2" d="M186.53109,74.17309c-4.70242,0-7.48386-3.54363-7.48386-8.411,0-4.90111,2.78144-8.44475,7.48386-8.44475,2.31728,0,3.84152.62923,6.2258,3.17962l-2.48411,2.18509c-1.78787-1.92-2.51607-2.18509-3.74169-2.18509a3.67433,3.67433,0,0,0-3.14532,1.49064,6.08445,6.08445,0,0,0-.86174,3.77449,6.04264,6.04264,0,0,0,.86174,3.74221,3.59542,3.59542,0,0,0,3.14532,1.48939c1.22562,0,1.95382-.26505,3.74169-2.15137l2.48411,2.18534c-2.38428,2.51638-3.90852,3.14543-6.2258,3.14543Z"/><path class="cls-2" d="M194.70206,57.54967V53.72892l3.24471-.55054v4.37129h2.88082v3.04669h-2.88082V69.6035a2.65978,2.65978,0,0,0,.03327.7282.3886.3886,0,0,0,.3963.23233h2.0878v3.344h-2.0878a3.50427,3.50427,0,0,1-3.07832-1.655,4.8599,4.8599,0,0,1-.596-2.64952V57.54967Z"/><polygon class="cls-2" points="16.87 77.995 30.665 71.993 37.34 52.213 30.585 41.486 16.87 77.995 16.87 77.995"/><polygon class="cls-2" points="30.277 38.06 47.2 64.828 57.151 60.51 41.284 38.06 30.277 38.06 30.277 38.06"/><polygon class="cls-2" points="64.356 59.132 14.131 80.907 24.836 88.213 58.498 73.908 64.356 59.132 64.356 59.132"/></svg>
@@ -13,7 +13,7 @@ new_case_study_styles: true
 heading_background: /images/case-studies/appdirect/banner1.jpg
 heading_title_logo: /images/appdirect_logo.png
 subheading: >
-  AppDirect: How AppDirect Supported the 10x Growth of Its Engineering Staff with Kubernetess
+  AppDirect: How AppDirect Supported the 10x Growth of Its Engineering Staff with Kubernetes
 case_study_details:
  - Company: AppDirect
  - Location: San Francisco, California
@@ -1 +1 @@
-<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:#fbfbfb;}</style></defs><title>kubernetes.io-logos</title><rect class="cls-1" x="-3.00001" y="-3.45836" width="223.25536" height="134.51136"/><g id="layer1"><path id="path14" d="M52.57518,56.39988v16.0054h8.94419l-1.2105,3.09349H48.6747V56.39991h3.90048"/><path id="path16" d="M127.89467,65.07506c2.48824-.94148,5.38-3.09349,5.38-6.65769,0-4.43848-3.83322-6.65769-9.28044-6.65769h-7.12845V75.49877h4.23673V65.74759h2.48823l6.59044,9.75118h4.77473l-7.0612-10.42368Zm-4.70747-2.35373h-2.08473V54.58412h2.62273c3.42972,0,5.31272,1.076,5.31272,4.035,0,3.16072-2.28649,4.10223-5.85071,4.10223"/><path id="path18" d="M68.98407,56.39988l.60525,1.47949L62.19187,75.49874h3.63148l1.614-3.90048h8.40619l1.68124,3.90048h4.304L73.3553,56.39988H68.98407Zm-.33625,12.30668,2.89174-7.06121,2.959,7.06121H68.64783"/><path id="path28" d="M41.27726,62.78859a5.40414,5.40414,0,0,0,4.10222-5.24547c0-5.78347-6.99395-5.78347-9.21318-5.78347H28.7016V75.49874h7.12845c4.16947,0,10.89442-1.00874,10.89442-6.85945C46.72447,65.41131,44.77423,63.32659,41.27726,62.78859Zm-5.649-8.13721c3.16072,0,5.31271.33625,5.31271,3.16073,0,3.83322-3.90047,3.63146-5.78345,3.63146h-2.152V54.58412l2.62273.06726Zm.06724,17.75387-2.69-.06726V64.537h2.48823c3.766,0,6.7922.538,6.7922,3.90048,0,3.29522-2.48823,3.96772-6.59046,3.96772"/><path id="path30" d="M98.84287,56.39988h3.96772V75.49874H98.84287Zm12.23941,0-8.06994,9.14595,8.13719,9.95294h4.50571L107.4508,65.34408l7.59921-8.9442h-3.96773"/><path id="path40" d="M80.95449,65.34405c0,6.25421,3.90048,10.49094,9.21318,10.49094a10.18329,10.18329,0,0,0,6.38871-1.95023l-.94149-1.883A9.90036,9.90036,0,0,1,91.17642,73.145c-3.42973,0-5.918-2.48823-5.918-7.66644,0-4.16946,1.54675-6.65769,4.50572-6.65769,1.614,0,2.959.538,3.497,2.152l3.42972-.87425c-.33625-1.345-2.152-3.96771-7.12845-3.96771-4.573,0-8.60794,3.09349-8.60794,9.21318"/><path id="path50" d="M171.67412,56.39988h3.90048V75.49874h-3.90048Zm12.23941,0-8.13721,9.14595,8.13721,9.95294h4.64021L180.282,65.34408l7.5992-8.9442h-3.96771"/><path id="path52" d="M153.71849,65.34405c0,6.25421,3.83322,10.49094,9.07869,10.49094a10.11878,10.11878,0,0,0,6.456-1.95023l-.94148-1.883A9.953,9.953,0,0,1,163.806,73.145c-3.42973,0-5.8507-2.48823-5.8507-7.66644,0-4.16946,1.54674-6.65769,4.573-6.65769a3.35862,3.35862,0,0,1,3.497,2.152l3.36248-.87425c-.33625-1.345-2.08475-3.96771-7.0612-3.96771-4.64021,0-8.60795,3.09349-8.60795,9.21318"/><path id="path54" d="M143.16032,56.13087c-5.17821,0-8.94419,3.09349-8.94419,9.81844s3.56423,9.88568,8.94419,9.88568,8.9442-3.16072,8.9442-9.88568S148.33853,56.13087,143.16032,56.13087Zm0,17.01414c-3.96771,0-4.90922-3.228-4.90922-7.1957,0-3.90048.94148-7.26295,4.90922-7.26295s4.90922,3.36247,4.90922,7.26295c0,3.96772-.94148,7.1957-4.90922,7.1957"/></g></svg>
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:transparent;}</style></defs><title>kubernetes.io-logos</title><rect class="cls-1" x="-3.00001" y="-3.45836" width="223.25536" height="134.51136"/><g id="layer1"><path id="path14" d="M52.57518,56.39988v16.0054h8.94419l-1.2105,3.09349H48.6747V56.39991h3.90048"/><path id="path16" d="M127.89467,65.07506c2.48824-.94148,5.38-3.09349,5.38-6.65769,0-4.43848-3.83322-6.65769-9.28044-6.65769h-7.12845V75.49877h4.23673V65.74759h2.48823l6.59044,9.75118h4.77473l-7.0612-10.42368Zm-4.70747-2.35373h-2.08473V54.58412h2.62273c3.42972,0,5.31272,1.076,5.31272,4.035,0,3.16072-2.28649,4.10223-5.85071,4.10223"/><path id="path18" d="M68.98407,56.39988l.60525,1.47949L62.19187,75.49874h3.63148l1.614-3.90048h8.40619l1.68124,3.90048h4.304L73.3553,56.39988H68.98407Zm-.33625,12.30668,2.89174-7.06121,2.959,7.06121H68.64783"/><path id="path28" d="M41.27726,62.78859a5.40414,5.40414,0,0,0,4.10222-5.24547c0-5.78347-6.99395-5.78347-9.21318-5.78347H28.7016V75.49874h7.12845c4.16947,0,10.89442-1.00874,10.89442-6.85945C46.72447,65.41131,44.77423,63.32659,41.27726,62.78859Zm-5.649-8.13721c3.16072,0,5.31271.33625,5.31271,3.16073,0,3.83322-3.90047,3.63146-5.78345,3.63146h-2.152V54.58412l2.62273.06726Zm.06724,17.75387-2.69-.06726V64.537h2.48823c3.766,0,6.7922.538,6.7922,3.90048,0,3.29522-2.48823,3.96772-6.59046,3.96772"/><path id="path30" d="M98.84287,56.39988h3.96772V75.49874H98.84287Zm12.23941,0-8.06994,9.14595,8.13719,9.95294h4.50571L107.4508,65.34408l7.59921-8.9442h-3.96773"/><path id="path40" d="M80.95449,65.34405c0,6.25421,3.90048,10.49094,9.21318,10.49094a10.18329,10.18329,0,0,0,6.38871-1.95023l-.94149-1.883A9.90036,9.90036,0,0,1,91.17642,73.145c-3.42973,0-5.918-2.48823-5.918-7.66644,0-4.16946,1.54675-6.65769,4.50572-6.65769,1.614,0,2.959.538,3.497,2.152l3.42972-.87425c-.33625-1.345-2.152-3.96771-7.12845-3.96771-4.573,0-8.60794,3.09349-8.60794,9.21318"/><path id="path50" d="M171.67412,56.39988h3.90048V75.49874h-3.90048Zm12.23941,0-8.13721,9.14595,8.13721,9.95294h4.64021L180.282,65.34408l7.5992-8.9442h-3.96771"/><path id="path52" d="M153.71849,65.34405c0,6.25421,3.83322,10.49094,9.07869,10.49094a10.11878,10.11878,0,0,0,6.456-1.95023l-.94148-1.883A9.953,9.953,0,0,1,163.806,73.145c-3.42973,0-5.8507-2.48823-5.8507-7.66644,0-4.16946,1.54674-6.65769,4.573-6.65769a3.35862,3.35862,0,0,1,3.497,2.152l3.36248-.87425c-.33625-1.345-2.08475-3.96771-7.0612-3.96771-4.64021,0-8.60795,3.09349-8.60795,9.21318"/><path id="path54" d="M143.16032,56.13087c-5.17821,0-8.94419,3.09349-8.94419,9.81844s3.56423,9.88568,8.94419,9.88568,8.9442-3.16072,8.9442-9.88568S148.33853,56.13087,143.16032,56.13087Zm0,17.01414c-3.96771,0-4.90922-3.228-4.90922-7.1957,0-3.90048.94148-7.26295,4.90922-7.26295s4.90922,3.36247,4.90922,7.26295c0,3.96772-.94148,7.1957-4.90922,7.1957"/></g></svg>
@@ -1 +1 @@
-<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:#fbfbfb;}</style></defs><title>kubernetes.io-logos</title><rect class="cls-1" x="-3.55202" y="-3.16104" width="223.25536" height="134.51136"/><path id="path5" d="M141.73824,63.05113H127.82345l2.25647-3.85486c.93932-1.59742,1.78587-2.16178,3.19675-2.16178,1.40994,0,1.88056,1.12776,1.12871,2.5377h14.197l1.4109-2.5377c1.034-1.88055,1.034-3.94955-1.50463-3.94955h-25.0097c-2.25648,0-4.04329.28218-5.35949,2.44492l-4.41922,7.52127c-1.2215,2.16274-1.12776,3.94956,1.88056,3.94956h14.3854L127.636,71.13676a3.487,3.487,0,0,1-3.47894,1.88056c-1.69307,0-1.59837-1.12776-.84654-2.25552H109.20724l-1.69307,2.82084c-1.034,1.78681-.18748,3.38519,2.25648,3.38519h24.82127a6.07295,6.07295,0,0,0,5.35949-2.91457l4.23078-7.05162C145.12343,65.21387,144.65281,63.05113,141.73824,63.05113Z"/><path id="path7" d="M202.14279,57.7299a2.01608,2.01608,0,1,0,2.06708,2.00395A2.04547,2.04547,0,0,0,202.14279,57.7299Zm0,3.63676a1.62259,1.62259,0,1,1,1.673-1.63377A1.64448,1.64448,0,0,1,202.14279,61.36666Z"/><path id="path9" d="M109.67787,53.08494H88.42834c-2.91458,0-6.67569,0-8.93217,3.94955l-9.4028,15.98283c-1.31619,2.16274-.18748,3.94956,2.25648,3.94956H97.07833c2.53865,0,4.137-.7528,6.11133-3.94956l9.40279-15.98283C113.814,54.96645,111.9334,53.08494,109.67787,53.08494ZM97.73642,59.19627l-7.05065,11.9405c-.94028,1.69307-1.78682,1.88055-3.94955,1.88055s-2.7271-.75184-2.068-1.88055l7.14535-11.94049a3.9896,3.9896,0,0,1,4.0433-2.16179C97.64364,57.03449,98.6767,57.59885,97.73642,59.19627Z"/><path id="path11" d="M72.44455,53.08494H45.17837L33.42631,73.01732h-25.95l.01294,3.94956H59.846c3.47893,0,4.60669-1.69308,5.35949-3.00928l2.81988-4.88887c.7528-1.3162.7528-2.82084-.75184-3.66642a5.94452,5.94452,0,0,0,4.79513-3.19676l2.82085-4.79514C76.30036,55.06019,74.60728,53.08494,72.44455,53.08494ZM58.436,69.16342,57.02512,71.419a2.662,2.662,0,0,1-2.6324,1.59837H47.43485l3.57554-6.01663H57.777C58.99942,67.00069,58.99942,68.12845,58.436,69.16342Zm5.9229-9.96715L62.948,61.45275a2.662,2.662,0,0,1-2.6324,1.59838H53.35774l3.57268-6.01664h6.76943C64.92327,57.03449,64.92327,58.16225,64.35891,59.19627Z"/><path id="path13" d="M203.20455,59.20871a.55269.55269,0,0,0-.25635-.485,1.13731,1.13731,0,0,0-.56532-.10809h-1.01107v2.25457h.34244V59.83046h.40462l.66193,1.0388h.3941l-.7021-1.0388C202.88507,59.81994,203.20455,59.6535,203.20455,59.20871Zm-1.11341.34244h-.37687V58.872h.5988c.29173,0,.54809.03922.54809.331C202.86211,59.60854,202.41636,59.55115,202.09114,59.55115Z"/><path id="polygon1317" d="M168.49306,57.03164h35.25806l-.011-3.94955H156.741L142.73151,76.964h32.71941l2.35022-3.94955H159.09122l3.57267-6.01664h18.70993l2.25649-3.94955H165.01508Z"/></svg>
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:transparent;}</style></defs><title>kubernetes.io-logos</title><rect class="cls-1" x="-3.55202" y="-3.16104" width="223.25536" height="134.51136"/><path id="path5" d="M141.73824,63.05113H127.82345l2.25647-3.85486c.93932-1.59742,1.78587-2.16178,3.19675-2.16178,1.40994,0,1.88056,1.12776,1.12871,2.5377h14.197l1.4109-2.5377c1.034-1.88055,1.034-3.94955-1.50463-3.94955h-25.0097c-2.25648,0-4.04329.28218-5.35949,2.44492l-4.41922,7.52127c-1.2215,2.16274-1.12776,3.94956,1.88056,3.94956h14.3854L127.636,71.13676a3.487,3.487,0,0,1-3.47894,1.88056c-1.69307,0-1.59837-1.12776-.84654-2.25552H109.20724l-1.69307,2.82084c-1.034,1.78681-.18748,3.38519,2.25648,3.38519h24.82127a6.07295,6.07295,0,0,0,5.35949-2.91457l4.23078-7.05162C145.12343,65.21387,144.65281,63.05113,141.73824,63.05113Z"/><path id="path7" d="M202.14279,57.7299a2.01608,2.01608,0,1,0,2.06708,2.00395A2.04547,2.04547,0,0,0,202.14279,57.7299Zm0,3.63676a1.62259,1.62259,0,1,1,1.673-1.63377A1.64448,1.64448,0,0,1,202.14279,61.36666Z"/><path id="path9" d="M109.67787,53.08494H88.42834c-2.91458,0-6.67569,0-8.93217,3.94955l-9.4028,15.98283c-1.31619,2.16274-.18748,3.94956,2.25648,3.94956H97.07833c2.53865,0,4.137-.7528,6.11133-3.94956l9.40279-15.98283C113.814,54.96645,111.9334,53.08494,109.67787,53.08494ZM97.73642,59.19627l-7.05065,11.9405c-.94028,1.69307-1.78682,1.88055-3.94955,1.88055s-2.7271-.75184-2.068-1.88055l7.14535-11.94049a3.9896,3.9896,0,0,1,4.0433-2.16179C97.64364,57.03449,98.6767,57.59885,97.73642,59.19627Z"/><path id="path11" d="M72.44455,53.08494H45.17837L33.42631,73.01732h-25.95l.01294,3.94956H59.846c3.47893,0,4.60669-1.69308,5.35949-3.00928l2.81988-4.88887c.7528-1.3162.7528-2.82084-.75184-3.66642a5.94452,5.94452,0,0,0,4.79513-3.19676l2.82085-4.79514C76.30036,55.06019,74.60728,53.08494,72.44455,53.08494ZM58.436,69.16342,57.02512,71.419a2.662,2.662,0,0,1-2.6324,1.59837H47.43485l3.57554-6.01663H57.777C58.99942,67.00069,58.99942,68.12845,58.436,69.16342Zm5.9229-9.96715L62.948,61.45275a2.662,2.662,0,0,1-2.6324,1.59838H53.35774l3.57268-6.01664h6.76943C64.92327,57.03449,64.92327,58.16225,64.35891,59.19627Z"/><path id="path13" d="M203.20455,59.20871a.55269.55269,0,0,0-.25635-.485,1.13731,1.13731,0,0,0-.56532-.10809h-1.01107v2.25457h.34244V59.83046h.40462l.66193,1.0388h.3941l-.7021-1.0388C202.88507,59.81994,203.20455,59.6535,203.20455,59.20871Zm-1.11341.34244h-.37687V58.872h.5988c.29173,0,.54809.03922.54809.331C202.86211,59.60854,202.41636,59.55115,202.09114,59.55115Z"/><path id="polygon1317" d="M168.49306,57.03164h35.25806l-.011-3.94955H156.741L142.73151,76.964h32.71941l2.35022-3.94955H159.09122l3.57267-6.01664h18.70993l2.25649-3.94955H165.01508Z"/></svg>
@@ -1 +1 @@
-<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:#fbfbfb;}.cls-2{fill:#0071f7;}</style></defs><title>kubernetes.io-logos</title><rect class="cls-1" x="-3.55202" y="-3.16104" width="223.25536" height="134.51136"/><path class="cls-2" d="M174.27862,87.85821a5.22494,5.22494,0,0,1-.67887,7.12815,5.66277,5.66277,0,0,1-7.46759-.67888l-11.88024-15.2746L142.71111,93.968c-1.69718,2.37606-5.09153,2.37606-7.46758.67888a4.96011,4.96011,0,0,1-1.01831-7.12815l13.57742-17.65065L134.22522,52.21747a5.32925,5.32925,0,0,1,8.48589-6.44927l11.54081,15.2746,11.88024-14.59574c1.69718-2.376,4.75211-2.71548,7.46759-1.0183,2.376,1.69717,2.376,5.09153.67887,7.46758l-13.238,17.31121Zm-61.77727-2.03662a15.64926,15.64926,0,0,1-15.95348-15.614,15.95716,15.95716,0,0,1,31.907,0A16.08546,16.08546,0,0,1,112.50135,85.82159Zm-46.84212,0a15.64926,15.64926,0,0,1-15.95347-15.614,15.95716,15.95716,0,0,1,31.907,0A15.64926,15.64926,0,0,1,65.65923,85.82159Zm46.84212-41.41114A26.28086,26.28086,0,0,0,89.41973,57.98787,26.42342,26.42342,0,0,0,65.99867,44.41045,27.90043,27.90043,0,0,0,50.0452,49.502V27.77811a5.22038,5.22038,0,0,0-5.09153-5.09154,5.297,5.297,0,0,0-5.431,5.09154V70.547A26.00829,26.00829,0,0,0,65.65923,96.00466,26.718,26.718,0,0,0,89.08029,82.0878a26.58818,26.58818,0,0,0,23.08162,13.91686,26.22192,26.22192,0,0,0,26.476-26.13654C138.97732,55.95126,127.09708,44.41045,112.50135,44.41045Z"/></svg>
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 215 127"><defs><style>.cls-1{fill:transparent;}.cls-2{fill:#0071f7;}</style></defs><title>kubernetes.io-logos</title><rect class="cls-1" x="-3.55202" y="-3.16104" width="223.25536" height="134.51136"/><path class="cls-2" d="M174.27862,87.85821a5.22494,5.22494,0,0,1-.67887,7.12815,5.66277,5.66277,0,0,1-7.46759-.67888l-11.88024-15.2746L142.71111,93.968c-1.69718,2.37606-5.09153,2.37606-7.46758.67888a4.96011,4.96011,0,0,1-1.01831-7.12815l13.57742-17.65065L134.22522,52.21747a5.32925,5.32925,0,0,1,8.48589-6.44927l11.54081,15.2746,11.88024-14.59574c1.69718-2.376,4.75211-2.71548,7.46759-1.0183,2.376,1.69717,2.376,5.09153.67887,7.46758l-13.238,17.31121Zm-61.77727-2.03662a15.64926,15.64926,0,0,1-15.95348-15.614,15.95716,15.95716,0,0,1,31.907,0A16.08546,16.08546,0,0,1,112.50135,85.82159Zm-46.84212,0a15.64926,15.64926,0,0,1-15.95347-15.614,15.95716,15.95716,0,0,1,31.907,0A15.64926,15.64926,0,0,1,65.65923,85.82159Zm46.84212-41.41114A26.28086,26.28086,0,0,0,89.41973,57.98787,26.42342,26.42342,0,0,0,65.99867,44.41045,27.90043,27.90043,0,0,0,50.0452,49.502V27.77811a5.22038,5.22038,0,0,0-5.09153-5.09154,5.297,5.297,0,0,0-5.431,5.09154V70.547A26.00829,26.00829,0,0,0,65.65923,96.00466,26.718,26.718,0,0,0,89.08029,82.0878a26.58818,26.58818,0,0,0,23.08162,13.91686,26.22192,26.22192,0,0,0,26.476-26.13654C138.97732,55.95126,127.09708,44.41045,112.50135,44.41045Z"/></svg>
@@ -1 +1 @@
-<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 215 127"><defs><style>.cls-1{fill:#fff;}.cls-2{fill:#fbfbfb;}.cls-3{fill:#192534;}.cls-4{mask:url(#mask);}.cls-5{mask:url(#mask-2-2);}.cls-6{mask:url(#mask-3);}</style><mask id="mask" x="14.75095" y="70.72244" width="32.48851" height="10.3304" maskUnits="userSpaceOnUse"><g id="mask-2"><polygon id="path-1" class="cls-1" points="14.751 81.053 14.751 70.722 47.239 70.722 47.239 81.053 14.751 81.053"/></g></mask><mask id="mask-2-2" x="14.75095" y="60.57418" width="32.48851" height="10.3304" maskUnits="userSpaceOnUse"><g id="mask-4"><polygon id="path-3" class="cls-1" points="14.751 70.905 14.751 60.574 47.239 60.574 47.239 70.905 14.751 70.905"/></g></mask><mask id="mask-3" x="14.75079" y="45.87467" width="32.48853" height="14.92678" maskUnits="userSpaceOnUse"><g id="mask-6"><polygon id="path-5" class="cls-1" points="14.751 60.801 14.751 45.875 47.239 45.875 47.239 60.801 14.751 60.801"/></g></mask></defs><title>kubernetes.io-logos</title><rect class="cls-2" x="-3.55202" y="-3.16104" width="223.25536" height="134.51136"/><g id="Symbols"><g id="Mobile-Header"><g id="Group-4"><g id="Mobile-Logo"><g id="Group"><g id="Group-3"><path id="Fill-1" class="cls-3" d="M190.28379,79.32487h-6.37906V54.34018h6.37906v3.41436a10.33414,10.33414,0,0,1,7.73637-4.03515v6.41449a8.21414,8.21414,0,0,0-1.75906-.15516c-2.10911,0-4.92212,1.24163-5.97731,2.84489ZM173.10344,64.47949a5.69574,5.69574,0,0,0-5.97731-5.53592c-3.96771,0-5.67637,3.05266-5.97732,5.53592ZM167.5278,79.94566c-7.38379,0-12.9595-5.12161-12.9595-13.13943,0-7.24181,5.174-13.08684,12.55783-13.08684,7.234,0,12.15607,5.58723,12.15607,13.76022v1.4481H161.24952c.40168,3.15535,2.86336,5.79371,6.98219,5.79371a10.15786,10.15786,0,0,0,6.47972-2.48319l2.813,4.24168c-2.41134,2.27549-6.22917,3.46575-9.99665,3.46575Zm-19.24929-.62079h-6.43068V60.08257h-4.018V54.34018h4.018V52.99592c0-5.32809,3.31543-8.68991,8.18843-8.68991a8.489,8.489,0,0,1,6.32989,2.32674l-2.41135,3.88a3.50681,3.50681,0,0,0-2.66312-1.03509c-1.75782,0-3.01317,1.19026-3.01317,3.51828v1.34426h4.92207v5.74239h-4.92207Zm-16.426,0H125.4218V60.08257h-4.0194V54.34018h4.0194V52.99592c0-5.32809,3.31541-8.68991,8.18843-8.68991a8.48418,8.48418,0,0,1,6.3286,2.32674l-2.41012,3.88a3.50683,3.50683,0,0,0-2.66312-1.03509c-1.75782,0-3.01318,1.19026-3.01318,3.51828v1.34426h4.92212v5.74239h-4.92212v19.2423Zm-14.43053,0h-6.38022V76.16952a11.21263,11.21263,0,0,1-8.53978,3.77613c-5.32385,0-7.83586-3.00012-7.83586-7.86259V54.34018h6.379v15.157c0,3.46574,1.75911,4.60345,4.4714,4.60345a7.07632,7.07632,0,0,0,5.52531-2.8449V54.34017H117.422v24.9847ZM71.87264,71.307a7.0348,7.0348,0,0,0,5.4762,2.79359c3.7171,0,6.17877-2.8975,6.17877-7.24181,0-4.34567-2.46168-7.29441-6.17877-7.29441a6.9786,6.9786,0,0,0-5.47621,2.8975V71.307Zm0,8.01788h-6.379V43.734h6.379V57.54807a9.2539,9.2539,0,0,1,7.48587-3.82868c6.17748,0,10.74961,4.96639,10.74961,13.13944,0,8.32821-4.62118,13.08683-10.74961,13.08683a9.33043,9.33043,0,0,1-7.48587-3.82745v3.20666Z"/></g><g id="Group-6"><g class="cls-4"><path id="Fill-4" class="cls-3" d="M47.00136,72.68049l-3.69521-1.83918a1.4583,1.4583,0,0,0-1.15329,0L31.57117,76.108a1.455,1.455,0,0,1-1.15194,0L19.83748,70.84131a1.45828,1.45828,0,0,0-1.15328,0L14.989,72.68049c-.31738.15841-.31738.416,0,.57308L30.41923,80.934a1.45553,1.45553,0,0,0,1.15194,0l15.43019-7.68046c.31738-.15706.31738-.41467,0-.57308"/></g></g><g id="Group-9"><g class="cls-5"><path id="Fill-7" class="cls-3" d="M47.00136,62.53223l-3.69521-1.83917a1.45872,1.45872,0,0,0-1.15329,0L31.57117,65.95984a1.46694,1.46694,0,0,1-1.15194,0L19.83748,60.69306a1.4587,1.4587,0,0,0-1.15328,0L14.989,62.53223c-.31738.15842-.31738.416,0,.5745l15.43024,7.67905a1.45553,1.45553,0,0,0,1.15194,0l15.43019-7.67905c.31738-.15848.31738-.41608,0-.5745"/></g></g><g id="Group-12"><g class="cls-6"><path id="Fill-10" class="cls-3" d="M14.98887,53.6031l15.43018,7.08965a1.58033,1.58033,0,0,0,1.15194,0L47.00123,53.6031c.31744-.14628.31744-.38408,0-.52907L31.571,45.98438a1.56676,1.56676,0,0,0-1.15194,0L14.98887,53.074c-.31744.145-.31744.38279,0,.52907"/></g></g></g></g></g></g></g></svg>
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 215 127"><defs><style>.cls-1{fill:#fff;}.cls-2{fill:transparent;}.cls-3{fill:#192534;}.cls-4{mask:url(#mask);}.cls-5{mask:url(#mask-2-2);}.cls-6{mask:url(#mask-3);}</style><mask id="mask" x="14.75095" y="70.72244" width="32.48851" height="10.3304" maskUnits="userSpaceOnUse"><g id="mask-2"><polygon id="path-1" class="cls-1" points="14.751 81.053 14.751 70.722 47.239 70.722 47.239 81.053 14.751 81.053"/></g></mask><mask id="mask-2-2" x="14.75095" y="60.57418" width="32.48851" height="10.3304" maskUnits="userSpaceOnUse"><g id="mask-4"><polygon id="path-3" class="cls-1" points="14.751 70.905 14.751 60.574 47.239 60.574 47.239 70.905 14.751 70.905"/></g></mask><mask id="mask-3" x="14.75079" y="45.87467" width="32.48853" height="14.92678" maskUnits="userSpaceOnUse"><g id="mask-6"><polygon id="path-5" class="cls-1" points="14.751 60.801 14.751 45.875 47.239 45.875 47.239 60.801 14.751 60.801"/></g></mask></defs><title>kubernetes.io-logos</title><rect class="cls-2" x="-3.55202" y="-3.16104" width="223.25536" height="134.51136"/><g id="Symbols"><g id="Mobile-Header"><g id="Group-4"><g id="Mobile-Logo"><g id="Group"><g id="Group-3"><path id="Fill-1" class="cls-3" d="M190.28379,79.32487h-6.37906V54.34018h6.37906v3.41436a10.33414,10.33414,0,0,1,7.73637-4.03515v6.41449a8.21414,8.21414,0,0,0-1.75906-.15516c-2.10911,0-4.92212,1.24163-5.97731,2.84489ZM173.10344,64.47949a5.69574,5.69574,0,0,0-5.97731-5.53592c-3.96771,0-5.67637,3.05266-5.97732,5.53592ZM167.5278,79.94566c-7.38379,0-12.9595-5.12161-12.9595-13.13943,0-7.24181,5.174-13.08684,12.55783-13.08684,7.234,0,12.15607,5.58723,12.15607,13.76022v1.4481H161.24952c.40168,3.15535,2.86336,5.79371,6.98219,5.79371a10.15786,10.15786,0,0,0,6.47972-2.48319l2.813,4.24168c-2.41134,2.27549-6.22917,3.46575-9.99665,3.46575Zm-19.24929-.62079h-6.43068V60.08257h-4.018V54.34018h4.018V52.99592c0-5.32809,3.31543-8.68991,8.18843-8.68991a8.489,8.489,0,0,1,6.32989,2.32674l-2.41135,3.88a3.50681,3.50681,0,0,0-2.66312-1.03509c-1.75782,0-3.01317,1.19026-3.01317,3.51828v1.34426h4.92207v5.74239h-4.92207Zm-16.426,0H125.4218V60.08257h-4.0194V54.34018h4.0194V52.99592c0-5.32809,3.31541-8.68991,8.18843-8.68991a8.48418,8.48418,0,0,1,6.3286,2.32674l-2.41012,3.88a3.50683,3.50683,0,0,0-2.66312-1.03509c-1.75782,0-3.01318,1.19026-3.01318,3.51828v1.34426h4.92212v5.74239h-4.92212v19.2423Zm-14.43053,0h-6.38022V76.16952a11.21263,11.21263,0,0,1-8.53978,3.77613c-5.32385,0-7.83586-3.00012-7.83586-7.86259V54.34018h6.379v15.157c0,3.46574,1.75911,4.60345,4.4714,4.60345a7.07632,7.07632,0,0,0,5.52531-2.8449V54.34017H117.422v24.9847ZM71.87264,71.307a7.0348,7.0348,0,0,0,5.4762,2.79359c3.7171,0,6.17877-2.8975,6.17877-7.24181,0-4.34567-2.46168-7.29441-6.17877-7.29441a6.9786,6.9786,0,0,0-5.47621,2.8975V71.307Zm0,8.01788h-6.379V43.734h6.379V57.54807a9.2539,9.2539,0,0,1,7.48587-3.82868c6.17748,0,10.74961,4.96639,10.74961,13.13944,0,8.32821-4.62118,13.08683-10.74961,13.08683a9.33043,9.33043,0,0,1-7.48587-3.82745v3.20666Z"/></g><g id="Group-6"><g class="cls-4"><path id="Fill-4" class="cls-3" d="M47.00136,72.68049l-3.69521-1.83918a1.4583,1.4583,0,0,0-1.15329,0L31.57117,76.108a1.455,1.455,0,0,1-1.15194,0L19.83748,70.84131a1.45828,1.45828,0,0,0-1.15328,0L14.989,72.68049c-.31738.15841-.31738.416,0,.57308L30.41923,80.934a1.45553,1.45553,0,0,0,1.15194,0l15.43019-7.68046c.31738-.15706.31738-.41467,0-.57308"/></g></g><g id="Group-9"><g class="cls-5"><path id="Fill-7" class="cls-3" d="M47.00136,62.53223l-3.69521-1.83917a1.45872,1.45872,0,0,0-1.15329,0L31.57117,65.95984a1.46694,1.46694,0,0,1-1.15194,0L19.83748,60.69306a1.4587,1.4587,0,0,0-1.15328,0L14.989,62.53223c-.31738.15842-.31738.416,0,.5745l15.43024,7.67905a1.45553,1.45553,0,0,0,1.15194,0l15.43019-7.67905c.31738-.15848.31738-.41608,0-.5745"/></g></g><g id="Group-12"><g class="cls-6"><path id="Fill-10" class="cls-3" d="M14.98887,53.6031l15.43018,7.08965a1.58033,1.58033,0,0,0,1.15194,0L47.00123,53.6031c.31744-.14628.31744-.38408,0-.52907L31.571,45.98438a1.56676,1.56676,0,0,0-1.15194,0L14.98887,53.074c-.31744.145-.31744.38279,0,.52907"/></g></g></g></g></g></g></g></svg>
--- a/Show More
+++ b/Show More