diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 0dd59e631a..9763b65c88 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -35,9 +35,11 @@ aliases: - divya-mohan0209 - jimangel - kbhawkey + - mehabhalodiya - onlydole - rajeshdeshpande02 - sftim + - shannonxtreme - tengqm sig-docs-es-owners: # Admins for Spanish content - raelga @@ -166,6 +168,7 @@ aliases: - xichengliudui # zhangxiaoyu-zidif sig-docs-pt-owners: # Admins for Portuguese content + - edsoncelio - femrtnz - jailton - jcjesus @@ -174,6 +177,7 @@ aliases: - rikatz - yagonobre sig-docs-pt-reviews: # PR reviews for Portugese content + - edsoncelio - femrtnz - jailton - jcjesus @@ -221,12 +225,12 @@ aliases: # authoritative source: git.k8s.io/community/OWNERS_ALIASES committee-steering: # provide PR approvals for announcements - cblecker - - derekwaynecarr - dims + - justaugustus - liggitt - mrbobbytables - - nikhita - parispittman + - tpepper # authoritative source: https://git.k8s.io/sig-release/OWNERS_ALIASES sig-release-leads: - cpanato # SIG Technical Lead diff --git a/README.md b/README.md index 74f99e2ffc..d7691e7f0d 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,8 @@ Learn more about SIG Docs Kubernetes community and meetings on the [community pa You can also reach the maintainers of this project at: -- [Slack](https://kubernetes.slack.com/messages/sig-docs) [Get an invite for this Slack](https://slack.k8s.io/) +- [Slack](https://kubernetes.slack.com/messages/sig-docs) + - [Get an invite for this Slack](https://slack.k8s.io/) - [Mailing List](https://groups.google.com/forum/#!forum/kubernetes-sig-docs) ## Contributing to the docs diff --git a/assets/scss/_base.scss b/assets/scss/_base.scss index 4113b49bee..97ce27fe28 100644 --- a/assets/scss/_base.scss +++ b/assets/scss/_base.scss @@ -810,11 +810,10 @@ section#cncf { } } -.td-search { - header > .header-filler { - height: $hero-padding-top; - background-color: black; - } +// Header filler size adjustment + +.header-hero.filler { + height: $hero-padding-top; } // Docs specific @@ -859,17 +858,6 @@ section#cncf { /* DOCUMENTATION */ -body.td-documentation { - header > .header-filler { - height: $hero-padding-top; - background-color: black; - } - /* Special case for if an announcement is active */ - header section#announcement ~ .header-filler { - display: none; - } -} - // nav-tabs and tab-content .nav-tabs { border-bottom: none !important; diff --git a/assets/scss/_custom.scss b/assets/scss/_custom.scss index 8047d4025b..28b6a8d6e6 100644 --- a/assets/scss/_custom.scss +++ b/assets/scss/_custom.scss @@ -26,6 +26,10 @@ $announcement-size-adjustment: 8px; } } +.header-hero #quickstartButton.button { + margin-top: 1em; +} + section { .main-section { @media only screen and (min-width: 1024px) { @@ -34,8 +38,11 @@ section { } } -.td-outer { - padding: 0 !important; +body { + header + .td-outer { + min-height: 50vh; + height: auto; + } } @@ -313,33 +320,40 @@ main { // blockquotes and callouts -.td-content, body { - blockquote.callout { +body { + .alert { + // Override Docsy styles padding: 0.4rem 0.4rem 0.4rem 1rem; - border: 1px solid #eee; - border-left-width: 0.5em; + border-top: 1px solid #eee; + border-bottom: 1px solid #eee; + border-right: 1px solid #eee; + border-radius: 0.25em; + border-left-width: 0.5em; // fallback in case calc() is missing background: #fff; color: #000; margin-top: 0.5em; margin-bottom: 0.5em; } - blockquote.callout { - border-radius: calc(1em/3); + // Set minimum width and radius for alert color + .alert { + border-left-width: calc(max(0.5em, 4px)); + border-top-left-radius: calc(max(0.5em, 4px)); + border-bottom-left-radius: calc(max(0.5em, 4px)); } - .callout.caution { + .alert.callout.caution { border-left-color: #f0ad4e; } - - .callout.note { + .alert.callout.note { border-left-color: #428bca; } - - .callout.warning { + .alert.callout.warning { border-left-color: #d9534f; } + .alert.third-party-content { + border-left-color: #444; + } - - h1:first-of-type + blockquote.callout { + h1:first-of-type + .alert.callout { margin-top: 1.5em; } } @@ -367,7 +381,7 @@ main { background: #f8f9cb; } -.deprecation-warning { +.deprecation-warning, .pageinfo.deprecation-warning { padding: 20px; margin: 20px 0; background-color: #faf5b6; @@ -554,34 +568,6 @@ main.content { } } -/* ANNOUNCEMENTS */ -section#fp-announcement ~ .header-hero { - padding: $announcement-size-adjustment 0; - - > div { - margin-top: $announcement-size-adjustment; - margin-bottom: $announcement-size-adjustment; - } - - h1, h2, h3, h4, h5 { - margin: $announcement-size-adjustment 0; - } -} - -section#announcement ~ .header-hero { - padding: #{$announcement-size-adjustment / 2} 0; - - > div { - margin-top: #{$announcement-size-adjustment / 2}; - margin-bottom: #{$announcement-size-adjustment / 2}; - padding-bottom: #{$announcement-size-adjustment / 2}; - } - - h1, h2, h3, h4, h5 { - margin: #{$announcement-size-adjustment / 2} 0; - } -} - /* DOCUMENTATION */ /* Don't show lead text */ @@ -607,12 +593,12 @@ body.td-documentation { @media print { /* Do not print announcements */ - #announcement, section#announcement, #fp-announcement, section#fp-announcement { + #announcement { display: none; } } -#announcement, #fp-announcement { +#announcement { > * { color: inherit; background: inherit; @@ -629,42 +615,90 @@ body.td-documentation { } } -#announcement { - padding-top: 105px; - padding-bottom: 25px; -} - .header-hero { padding-top: 40px; } -/* Extra announcement height only for landscape viewports */ -@media (min-aspect-ratio: 8/9) { - #fp-announcement { - min-height: 25vh; - } -} - -#fp-announcement aside { - padding-top: 115px; - padding-bottom: 25px; -} - -.announcement { - .content { +#announcement { + .announcement-main { + margin-left: auto; + margin-right: auto; margin-bottom: 0px; + + // for padding-top see _size.scss + padding-bottom: calc(max(2em, 2rem)); + + max-width: calc(min(1200px - 8em, 80vw)); } - > p { - .gridPage #announcement .content p, - .announcement > h4, - .announcement > h3 { - color: #ffffff; + /* always white */ + h1, h2, h3, h4, h5, h6, p * { + color: #ffffff; + background: transparent; + + img.event-logo { + display: inline-block; + max-height: calc(min(80px, 8em)); + max-width: calc(min(240px, 33vw)); + float: right; } } } +#announcement + .header-hero { + padding-top: 2em; +} + +// Extra padding for anything except wide viewports +@media (min-width: 992px) { + #announcement aside { // more specific + .announcement-main { + padding-top: calc(max(8em, 8rem)); + } + } +} + +@media (max-width: 768px) { + #announcement { + padding-top: 4rem; + padding-bottom: 4rem; + .announcement-main, aside .announcement-main { + padding-top: calc(min(2rem,2em)); + } + } +} + +@media (max-width: 480px) { + #announcement { + padding-bottom: 0.5em; + } + #announcement aside { + h1, h2, h3, h4, h5, h6 { + img.event-logo { + margin-left: auto; + margin-right: auto; + margin-bottom: 0.75em; + display: block; + max-height: initial; + max-width: calc(min(calc(100vw - 2em), 240px)); + float: initial; + } + } + } +} + +#announcement + .header-hero.filler { + display: none; +} + +@media (min-width: 768px) { + #announcement + .header-hero { + display: none; + } +} + + // Match Docsy-imposed max width on text body @media (min-width: 1200px) { body.td-blog main .td-content > figure { @@ -721,3 +755,13 @@ figure { } } } + +// Indent definition lists +dl { + padding-left: 1.5em; + + // Add vertical space before definitions + > *:not(dt) + dt, dt:first-child { + margin-top: 1.5em; + } +} diff --git a/assets/scss/_size.scss b/assets/scss/_size.scss index 9f0b7f655f..14d56201c3 100644 --- a/assets/scss/_size.scss +++ b/assets/scss/_size.scss @@ -18,3 +18,11 @@ section, line-height: $vendor-strip-height; font-size: $vendor-strip-font-size; } + +#announcement { + min-height: $hero-padding-top; + + .announcement-main { + padding-top: calc(max(8em, 8rem, #{$hero-padding-top} / 3)); + } +} diff --git a/content/de/training/_index.html b/content/de/training/_index.html new file mode 100644 index 0000000000..6ba21e7861 --- /dev/null +++ b/content/de/training/_index.html @@ -0,0 +1,137 @@ +--- +title: Schulungen +bigheader: Kubernetes Schulungen und Zertifizierungen +abstract: Schulungsprogramme, Zertifizierungen und Partner. +layout: basic +cid: training +class: training +--- + +
+
+
+
+

Gestalte deine Cloud Native Karriere

+

Kubernetes ist das Herzstück der Cloud Native-Bewegung. Mit den Schulungen und Zertifizierungen der Linux Foundation und unserer Schulungspartner kannst Du in deine Karriere investieren, Kubernetes lernen und deine Cloud Native-Projekte zum Erfolg führen.

+
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+ +
+
+
+

Nimm an einen kostenlosen Kurs bei edX teil

+
+
+
+
+
+ Einführung in Kubernetes
 
+
+

Möchtest Du Kubernetes lernen? Erfahre alles über dieses leistungsstarke System zur Verwaltung von Containeranwendungen.

+
+ Zum Kurs +
+
+
+
+
+ Einführung in Cloud-Infrastruktur Technologien +
+

Lerne die Grundlagen für den Aufbau und die Verwaltung von Cloud-Technologien direkt von der Linux Foundation, dem Marktführer im Bereich Open Source.

+
+ Zum Kurs +
+
+
+
+
+ Einführung in Linux +
+

Du hast nie Linux gelernt? Willst du eine Auffrischung? Erarbeite dir gute Linux-Kenntnisse über die grafische Oberfläche und die Kommandozeile der wichtigsten Linux-Distributionen.

+
+ Zum Kurs +
+
+
+
+ +
+
+
+

Mit der Linux Foundation lernen

+

Die Linux Foundation bietet Kurse für alle Aspekte der Entwicklung und des Betriebs von Kubernetes-Anwendungen an, die entweder von Lehrkräften geleitet werden oder zum Selbststudium geeignet sind.

+

+ Kurse anzeigen +
+
+
+ +
+
+

Werde Kubernetes zertifiziert

+
+
+
+ Kubernetes and Cloud Native Associate (KCNA) +
+

Die Prüfung zum Kubernetes and Cloud Native Associate (KCNA) weist die grundlegenden Kenntnisse und Fähigkeiten eines Benutzers in Kubernetes und dem breiteren Cloud Native-Ökosystem nach.

+

Ein zertifizierter KCNA bestätigt konzeptionelles Wissen über das gesamte Cloud Native Ecosystem, mit besonderem Fokus auf Kubernetes.

+
+ Zur Zertifizierung +
+
+
+ Certified Kubernetes Application Developer (CKAD) +
+

Die Prüfung zum Certified Kubernetes Application Developer (Zertifizierter Kubernetes-Anwendungsentwickler) bescheinigt, dass Teilnehmer Cloud Native-Anwendungen für Kubernetes entwerfen, erstellen, konfigurieren und bereitstellen können.

+

Ein CKAD kann Anwendungsressourcen definieren und zentrale Elemente verwenden, um skalierbare Anwendungen und Tools in Kubernetes zu erstellen, zu überwachen und Fehler zu beheben.

+
+ Zur Zertifizierung +
+
+
+ Certified Kubernetes Administrator (CKA) +
+

Das Certified Kubernetes Administrator (CKA)-Programm garantiert, dass CKAs die Fähigkeiten, das Wissen und die Kompetenz besitzen, um die Aufgaben eines Kubernetes-Administrators zu erfüllen.

+

Ein zertifizierter Kubernetes-Administrator hat nachgewiesen, dass er in der Lage ist, grundlegende Installationen durchzuführen sowie Kubernetes-Cluster in einer Produktionsumgebung zu konfigurieren und zu verwalten.

+
+ Zur Zertifizierung +
+
+
+ Certified Kubernetes Security Specialist (CKS) +
+

Das Programm Certified Kubernetes Security Specialist (CKS) bietet die Gewissheit, dass der Zertifikatsinhaber mit einem breiten Spektrum an Best Practices vertraut ist und diese beherrscht. Die CKS-Zertifizierung umfasst Fähigkeiten zur Sicherung von Container-basierten Anwendungen und Kubernetes-Plattformen während der Erstellung, Bereitstellung und Laufzeit.

+

Kandidaten für den CKS müssen über eine aktuelle Zertifizierung als Certified Kubernetes Administrator (CKA) verfügen, um nachzuweisen, dass sie über ausreichende Kubernetes-Kenntnisse verfügen, bevor sie sich für den CKS anmelden.

+
+ Zur Zertifizierung +
+
+
+
+ +
+
+
+

Kubernetes Schulungspartner

+

Unser Netzwerk von Kubernetes-Schulungspartnern bietet Schulungsangebote für Kubernetes- und Cloud Native-Projekte.

+
+
+
+ {{< cncf-landscape helpers=false category="kubernetes-training-partner" >}} +
+
diff --git a/content/en/blog/_posts/2020-10-12-steering-committee-results.md b/content/en/blog/_posts/2020-10-12-steering-committee-results.md index 2acbb6d6e4..b47d9674f3 100644 --- a/content/en/blog/_posts/2020-10-12-steering-committee-results.md +++ b/content/en/blog/_posts/2020-10-12-steering-committee-results.md @@ -29,7 +29,7 @@ They join continuing members Christoph Blecker ([@cblecker](https://github.com/c * Josh Berkus ([@jberkus](https://github.com/jberkus)), Red Hat * Thanks to the Emeritus Steering Committee Members. Your prior service is appreciated by the community: * Aaron Crickenberger ([@spiffxp](https://github.com/spiffxp)), Google - * and Lachlan Evenson([@lachie8e)](https://github.com/lachie8e)), Microsoft + * and Lachlan Evenson([@lachie83)](https://github.com/lachie83)), Microsoft * And thank you to all the candidates who came forward to run for election. As [Jorge Castro put it](https://twitter.com/castrojo/status/1315718627639820288?s=20): we are spoiled with capable, kind, and selfless volunteers who put the needs of the project first. ## Get Involved with the Steering Committee diff --git a/content/en/blog/_posts/2021-08-06-server-side-apply-ga.md b/content/en/blog/_posts/2021-08-06-server-side-apply-ga.md index eca57a561e..01403715cc 100644 --- a/content/en/blog/_posts/2021-08-06-server-side-apply-ga.md +++ b/content/en/blog/_posts/2021-08-06-server-side-apply-ga.md @@ -121,7 +121,7 @@ deploymentApplyConfig.Spec.Template.Spec.WithContainers(corev1ac.Container(). ) // apply -applied, err := deploymentClient.Apply(ctx, extractedDeployment, metav1.ApplyOptions{FieldManager: fieldMgr}) +applied, err := deploymentClient.Apply(ctx, deploymentApplyConfig, metav1.ApplyOptions{FieldManager: fieldMgr}) ``` For developers using Custom Resource Definitions (CRDs), the Kubebuilder apply support will provide the same capabilities. Documentation will be included in the Kubebuilder book when available. diff --git a/content/en/blog/_posts/2021-11-08-steering-committee-results-2021.md b/content/en/blog/_posts/2021-11-08-steering-committee-results-2021.md new file mode 100644 index 0000000000..922824e390 --- /dev/null +++ b/content/en/blog/_posts/2021-11-08-steering-committee-results-2021.md @@ -0,0 +1,56 @@ +--- +layout: blog +title: "Announcing the 2021 Steering Committee Election Results" +date: 2021-11-08 +slug: steering-committee-results-2021 +--- + +**Author**: Kaslin Fields + +The [2021 Steering Committee Election](https://github.com/kubernetes/community/tree/master/events/elections/2021) is now complete. The Kubernetes Steering Committee consists of 7 seats, 4 of which were up for election in 2021. Incoming committee members serve a term of 2 years, and all members are elected by the Kubernetes Community. + +This community body is significant since it oversees the governance of the entire Kubernetes project. With that great power comes great responsibility. You can learn more about the steering committee’s role in their [charter](https://github.com/kubernetes/steering/blob/master/charter.md). + +## Results + +Congratulations to the elected committee members whose two year terms begin immediately (listed in alphabetical order by GitHub handle): + +* **Christoph Blecker ([@cblecker](https://github.com/cblecker)), Red Hat** +* **Stephen Augustus ([@justaugustus](https://github.com/justaugustus)), Cisco** +* **Paris Pittman ([@parispittman](https://github.com/parispittman)), Apple** +* **Tim Pepper ([@tpepper](https://github.com/tpepper)), VMware** + +They join continuing members: + +* **Davanum Srinivas ([@dims](https://github.com/dims)), VMware** +* **Jordan Liggitt ([@liggitt](https://github.com/liggitt)), Google** +* **Bob Killen ([@mrbobbytables](https://github.com/mrbobbytables)), Google** + +Paris Pittman and Christoph Blecker are returning Steering Committee Members. + +## Big Thanks + +Thank you and congratulations on a successful election to this round’s election officers: + +* Alison Dowdney, ([@alisondy](https://github.com/alisondy)) +* Noah Kantrowitz ([@coderanger](https://github.com/coderanger)) +* Josh Berkus ([@jberkus](https://github.com/jberkus)) + +Special thanks to Arnaud Meukam ([@ameukam](https://github.com/ameukam)), k8s-infra liaison, who enabled our voting software on community-owned infrastructure. + +Thanks to the Emeritus Steering Committee Members. Your prior service is appreciated by the community: + +* Derek Carr ([@derekwaynecarr](https://github.com/derekwaynecarr)) +* Nikhita Raghunath ([@nikhita](https://github.com/nikhita)) + +And thank you to all the candidates who came forward to run for election. + +## Get Involved with the Steering Committee + +This governing body, like all of Kubernetes, is open to all. You can follow along with Steering Committee [backlog items](https://github.com/kubernetes/steering/projects/1) and weigh in by filing an issue or creating a PR against their [repo](https://github.com/kubernetes/steering). They have an open meeting on [the first Monday at 9:30am PT of every month](https://github.com/kubernetes/steering) and regularly attend Meet Our Contributors. They can also be contacted at their public mailing list steering@kubernetes.io. + +You can see what the Steering Committee meetings are all about by watching past meetings on the [YouTube Playlist](https://www.youtube.com/playlist?list=PL69nYSiGNLP1yP1B_nd9-drjoxp0Q14qM). + +--- + +_This post was written by the [Upstream Marketing Working Group](https://github.com/kubernetes/community/tree/master/communication/marketing-team#contributor-marketing). If you want to write stories about the Kubernetes community, learn more about us._ diff --git a/content/en/blog/_posts/2021-11-09-non-root-containers-and-devices.md b/content/en/blog/_posts/2021-11-09-non-root-containers-and-devices.md new file mode 100644 index 0000000000..8fd5cc6ba1 --- /dev/null +++ b/content/en/blog/_posts/2021-11-09-non-root-containers-and-devices.md @@ -0,0 +1,238 @@ +--- +layout: blog +title: 'Non-root Containers And Devices' +date: 2021-11-09 +slug: non-root-containers-and-devices +--- + +**Author:** Mikko Ylinen (Intel) + +The user/group ID related security settings in Pod's `securityContext` trigger a problem when users want to +deploy containers that use accelerator devices (via [Kubernetes Device Plugins](/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/)) on Linux. In this blog +post I talk about the problem and describe the work done so far to address it. It's not meant to be a long story about getting the [k/k issue](https://github.com/kubernetes/kubernetes/issues/92211) fixed. + +Instead, this post aims to raise awareness of the issue and to highlight important device use-cases too. This is needed as Kubernetes works on new related features such as support for user namespaces. + +## Why non-root containers can't use devices and why it matters +One of the key security principles for running containers in Kubernetes is the +principle of least privilege. The Pod/container `securityContext` specifies the config +options to set, e.g., Linux capabilities, MAC policies, and user/group ID values to achieve this. + +Furthermore, the cluster admins are supported with tools like [PodSecurityPolicy](/docs/concepts/policy/pod-security-policy/) (deprecated) or +[Pod Security Admission](/docs/concepts/security/pod-security-admission/) (alpha) to enforce the desired security settings for pods that are being deployed in +the cluster. These settings could, for instance, require that containers must be `runAsNonRoot` or +that they are forbidden from running with root's group ID in `runAsGroup` or `supplementalGroups`. + +In Kubernetes, the kubelet builds the list of [`Device`](https://pkg.go.dev/k8s.io/cri-api@v0.22.1/pkg/apis/runtime/v1#Device) resources to be made available to a container +(based on inputs from the Device Plugins) and the list is included in the CreateContainer CRI message +sent to the CRI container runtime. Each `Device` contains little information: host/container device +paths and the desired devices cgroups permissions. + +The [OCI Runtime Spec for Linux Container Configuration](https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md) +expects that in addition to the devices cgroup fields, more detailed information about the devices +must be provided: + +```yaml +{ + "type": "", + "path": "", + "major": , + "minor": , + "fileMode": , + "uid": , + "gid": +}, +``` + +The CRI container runtimes (containerd, CRI-O) are responsible for obtaining this information +from the host for each `Device`. By default, the runtimes copy the host device's user and group IDs: + +- `uid` (uint32, OPTIONAL) - id of device owner in the container namespace. +- `gid` (uint32, OPTIONAL) - id of device group in the container namespace. + +Similarly, the runtimes prepare other mandatory `config.json` sections based on the CRI fields, +including the ones defined in `securityContext`: `runAsUser`/`runAsGroup`, which become part of the POSIX +platforms user structure via: + +- `uid` (int, REQUIRED) specifies the user ID in the container namespace. +- `gid` (int, REQUIRED) specifies the group ID in the container namespace. +- `additionalGids` (array of ints, OPTIONAL) specifies additional group IDs in the container namespace to be added to the process. + +However, the resulting `config.json` triggers a problem when trying to run containers with +both devices added and with non-root uid/gid set via `runAsUser`/`runAsGroup`: the container user process +has no permission to use the device even when its group id (gid, copied from host) was permissive to +non-root groups. This is because the container user does not belong to that host group (e.g., via `additionalGids`). + +Being able to run applications that use devices as non-root user is normal and expected to work so that +the security principles can be met. Therefore, several alternatives were considered to get the gap filled with what the PodSec/CRI/OCI supports today. + +## What was done to solve the issue? +You might have noticed from the problem definition that it would at least be possible to workaround +the problem by manually adding the device gid(s) to `supplementalGroups`, or in +the case of just one device, set `runAsGroup` to the device's group id. However, this is problematic because the device gid(s) may have +different values depending on the nodes' distro/version in the cluster. For example, with GPUs the following commands for different distros and versions return different gids: + +Fedora 33: +``` +$ ls -l /dev/dri/ +total 0 +drwxr-xr-x. 2 root root 80 19.10. 10:21 by-path +crw-rw----+ 1 root video 226, 0 19.10. 10:42 card0 +crw-rw-rw-. 1 root render 226, 128 19.10. 10:21 renderD128 +$ grep -e video -e render /etc/group +video:x:39: +render:x:997: +``` + +Ubuntu 20.04: +``` +$ ls -l /dev/dri/ +total 0 +drwxr-xr-x 2 root root 80 19.10. 17:36 by-path +crw-rw---- 1 root video 226, 0 19.10. 17:36 card0 +crw-rw---- 1 root render 226, 128 19.10. 17:36 renderD128 +$ grep -e video -e render /etc/group +video:x:44: +render:x:133: +``` + +Which number to choose in your `securityContext`? Also, what if the `runAsGroup`/`runAsUser` values cannot be hard-coded because +they are automatically assigned during pod admission time via external security policies? + +Unlike volumes with `fsGroup`, the devices have no official notion of `deviceGroup`/`deviceUser` that the CRI runtimes (or kubelet) +would be able to use. We considered using container annotations set by the device plugins (e.g., `io.kubernetes.cri.hostDeviceSupplementalGroup/`) to get custom OCI `config.json` uid/gid values. +This would have required changes to all existing device plugins which was not ideal. + +Instead, a solution that is *seamless* to end-users without getting the device plugin vendors involved was preferred. The selected approach was +to re-use `runAsUser` and `runAsGroup` values in `config.json` for devices: + +```yaml +{ + "type": "c", + "path": "/dev/foo", + "major": 123, + "minor": 4, + "fileMode": 438, + "uid": , + "gid": +}, +``` + +With `runc` OCI runtime (in non-rootless mode), the device is created (`mknod(2)`) in +the container namespace and the ownership is changed to `runAsUser`/`runAsGroup` using `chmod(2)`. + +{{< note >}} +[Rootless mode](/docs/tasks/administer-cluster/kubelet-in-userns/) and devices is not supported. +{{}} +Having the ownership updated in the container namespace is justified as the user process is the only one accessing the device. Only `runAsUser`/`runAsGroup` +are taken into account, and, e.g., the `USER` setting in the container is currently ignored. + +While it is likely that the "faulty" deployments (i.e., non-root `securityContext` + devices) do not exist, to be absolutely sure no +deployments break, an opt-in config entry in both containerd and CRI-O to enable the new behavior was added. The following: + +`device_ownership_from_security_context (bool)` + +defaults to `false` and must be enabled to use the feature. + +## See non-root containers using devices after the fix +To demonstrate the new behavior, let's use a Data Plane Development Kit (DPDK) application using hardware accelerators, Kubernetes CPU manager, and HugePages as an example. The cluster runs containerd with: + +```toml +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + device_ownership_from_security_context = true +``` + +or CRI-O with: + +```toml +[crio.runtime] +device_ownership_from_security_context = true +``` + +and the `Guaranteed` QoS Class Pod that runs DPDK's crypto-perf test utility with this YAML: + +```yaml +... +metadata: + name: qat-dpdk +spec: + securityContext: + runAsUser: 1000 + runAsGroup: 2000 + fsGroup: 3000 + containers: + - name: crypto-perf + image: intel/crypto-perf:devel + ... + resources: + requests: + cpu: "3" + memory: "128Mi" + qat.intel.com/generic: '4' + hugepages-2Mi: "128Mi" + limits: + cpu: "3" + memory: "128Mi" + qat.intel.com/generic: '4' + hugepages-2Mi: "128Mi" + ... +``` + +To verify the results, check the user and group ID that the container runs as: + +``` +$ kubectl exec -it qat-dpdk -c crypto-perf -- id +``` + +They are set to non-zero values as expected: + +``` +uid=1000 gid=2000 groups=2000,3000 +``` + +Next, check the device node permissions (`qat.intel.com/generic` exposes `/dev/vfio/` devices) are accessible to `runAsUser`/`runAsGroup`: + +``` +$ kubectl exec -it qat-dpdk -c crypto-perf -- ls -la /dev/vfio +total 0 +drwxr-xr-x 2 root root 140 Sep 7 10:55 . +drwxr-xr-x 7 root root 380 Sep 7 10:55 .. +crw------- 1 1000 2000 241, 0 Sep 7 10:55 58 +crw------- 1 1000 2000 241, 2 Sep 7 10:55 60 +crw------- 1 1000 2000 241, 10 Sep 7 10:55 68 +crw------- 1 1000 2000 241, 11 Sep 7 10:55 69 +crw-rw-rw- 1 1000 2000 10, 196 Sep 7 10:55 vfio +``` + +Finally, check the non-root container is also allowed to create HugePages: + +``` +$ kubectl exec -it qat-dpdk -c crypto-perf -- ls -la /dev/hugepages/ +``` + +`fsGroup` gives a `runAsUser` writable HugePages emptyDir mountpoint: + +``` +total 0 +drwxrwsr-x 2 root 3000 0 Sep 7 10:55 . +drwxr-xr-x 7 root root 380 Sep 7 10:55 .. +``` + +## Help us test it and provide feedback! +The functionality described here is expected to help with cluster security and the configurability of device permissions. To allow +non-root containers to use devices requires cluster admins to opt-in to the functionality by setting +`device_ownership_from_security_context = true`. To make it a default setting, please test it and provide your feedback (via SIG-Node meetings or issues)! +The flag is available in CRI-O v1.22 release and queued for containerd v1.6. + +More work is needed to get it *properly* supported. It is known to work with `runc` but it also needs to be made to function +with other OCI runtimes too, where applicable. For instance, Kata Containers supports device passthrough and allows it to make devices +available to containers in VM sandboxes too. + +Moreover, the additional challenge comes with support of user names and devices. This problem is still [open](https://github.com/kubernetes/enhancements/pull/2101) +and requires more brainstorming. + +Finally, it needs to be understood whether `runAsUser`/`runAsGroup` are enough or if device specific settings similar to `fsGroups` are needed in PodSpec/CRI v2. + +## Thanks +My thanks goes to Mike Brown (IBM, containerd), Peter Hunt (Redhat, CRI-O), and Alexander Kanevskiy (Intel) for providing all the feedback and good conversations. diff --git a/content/en/blog/_posts/2021-11-12-are-you-ready-for-dockershim-removal/index.md b/content/en/blog/_posts/2021-11-12-are-you-ready-for-dockershim-removal/index.md new file mode 100644 index 0000000000..b372e7e7b4 --- /dev/null +++ b/content/en/blog/_posts/2021-11-12-are-you-ready-for-dockershim-removal/index.md @@ -0,0 +1,59 @@ +--- +layout: blog +title: "Dockershim removal is coming. Are you ready?" +date: 2021-11-12 +slug: are-you-ready-for-dockershim-removal +--- + +**Author:** Sergey Kanzhelev, Google. With reviews from Davanum Srinivas, Elana Hashman, Noah Kantrowitz, Rey Lejano. + +Last year we announced that Dockershim is being deprecated: [Dockershim Deprecation FAQ](/blog/2020/12/02/dockershim-faq/). +Our current plan is to remove dockershim from the Kubernetes codebase soon. +We are looking for feedback from you whether you are ready for dockershim +removal and to ensure that you are ready when the time comes. +**Please fill out this survey: https://forms.gle/svCJmhvTv78jGdSx8**. + +The dockershim component that enables Docker as a Kubernetes container runtime is +being deprecated in favor of runtimes that directly use the [Container Runtime Interface](/blog/2016/12/container-runtime-interface-cri-in-kubernetes/) +created for Kubernetes. Many Kubernetes users have migrated to +other container runtimes without problems. However we see that dockershim is +still very popular. You may see some public numbers in recent [Container Report](https://www.datadoghq.com/container-report/#8) from DataDog. +Some Kubernetes hosting vendors just recently enabled other runtimes support +(especially for Windows nodes). And we know that many third party tools vendors +are still not ready: [migrating telemetry and security agents](/docs/tasks/administer-cluster/migrating-from-dockershim/migrating-telemetry-and-security-agents/#telemetry-and-security-agent-vendors). + +At this point, we believe that there is feature parity between Docker and the +other runtimes. Many end-users have used our [migration guide](/docs/tasks/administer-cluster/migrating-from-dockershim/) +and are running production workload using these different runtimes. The plan of +record today is that dockershim will be removed in version 1.24, slated for +release around April of next year. For those developing or running alpha and +beta versions, dockershim will be removed in December at the beginning of the +1.24 release development cycle. + +There is only one month left to give us feedback. We want you to tell us how +ready you are. + +**We are collecting opinions through this survey: [https://forms.gle/svCJmhvTv78jGdSx8](https://forms.gle/svCJmhvTv78jGdSx8)** +To better understand preparedness for the dockershim removal, our survey is +asking the version of Kubernetes you are currently using, and an estimate of +when you think you will adopt Kubernetes 1.24. All the aggregated information +on dockershim removal readiness will be published. +Free form comments will be reviewed by SIG Node leadership. If you want to +discuss any details of migrating from dockershim, report bugs or adoption +blockers, you can use one of the SIG Node contact options any time: +https://github.com/kubernetes/community/tree/master/sig-node#contact + +Kubernetes is a mature project. This deprecation is another +step in the effort to get away from permanent beta features and providing more +stability and compatibility guarantees. With the migration from dockershim you +will get more flexibility and choice of container runtime features as well as +less dependencies of your apps on specific underlying technology. Please take +time to review the [dockershim migration documentation](/docs/tasks/administer-cluster/migrating-from-dockershim/) +and consult your Kubernetes hosting vendor (if you have one) what container runtime options are available for you. +Read up [container runtime documentation with instructions on how to use containerd and CRI-O](/docs/setup/production-environment/container-runtimes/#container-runtimes) +to help prepare you when you're ready to upgrade to 1.24. CRI-O, containerd, and +Docker with [Mirantis cri-dockerd](https://github.com/Mirantis/cri-dockerd) are +not the only container runtime options, we encourage you to explore the [CNCF landscape on container runtimes](https://landscape.cncf.io/card-mode?category=container-runtime&grouping=category) +in case another suits you better. + +Thank you! diff --git a/content/en/community/_index.html b/content/en/community/_index.html index b41323c69e..c08aa25ae0 100644 --- a/content/en/community/_index.html +++ b/content/en/community/_index.html @@ -19,6 +19,7 @@ cid: community
+Contributor Community      Community Values      Code of conduct       Videos      diff --git a/content/en/docs/concepts/architecture/cloud-controller.md b/content/en/docs/concepts/architecture/cloud-controller.md index 229cc489f9..552f63d948 100644 --- a/content/en/docs/concepts/architecture/cloud-controller.md +++ b/content/en/docs/concepts/architecture/cloud-controller.md @@ -43,11 +43,11 @@ The controllers inside the cloud controller manager include: ### Node controller -The node controller is responsible for creating {{< glossary_tooltip text="Node" term_id="node" >}} objects +The node controller is responsible for updating {{< glossary_tooltip text="Node" term_id="node" >}} objects when new servers are created in your cloud infrastructure. The node controller obtains information about the hosts running inside your tenancy with the cloud provider. The node controller performs the following functions: -1. Initialize a Node object for each server that the controller discovers through the cloud provider API. +1. Update a Node object with the corresponding server's unique identifier obtained from the cloud provider API. 2. Annotating and labelling the Node object with cloud-specific information, such as the region the node is deployed into and the resources (CPU, memory, etc) that it has available. 3. Obtain the node's hostname and network addresses. diff --git a/content/en/docs/concepts/architecture/nodes.md b/content/en/docs/concepts/architecture/nodes.md index 1d4f6455b7..a57d47219e 100644 --- a/content/en/docs/concepts/architecture/nodes.md +++ b/content/en/docs/concepts/architecture/nodes.md @@ -72,7 +72,8 @@ The name of a Node object must be a valid The [name](/docs/concepts/overview/working-with-objects/names#names) identifies a Node. Two Nodes cannot have the same name at the same time. Kubernetes also assumes that a resource with the same name is the same object. In case of a Node, it is implicitly assumed that an instance using the -same name will have the same state (e.g. network settings, root disk contents). This may lead to +same name will have the same state (e.g. network settings, root disk contents) +and attributes like node labels. This may lead to inconsistencies if an instance was modified without changing its name. If the Node needs to be replaced or updated significantly, the existing Node object needs to be removed from API server first and re-added after the update. @@ -98,6 +99,21 @@ When the [Node authorization mode](/docs/reference/access-authn-authz/node/) and [NodeRestriction admission plugin](/docs/reference/access-authn-authz/admission-controllers/#noderestriction) are enabled, kubelets are only authorized to create/modify their own Node resource. +{{< note >}} +As mentioned in the [Node name uniqueness](#node-name-uniqueness) section, +when Node configuration needs to be updated, it is a good practice to re-register +the node with the API server. For example, if the kubelet being restarted with +the new set of `--node-labels`, but the same Node name is used, the change will +not take an effect, as labels are being set on the Node registration. + +Pods already scheduled on the Node may misbehave or cause issues if the Node +configuration will be changed on kubelet restart. For example, already running +Pod may be tainted against the new labels assigned to the Node, while other +Pods, that are incompatible with that Pod will be scheduled based on this new +label. Node re-registration ensures all Pods will be drained and properly +re-scheduled. +{{< /note >}} + ### Manual Node administration You can create and modify Node objects using diff --git a/content/en/docs/concepts/cluster-administration/addons.md b/content/en/docs/concepts/cluster-administration/addons.md index 5ed93ad20b..f2743064f2 100644 --- a/content/en/docs/concepts/cluster-administration/addons.md +++ b/content/en/docs/concepts/cluster-administration/addons.md @@ -25,7 +25,7 @@ This page lists some of the available add-ons and links to their respective inst * [Contrail](https://www.juniper.net/us/en/products-services/sdn/contrail/contrail-networking/), based on [Tungsten Fabric](https://tungsten.io), is an open source, multi-cloud network virtualization and policy management platform. Contrail and Tungsten Fabric are integrated with orchestration systems such as Kubernetes, OpenShift, OpenStack and Mesos, and provide isolation modes for virtual machines, containers/pods and bare metal workloads. * [Flannel](https://github.com/flannel-io/flannel#deploying-flannel-manually) is an overlay network provider that can be used with Kubernetes. * [Knitter](https://github.com/ZTE/Knitter/) is a plugin to support multiple network interfaces in a Kubernetes pod. -* [Multus](https://github.com/Intel-Corp/multus-cni) is a Multi plugin for multiple network support in Kubernetes to support all CNI plugins (e.g. Calico, Cilium, Contiv, Flannel), in addition to SRIOV, DPDK, OVS-DPDK and VPP based workloads in Kubernetes. +* Multus is a Multi plugin for multiple network support in Kubernetes to support all CNI plugins (e.g. Calico, Cilium, Contiv, Flannel), in addition to SRIOV, DPDK, OVS-DPDK and VPP based workloads in Kubernetes. * [OVN-Kubernetes](https://github.com/ovn-org/ovn-kubernetes/) is a networking provider for Kubernetes based on [OVN (Open Virtual Network)](https://github.com/ovn-org/ovn/), a virtual networking implementation that came out of the Open vSwitch (OVS) project. OVN-Kubernetes provides an overlay based networking implementation for Kubernetes, including an OVS based implementation of load balancing and network policy. * [OVN4NFV-K8S-Plugin](https://github.com/opnfv/ovn4nfv-k8s-plugin) is OVN based CNI controller plugin to provide cloud native based Service function chaining(SFC), Multiple OVN overlay networking, dynamic subnet creation, dynamic creation of virtual networks, VLAN Provider network, Direct provider network and pluggable with other Multi-network plugins, ideal for edge based cloud native workloads in Multi-cluster networking * [NSX-T](https://docs.vmware.com/en/VMware-NSX-T/2.0/nsxt_20_ncp_kubernetes.pdf) Container Plug-in (NCP) provides integration between VMware NSX-T and container orchestrators such as Kubernetes, as well as integration between NSX-T and container-based CaaS/PaaS platforms such as Pivotal Container Service (PKS) and OpenShift. diff --git a/content/en/docs/concepts/cluster-administration/networking.md b/content/en/docs/concepts/cluster-administration/networking.md index d8fec56d9d..da9aeed0ba 100644 --- a/content/en/docs/concepts/cluster-administration/networking.md +++ b/content/en/docs/concepts/cluster-administration/networking.md @@ -246,7 +246,7 @@ Lars Kellogg-Stedman. ### Multus (a Multi Network plugin) -[Multus](https://github.com/Intel-Corp/multus-cni) is a Multi CNI plugin to support the Multi Networking feature in Kubernetes using CRD based network objects in Kubernetes. +Multus is a Multi CNI plugin to support the Multi Networking feature in Kubernetes using CRD based network objects in Kubernetes. Multus supports all [reference plugins](https://github.com/containernetworking/plugins) (eg. [Flannel](https://github.com/containernetworking/cni.dev/blob/main/content/plugins/v0.9/meta/flannel.md), [DHCP](https://github.com/containernetworking/plugins/tree/master/plugins/ipam/dhcp), [Macvlan](https://github.com/containernetworking/plugins/tree/master/plugins/main/macvlan)) that implement the CNI specification and 3rd party plugins (eg. [Calico](https://github.com/projectcalico/cni-plugin), [Weave](https://github.com/weaveworks/weave), [Cilium](https://github.com/cilium/cilium), [Contiv](https://github.com/contiv/netplugin)). In addition to it, Multus supports [SRIOV](https://github.com/hustcat/sriov-cni), [DPDK](https://github.com/Intel-Corp/sriov-cni), [OVS-DPDK & VPP](https://github.com/intel/vhost-user-net-plugin) workloads in Kubernetes with both cloud native and NFV based applications in Kubernetes. diff --git a/content/en/docs/concepts/configuration/manage-resources-containers.md b/content/en/docs/concepts/configuration/manage-resources-containers.md index cbcfff67ef..e21173bfd3 100644 --- a/content/en/docs/concepts/configuration/manage-resources-containers.md +++ b/content/en/docs/concepts/configuration/manage-resources-containers.md @@ -99,9 +99,10 @@ resource requests/limits of that type for each Container in the Pod. Limits and requests for CPU resources are measured in *cpu* units. One cpu, in Kubernetes, is equivalent to **1 vCPU/Core** for cloud providers and **1 hyperthread** on bare-metal Intel processors. -Fractional requests are allowed. A Container with -`spec.containers[].resources.requests.cpu` of `0.5` is guaranteed half as much -CPU as one that asks for 1 CPU. The expression `0.1` is equivalent to the +Fractional requests are allowed. When you define a container with +`spec.containers[].resources.requests.cpu` set to `0.5`, you are requesting half +as much CPU time compared to if you asked for `1.0` CPU. +For CPU resource units, the expression `0.1` is equivalent to the expression `100m`, which can be read as "one hundred millicpu". Some people say "one hundred millicores", and this is understood to mean the same thing. A request with a decimal point, like `0.1`, is converted to `100m` by the API, and @@ -236,7 +237,7 @@ The kubelet also uses this kind of storage to hold container images, and the writable layers of running containers. {{< caution >}} -If a node fails, the data in its ephemeral storage can be lost. +If a node fails, the data in its ephemeral storage can be lost. Your applications cannot expect any performance SLAs (disk IOPS for example) from local ephemeral storage. {{< /caution >}} @@ -440,7 +441,7 @@ Kubernetes does not use them. Quotas are faster and more accurate than directory scanning. When a directory is assigned to a project, all files created under a directory are created in that project, and the kernel merely has to -keep track of how many blocks are in use by files in that project. +keep track of how many blocks are in use by files in that project. If a file is created and deleted, but has an open file descriptor, it continues to consume space. Quota tracking records that space accurately whereas directory scans overlook the storage used by deleted files. diff --git a/content/en/docs/concepts/configuration/overview.md b/content/en/docs/concepts/configuration/overview.md index 36eebe3abc..a4dd5c5901 100644 --- a/content/en/docs/concepts/configuration/overview.md +++ b/content/en/docs/concepts/configuration/overview.md @@ -55,7 +55,7 @@ DNS server watches the Kubernetes API for new `Services` and creates a set of DN If you only need access to the port for debugging purposes, you can use the [apiserver proxy](/docs/tasks/access-application-cluster/access-cluster/#manually-constructing-apiserver-proxy-urls) or [`kubectl port-forward`](/docs/tasks/access-application-cluster/port-forward-access-application-cluster/). - If you explicitly need to expose a Pod's port on the node, consider using a [NodePort](/docs/concepts/services-networking/service/#nodeport) Service before resorting to `hostPort`. + If you explicitly need to expose a Pod's port on the node, consider using a [NodePort](/docs/concepts/services-networking/service/#type-nodeport) Service before resorting to `hostPort`. - Avoid using `hostNetwork`, for the same reasons as `hostPort`. diff --git a/content/en/docs/concepts/configuration/secret.md b/content/en/docs/concepts/configuration/secret.md index 2f630be2b9..5f1b5e1f52 100644 --- a/content/en/docs/concepts/configuration/secret.md +++ b/content/en/docs/concepts/configuration/secret.md @@ -244,7 +244,7 @@ on the fly: The `kubernetes.io/basic-auth` type is provided for storing credentials needed for basic authentication. When using this Secret type, the `data` field of the -Secret must contain the following two keys: +Secret must contain one of the following two keys: - `username`: the user name for authentication; - `password`: the password or token for authentication. diff --git a/content/en/docs/concepts/containers/container-lifecycle-hooks.md b/content/en/docs/concepts/containers/container-lifecycle-hooks.md index 96569f9518..78c0a43b99 100644 --- a/content/en/docs/concepts/containers/container-lifecycle-hooks.md +++ b/content/en/docs/concepts/containers/container-lifecycle-hooks.md @@ -59,7 +59,7 @@ Resources consumed by the command are counted against the Container. ### Hook handler execution When a Container lifecycle management hook is called, -the Kubernetes management system execute the handler according to the hook action, +the Kubernetes management system executes the handler according to the hook action, `httpGet` and `tcpSocket` are executed by the kubelet process, and `exec` is executed in the container. Hook handler calls are synchronous within the context of the Pod containing the Container. diff --git a/content/en/docs/concepts/containers/images.md b/content/en/docs/concepts/containers/images.md index 5dfd349f80..3512ed2098 100644 --- a/content/en/docs/concepts/containers/images.md +++ b/content/en/docs/concepts/containers/images.md @@ -265,6 +265,73 @@ template needs to include the `.docker/config.json` or mount a drive that contai All pods will have read access to images in any private registry once private registry keys are added to the `.docker/config.json`. +### Interpretation of config.json {#config-json} + +The interpretation of `config.json` varies between the original Docker +implementation and the Kubernetes interpretation. In Docker, the `auths` keys +can only specify root URLs, whereas Kubernetes allows glob URLs as well as +prefix-matched paths. This means that a `config.json` like this is valid: + +```json +{ + "auths": { + "*my-registry.io/images": { + "auth": "…" + } + } +} +``` + +The root URL (`*my-registry.io`) is matched by using the following syntax: + +``` +pattern: + { term } + +term: + '*' matches any sequence of non-Separator characters + '?' matches any single non-Separator character + '[' [ '^' ] { character-range } ']' + character class (must be non-empty) + c matches character c (c != '*', '?', '\\', '[') + '\\' c matches character c + +character-range: + c matches character c (c != '\\', '-', ']') + '\\' c matches character c + lo '-' hi matches character c for lo <= c <= hi +``` + +Image pull operations would now pass the credentials to the CRI container +runtime for every valid pattern. For example the following container image names +would match successfully: + +- `my-registry.io/images` +- `my-registry.io/images/my-image` +- `my-registry.io/images/another-image` +- `sub.my-registry.io/images/my-image` +- `a.sub.my-registry.io/images/my-image` + +The kubelet performs image pulls sequentially for every found credential. This +means, that multiple entries in `config.json` are possible, too: + +```json +{ + "auths": { + "my-registry.io/images": { + "auth": "…" + }, + "my-registry.io/images/subpath": { + "auth": "…" + } + } +} +``` + +If now a container specifies an image `my-registry.io/images/subpath/my-image` +to be pulled, then the kubelet will try to download them from both +authentication sources if one of them fails. + ### Pre-pulled images {{< note >}} @@ -390,3 +457,4 @@ Kubelet will merge any `imagePullSecrets` into a single virtual `.docker/config. * Read the [OCI Image Manifest Specification](https://github.com/opencontainers/image-spec/blob/master/manifest.md). * Learn about [container image garbage collection](/docs/concepts/architecture/garbage-collection/#container-image-garbage-collection). +* Learn more about [pulling an Image from a Private Registry](/docs/tasks/configure-pod-container/pull-image-private-registry). diff --git a/content/en/docs/concepts/extend-kubernetes/_index.md b/content/en/docs/concepts/extend-kubernetes/_index.md index 083dca6964..825484f50d 100644 --- a/content/en/docs/concepts/extend-kubernetes/_index.md +++ b/content/en/docs/concepts/extend-kubernetes/_index.md @@ -52,9 +52,9 @@ Flags and configuration files may not always be changeable in a hosted Kubernete Extensions are software components that extend and deeply integrate with Kubernetes. They adapt it to support new types and new kinds of hardware. -Most cluster administrators will use a hosted or distribution -instance of Kubernetes. As a result, most Kubernetes users will not need to -install extensions and fewer will need to author new ones. +Many cluster administrators use a hosted or distribution instance of Kubernetes. +These clusters come with extensions pre-installed. As a result, most Kubernetes +users will not need to install extensions and even fewer users will need to author new ones. ## Extension Patterns diff --git a/content/en/docs/concepts/extend-kubernetes/api-extension/custom-resources.md b/content/en/docs/concepts/extend-kubernetes/api-extension/custom-resources.md index b3fa73e5f0..365590e657 100644 --- a/content/en/docs/concepts/extend-kubernetes/api-extension/custom-resources.md +++ b/content/en/docs/concepts/extend-kubernetes/api-extension/custom-resources.md @@ -148,8 +148,8 @@ and use a controller to handle events. Usually, each resource in the Kubernetes API requires code that handles REST requests and manages persistent storage of objects. The main Kubernetes API server handles built-in resources like *pods* and *services*, and can also generically handle custom resources through [CRDs](#customresourcedefinitions). The [aggregation layer](/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/) allows you to provide specialized -implementations for your custom resources by writing and deploying your own standalone API server. -The main API server delegates requests to you for the custom resources that you handle, +implementations for your custom resources by writing and deploying your own API server. +The main API server delegates requests to your API server for the custom resources that you handle, making them available to all of its clients. ## Choosing a method for adding custom resources diff --git a/content/en/docs/concepts/extend-kubernetes/operator.md b/content/en/docs/concepts/extend-kubernetes/operator.md index 72fe12f1e7..7245c45bfc 100644 --- a/content/en/docs/concepts/extend-kubernetes/operator.md +++ b/content/en/docs/concepts/extend-kubernetes/operator.md @@ -31,9 +31,7 @@ built-in automation from the core of Kubernetes. You can use Kubernetes to automate deploying and running workloads, *and* you can automate how Kubernetes does that. -Kubernetes' {{< glossary_tooltip text="controllers" term_id="controller" >}} -concept lets you extend the cluster's behaviour without modifying the code -of Kubernetes itself. +Kubernetes' {{< glossary_tooltip text="operator pattern" term_id="operator-pattern" >}} concept lets you extend the cluster's behaviour without modifying the code of Kubernetes itself by linking {{< glossary_tooltip text="controllers" term_id="controller" >}} to one or more custom resources. Operators are clients of the Kubernetes API that act as controllers for a [Custom Resource](/docs/concepts/extend-kubernetes/api-extension/custom-resources/). diff --git a/content/en/docs/concepts/security/overview.md b/content/en/docs/concepts/security/overview.md index 0028f873b4..9519ef1eaf 100644 --- a/content/en/docs/concepts/security/overview.md +++ b/content/en/docs/concepts/security/overview.md @@ -74,10 +74,10 @@ Suggestions for securing your infrastructure in a Kubernetes cluster: Area of Concern for Kubernetes Infrastructure | Recommendation | --------------------------------------------- | -------------- | Network access to API Server (Control plane) | All access to the Kubernetes control plane is not allowed publicly on the internet and is controlled by network access control lists restricted to the set of IP addresses needed to administer the cluster.| -Network access to Nodes (nodes) | Nodes should be configured to _only_ accept connections (via network access control lists)from the control plane on the specified ports, and accept connections for services in Kubernetes of type NodePort and LoadBalancer. If possible, these nodes should not be exposed on the public internet entirely. +Network access to Nodes (nodes) | Nodes should be configured to _only_ accept connections (via network access control lists) from the control plane on the specified ports, and accept connections for services in Kubernetes of type NodePort and LoadBalancer. If possible, these nodes should not be exposed on the public internet entirely. Kubernetes access to Cloud Provider API | Each cloud provider needs to grant a different set of permissions to the Kubernetes control plane and nodes. It is best to provide the cluster with cloud provider access that follows the [principle of least privilege](https://en.wikipedia.org/wiki/Principle_of_least_privilege) for the resources it needs to administer. The [Kops documentation](https://github.com/kubernetes/kops/blob/master/docs/iam_roles.md#iam-roles) provides information about IAM policies and roles. Access to etcd | Access to etcd (the datastore of Kubernetes) should be limited to the control plane only. Depending on your configuration, you should attempt to use etcd over TLS. More information can be found in the [etcd documentation](https://github.com/etcd-io/etcd/tree/master/Documentation). -etcd Encryption | Wherever possible it's a good practice to encrypt all drives at rest, but since etcd holds the state of the entire cluster (including Secrets) its disk should especially be encrypted at rest. +etcd Encryption | Wherever possible it's a good practice to encrypt all storage at rest, and since etcd holds the state of the entire cluster (including Secrets) its disk should especially be encrypted at rest. {{< /table >}} @@ -99,7 +99,7 @@ good information practices, read and follow the advice about Depending on the attack surface of your application, you may want to focus on specific aspects of security. For example: If you are running a service (Service A) that is critical in a chain of other resources and a separate workload (Service B) which is -vulnerable to a resource exhaustion attack then the risk of compromising Service A +vulnerable to a resource exhaustion attack, then the risk of compromising Service A is high if you do not limit the resources of Service B. The following table lists areas of security concerns and recommendations for securing workloads running in Kubernetes: @@ -108,10 +108,10 @@ Area of Concern for Workload Security | Recommendation | RBAC Authorization (Access to the Kubernetes API) | https://kubernetes.io/docs/reference/access-authn-authz/rbac/ Authentication | https://kubernetes.io/docs/concepts/security/controlling-access/ Application secrets management (and encrypting them in etcd at rest) | https://kubernetes.io/docs/concepts/configuration/secret/
https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/ -Pod Security Policies | https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +Ensuring that pods meet defined Pod Security Standards | https://kubernetes.io/docs/concepts/security/pod-security-standards/#policy-instantiation Quality of Service (and Cluster resource management) | https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/ Network Policies | https://kubernetes.io/docs/concepts/services-networking/network-policies/ -TLS For Kubernetes Ingress | https://kubernetes.io/docs/concepts/services-networking/ingress/#tls +TLS for Kubernetes Ingress | https://kubernetes.io/docs/concepts/services-networking/ingress/#tls ## Container @@ -137,7 +137,7 @@ are recommendations to protect application code: Area of Concern for Code | Recommendation | -------------------------| -------------- | -Access over TLS only | If your code needs to communicate by TCP, perform a TLS handshake with the client ahead of time. With the exception of a few cases, encrypt everything in transit. Going one step further, it's a good idea to encrypt network traffic between services. This can be done through a process known as mutual or [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication) which performs a two sided verification of communication between two certificate holding services. | +Access over TLS only | If your code needs to communicate by TCP, perform a TLS handshake with the client ahead of time. With the exception of a few cases, encrypt everything in transit. Going one step further, it's a good idea to encrypt network traffic between services. This can be done through a process known as mutual TLS authentication or [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication) which performs a two sided verification of communication between two certificate holding services. | Limiting port ranges of communication | This recommendation may be a bit self-explanatory, but wherever possible you should only expose the ports on your service that are absolutely essential for communication or metric gathering. | 3rd Party Dependency Security | It is a good practice to regularly scan your application's third party libraries for known security vulnerabilities. Each programming language has a tool for performing this check automatically. | Static Code Analysis | Most languages provide a way for a snippet of code to be analyzed for any potentially unsafe coding practices. Whenever possible you should perform checks using automated tooling that can scan codebases for common security errors. Some of the tools can be found at: https://owasp.org/www-community/Source_Code_Analysis_Tools | diff --git a/content/en/docs/concepts/security/pod-security-standards.md b/content/en/docs/concepts/security/pod-security-standards.md index 62e8943479..205d4ddc8e 100644 --- a/content/en/docs/concepts/security/pod-security-standards.md +++ b/content/en/docs/concepts/security/pod-security-standards.md @@ -484,7 +484,7 @@ of individual policies are not defined here. - {{< example file="security/podsecurity-baseline.yaml" >}}Baseline namespace{{< /example >}} - {{< example file="security/podsecurity-restricted.yaml" >}}Restricted namespace{{< /example >}} -[**PodSecurityPolicy**](/docs/concepts/profile/pod-security-profile/) (Deprecated) +[**PodSecurityPolicy**](/docs/concepts/policy/pod-security-policy/) (Deprecated) - {{< example file="policy/privileged-psp.yaml" >}}Privileged{{< /example >}} - {{< example file="policy/baseline-psp.yaml" >}}Baseline{{< /example >}} diff --git a/content/en/docs/concepts/services-networking/connect-applications-service.md b/content/en/docs/concepts/services-networking/connect-applications-service.md index 89d2daddb2..07bab7965b 100644 --- a/content/en/docs/concepts/services-networking/connect-applications-service.md +++ b/content/en/docs/concepts/services-networking/connect-applications-service.md @@ -54,7 +54,7 @@ kubectl get pods -l run=my-nginx -o yaml | grep podIP You should be able to ssh into any node in your cluster and curl both IPs. Note that the containers are *not* using port 80 on the node, nor are there any special NAT rules to route traffic to the pod. This means you can run multiple nginx pods on the same node all using the same containerPort and access them from any other pod or node in your cluster using IP. Like Docker, ports can still be published to the host node's interfaces, but the need for this is radically diminished because of the networking model. -You can read more about [how we achieve this](/docs/concepts/cluster-administration/networking/#how-to-achieve-this) if you're curious. +You can read more about the [Kubernetes Networking Model](/docs/concepts/cluster-administration/networking/#the-kubernetes-network-model) if you're curious. ## Creating a Service diff --git a/content/en/docs/concepts/services-networking/ingress-controllers.md b/content/en/docs/concepts/services-networking/ingress-controllers.md index 0ee1d53ef9..b19c973fc7 100644 --- a/content/en/docs/concepts/services-networking/ingress-controllers.md +++ b/content/en/docs/concepts/services-networking/ingress-controllers.md @@ -28,6 +28,7 @@ Kubernetes as a project supports and maintains [AWS](https://github.com/kubernet controller. * [Apache APISIX ingress controller](https://github.com/apache/apisix-ingress-controller) is an [Apache APISIX](https://github.com/apache/apisix)-based ingress controller. * [Avi Kubernetes Operator](https://github.com/vmware/load-balancer-and-ingress-services-for-kubernetes) provides L4-L7 load-balancing using [VMware NSX Advanced Load Balancer](https://avinetworks.com/). +* [BFE Ingress Controller](https://github.com/bfenetworks/ingress-bfe) is a [BFE](https://www.bfe-networks.net)-based ingress controller. * The [Citrix ingress controller](https://github.com/citrix/citrix-k8s-ingress-controller#readme) works with Citrix Application Delivery Controller. * [Contour](https://projectcontour.io/) is an [Envoy](https://www.envoyproxy.io/) based ingress controller. diff --git a/content/en/docs/concepts/services-networking/ingress.md b/content/en/docs/concepts/services-networking/ingress.md index fbeb5289fb..5f4837fa29 100644 --- a/content/en/docs/concepts/services-networking/ingress.md +++ b/content/en/docs/concepts/services-networking/ingress.md @@ -51,7 +51,7 @@ graph LR; An Ingress may be configured to give Services externally-reachable URLs, load balance traffic, terminate SSL / TLS, and offer name-based virtual hosting. An [Ingress controller](/docs/concepts/services-networking/ingress-controllers) is responsible for fulfilling the Ingress, usually with a load balancer, though it may also configure your edge router or additional frontends to help handle the traffic. An Ingress does not expose arbitrary ports or protocols. Exposing services other than HTTP and HTTPS to the internet typically -uses a service of type [Service.Type=NodePort](/docs/concepts/services-networking/service/#nodeport) or +uses a service of type [Service.Type=NodePort](/docs/concepts/services-networking/service/#type-nodeport) or [Service.Type=LoadBalancer](/docs/concepts/services-networking/service/#loadbalancer). ## Prerequisites diff --git a/content/en/docs/concepts/services-networking/service.md b/content/en/docs/concepts/services-networking/service.md index 4f8eb45f38..cb3686dd21 100644 --- a/content/en/docs/concepts/services-networking/service.md +++ b/content/en/docs/concepts/services-networking/service.md @@ -568,7 +568,7 @@ The default is `ClusterIP`. You can also use [Ingress](/docs/concepts/services-networking/ingress/) to expose your Service. Ingress is not a Service type, but it acts as the entry point for your cluster. It lets you consolidate your routing rules into a single resource as it can expose multiple services under the same IP address. -### Type NodePort {#nodeport} +### Type NodePort {#type-nodeport} If you set the `type` field to `NodePort`, the Kubernetes control plane allocates a port from a range specified by `--service-node-port-range` flag (default: 30000-32767). diff --git a/content/en/docs/concepts/storage/projected-volumes.md b/content/en/docs/concepts/storage/projected-volumes.md new file mode 100644 index 0000000000..a5914adb6f --- /dev/null +++ b/content/en/docs/concepts/storage/projected-volumes.md @@ -0,0 +1,123 @@ +--- +reviewers: +- sftim +- marosset +- jsturtevant +- zshihang +title: Projected Volumes +content_type: concept +--- + + + +This document describes the current state of _projected volumes_ in Kubernetes. Familiarity with [volumes](/docs/concepts/storage/volumes/) is suggested. + + + +## Introduction + +A `projected` volume maps several existing volume sources into the same directory. + +Currently, the following types of volume sources can be projected: + +* [`secret`](/docs/concepts/storage/volumes/#secret) +* [`downwardAPI`](/docs/concepts/storage/volumes/#downwardapi) +* [`configMap`](/docs/concepts/storage/volumes/#configmap) +* `serviceAccountToken` + +All sources are required to be in the same namespace as the Pod. For more details, +see the [all-in-one volume design document](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/all-in-one-volume.md). + +### Example configuration with a secret, a downwardAPI, and a configMap {#example-configuration-secret-downwardapi-configmap} + +{{< codenew file="pods/storage/projected-secret-downwardapi-configmap.yaml" >}} + +### Example configuration: secrets with a non-default permission mode set {#example-configuration-secrets-nondefault-permission-mode} + +{{< codenew file="pods/storage/projected-secrets-nondefault-permission-mode.yaml" >}} + +Each projected volume source is listed in the spec under `sources`. The +parameters are nearly the same with two exceptions: + +* For secrets, the `secretName` field has been changed to `name` to be consistent + with ConfigMap naming. +* The `defaultMode` can only be specified at the projected level and not for each + volume source. However, as illustrated above, you can explicitly set the `mode` + for each individual projection. + +When the `TokenRequestProjection` feature is enabled, you can inject the token +for the current [service account](/docs/reference/access-authn-authz/authentication/#service-account-tokens) +into a Pod at a specified path. For example: + +{{< codenew file="pods/storage/projected-service-account-token.yaml" >}} + +The example Pod has a projected volume containing the injected service account +token. This token can be used by a Pod's containers to access the Kubernetes API +server. The `audience` field contains the intended audience of the +token. A recipient of the token must identify itself with an identifier specified +in the audience of the token, and otherwise should reject the token. This field +is optional and it defaults to the identifier of the API server. + +The `expirationSeconds` is the expected duration of validity of the service account +token. It defaults to 1 hour and must be at least 10 minutes (600 seconds). An administrator +can also limit its maximum value by specifying the `--service-account-max-token-expiration` +option for the API server. The `path` field specifies a relative path to the mount point +of the projected volume. + +{{< note >}} +A container using a projected volume source as a [`subPath`](/docs/concepts/storage/volumes/#using-subpath) +volume mount will not receive updates for those volume sources. +{{< /note >}} + +## SecurityContext interactions + +The [proposal for file permission handling in projected service account volume](https://github.com/kubernetes/enhancements/pull/1598) +enhancement introduced the projected files having the the correct owner +permissions set. + +### Linux + +In Linux pods that have a projected volume and `RunAsUser` set in the Pod +[`SecurityContext`](/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context), +the projected files have the correct ownership set including container user +ownership. + +### Windows + +In Windows pods that have a projected volume and `RunAsUsername` set in the +Pod `SecurityContext`, the ownership is not enforced due to the way user +accounts are managed in Windows. Windows stores and manages local user and group +accounts in a database file called Security Account Manager (SAM). Each +container maintains its own instance of the SAM database, to which the host has +no visibility into while the container is running. Windows containers are +designed to run the user mode portion of the OS in isolation from the host, +hence the maintenance of a virtual SAM database. As a result, the kubelet running +on the host does not have the ability to dynamically configure host file +ownership for virtualized container accounts. It is recommended that if files on +the host machine are to be shared with the container then they should be placed +into their own volume mount outside of `C:\`. + +By default, the projected files will have the following ownership as shown for +an example projected volume file: +```powershell +Path : Microsoft.PowerShell.Core\FileSystem::C:\var\run\secrets\kubernetes.io\serviceaccount\..2021_08_31_22_22_18.318230061\ca.crt +Owner : BUILTIN\Administrators +Group : NT AUTHORITY\SYSTEM +Access : NT AUTHORITY\SYSTEM Allow FullControl + BUILTIN\Administrators Allow FullControl + BUILTIN\Users Allow ReadAndExecute, Synchronize +Audit : +Sddl : O:BAG:SYD:AI(A;ID;FA;;;SY)(A;ID;FA;;;BA)(A;ID;0x1200a9;;;BU) +``` +This implies all administrator users like `ContainerAdministrator` will have +read, write and execute access while, non-administrator users will have read and +execute access. + +{{< note >}} +In general, granting the container access to the host is discouraged as it can +open the door for potential security exploits. + +Creating a Windows Pod with `RunAsUser` in it's `SecurityContext` will result in +the Pod being stuck at `ContainerCreating` forever. So it is advised to not use +the Linux only `RunAsUser` option with Windows Pods. +{{< /note >}} diff --git a/content/en/docs/concepts/storage/volumes.md b/content/en/docs/concepts/storage/volumes.md index 56694dee66..7506edf556 100644 --- a/content/en/docs/concepts/storage/volumes.md +++ b/content/en/docs/concepts/storage/volumes.md @@ -33,8 +33,8 @@ drivers, but the functionality is somewhat limited. Kubernetes supports many types of volumes. A {{< glossary_tooltip term_id="pod" text="Pod" >}} can use any number of volume types simultaneously. Ephemeral volume types have a lifetime of a pod, but persistent volumes exist beyond -the lifetime of a pod. When a pod ceases to exist, Kubernetes destroys ephemeral volumes; -however, Kubernetes does not destroy persistent volumes. +the lifetime of a pod. When a pod ceases to exist, Kubernetes destroys ephemeral volumes; +however, Kubernetes does not destroy persistent volumes. For any kind of volume in a given pod, data is preserved across container restarts. At its core, a volume is a directory, possibly with some data in it, which @@ -44,12 +44,21 @@ volume type used. To use a volume, specify the volumes to provide for the Pod in `.spec.volumes` and declare where to mount those volumes into containers in `.spec.containers[*].volumeMounts`. -A process in a container sees a filesystem view composed from their Docker -image and volumes. The [Docker image](https://docs.docker.com/userguide/dockerimages/) -is at the root of the filesystem hierarchy. Volumes mount at the specified paths within -the image. Volumes can not mount onto other volumes or have hard links to -other volumes. Each Container in the Pod's configuration must independently specify where to -mount each volume. +A process in a container sees a filesystem view composed from the initial contents of +the {{< glossary_tooltip text="container image" term_id="image" >}}, plus volumes +(if defined) mounted inside the container. +The process sees a root filesystem that initially matches the contents of the container +image. +Any writes to within that filesystem hierarchy, if allowed, affect what that process views +when it performs a subsequent filesystem access. +Volumes mount at the [specified paths](#using-subpath) within +the image. +For each container defined within a Pod, you must independently specify where +to mount each volume that the container uses. + +Volumes cannot mount within other volumes (but see [Using subPath](#using-subpath) +for a related mechanism). Also, a volume cannot contain a hard link to anything in +a different volume. ## Types of Volumes {#volume-types} @@ -217,7 +226,7 @@ It redirects all plugin operations from the existing in-tree plugin to the `cinder.csi.openstack.org` Container Storage Interface (CSI) Driver. [OpenStack Cinder CSI Driver](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) must be installed on the cluster. -You can disable Cinder CSI migration for your cluster by setting the `CSIMigrationOpenStack` +You can disable Cinder CSI migration for your cluster by setting the `CSIMigrationOpenStack` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) to `false`. If you disable the `CSIMigrationOpenStack` feature, the in-tree Cinder volume plugin takes responsibility for all aspects of Cinder volume storage management. @@ -801,143 +810,8 @@ For more details, see the [Portworx volume](https://github.com/kubernetes/exampl ### projected -A `projected` volume maps several existing volume sources into the same directory. - -Currently, the following types of volume sources can be projected: - -* [`secret`](#secret) -* [`downwardAPI`](#downwardapi) -* [`configMap`](#configmap) -* `serviceAccountToken` - -All sources are required to be in the same namespace as the Pod. For more details, -see the [all-in-one volume design document](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/all-in-one-volume.md). - -#### Example configuration with a secret, a downwardAPI, and a configMap {#example-configuration-secret-downwardapi-configmap} - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: volume-test -spec: - containers: - - name: container-test - image: busybox - volumeMounts: - - name: all-in-one - mountPath: "/projected-volume" - readOnly: true - volumes: - - name: all-in-one - projected: - sources: - - secret: - name: mysecret - items: - - key: username - path: my-group/my-username - - downwardAPI: - items: - - path: "labels" - fieldRef: - fieldPath: metadata.labels - - path: "cpu_limit" - resourceFieldRef: - containerName: container-test - resource: limits.cpu - - configMap: - name: myconfigmap - items: - - key: config - path: my-group/my-config -``` - -#### Example configuration: secrets with a non-default permission mode set {#example-configuration-secrets-nondefault-permission-mode} - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: volume-test -spec: - containers: - - name: container-test - image: busybox - volumeMounts: - - name: all-in-one - mountPath: "/projected-volume" - readOnly: true - volumes: - - name: all-in-one - projected: - sources: - - secret: - name: mysecret - items: - - key: username - path: my-group/my-username - - secret: - name: mysecret2 - items: - - key: password - path: my-group/my-password - mode: 511 -``` - -Each projected volume source is listed in the spec under `sources`. The -parameters are nearly the same with two exceptions: - -* For secrets, the `secretName` field has been changed to `name` to be consistent - with ConfigMap naming. -* The `defaultMode` can only be specified at the projected level and not for each - volume source. However, as illustrated above, you can explicitly set the `mode` - for each individual projection. - -When the `TokenRequestProjection` feature is enabled, you can inject the token -for the current [service account](/docs/reference/access-authn-authz/authentication/#service-account-tokens) -into a Pod at a specified path. For example: - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: sa-token-test -spec: - containers: - - name: container-test - image: busybox - volumeMounts: - - name: token-vol - mountPath: "/service-account" - readOnly: true - volumes: - - name: token-vol - projected: - sources: - - serviceAccountToken: - audience: api - expirationSeconds: 3600 - path: token -``` - -The example Pod has a projected volume containing the injected service account -token. This token can be used by a Pod's containers to access the Kubernetes API -server. The `audience` field contains the intended audience of the -token. A recipient of the token must identify itself with an identifier specified -in the audience of the token, and otherwise should reject the token. This field -is optional and it defaults to the identifier of the API server. - -The `expirationSeconds` is the expected duration of validity of the service account -token. It defaults to 1 hour and must be at least 10 minutes (600 seconds). An administrator -can also limit its maximum value by specifying the `--service-account-max-token-expiration` -option for the API server. The `path` field specifies a relative path to the mount point -of the projected volume. - -{{< note >}} -A container using a projected volume source as a [`subPath`](#using-subpath) volume mount will not -receive updates for those volume sources. -{{< /note >}} +A projected volume maps several existing volume sources into the same +directory. For more details, see [projected volumes](/docs/concepts/storage/projected-volumes/) ### quobyte (deprecated) {#quobyte} diff --git a/content/en/docs/concepts/workloads/controllers/cron-jobs.md b/content/en/docs/concepts/workloads/controllers/cron-jobs.md index 0e95f72011..62cac0f001 100644 --- a/content/en/docs/concepts/workloads/controllers/cron-jobs.md +++ b/content/en/docs/concepts/workloads/controllers/cron-jobs.md @@ -17,8 +17,6 @@ A _CronJob_ creates {{< glossary_tooltip term_id="job" text="Jobs" >}} on a repe One CronJob object is like one line of a _crontab_ (cron table) file. It runs a job periodically on a given schedule, written in [Cron](https://en.wikipedia.org/wiki/Cron) format. -In addition, the CronJob schedule supports timezone handling, you can specify the timezone by adding "CRON_TZ=