From 461b0889e6485c23b7cade1fed4dffb00074fb23 Mon Sep 17 00:00:00 2001 From: Scott Sorell Date: Sat, 3 Oct 2026 09:43:45 -0600 Subject: [PATCH] A number of fixes to get things running --- .github/copilot-instructions.md | 116 ++ .mcp.json | 14 + GITOPS.md | 82 +- README.md | 27 +- TALOS.md | 47 + cilium_test/cilium.yaml | 1665 +++++++++++++++++ .../cilium/templates/namespace.yaml | 8 + infrastructure/cilium/values.yaml | 38 +- terraform/main.tf | 17 + terraform/outputs.tf | 31 +- 10 files changed, 2035 insertions(+), 10 deletions(-) create mode 100644 .github/copilot-instructions.md create mode 100644 .mcp.json create mode 100644 cilium_test/cilium.yaml create mode 100644 infrastructure/cilium/templates/namespace.yaml diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..f1c6d4a --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,116 @@ +# Copilot instructions for home-services + +## Repository overview + +This repository manages a home lab through two delivery paths: + +- Host services via Docker/Podman Compose under `compose-services/` +- A Talos Kubernetes cluster bootstrapped with Terraform and reconciled by Argo CD + +The core directories are: + +- `compose-services/` — host-specific compose stacks (`media_nas1`, `media_rhel1`) +- `bootstrap/applicationsets/` — Argo CD ApplicationSets for infrastructure and app workloads +- `infrastructure/` — cluster-level Helm apps such as Cilium and Longhorn +- `services/` — application Helm charts such as Vaultwarden +- `terraform/` — Talos bootstrap, cluster provisioning, and Argo CD installation +- `.gitea/workflows/deploy.yaml` — deployment automation for the host stacks + +## Build, test, and validation commands + +There is no repo-wide lint step in this project. Validation is a combination of the Python unit tests for `composectl.py` and Terraform validation for the bootstrap path. + +Run the unit tests: + +```bash +python3 -m unittest tests.test_composectl +``` + +Run a single test by name: + +```bash +python3 -m unittest tests.test_composectl.ComposeCtlTests.test_build_compose_command_uses_selected_runtime +``` + +Inspect the CLI and supported actions: + +```bash +python3 composectl.py --help +``` + +Dry-run a stack refresh before changing live workloads: + +```bash +python3 composectl.py --dry-run --stack media_nas1 refresh +python3 composectl.py --dry-run --stack media_rhel1 refresh +``` + +Terraform bootstrap validation (from `terraform/`): + +```bash +cd terraform +terraform init +terraform fmt -check -diff +terraform validate +``` + +## Architecture and operational flow + +### 1) Compose deployment path + +`composectl.py` is the entry point for runtime-agnostic stack operations. It detects Docker or Podman at runtime and runs the same `docker compose`/`podman compose` flow against a stack under `compose-services//compose.yaml`. + +The default maintenance flow for a stack is: + +1. `compose config` +2. `pull` +3. `up -d --remove-orphans` +4. optional `image prune -f` + +This is the path used by the Gitea workflow in `.gitea/workflows/deploy.yaml` for `media_nas1` and `media_rhel1`. + +### 2) Talos + GitOps path + +`terraform/main.tf` bootstraps the Talos cluster and installs Argo CD. It creates: + +- Talos machine secrets and client configuration +- per-role machine configs for control-plane and worker nodes +- bootstrap of the selected control-plane node +- a kubeconfig for the cluster +- a Helm release for Argo CD +- the `gitea-repository` secret used by Argo CD to pull from this repo + +Once the cluster is ready, Argo CD discovers the application manifests via `bootstrap/applicationsets/`, which fan out into `infrastructure/*` and `services/*` chart definitions. + +### 3) Secret handling and environment boundaries + +Sensitive values are intentionally not committed to the repo. The project expects: + +- `.sops.yaml` as the repository secret policy +- age-based decryption for encrypted secrets +- local-only Terraform `.tfvars` files and environment variables such as `TF_VAR_gitea_username` / `TF_VAR_gitea_password` +- non-committed kubeconfig and talosconfig outputs saved to local paths + +## MCP server + +VS Code loads the portable workspace MCP configuration from `.mcp.json`. It runs the pinned HashiCorp Terraform MCP Server image through Podman and exposes only read-only public Registry lookups for providers and modules. Podman must be installed; the image is pulled on first use. The configuration does not provide HCP Terraform credentials or enable workspace operations. + +## Conventions specific to this repo + +- Treat `compose-services//compose.yaml` as the source of truth for host workloads; keep stack naming aligned with the folder name. +- Prefer `composectl.py --dry-run` before any live deployment or refresh. +- Keep the runtime abstraction intact: avoid writing Docker-only or Podman-only commands into repo scripts when `composectl.py` can handle both. +- Use the existing stack naming pattern: `media_nas1` maps to `media_nas1_services`, and similarly for `media_rhel1`. +- Keep Terraform inputs site-specific and local; `terraform.tfvars.example` is a template, not a checked-in secret file. +- Do not commit credentials, tokens, kubeconfigs, or local state artifacts. If a file is sensitive, keep it outside the repo or encrypted under the SOPS policy. +- The Gitea workflow is keyed to the `main` branch and the first Friday of the month schedule; do not change that deployment cadence without understanding the automated rollback behavior. +- When editing cluster bootstrapping or Argo CD setup, keep the GitOps boundary clear: Terraform provisions the platform, while chart directories under `infrastructure/` and `services/` hold the workloads. + +## Practical guidance for Copilot sessions + +When making changes here, favor the repository's two-layer model: + +- Compose changes belong in `compose-services/*/compose.yaml` or `composectl.py` +- Cluster or app delivery changes belong in `terraform/`, `bootstrap/applicationsets/`, `infrastructure/`, or `services/` + +Before proposing deploy commands, check whether the change targets the host Compose path or the Kubernetes/GitOps path. The repository intentionally keeps those flows separate. diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..6075538 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,14 @@ +{ + "mcpServers": { + "terraform-registry": { + "command": "podman", + "args": [ + "run", + "-i", + "--rm", + "docker.io/hashicorp/terraform-mcp-server:1.3.0", + "--tools=search_providers,get_provider_details,get_latest_provider_version,get_provider_capabilities,search_modules,get_module_details,get_latest_module_version" + ] + } + } +} diff --git a/GITOPS.md b/GITOPS.md index e6cee6b..f1f89e7 100644 --- a/GITOPS.md +++ b/GITOPS.md @@ -15,6 +15,68 @@ Terraform owns Talos machine configuration, cluster bootstrap, and the Argo CD H Terraform generates one machine-secrets bundle, renders one machine configuration per role, applies the matching configuration to every node, and bootstraps only the selected control-plane node. For more than one control-plane node, `cluster_endpoint` must resolve to a stable VIP or load balancer rather than an individual node address. +## Talos 1.13 to 1.14 upgrade + +Talos 1.14 uses the Image Factory installer image. The old image `ghcr.io/siderolabs/installer:v1.14.0` is not published and returns `NotFound`. For these amd64 KVM nodes, use the configured schematic: + +```bash +export TALOSCONFIG="$HOME/.talos/home-services/talosconfig" +export TALOS_INSTALLER_IMAGE="factory.talos.dev/metal-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.14.0" +``` + +Upgrade one node at a time, worker first: + +```bash +talosctl upgrade --endpoints 10.0.0.246 --nodes 10.0.0.246 --image "$TALOS_INSTALLER_IMAGE" --wait +talosctl version --endpoints 10.0.0.246 --nodes 10.0.0.246 + +talosctl upgrade --endpoints 10.0.0.99 --nodes 10.0.0.99 --image "$TALOS_INSTALLER_IMAGE" --drain=false --wait +talosctl version --endpoints 10.0.0.99 --nodes 10.0.0.99 +talosctl health --endpoints 10.0.0.99 --nodes 10.0.0.99,10.0.0.246 +``` + +Set `--endpoints` to the same address as `--nodes` for each upgrade. This prevents Talosctl from attempting request forwarding through the other node during the version-compatibility check. The control-plane upgrade uses `--drain=false` because this single-control-plane cluster cannot safely drain through a Kubernetes kubeconfig during the Talos transition. + +After both nodes report 1.14, set `talos_version = "v1.14.0"` in `terraform.tfvars`, then run `terraform plan` and `terraform apply` to reconcile the generated machine configurations and GitOps workloads. Do not reset or recreate the VMs during an in-place upgrade. + +## CLI credentials and outputs + +After `terraform apply` completes, export the sensitive Terraform outputs to local files for the Talos and Kubernetes CLI tools. Do not commit or share these files: + +```bash +cd terraform + +export KUBECONFIG="$HOME/.kube/home-services" +mkdir -p "$(dirname "$KUBECONFIG")" +terraform output -raw kubeconfig > "$KUBECONFIG" +chmod 600 "$KUBECONFIG" +kubectl cluster-info + +export TALOSCONFIG="$HOME/.talos/home-services/talosconfig" +mkdir -p "$(dirname "$TALOSCONFIG")" +terraform output -raw talosconfig > "$TALOSCONFIG" +chmod 600 "$TALOSCONFIG" +talosctl --nodes "$(terraform output -json talos_nodes | jq -r 'join(",")')" version +``` + +Terraform state contains the Talos client credentials and Kubernetes client certificates. Protect the state backend and local state files. The `kubeconfig` and `talosconfig` outputs are marked sensitive, but `terraform output -raw` intentionally writes their contents to the terminal or redirected file. + +Argo CD is installed as an internal `ClusterIP` service. Use the generated port-forward command to connect the Argo CD CLI: + +```bash +argocd login --port-forward --port-forward-namespace argocd argocd-server +argocd app list +``` + +The initial admin password can be retrieved with: + +```bash +kubectl -n argocd get secret argocd-initial-admin-secret \ + -o jsonpath='{.data.password}' | base64 -d +``` + +Useful output values include `talos_nodes`, `argocd_namespace`, `argocd_server_service`, `argocd_cli_port_forward_command`, and `argocd_admin_password_command`. + ## Talos certificate mismatch `talos_machine_secrets.cluster` generates the Talos CA and client certificate used by Terraform. The nodes must have been configured from that same secrets bundle. An error such as `x509: certificate signed by unknown authority` means the node at the configured IP was initialized with a different Talos CA, or the Terraform state was replaced. @@ -23,7 +85,25 @@ Do not disable TLS verification and do not regenerate `talos_machine_secrets` wh ## Reconciliation order -Cilium must become healthy before workloads depending on its networking features. Longhorn must have a usable disk before PVC-backed workloads are considered ready. ApplicationSets use automated sync, pruning, self-healing, and namespace creation, so deleting a chart directory intentionally removes its generated Application resources. +Cilium must become healthy before workloads depending on its networking features. Cilium requires host networking, host paths, privileged containers, and additional Linux capabilities, so `infrastructure/cilium/templates/namespace.yaml` labels only the Cilium namespace with Pod Security `privileged` enforcement. On Talos, the machine configuration also removes the default Flannel CNI and kube-proxy, disables Cilium cgroup auto-mounting, uses KubePrism on `localhost:7445`, drops `SYS_MODULE`, and enables legacy host routing for DNS compatibility. + +If Cilium is unhealthy, it adds `node.cilium.io/agent-not-ready` taints and CoreDNS may remain Pending. That leaves Argo CD unable to resolve `argocd-repo-server`. After the corrected Cilium configuration is committed and pushed, temporarily allow CoreDNS to schedule so Argo CD can reconcile: + +```bash +export KUBECONFIG="$HOME/.kube/home-services" +kubectl taint nodes --all node.cilium.io/agent-not-ready:NoSchedule- +kubectl -n kube-system rollout status deployment/coredns +``` + +Then refresh the Cilium Application and verify the agents before relying on Argo CD again: + +```bash +kubectl -n argocd annotate application infrastructure-cilium argocd.argoproj.io/refresh=hard --overwrite +kubectl -n cilium get pods -o wide +kubectl -n argocd get application infrastructure-cilium +``` + +Longhorn must have a usable disk before PVC-backed workloads are considered ready. ApplicationSets use automated sync, pruning, self-healing, and namespace creation, so deleting a chart directory intentionally removes its generated Application resources. ## SOPS workflow diff --git a/README.md b/README.md index 30e2149..6fb5159 100644 --- a/README.md +++ b/README.md @@ -147,8 +147,31 @@ The Kubernetes path accepts a `nodes` list containing any number of control-plan ``` Set `KUBECONFIG` only after `terraform apply` has completed and the `kubeconfig` output exists. This file is a cluster credential and must not be committed or shared. -7. Apply both ApplicationSets with `kubectl apply -f bootstrap/applicationsets/`. -8. Confirm Argo CD discovers `infrastructure/cilium`, `infrastructure/longhorn`, and `services/vaultwarden`. +7. Export the Talos client configuration for `talosctl`: + + ```bash + export TALOSCONFIG="$HOME/.talos/home-services/talosconfig" + mkdir -p "$(dirname "$TALOSCONFIG")" + terraform output -raw talosconfig > "$TALOSCONFIG" + chmod 600 "$TALOSCONFIG" + talosctl --nodes "$(terraform output -json talos_nodes | jq -r 'join(",")')" version + ``` + +8. Apply both ApplicationSets with `kubectl apply -f bootstrap/applicationsets/`. +9. Connect the Argo CD CLI through the internal service: + + ```bash + argocd login --port-forward --port-forward-namespace argocd argocd-server + argocd app list + ``` + + Retrieve the initial admin password with: + + ```bash + kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d + ``` + +10. Confirm Argo CD discovers `infrastructure/cilium`, `infrastructure/longhorn`, and `services/vaultwarden`. Talos uses its default CNI during bootstrap so Argo CD can start. Cilium is then reconciled as the first infrastructure application. Confirm the LAN interface and address pool before enabling production LoadBalancer services. Longhorn also requires a validated disk or mount on the storage node. diff --git a/TALOS.md b/TALOS.md index 456cc95..b980cd3 100644 --- a/TALOS.md +++ b/TALOS.md @@ -23,3 +23,50 @@ sudo virt-install --virt-type kvm --name worker-node-1 --ram 4096 --vcpus 4 --di ``` Note that the VMs are booted into the ISO and terraform initiates the install and cluster build. + +## Upgrade Talos from 1.13 to 1.14 + +Talos 1.14 no longer publishes the release installer image at `ghcr.io/siderolabs/installer:v1.14.0`. Use the Talos Image Factory installer image with the generic amd64 KVM schematic: + +```bash +export TALOSCONFIG="$HOME/.talos/home-services/talosconfig" + +export TALOS_INSTALLER_IMAGE="factory.talos.dev/metal-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.14.0" +``` + +Upgrade the worker first: + +```bash +talosctl upgrade \ + --endpoints 10.0.0.246 \ + --nodes 10.0.0.246 \ + --image "$TALOS_INSTALLER_IMAGE" \ + --wait +``` + +After the worker is healthy, upgrade the control-plane node: + +```bash +talosctl upgrade \ + --endpoints 10.0.0.99 \ + --nodes 10.0.0.99 \ + --image "$TALOS_INSTALLER_IMAGE" \ + --wait +``` + +Verify both nodes before reconciling Terraform: + +```bash +talosctl version --endpoints 10.0.0.246 --nodes 10.0.0.246 +talosctl version --endpoints 10.0.0.99 --nodes 10.0.0.99 +talosctl health --endpoints 10.0.0.99 --nodes 10.0.0.99,10.0.0.246 +``` + +Then update `terraform.tfvars` to `talos_version = "v1.14.0"` if needed and run: + +```bash +terraform plan +terraform apply +``` + +Do not use `talosctl reset` or recreate the VMs for this upgrade. The installer preserves the Talos state and Kubernetes data. diff --git a/cilium_test/cilium.yaml b/cilium_test/cilium.yaml new file mode 100644 index 0000000..c818e27 --- /dev/null +++ b/cilium_test/cilium.yaml @@ -0,0 +1,1665 @@ +--- +# Source: cilium/templates/cilium-secrets-namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + annotations: + +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system + +--- +# Source: cilium/templates/cilium-envoy/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-envoy" + namespace: kube-system + +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system + +--- +# Source: cilium/templates/cilium-ca-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: cilium-ca + namespace: kube-system +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUtsR3ZkR0lmZGRsRGx2cm1Vb1p4UVF3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkyTURreU5EQXdNelkwTjFvWERUSTVNRGt5TXpBdwpNelkwTjFvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUEzUDRiRFArNnYrYURWMTJ1R0UrQmNFeU9JQWpHSXZJcExQSmxyWkpVMFZteXZyMHIKamZNRWl4Z1VsVGFxcndSdWtzcjFCUWNJZkZheUIrYlRlYkk4Tm1LZjlUb21IdWhFRTFCQk5ua011SjN3eE5JMQpteWVUQ2RFY1RiYzMvaU5EK3U0bzdGQ1hHUi83N1pxMTJBUkZFMm03eFpmRzh3bElyYXpIc3VrMkpwaDlWZzZyCmZrM3VGVE1GYWJEUVRKWUhDbnBLTEJEa2YyV1k5OThPeXBjZ2VST3dYSmNpVWlEYlZpSXVJQkcrWG8xNHh4OGsKOTNFU21GSVFYazljYzcyaXJiZmVFdGQ0eUNieS9EcFdmaDZndGVWYTk3dFhGRjZlZndoazU4eHg5VTF3VFEzMQp6NFFjK2VsSStrbW91SzFJRU02dE4wNklmQzV4Kzc4U2JYSGpPUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRytjNVZ4T1RNaGNPeXJmR01TN3NGS3d3SE5wTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQlVQN0xIM0hzRGYraDdKdHl3VjczditicWJzL1U1K2tMSW5LeE9tN05OQ3VRbXVYL2hWaHNpCkIyRU1hNjdVdXNseWJsOFZDQTByOHNCRjZuWDZXWHlTVWtSaVFWRlltNWUzaXdUVWcvZFFQeDZFOW9oY0NFdnEKdW1hbklkTDNaRXR1YTZxdVZJbGdySWcyeW1LUytNd0FvK1hPclVYK0lkTC9MdWFwSm9wcXhOOFQ1WHVpa0FoRQp3c0xHYjlmQnlMbllra0JQaG5PQ0RrWkJTeTduUFFiL0pqUXVWQ1hBVVJQa1M3dlFoMExMY2ZueFdvRWlQbHJiCndVdlEzbGhvdHdnVnBxZkJ3NWtmVjRYOU9iQ2VNb1dYenVLRndCeEFNaDUzQzR3eHNBZnJGS1I1TEZYNWIvYjAKOFRBL2RpWm11aDdsTkgzV3UrR1d3VDJSa2lxMHkvN0YKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBM1A0YkRQKzZ2K2FEVjEydUdFK0JjRXlPSUFqR0l2SXBMUEpsclpKVTBWbXl2cjByCmpmTUVpeGdVbFRhcXJ3UnVrc3IxQlFjSWZGYXlCK2JUZWJJOE5tS2Y5VG9tSHVoRUUxQkJObmtNdUozd3hOSTEKbXllVENkRWNUYmMzL2lORCt1NG83RkNYR1IvNzdacTEyQVJGRTJtN3haZkc4d2xJcmF6SHN1azJKcGg5Vmc2cgpmazN1RlRNRmFiRFFUSllIQ25wS0xCRGtmMldZOTk4T3lwY2dlUk93WEpjaVVpRGJWaUl1SUJHK1hvMTR4eDhrCjkzRVNtRklRWGs5Y2M3MmlyYmZlRXRkNHlDYnkvRHBXZmg2Z3RlVmE5N3RYRkY2ZWZ3aGs1OHh4OVUxd1RRMzEKejRRYytlbEkra21vdUsxSUVNNnROMDZJZkM1eCs3OFNiWEhqT1FJREFRQUJBb0lCQUUvTFFwb3hkazFYSHNKNwpkV0theG9wMVlmWUlKUDYyUUk3Vm9VZ0laNjdGdkFncUJ1RjB4NFd2dHJMZjc4Wk5DOTNlQVpXRW9oZy9OMzRwCjV0N3dyOVFWVFZ0KzJRZmM1c093ZlROOWhzM2NYWFM3MEJaTE5ramJvK1FkelU1cWpsKzdBYlMrTHFpb1IwOVcKUnVXT09TRFcwekYzNmFLVWpoVURwNEdldW1vNWhsbklqVSs0bkFDY1BkbkRseUZEZ3dyZSsvWkdqeERITFptdwpwZG1rUlRWUXQzOG13amo3TmhmSnRrcERMT003NFhLNEVtUGN6SS8wK2tubWhZcTFUc2hsM1M5aSttbXd4bWxLCnpDL1RpaVFxNHo3YnhucEZyNFE2d28xa1RpRXp2SWlHWE9ndkNtTHo4UFN2NGw4d0h6a0lWTW9UdXpsNTJPVUgKVEJuRTY4Y0NnWUVBNVRqS0FxcXBEVFBtNmVZVnlVenBkVDBxbW5EK1BPSERteWx0WTEvR2RpMCsyMndrY3NvawpUOFl5aWhVMmNMSURVdElnREZLMERYNG5WaUdhbWtSMlNqbDRib3E4ems1UzBZT2FoajNMbURqNWpRdDRaQkh1CjluRUt4ZEN0UUIxcmI3bmMzcytNeHR5QWZsYStJaVZaQUI5WktSc1c1b2FyeXQ3NTZLckdXSmNDZ1lFQTlzODEKbnZaOXBoQlJEVWhROWUzUHZKYi9XZWZ1VVZCcmFnZDV4YWpRVWhaL2F1VFF4TkJPMFdKR0lmVWJVeVRGbVRtMgo0MVRXWDZDY2kzQXl0TlVWRkFKMEVrcC9UNm96YXV2d3NSSktNZVg3c280aTRVbmVNZnFNRWFweW1wdXN2RTJPCkgzc2ZjSEppU1FuL0c3ak4xa01uT2NCVTJZazZqaWZpU2tZMXpLOENnWUVBemMxTUNsTGhqem9QVStmQm5jZlIKUC9MWUFtdjZ4L3pIVzlNNUpHVEVVaXhUYUR4VENVRVdrdzFYS1FUY3FSZ2F4Y3hrKzh0eFh6L3AzM295VGRtegovdlpVZTZOUzIwUVZOWWtjUHZUU2kreHpta0EzS2xtem5FNW9KcTB5TUFONkI0anZscGp3UTBlTWttSWRPWmtnCnJHeDQ1K0Fya25STlg4c013K280UlNjQ2dZQkVjVVhQWlUrR3c3T1dXVjJLTlg2TUw5dDBGU3N6aFhla0dacWQKUTZreGVtZXZQdkRlQTE0NEZtYVllQjR4dWxySm1RV3NtUlF4eEpwazJLSlhIMWdQQ25jaHMyTUd0bzQzc21JMgoySG1XeU00NXY4elFhT21qTm5XdFdIVWJodHdxQTkvTHZ4c0MxZFdIcDZ3RkJnU2V6RlNPWVZXbVZiWFZRaDR3CnRyT29wUUtCZ0dOaHJoMHJmdnpkL3d0VmNQYkswY1NIYnFrTFRtZXJ1TjJTY3pCY1hqSnhLeFBXUFlXVEIzV0gKdjVnSVRIZy9tZ3dOMWlURXdsR2U1dkJha2ZwTkI1S2N2R1R2OXg3Z0I3RkNjV2hQbzZYY2M0L1VPQmJwYndIUgp3Z1RTd25OMmR1YmNheThWYXcyTDhtclJWNjZzZTFiRUxyQ1R5T0R4ZW9tWGM0N3JaQllBCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + +--- +# Source: cilium/templates/hubble/tls-helm/server-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: hubble-server-certs + namespace: kube-system +type: kubernetes.io/tls +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUtsR3ZkR0lmZGRsRGx2cm1Vb1p4UVF3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkyTURreU5EQXdNelkwTjFvWERUSTVNRGt5TXpBdwpNelkwTjFvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUEzUDRiRFArNnYrYURWMTJ1R0UrQmNFeU9JQWpHSXZJcExQSmxyWkpVMFZteXZyMHIKamZNRWl4Z1VsVGFxcndSdWtzcjFCUWNJZkZheUIrYlRlYkk4Tm1LZjlUb21IdWhFRTFCQk5ua011SjN3eE5JMQpteWVUQ2RFY1RiYzMvaU5EK3U0bzdGQ1hHUi83N1pxMTJBUkZFMm03eFpmRzh3bElyYXpIc3VrMkpwaDlWZzZyCmZrM3VGVE1GYWJEUVRKWUhDbnBLTEJEa2YyV1k5OThPeXBjZ2VST3dYSmNpVWlEYlZpSXVJQkcrWG8xNHh4OGsKOTNFU21GSVFYazljYzcyaXJiZmVFdGQ0eUNieS9EcFdmaDZndGVWYTk3dFhGRjZlZndoazU4eHg5VTF3VFEzMQp6NFFjK2VsSStrbW91SzFJRU02dE4wNklmQzV4Kzc4U2JYSGpPUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRytjNVZ4T1RNaGNPeXJmR01TN3NGS3d3SE5wTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQlVQN0xIM0hzRGYraDdKdHl3VjczditicWJzL1U1K2tMSW5LeE9tN05OQ3VRbXVYL2hWaHNpCkIyRU1hNjdVdXNseWJsOFZDQTByOHNCRjZuWDZXWHlTVWtSaVFWRlltNWUzaXdUVWcvZFFQeDZFOW9oY0NFdnEKdW1hbklkTDNaRXR1YTZxdVZJbGdySWcyeW1LUytNd0FvK1hPclVYK0lkTC9MdWFwSm9wcXhOOFQ1WHVpa0FoRQp3c0xHYjlmQnlMbllra0JQaG5PQ0RrWkJTeTduUFFiL0pqUXVWQ1hBVVJQa1M3dlFoMExMY2ZueFdvRWlQbHJiCndVdlEzbGhvdHdnVnBxZkJ3NWtmVjRYOU9iQ2VNb1dYenVLRndCeEFNaDUzQzR3eHNBZnJGS1I1TEZYNWIvYjAKOFRBL2RpWm11aDdsTkgzV3UrR1d3VDJSa2lxMHkvN0YKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWekNDQWorZ0F3SUJBZ0lSQU1tMkpuRlNjK0dwaXRvUi9YTGRlbTR3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkyTURreU5EQXdNelkwTjFvWERUSTNNRGt5TkRBdwpNelkwTjFvd0tqRW9NQ1lHQTFVRUF3d2ZLaTVrWldaaGRXeDBMbWgxWW1Kc1pTMW5jbkJqTG1OcGJHbDFiUzVwCmJ6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU0vNzVQNUtBNWw4ai9iQ3pLdWwKSHpqdzZVU1pGVVNnNHgzajZUdHlDS3ZrZDZWSUtrYUY2Y3I1M2tvYTdqaVZmUUc4dFBYQ3B3MzdINCs3SEdCYQpQeXRndTRiVzFEUnpQU3ZuTlIyeUMwQStpQjhqbmc4bHZyaGpwenlFajFZN1dWbkNOUllTcTk2ckRRd0d4SmR5CmJlSStJdEk4QnVWaTl1Z0prRXpBdWVoK3A5UFA2NTlyOHQ1T1VEQXB6cUlYbjhCc01JTTQ4WFJkUHVZUXFHM2oKOVhFSFU0d0tBRnlJSE1FbWRMS3FHbE5zRS9ENXZEMmhMRXNmZXpKcWladlNReklTVTQ1b1VkaTg2bkZJUS95RwpTblBKdWlqRm1GRjdSM3BjQU1DMHk4ZGlWdFhtNXZVQ1ZqKzBnY1VLa0l5UmZMSTdxcWtiblNtbGVZQmlMUHpVCjhla0NBd0VBQWFPQmpUQ0JpakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdIUVlEVlIwbEJCWXdGQVlJS3dZQkJRVUgKQXdFR0NDc0dBUVVGQndNQ01Bd0dBMVVkRXdFQi93UUNNQUF3SHdZRFZSMGpCQmd3Rm9BVWI1emxYRTVNeUZ3NwpLdDhZeEx1d1VyREFjMmt3S2dZRFZSMFJCQ013SVlJZktpNWtaV1poZFd4MExtaDFZbUpzWlMxbmNuQmpMbU5wCmJHbDFiUzVwYnpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQWM3cVRYdHRjdGRJL2xFMGpYMTh3WERhOUpJdzYKb2tnUXg2d0hPeUdPL0t5cGRsWUFwS1lKNkxhK2YrZGlIbmljQmEwME56cU1BZnR4UnVtZjhOY2U1U2l6SU9raApFR29kM0VtVFJZL09qdGFLbzZrdEhkYTFJeHZ4K2ZiUzNTaDZPdGMwOGlzYzlWdzZJYkNFWXI1WXdLZGl0QzhOCmRvLzdxSFVvazNGclZYaEZEMzkycVlFZkVKMnJVdWNZazE0blpZdTEvZ2ZYR0F4UWI5anJOd0ZaOTVzT1BJazIKWXd6M1F4dmpWUG5JL1dBZzRTcmN4VlRWWG9rbm5iMGs5OC9idExGdFQyWnR5K0prYWQ5UVhWdUcyaUdKdFVuQQoyUXNrQWpkUnUyc2FmeWwvN3owbzNUOUpDYlVHZXJ3bGFuTTBmR2YzdEgydGt1MXFTQi9MS21DY2ZBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBei92ay9rb0RtWHlQOXNMTXE2VWZPUERwUkprVlJLRGpIZVBwTzNJSXErUjNwVWdxClJvWHB5dm5lU2hydU9KVjlBYnkwOWNLbkRmc2ZqN3NjWUZvL0syQzdodGJVTkhNOUsrYzFIYklMUUQ2SUh5T2UKRHlXK3VHT25QSVNQVmp0WldjSTFGaEtyM3FzTkRBYkVsM0p0NGo0aTBqd0c1V0wyNkFtUVRNQzU2SDZuMDgvcgpuMnZ5M2s1UU1Dbk9vaGVmd0d3d2d6anhkRjArNWhDb2JlUDFjUWRUakFvQVhJZ2N3U1owc3FvYVUyd1Q4UG04ClBhRXNTeDk3TW1xSm05SkRNaEpUam1oUjJMenFjVWhEL0laS2M4bTZLTVdZVVh0SGVsd0F3TFRMeDJKVzFlYm0KOVFKV1A3U0J4UXFRakpGOHNqdXFxUnVkS2FWNWdHSXMvTlR4NlFJREFRQUJBb0lCQUFRTkQzcEdmY2gxRFNiTQp4NGpHV0JQWFdXdk5JWVZyTHRGTHVhUy9SVnozdjc3WDUzN2xKOWNjdzlBa3N5UW1BVC9SNmFKcHpyQXVob0JaCnNDVk9UTEd2R2lmM3NqMXRTekU2T1NZaGJDZXRzQ0FQSlFwOWZUZ0JOT1l3WE14Y3JwSlpmb2cyQitQMzUvS20KaFJqQUVWVFpTZnRYVVBrUjB1WlZIVnJwWEJXanJPY1FieFFBa3pFNGtvd2RvVFVTLzJEanczbExLWG9TMjd6eQp4bjIwb0JrUUN3Mm9nTFhkWUdmWnZhUkVpVFRzenZCMkp4Rjl0TmtRekVWdFJmbnpqcE4yU29URVFob0ZORkxtCkFibDVhNTkvUGFNeUNSVmdvWEdUUzcxNjNpcVVlVlpqcVFqZ0VaWVhpUThOWVVRZDBGUWZiSXV3MER2T1dHN0MKTUE2LzdCc0NnWUVBOUtwYWE0Y0FydTVCN2NaN2VRNmtMOWRkOWtzYlo1d2dvTmFBMUpGeHlnb3EzQmNiTEk3NQpRNXE3Qkh5Ky9oZmNkc2RZb3NUMDFHcTkyTkd4TzJvUVk5WEQ5MTcyUk9qcENxOFJ5ZzhBekpNQjMwcGFIbkJ2Clc5bFBGQzNqdFZJbnpGVDRzOU5UekZySTlSZ2xxcnRiaU53MEpoWEcrbm5xaHBWcklncEI0YmNDZ1lFQTJaNkQKQWtPYlFma2V4L1BzY0lKRFVLVytQR25zcHBON1U1aUlucmc1UU5LZ3JoRFpHQjNDaVJYeUFnSHcxTDYzY3pHaAovdmhlRDdtTHlEeUVGRXFFc3lMR2ZoRUJ4UldyZnF3dnU3WWtHeC9oWmR5VWlGWUdDa1dOYVk5aXdieVlnN1pnCmh3M09pS2loaEZYRXBWZDJQVC9lUlhya1YvVk9BTlhJYzc4alNWOENnWUJlcHhENERIQjlQSHVSQzNwMUtPN24KK3ovRXF1WGRQWTk1NmNKSmVrTTFVSjJvSytBOWFzcVFGdXBISUNPYmxZdWJBQ0FRU0hZWFAzeFd4emp6cldMTgozTU5Gamgxa0oxOFEzUTNheDljZFRib3BzbEFPK3B4NlFpWFB2cmtiU3I5TzA0aXJ1UlB3T1NjSmV1QUhueGdZCmFHaGNXNHh4Rm4xZXQ2dU1yYjh6SFFLQmdGMGRjQjBsNUdia080Ri9JNm80Y3FrdFlpcmpmL2orbmdSMC9mQTgKRDNObkxJMk9jL1hYdGdlR05sa1pqM3pZWmhLS0tEQzdBVFJsUmlSVmtJWkdKR1dwK1ppem1qaEVQMjhpS2dKOQp1eEZJSU4rKzlHY3B0OGZwWHNyT0tZbVFNd29ZblZEc1hYVW9Bb1JuVm84UXoyMEJ5UStkdDlyVEFaOWwyakY4CmxaS3RBb0dBVVBCMnJvSm0rcTFxN01TZVZkc21GVWY0dE56NnZnRHltUDIwUHhZSVpYZUFwZzBpZHpzM2VlYUIKWTFUb3NBWUxQRlNoTGs2eDk1ZisrMTlXclU5aFkybFVieUZJNzhNYjJqVkNtaEZrRzVzT0ZUK0lOUW02TEg2OApiakg4d01kZmc1UTBuanNSUit1UVVEcHNYTzJUTFNpMjlJSjZtMU1tSWpoRzF5amp2ZU09Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd", "kvstore" or + # "doublewrite-readkvstore" / "doublewrite-readcrd". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + # - "doublewrite" modes store identities in both the kvstore and CRDs. This is useful + # for seamless migrations from the kvstore mode to the crd mode. Consult the + # documentation for more information on how to perform the migration. + identity-allocation-mode: crd + + identity-heartbeat-timeout: "30m0s" + identity-gc-interval: "15m0s" + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + metrics-sampling-interval: "5m" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes + enable-policy: "default" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + enable-policy-secrets-sync: "true" + policy-secrets-only-from-secrets-namespace: "true" + policy-secrets-namespace: "cilium-secrets" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "false" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "false" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: "5s" + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-policy-stats-map-max specifies the maximum number of entries in global + # policy stats map + bpf-policy-stats-map-max: "65536" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + bpf-lb-external-clusterip: "false" + bpf-lb-source-range-all-types: "false" + bpf-lb-algorithm-annotation: "false" + bpf-lb-mode-annotation: "false" + + bpf-distributed-lru: "false" + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: "default" + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + + routing-mode: "tunnel" + tunnel-protocol: "vxlan" + tunnel-source-port-range: "0-0" + service-no-backend-response: "reject" + + + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + enable-ipv4-masquerade: "true" + enable-ipv4-big-tcp: "false" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-tcx: "true" + datapath-mode: "veth" + enable-masquerade-to-route-source: "false" + + enable-xt-socket-fallback: "true" + install-no-conntrack-iptables-rules: "false" + iptables-random-fully: "false" + + auto-direct-node-routes: "false" + direct-routing-skip-unreachable: "false" + + + + kube-proxy-replacement: "true" + kube-proxy-replacement-healthz-bind-address: "" + bpf-lb-sock: "false" + enable-health-check-nodeport: "true" + enable-health-check-loadbalancer-ip: "false" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + bpf-lb-acceleration: "disabled" + enable-svc-source-range-check: "true" + enable-l2-neigh-discovery: "false" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + enable-k8s-networkpolicy: "true" + enable-endpoint-lockdown-on-policy-overflow: "false" + # Tell the agent to generate and write a CNI configuration file + write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist + cni-exclusive: "true" + cni-log-file: "/var/run/cilium/cilium-cni.log" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + health-check-icmp-failure-threshold: "3" + enable-well-known-identities: "false" + enable-node-selector-labels: "false" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + + enable-hubble: "true" + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: "/var/run/cilium/hubble.sock" + hubble-network-policy-correlation-enabled: "true" + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: ":4244" + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + ipam: "kubernetes" + ipam-cilium-node-update-rate: "15s" + + default-lb-service-ipam: "lbipam" + egress-gateway-reconciliation-trigger-interval: "1s" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/sys/fs/cgroup" + + identity-management-mode: "agent" + enable-sctp: "false" + remove-cilium-node-taints: "true" + set-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15" + # default DNS proxy to transparent mode in non-chaining modes + dnsproxy-enable-transparent-mode: "true" + dnsproxy-socket-linger-timeout: "10" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-proxy-response-max-delay: "100ms" + tofqdns-preallocate-identities: "true" + agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" + + mesh-auth-enabled: "true" + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + mesh-auth-gc-interval: "5m0s" + + proxy-xff-num-trusted-hops-ingress: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-connect-timeout: "2" + proxy-initial-fetch-timeout: "30" + proxy-max-requests-per-connection: "0" + proxy-max-connection-duration-seconds: "0" + proxy-idle-timeout-seconds: "60" + proxy-max-concurrent-retries: "128" + http-retry-count: "3" + + external-envoy-proxy: "true" + envoy-base-id: "0" + envoy-access-log-buffer-size: "4096" + envoy-keep-cap-netbindservice: "false" + max-connected-clusters: "255" + clustermesh-enable-endpoint-sync: "false" + clustermesh-enable-mcs-api: "false" + policy-default-local-cluster: "false" + + nat-map-stats-entries: "32" + nat-map-stats-interval: "30s" + enable-internal-traffic-policy: "true" + enable-lb-ipam: "true" + enable-non-default-deny-policies: "true" + enable-source-ip-verification: "true" + +# Extra config allows adding arbitrary properties to the cilium config. +# By putting it at the end of the ConfigMap, it's also possible to override existing properties. + +--- +# Source: cilium/templates/cilium-envoy/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-envoy-config + namespace: kube-system +data: + # Keep the key name as bootstrap-config.json to avoid breaking changes + bootstrap-config.json: | + {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} + +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints/status + - ciliumendpoints + - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status + verbs: + - patch + +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cilium-config + verbs: + # allow patching of the configmap to set annotations + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs + verbs: + - create + - update + - get + - list + - watch + - delete + - patch +- apiGroups: + - cilium.io + resources: + - ciliumbgpclusterconfigs/status + - ciliumbgppeerconfigs/status + verbs: + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumloadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumcidrgroups.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumpodippools.cilium.io + - ciliumgatewayclassconfigs.cilium.io +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides + - ciliumbgppeerconfigs + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumpodippools + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools/status + verbs: + - patch +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update + +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system + +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system + +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + +--- +# Source: cilium/templates/cilium-operator/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-operator-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - update + - patch + +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-config-agent +subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system + +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-tlsinterception-secrets +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system + +--- +# Source: cilium/templates/cilium-operator/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-operator-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-tlsinterception-secrets +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system + +--- +# Source: cilium/templates/cilium-envoy/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-envoy + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + io.cilium/app: proxy +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium-envoy + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: envoy-metrics + +--- +# Source: cilium/templates/hubble/peer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: hubble-peer + +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 + internalTrafficPolicy: Local + +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-agent +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: cilium-agent + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + spec: + securityContext: + appArmorProfile: + type: Unconfined + seccompProfile: + type: Unconfined + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.18.0@sha256:dfea023972d06ec183cfa3c9e7809716f85daaff042e573ef366e9ec6a0c0ab2" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 300 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + - name: "require-k8s-connectivity" + value: "false" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + divisor: '1' + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + - name: KUBE_CLIENT_BACKOFF_BASE + value: "1" + - name: KUBE_CLIENT_BACKOFF_DURATION + value: "120" + lifecycle: + postStart: + exec: + command: + - "bash" + - "-c" + - | + set -o errexit + set -o pipefail + set -o nounset + + # When running in AWS ENI mode, it's likely that 'aws-node' has + # had a chance to install SNAT iptables rules. These can result + # in dropped traffic, so we should attempt to remove them. + # We do it using a 'postStart' hook since this may need to run + # for nodes which might have already been init'ed but may still + # have dangling rules. This is safe because there are no + # dependencies on anything that is part of the startup script + # itself, and can be safely run multiple times per node (e.g. in + # case of a restart). + if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; + then + echo 'Deleting iptables rules created by the AWS CNI VPC plugin' + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore + fi + echo 'Done!' + + preStop: + exec: + command: + - /cni-uninstall.sh + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + # Check for duplicate mounts before mounting + - name: cilium-cgroup + mountPath: /sys/fs/cgroup + - name: cilium-run + mountPath: /var/run/cilium + - name: cilium-netns + mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + - name: tmp + mountPath: /tmp + + initContainers: + - name: config + image: "quay.io/cilium/cilium:v1.18.0@sha256:dfea023972d06ec183cfa3c9e7809716f85daaff042e573ef366e9ec6a0c0ab2" + imagePullPolicy: IfNotPresent + command: + - cilium-dbg + - build-config + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + volumeMounts: + - name: tmp + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.18.0@sha256:dfea023972d06ec183cfa3c9e7809716f85daaff042e573ef366e9ec6a0c0ab2" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.18.0@sha256:dfea023972d06ec183cfa3c9e7809716f85daaff042e573ef366e9ec6a0c0ab2" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.18.0@sha256:dfea023972d06ec183cfa3c9e7809716f85daaff042e573ef366e9ec6a0c0ab2" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + name: cilium-config + key: write-cni-conf-when-ready + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /sys/fs/cgroup + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.18.0@sha256:dfea023972d06ec183cfa3c9e7809716f85daaff042e573ef366e9ec6a0c0ab2" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin # .Values.cni.install + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # For sharing configuration between the "config" initContainer and the agent + - name: tmp + emptyDir: {} + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To exec into pod network namespaces + - name: cilium-netns + hostPath: + path: /var/run/netns + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host or apply sysctlfix + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /sys/fs/cgroup + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Sharing socket with Cilium Envoy on the same node by using a host path + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: cilium-clustermesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of all agents + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + - key: ca.crt + path: common-etcd-client-ca.crt + # note: we configure the volume for the kvstoremesh-specific certificate + # regardless of whether KVStoreMesh is enabled or not, so that it can be + # automatically mounted in case KVStoreMesh gets subsequently enabled, + # without requiring an agent restart. + - secret: + name: clustermesh-apiserver-local-cert + optional: true + items: + - key: tls.key + path: local-etcd-client.key + - key: tls.crt + path: local-etcd-client.crt + - key: ca.crt + path: local-etcd-client-ca.crt + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + - key: ca.crt + path: client-ca.crt + + +--- +# Source: cilium/templates/cilium-envoy/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium-envoy + namespace: kube-system + labels: + k8s-app: cilium-envoy + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy + name: cilium-envoy +spec: + selector: + matchLabels: + k8s-app: cilium-envoy + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: cilium-envoy + name: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + spec: + securityContext: + appArmorProfile: + type: Unconfined + containers: + - name: cilium-envoy + image: "quay.io/cilium/cilium-envoy:v1.34.4-1753677767-266d5a01d1d55bd1d60148f991b98dac0390d363@sha256:231b5bd9682dfc648ae97f33dcdc5225c5a526194dda08124f5eded833bf02bf" + imagePullPolicy: IfNotPresent + command: + - /usr/bin/cilium-envoy-starter + args: + - '--' + - '-c /var/run/cilium/envoy/bootstrap-config.json' + - '--base-id 0' + - '--log-level info' + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + ports: + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + - name: envoy-artifacts + mountPath: /var/run/cilium/envoy/artifacts + readOnly: true + - name: envoy-config + mountPath: /var/run/cilium/envoy/ + readOnly: true + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium-envoy" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium-envoy + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + - name: envoy-artifacts + hostPath: + path: "/var/run/cilium/envoy/artifacts" + type: DirectoryOrCreate + - name: envoy-config + configMap: + name: "cilium-envoy-config" + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: bootstrap-config.json + path: bootstrap-config.json + # To keep state between restarts / upgrades + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case + # of one replica and no user configured Recreate strategy. + # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the + # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 50% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.18.0@sha256:398378b4507b6e9db22be2f4455d8f8e509b189470061b0f813f0fabaf944f51" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + ports: + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 5 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node.cilium.io/agent-not-ready + operator: Exists + + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config diff --git a/infrastructure/cilium/templates/namespace.yaml b/infrastructure/cilium/templates/namespace.yaml new file mode 100644 index 0000000..4f8927a --- /dev/null +++ b/infrastructure/cilium/templates/namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cilium + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged diff --git a/infrastructure/cilium/values.yaml b/infrastructure/cilium/values.yaml index b9c5613..75d4ff5 100644 --- a/infrastructure/cilium/values.yaml +++ b/infrastructure/cilium/values.yaml @@ -1,12 +1,42 @@ # Confirm the interface and address pool before enabling L2 announcements. cilium: kubeProxyReplacement: true - k8sServiceHost: 10.0.0.99 - k8sServicePort: 6443 + k8sServiceHost: localhost + k8sServicePort: 7445 ipam: mode: kubernetes + bpf: + hostLegacyRouting: true + + cgroup: + autoMount: + enabled: false + hostRoot: /sys/fs/cgroup + + extraConfig: + direct-routing-device: enp1s0 + + securityContext: + capabilities: + ciliumAgent: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + cleanCiliumState: + - NET_ADMIN + - SYS_ADMIN + - SYS_RESOURCE + l2announcements: enabled: true @@ -16,9 +46,9 @@ cilium: operator: replicas: 1 -# Replace with the LAN interface on both Talos nodes. + # The LAN interface is enp1s0 on both Talos nodes. devices: - - "" + - enp1s0 # Configure a CiliumLoadBalancerIPPool and L2AnnouncementPolicy after # confirming the unused address range in the cluster LAN. diff --git a/terraform/main.tf b/terraform/main.tf index 1a26d5e..dd43ce3 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -33,6 +33,13 @@ resource "talos_machine_secrets" "cluster" { talos_version = var.talos_version } +data "talos_client_configuration" "cluster" { + cluster_name = var.cluster_name + client_configuration = talos_machine_secrets.cluster.client_configuration + endpoints = [for node in var.nodes : node.ip] + nodes = [for node in var.nodes : node.ip] +} + data "talos_machine_configuration" "role" { for_each = toset(["controlplane", "worker"]) @@ -50,6 +57,16 @@ data "talos_machine_configuration" "role" { disk = var.install_disk } } + cluster = { + network = { + cni = { + name = "none" + } + } + proxy = { + disabled = true + } + } }) ] } diff --git a/terraform/outputs.tf b/terraform/outputs.tf index e51204f..7ab8849 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -4,12 +4,37 @@ output "kubeconfig" { sensitive = true } -output "talos_client_configuration" { - description = "Talos client configuration generated for this cluster." - value = talos_machine_secrets.cluster.client_configuration +output "talosconfig" { + description = "Raw talosconfig for use with talosctl." + value = data.talos_client_configuration.cluster.talos_config sensitive = true } +output "talos_nodes" { + description = "Talos node addresses configured by this module." + value = [for node in var.nodes : node.ip] +} + +output "argocd_namespace" { + description = "Namespace containing Argo CD." + value = helm_release.argocd.namespace +} + +output "argocd_server_service" { + description = "Internal Argo CD server service address." + value = "argocd-server.${helm_release.argocd.namespace}.svc.cluster.local:443" +} + +output "argocd_cli_port_forward_command" { + description = "Command to connect argocd CLI to the internal Argo CD server." + value = "argocd login --port-forward --port-forward-namespace ${helm_release.argocd.namespace} argocd-server" +} + +output "argocd_admin_password_command" { + description = "Command to retrieve the initial Argo CD admin password." + value = "kubectl -n ${helm_release.argocd.namespace} get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d" +} + output "argocd_repository_secret_name" { description = "Argo CD repository credential Secret name." value = kubernetes_secret_v1.gitea.metadata[0].name