diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index c4dbd2b6aa102..938331800ce77 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -945,11 +945,11 @@ func TestKarpenter(t *testing.T) { withOIDCDiscovery(). withDefaults24(). withAddons("karpenter.sh-k8s-1.19"). - withServiceAccountRole("aws-node-termination-handler.kube-system", true). + withoutNTH(). withServiceAccountRole("karpenter.kube-system", true) test.expectTerraformFilenames = append(test.expectTerraformFilenames, - "aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data", - "aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data", + "aws_s3_object_nodeupscript-karpenter-nodes-single-machinetype_content", + "aws_s3_object_nodeupscript-karpenter-nodes-default_content", "aws_s3_object_nodeupconfig-karpenter-nodes-single-machinetype_content", "aws_s3_object_nodeupconfig-karpenter-nodes-default_content", ) diff --git a/docs/operations/karpenter.md b/docs/operations/karpenter.md index 7841e99afe1e7..0198b4c9c329c 100644 --- a/docs/operations/karpenter.md +++ b/docs/operations/karpenter.md @@ -1,69 +1,122 @@ # Karpenter -[Karpenter](https://karpenter.sh) is a Kubernetes-native capacity manager that directly provisions Nodes and underlying instances based on Pod requirements. On AWS, kOps supports managing an InstanceGroup with either Karpenter or an AWS Auto Scaling Group (ASG). +[Karpenter](https://karpenter.sh) is an open-source node lifecycle management project built for Kubernetes. +Adding Karpenter to a Kubernetes cluster can dramatically improve the efficiency and cost of running workloads on that cluster. + +On AWS, kOps supports managing an InstanceGroup with either Karpenter or an AWS Auto Scaling Group (ASG). + +## Prerequisites + +Managed Karpenter requires kOps 1.34+ and that [IAM Roles for Service Accounts (IRSA)](/cluster_spec#service-account-issuer-discovery-and-aws-iam-roles-for-service-accounts-irsa) be enabled for the cluster. + +If an older version of Karpenter was installed, it must be uninstalled before installing the new version. ## Installing -If using kOps 1.26 or older, enable the Karpenter feature flag : +### New clusters ```sh -export KOPS_FEATURE_FLAGS="Karpenter" -``` +export KOPS_STATE_STORE="s3://my-state-store" +export KOPS_DISCOVERY_STORE="s3://my-discovery-store" +export NAME="my-cluster.example.com" +export ZONES="eu-central-1a" -Karpenter requires that external permissions for ServiceAccounts be enabled for the cluster. See [AWS IAM roles for ServiceAccounts documentation](/cluster_spec#service-account-issuer-discovery-and-aws-iam-roles-for-service-accounts-irsa) for how to enable this. +kops create cluster --name ${NAME} \ + --cloud=aws \ + --instance-manager=karpenter \ + --discovery-store=${KOPS_DISCOVERY_STORE} \ + --zones=${ZONES} \ + --yes + +kops validate cluster --name ${NAME} --wait=10m + +kops export kubeconfig --name ${NAME} --admin +``` ### Existing clusters -On existing clusters, you can create a Karpenter InstanceGroup by adding the following to its InstanceGroup spec: +The Karpenter addon must be enabled in the cluster spec: ```yaml spec: - manager: Karpenter + karpenter: + enabled: true ``` -You also need to enable the Karpenter addon in the cluster spec: +To create a Karpenter InstanceGroup, set the following in its InstanceGroup spec: ```yaml spec: - karpenter: - enabled: true + manager: Karpenter ``` -### New clusters - -On new clusters, you can simply add the `--instance-manager=karpenter` flag: +### EC2NodeClass and NodePool ```sh -kops create cluster --name mycluster.example.com --cloud aws --networking=amazonvpc --zones=eu-central-1a,eu-central-1b --master-count=3 --yes --discovery-store=s3://discovery-store/ +USER_DATA=$(aws s3 cp ${KOPS_STATE_STORE}/${NAME}/igconfig/node/nodes/nodeupscript.sh -) +USER_DATA=${USER_DATA//$'\n'/$'\n '} + +kubectl apply -f - <: "true"` for each InstanceGroup the subnet is assigned to. If you enable manual tagging of subnets, you have to ensure these tags are added, if not Karpenter will fail to provision any instances. - -## Instance Types - -If you do not specify a mixed instances policy, only the instance type specified by `spec.machineType` will be used. With Karpenter, one typically wants a wider range of instances to choose from. kOps supports both providing a list of instance types through `spec.mixedInstancesPolicy.instances` and providing instance type requirements through `spec.mixedInstancesPolicy.instanceRequirements`. See (/instance_groups)[InstanceGroup documentation] for more details. +A Karpenter-managed InstanceGroup controls the bootstrap script. kOps will ensure the correct AWS security groups, subnets and permissions. +`EC2NodeClass` and `NodePool` objects must be created by the cluster operator. ## Known limitations -### Karpenter-managed Launch Templates - -On EKS, Karpener creates its own launch templates for Provisioners. These launch templates will not work with a kOps cluster for a number of reasons. Most importantly, they do not use supported AMIs and they do not install and configure nodeup, the instance-side kOps component. The Karpenter features that require Karpenter to directly manage launch templates will not be available on kOps. - -### Unmanaged Provisioner resources - -As mentioned above, kOps will manage a Provisioner resource per InstanceGroup. It is technically possible to create Provsioner resources directly, but you have to ensure that you configure Provisioners according to kOps requirements. As mentioned above, Karpenter-managed launch templates do not work and you have to maintain your own kOps-compatible launch templates. - -### Other minor limitations - -* Control plane nodes must be provisioned with an ASG, not Karpenter. -* Provisioners will unconditionally use spot with a fallback on ondemand instances. -* Provisioners will unconditionally include burstable instance groups such as the T3 instance family. -* kOps will not allow mixing arm64 and amd64 instances in the same Provider. \ No newline at end of file +* **Upgrade is not supported** from the previous version of managed Karpenter. +* Control plane nodes must be provisioned with an ASG. +* All `EC2NodeClass` objects must have the `spec.amiFamily` set to `Custom`. +* `spec.instanceStorePolicy` configuration is not supported in `EC2NodeClass`. +* `spec.kubelet`, `spec.taints` and `spec.labels` configuration are not supported in `EC2NodeClass`, but they can be configured in the `Cluster` or `InstanceGroup` spec. diff --git a/docs/releases/1.34-NOTES.md b/docs/releases/1.34-NOTES.md index 8623bf5d9d14c..b3eb1aaec673b 100644 --- a/docs/releases/1.34-NOTES.md +++ b/docs/releases/1.34-NOTES.md @@ -15,7 +15,7 @@ This is a document to gather the release notes prior to the release. ## AWS -* TODO +* Karpenter has been upgraded to v1.6.2. ([17567](https://github.com/kubernetes/kops/pull/17567) ## GCP @@ -33,7 +33,9 @@ This is a document to gather the release notes prior to the release. ## Other breaking changes -* Legacy addons have been removed from the kOps repo. These were only referenced by kOps <1.22 ([17322](https://github.com/kubernetes/kops/pull/17332)) +* Legacy addons have been removed from the kOps repo. These were only referenced by kOps <1.22. ([17322](https://github.com/kubernetes/kops/pull/17332)) + +* If an older version of Karpenter was installed, it must be uninstalled before upgrading. ([17567](https://github.com/kubernetes/kops/pull/17567) # Known Issues diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 876126e849997..6c6213f350375 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -1898,6 +1898,9 @@ func validateMetricsServer(cluster *kops.Cluster, spec *kops.MetricsServerConfig } func validateNodeTerminationHandler(cluster *kops.Cluster, spec *kops.NodeTerminationHandlerSpec, fldPath *field.Path) (allErrs field.ErrorList) { + if (spec.Enabled == nil || *spec.Enabled) && cluster.Spec.Karpenter != nil && cluster.Spec.Karpenter.Enabled { + allErrs = append(allErrs, field.Forbidden(fldPath, "nodeTerminationHandler cannot be used in conjunction with Karpenter")) + } if spec.IsQueueMode() { if spec.EnableSpotInterruptionDraining != nil && !*spec.EnableSpotInterruptionDraining { allErrs = append(allErrs, field.Forbidden(fldPath.Child("enableSpotInterruptionDraining"), "spot interruption draining cannot be disabled in Queue Processor mode")) diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index c4114fea16644..93320d0be8c5a 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -74,19 +74,26 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) e } } - task, err := b.buildLaunchTemplateTask(c, name, ig) + // Always create the user data, even for Karpenter manged instance groups + // Kaprenter expects the user data to be available in the state store: + // ${KOPS_STATE_STORE}/${CLUSTER_NAME}/igconfig/node/${IG_NAME}/nodeupscript.sh + userData, err := b.BootstrapScriptBuilder.ResourceNodeUp(c, ig) if err != nil { return err } - c.AddTask(task) - // @step: now lets build the autoscaling group task if ig.Spec.Manager != "Karpenter" { + lt, err := b.buildLaunchTemplateTask(c, name, ig, userData) + if err != nil { + return err + } + c.AddTask(lt) + asg, err := b.buildAutoScalingGroupTask(c, name, ig) if err != nil { return err } - asg.LaunchTemplate = task + asg.LaunchTemplate = lt c.AddTask(asg) warmPool := b.Cluster.Spec.CloudProvider.AWS.WarmPool.ResolveDefaults(ig) @@ -136,7 +143,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) e } // buildLaunchTemplateTask is responsible for creating the template task into the aws model -func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.CloudupModelBuilderContext, name string, ig *kops.InstanceGroup) (*awstasks.LaunchTemplate, error) { +func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.CloudupModelBuilderContext, name string, ig *kops.InstanceGroup, userData fi.Resource) (*awstasks.LaunchTemplate, error) { // @step: add the iam instance profile link, err := b.LinkToIAMInstanceProfile(ig) if err != nil { @@ -180,11 +187,6 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.CloudupMode return nil, fmt.Errorf("error building cloud tags: %v", err) } - userData, err := b.BootstrapScriptBuilder.ResourceNodeUp(c, ig) - if err != nil { - return nil, err - } - lt := &awstasks.LaunchTemplate{ Name: fi.PtrTo(name), Lifecycle: b.Lifecycle, diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 3bcaefa769a9d..21bacfeb4e18b 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -27,11 +27,13 @@ import ( "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/assets" "k8s.io/kops/pkg/model/resources" + "k8s.io/kops/pkg/nodemodel/wellknownassets" "k8s.io/kops/pkg/wellknownservices" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/fitasks" "k8s.io/kops/upup/pkg/fi/utils" "k8s.io/kops/util/pkg/architectures" + "k8s.io/kops/util/pkg/vfs" ) type NodeUpConfigBuilder interface { @@ -65,6 +67,8 @@ type BootstrapScript struct { // nodeupConfig contains the nodeup config. nodeupConfig fi.CloudupTaskDependentResource + // nodeupScript contains the nodeup bootstrap script, for use with Karpenter. + nodeupScript fi.CloudupTaskDependentResource } var ( @@ -74,7 +78,7 @@ var ( ) // kubeEnv returns the boot config for the instance group -func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.CloudupContext) (*nodeup.BootConfig, error) { +func (b *BootstrapScript) kubeEnv(cluster *kops.Cluster, ig *kops.InstanceGroup, c *fi.CloudupContext) (*nodeup.BootConfig, error) { wellKnownAddresses := make(WellKnownAddresses) for _, hasAddress := range b.hasAddressTasks { @@ -121,6 +125,40 @@ func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.CloudupContext) bootConfig.NodeupConfigHash = base64.StdEncoding.EncodeToString(sum256[:]) b.nodeupConfig.Resource = fi.NewBytesResource(configData) + if ig.Spec.Manager == kops.InstanceManagerKarpenter { + assetBuilder := assets.NewAssetBuilder(vfs.NewVFSContext(), cluster.Spec.Assets, false) + nodeUpAssets := make(map[architectures.Architecture]*assets.MirroredAsset) + for _, arch := range architectures.GetSupported() { + asset, err := wellknownassets.NodeUpAsset(assetBuilder, arch) + if err != nil { + return nil, err + } + nodeUpAssets[arch] = asset + } + + var nodeupScript resources.NodeUpScript + nodeupScript.NodeUpAssets = nodeUpAssets + nodeupScript.BootConfig = bootConfig + + nodeupScript.WithEnvironmentVariables(cluster, ig) + nodeupScript.WithProxyEnv(cluster) + nodeupScript.WithSysctls() + + nodeupScript.CompressUserData = fi.ValueOf(ig.Spec.CompressUserData) + + nodeupScript.CloudProvider = string(cluster.GetCloudProvider()) + + scriptResource, err := nodeupScript.Build() + if err != nil { + return nil, err + } + scriptData, err := fi.ResourceAsBytes(scriptResource) + if err != nil { + return nil, err + } + b.nodeupScript.Resource = fi.NewBytesResource(scriptData) + } + return bootConfig, nil } @@ -194,6 +232,7 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.CloudupModelBuilderContext } task.resource.Task = task task.nodeupConfig.Task = task + task.nodeupScript.Task = task c.AddTask(task) c.AddTask(&fitasks.ManagedFile{ @@ -202,6 +241,14 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.CloudupModelBuilderContext Location: fi.PtrTo("igconfig/" + ig.Spec.Role.ToLowerString() + "/" + ig.Name + "/nodeupconfig.yaml"), Contents: &task.nodeupConfig, }) + if ig.Spec.Manager == kops.InstanceManagerKarpenter { + c.AddTask(&fitasks.ManagedFile{ + Name: fi.PtrTo("nodeupscript-" + ig.Name), + Lifecycle: b.Lifecycle, + Location: fi.PtrTo("igconfig/" + ig.Spec.Role.ToLowerString() + "/" + ig.Name + "/nodeupscript.sh"), + Contents: &task.nodeupScript, + }) + } return &task.resource, nil } @@ -231,7 +278,7 @@ func (b *BootstrapScript) Run(c *fi.CloudupContext) error { return nil } - bootConfig, err := b.kubeEnv(b.ig, c) + bootConfig, err := b.kubeEnv(b.cluster, b.ig, c) if err != nil { return err } diff --git a/pkg/model/components/addonmanifests/karpenter/iam.go b/pkg/model/components/addonmanifests/karpenter/iam.go index 4cdb2fb7c6081..a4aaf2fff6365 100644 --- a/pkg/model/components/addonmanifests/karpenter/iam.go +++ b/pkg/model/components/addonmanifests/karpenter/iam.go @@ -47,26 +47,45 @@ func (r *ServiceAccount) ServiceAccount() (types.NamespacedName, bool) { } func addKarpenterPermissions(p *iam.Policy) { + // https://karpenter.sh/v1.6/getting-started/migrating-from-cas/#create-iam-roles + // Karpenter p.AddUnconditionalActions( - // Not included because we require Karpenter - // use existing kOps instance group launch templates - // "ec2:CreateLaunchTemplate", - // "ec2:DeleteLaunchTemplate", - "ec2:CreateFleet", - "ec2:CreateTags", - "ec2:DescribeAvailabilityZones", + "ssm:GetParameter", "ec2:DescribeImages", - "ec2:DescribeInstanceTypeOfferings", - "ec2:DescribeInstanceTypes", - "ec2:DescribeInstances", - "ec2:DescribeLaunchTemplates", + "ec2:RunInstances", + "ec2:DescribeSubnets", "ec2:DescribeSecurityGroups", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DeleteLaunchTemplate", + "ec2:CreateTags", + "ec2:CreateLaunchTemplate", + "ec2:CreateFleet", "ec2:DescribeSpotPriceHistory", - "ec2:DescribeSubnets", - "ec2:RunInstances", + "pricing:GetProducts", + ) + // ConditionalEC2Termination + p.AddUnconditionalActions( "ec2:TerminateInstances", + ) + // PassNodeIAMRole + p.AddUnconditionalActions( "iam:PassRole", - "pricing:GetProducts", - "ssm:GetParameter", + ) + // AllowScopedInstanceProfileTagActions + p.AddUnconditionalActions( + "iam:TagInstanceProfile", + ) + // AllowScopedInstanceProfileActions + p.AddUnconditionalActions( + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile", + ) + // AllowInstanceProfileReadActions + p.AddUnconditionalActions( + "iam:GetInstanceProfile", ) } diff --git a/pkg/model/components/karpenter.go b/pkg/model/components/karpenter.go index c49df14b8bea5..4746c09ca8650 100644 --- a/pkg/model/components/karpenter.go +++ b/pkg/model/components/karpenter.go @@ -36,7 +36,7 @@ func (b *KarpenterOptionsBuilder) BuildOptions(o *kops.Cluster) error { } if c.Image == "" { - c.Image = "public.ecr.aws/karpenter/controller:v0.31.3" + c.Image = "public.ecr.aws/karpenter/controller:1.6.2" } if c.LogEncoding == "" { diff --git a/pkg/model/components/nodeterminationhandler.go b/pkg/model/components/nodeterminationhandler.go index cf04bf858e96c..394b87117ed3c 100644 --- a/pkg/model/components/nodeterminationhandler.go +++ b/pkg/model/components/nodeterminationhandler.go @@ -35,16 +35,24 @@ func (b *NodeTerminationHandlerOptionsBuilder) BuildOptions(o *kops.Cluster) err if clusterSpec.CloudProvider.AWS == nil { return nil } + if clusterSpec.Karpenter != nil && clusterSpec.Karpenter.Enabled { + // Karpenter manages its own NTH, so we disable the NTH addon. + // https://karpenter.sh/docs/troubleshooting/#aws-node-termination-handler-nth-interactions + return nil + } if clusterSpec.CloudProvider.AWS.NodeTerminationHandler == nil { clusterSpec.CloudProvider.AWS.NodeTerminationHandler = &kops.NodeTerminationHandlerSpec{} } nth := clusterSpec.CloudProvider.AWS.NodeTerminationHandler - if nth.DeleteSQSMsgIfNodeNotFound == nil { - nth.DeleteSQSMsgIfNodeNotFound = fi.PtrTo(false) - } if nth.Enabled == nil { nth.Enabled = fi.PtrTo(true) } + if !fi.ValueOf(nth.Enabled) { + return nil + } + if nth.DeleteSQSMsgIfNodeNotFound == nil { + nth.DeleteSQSMsgIfNodeNotFound = fi.PtrTo(false) + } if nth.EnableSpotInterruptionDraining == nil { nth.EnableSpotInterruptionDraining = fi.PtrTo(true) } diff --git a/tests/e2e/scenarios/karpenter/run-test.sh b/tests/e2e/scenarios/karpenter/run-test.sh new file mode 100755 index 0000000000000..0ab15aa61f8d4 --- /dev/null +++ b/tests/e2e/scenarios/karpenter/run-test.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +# Copyright 2025 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +REPO_ROOT=$(git rev-parse --show-toplevel); +source "${REPO_ROOT}"/tests/e2e/scenarios/lib/common.sh + +kops-acquire-latest + +NETWORKING=cilium +OVERRIDES="${OVERRIDES-} --instance-manager=karpenter" +OVERRIDES="${OVERRIDES} --control-plane-size=c6g.large" + +kops-up + +# Get the node instance group user data script from the kOps state store +USER_DATA=$(aws s3 cp "${KOPS_STATE_STORE-}/${CLUSTER_NAME}/igconfig/node/nodes/nodeupscript.sh" -) +# Indent the user data script for embedding in the EC2NodeClass +USER_DATA=${USER_DATA//$'\n'/$'\n '} + +# Create a EC2NodeClass for Karpenter +kubectl apply -f - < self.imageGCLowThresholdPercent : + true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching + evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, + (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. properties: - apiVersion: - description: API version of the referent + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional type: string - required: - - name type: object - requirements: - description: Requirements are layered with Labels and applied to every - node. + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of security group + selector terms. The terms are ORed. items: - description: A node selector requirement is a selector that contains - values, a key, and an operator that relates the key and values. + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: - key: - description: The label key that the selector applies to. + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ type: string - operator: - description: Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and - Lt. + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. type: string - values: - description: An array of string values. If the operator is In - or NotIn, the values array must be non-empty. If the operator - is Exists or DoesNotExist, the values array must be empty. - If the operator is Gt or Lt, the values array must have a - single element, which will be interpreted as an integer. This - array is replaced during a strategic merge patch. - items: + tags: + additionalProperties: type: string - type: array - required: - - key - - operator + description: |- + Tags is a map of key/value tags used to select security groups. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') type: object + maxItems: 30 type: array - startupTaints: - description: StartupTaints are taints that are applied to nodes upon - startup which are expected to be removed automatically within a - short period of time, typically by a DaemonSet that tolerates the - taint. These are commonly used by daemonsets to allow initialization - and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint - in order to have nodes provisioned for them. + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in a security group selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination + of other fields in a security group selector term' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of subnet selector terms. + The terms are ORed. items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ type: string - required: - - effect - - key + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') type: object + maxItems: 30 type: array - taints: - description: Taints will be applied to every node launched by the - Provisioner. If specified, the provisioner will not provision nodes - for pods that do not have matching tolerations. Additional taints - will be created that match pod tolerations on a per-node basis. + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in a subnet selector term' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and + launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) + && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. + You must delete and recreate this node class if you want to change + this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) + && has(self.instanceProfile)) + - message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 + alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) + && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' + || self.amiFamily == ''AL2'') : true)' + - message: if set, amiFamily must be 'AL2023' or 'Custom' when using an + AL2023 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) + && x.alias.find(''^[^@]+'') == ''al2023'') ? (self.amiFamily == ''Custom'' + || self.amiFamily == ''AL2023'') : true)' + - message: if set, amiFamily must be 'Bottlerocket' or 'Custom' when using + a Bottlerocket alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) + && x.alias.find(''^[^@]+'') == ''bottlerocket'') ? (self.amiFamily + == ''Custom'' || self.amiFamily == ''Bottlerocket'') : true)' + - message: if set, amiFamily must be 'Windows2019' or 'Custom' when using + a Windows2019 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) + && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily + == ''Custom'' || self.amiFamily == ''Windows2019'') : true)' + - message: if set, amiFamily must be 'Windows2022' or 'Custom' when using + a Windows2022 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) + && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily + == ''Custom'' || self.amiFamily == ''Windows2022'') : true)' + - message: must specify amiFamily if amiSelectorTerms does not contain + an alias + rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)' + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + description: AMI contains resolved AMI selector values utilized + for node launch properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time + deprecated: + description: Deprecation status of the AMI + type: boolean + id: + description: ID of the AMI type: string - value: - description: The taint value corresponding to the taint key. + name: + description: Name of the AMI type: string + requirements: + description: Requirements of the AMI to be utilized on an instance + type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array required: - - effect - - key + - id + - requirements type: object type: array - ttlSecondsAfterEmpty: - description: "TTLSecondsAfterEmpty is the number of seconds the controller - will wait before attempting to delete a node, measured from when - the node is detected to be empty. A Node is considered to be empty - when it does not have pods scheduled to it, excluding daemonsets. - \n Termination due to no utilization is disabled if this field is - not set." - format: int64 - type: integer - ttlSecondsUntilExpired: - description: "TTLSecondsUntilExpired is the number of seconds the - controller will wait before terminating a node, measured from when - the node is created. This is useful to implement features like eventually - consistent node upgrade, memory leak protection, and disruption - testing. \n Termination due to expiration is disabled if this field - is not set." - format: int64 - type: integer - weight: - description: Weight is the priority given to the provisioner during - scheduling. A higher numerical weight indicates that this provisioner - will be ordered ahead of other provisioners with lower weights. - A provisioner with no weight will be treated as if it is a provisioner - with a weight of 0. - format: int32 - maximum: 100 - minimum: 1 - type: integer - type: object - status: - description: ProvisionerStatus defines the observed state of Provisioner - properties: - conditions: - description: Conditions is the set of conditions required for this - provisioner to scale its target, and indicates whether or not those - conditions are met. + capacityReservations: + description: |- + CapacityReservations contains the current capacity reservation values that are available to this NodeClass under the + CapacityReservation selectors. items: - description: 'Condition defines a readiness condition for a Knative - resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' properties: - lastTransitionTime: - description: LastTransitionTime is the last time the condition - transitioned from one status to another. We use VolatileTime - in place of metav1.Time to exclude this from creating equality.Semantic - differences (all other things held constant). + availabilityZone: + description: The availability zone the capacity reservation + is available in. type: string - message: - description: A human readable message indicating details about - the transition. + endTime: + description: |- + The time at which the capacity reservation expires. Once expired, the reserved capacity is released and Karpenter + will no longer be able to launch instances into that reservation. + format: date-time type: string - reason: - description: The reason for the condition's last transition. + id: + description: The id for the capacity reservation. + pattern: ^cr-[0-9a-z]+$ type: string - severity: - description: Severity with which to treat failures of this type - of condition. When this is not specified, it defaults to Error. + instanceMatchCriteria: + description: Indicates the type of instance launches the capacity + reservation accepts. + enum: + - open + - targeted type: string - status: - description: Status of the condition, one of True, False, Unknown. + instanceType: + description: The instance type for the capacity reservation. type: string - type: - description: Type of condition. + ownerID: + description: The ID of the AWS account that owns the capacity + reservation. + pattern: ^[0-9]{12}$ type: string - required: - - status - - type - type: object - type: array - lastScaleTime: - description: LastScaleTime is the last time the Provisioner scaled - the number of nodes - format: date-time - type: string - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Resources is the list of resources that have been provisioned. - type: object - type: object - type: object - served: true - storage: true - subresources: - status: {} - ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/managed-by: kops - k8s-addon: karpenter.sh - name: awsnodetemplates.karpenter.k8s.aws -spec: - group: karpenter.k8s.aws - names: - categories: - - karpenter - kind: AWSNodeTemplate - listKind: AWSNodeTemplateList - plural: awsnodetemplates - singular: awsnodetemplate - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: AWSNodeTemplate is the Schema for the AWSNodeTemplate API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: AWSNodeTemplateSpec is the top level specification for the - AWS Karpenter Provider. This will contain configuration necessary to - launch instances in AWS. - properties: - amiFamily: - description: AMIFamily is the AMI family that instances use. - type: string - amiSelector: - additionalProperties: - type: string - description: AMISelector discovers AMIs to be used by Amazon EC2 tags. - type: object - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - blockDeviceMappings: - description: BlockDeviceMappings to be applied to provisioned nodes. - items: - properties: - deviceName: - description: The device name (for example, /dev/sdh or xvdh). + reservationType: + default: default + description: The type of capacity reservation. + enum: + - default + - capacity-block type: string - ebs: - description: EBS contains parameters used to automatically set - up EBS volumes when an instance is launched. - properties: - deleteOnTermination: - description: DeleteOnTermination indicates whether the EBS - volume is deleted on instance termination. - type: boolean - encrypted: - description: Encrypted indicates whether the EBS volume - is encrypted. Encrypted volumes can only be attached to - instances that support Amazon EBS encryption. If you are - creating a volume from a snapshot, you can't specify an - encryption value. - type: boolean - iops: - description: "IOPS is the number of I/O operations per second - (IOPS). For gp3, io1, and io2 volumes, this represents - the number of IOPS that are provisioned for the volume. - For gp2 volumes, this represents the baseline performance - of the volume and the rate at which the volume accumulates - I/O credits for bursting. \n The following are the supported - values for each volume type: \n * gp3: 3,000-16,000 IOPS - \n * io1: 100-64,000 IOPS \n * io2: 100-64,000 IOPS \n - For io1 and io2 volumes, we guarantee 64,000 IOPS only - for Instances built on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). - Other instance families guarantee performance up to 32,000 - IOPS. \n This parameter is supported for io1, io2, and - gp3 volumes only. This parameter is not supported for - gp2, st1, sc1, or standard volumes." - format: int64 - type: integer - kmsKeyID: - description: KMSKeyID (ARN) of the symmetric Key Management - Service (KMS) CMK used for encryption. - type: string - snapshotID: - description: SnapshotID is the ID of an EBS snapshot - type: string - throughput: - description: 'Throughput to provision for a gp3 volume, - with a maximum of 1,000 MiB/s. Valid Range: Minimum value - of 125. Maximum value of 1000.' - format: int64 - type: integer - volumeSize: - anyOf: - - type: integer - - type: string - description: "VolumeSize in GiBs. You must specify either - a snapshot ID or a volume size. The following are the - supported volumes sizes for each volume type: \n * gp2 - and gp3: 1-16,384 \n * io1 and io2: 4-16,384 \n * st1 - and sc1: 125-16,384 \n * standard: 1-1,024" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - volumeType: - description: VolumeType of the block device. For more information, - see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) - in the Amazon Elastic Compute Cloud User Guide. - type: string - type: object + state: + default: active + description: |- + The state of the capacity reservation. A capacity reservation is considered to be expiring if it is within the EC2 + reclaimation window. Only capacity-block reservations may be in this state. + enum: + - active + - expiring + type: string + required: + - availabilityZone + - id + - instanceMatchCriteria + - instanceType + - ownerID type: object type: array - context: - description: Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html - type: string - detailedMonitoring: - description: DetailedMonitoring controls if detailed monitoring is - enabled for instances that are launched - type: boolean - instanceProfile: - description: InstanceProfile is the AWS identity that instances use. - type: string - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - launchTemplate: - description: 'LaunchTemplateName for the node. If not specified, a - launch template will be generated. NOTE: This field is for specifying - a custom launch template and is exposed in the Spec as `launchTemplate` - for backwards compatibility.' - type: string - metadataOptions: - description: "MetadataOptions for the generated launch template of - provisioned nodes. \n This specifies the exposure of the Instance - Metadata Service to provisioned EC2 nodes. For more information, - see Instance Metadata and User Data (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) - in the Amazon Elastic Compute Cloud User Guide. \n Refer to recommended, - security best practices (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) - for limiting exposure of Instance Metadata and User Data to pods. - If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - disabled, with httpPutResponseLimit of 2, and with httpTokens required." - properties: - httpEndpoint: - description: "HTTPEndpoint enables or disables the HTTP metadata - endpoint on provisioned nodes. If metadata options is non-nil, - but this parameter is not specified, the default state is \"enabled\". - \n If you specify a value of \"disabled\", instance metadata - will not be accessible on the node." - type: string - httpProtocolIPv6: - description: HTTPProtocolIPv6 enables or disables the IPv6 endpoint - for the instance metadata service on provisioned nodes. If metadata - options is non-nil, but this parameter is not specified, the - default state is "disabled". - type: string - httpPutResponseHopLimit: - description: HTTPPutResponseHopLimit is the desired HTTP PUT response - hop limit for instance metadata requests. The larger the number, - the further instance metadata requests can travel. Possible - values are integers from 1 to 64. If metadata options is non-nil, - but this parameter is not specified, the default value is 1. - format: int64 - type: integer - httpTokens: - description: "HTTPTokens determines the state of token usage for - instance metadata requests. If metadata options is non-nil, - but this parameter is not specified, the default state is \"optional\". - \n If the state is optional, one can choose to retrieve instance - metadata with or without a signed token header on the request. - If one retrieves the IAM role credentials without a token, the - version 1.0 role credentials are returned. If one retrieves - the IAM role credentials using a valid signed token, the version - 2.0 role credentials are returned. \n If the state is \"required\", - one must send a signed token header with any instance metadata - retrieval requests. In this state, retrieving the IAM role credentials - always returns the version 2.0 credentials; the version 1.0 - credentials are not available." - type: string - type: object - securityGroupSelector: - additionalProperties: - type: string - description: SecurityGroups specify the names of the security groups. - type: object - subnetSelector: - additionalProperties: - type: string - description: SubnetSelector discovers subnets by tags. A value of - "" is a wildcard. - type: object - tags: - additionalProperties: - type: string - description: Tags to be applied on ec2 resources like instances and - launch templates. - type: object - userData: - description: UserData to be applied to the provisioned nodes. It must - be in the appropriate format based on the AMIFamily in use. Karpenter - will merge certain fields into this UserData to ensure nodes are - being provisioned with the correct configuration. - type: string - type: object - status: - description: AWSNodeTemplateStatus contains the resolved state of the - AWSNodeTemplate - properties: - amis: - description: AMI contains the current AMI values that are available - to the cluster under the AMI selectors. + conditions: + description: Conditions contains signals for health and readiness items: - description: AMI contains resolved AMI selector values utilized - for node launch + description: Condition aliases the upstream type and adds additional + helper methods properties: - id: - description: ID of the AMI + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time type: string - name: - description: Name of the AMI + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string - requirements: - description: Requirements of the AMI to be utilized on an instance - type - items: - description: A node selector requirement is a selector that - contains values, a key, and an operator that relates the - key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set - of values. Valid operators are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. If the operator - is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. If the operator is Gt or Lt, the - values array must have a single element, which will - be interpreted as an integer. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array required: - - id - - requirements + - lastTransitionTime + - message + - reason + - status + - type type: object type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile + for the role + type: string securityGroups: - description: SecurityGroups contains the current Security Groups values - that are available to the cluster under the SecurityGroups selectors. + description: |- + SecurityGroups contains the current security group values that are available to the + cluster under the SecurityGroups selectors. items: description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch @@ -680,8 +894,9 @@ spec: type: object type: array subnets: - description: Subnets contains the current Subnet values that are available - to the cluster under the subnet selectors. + description: |- + Subnets contains the current subnet values that are available to the + cluster under the subnet selectors. items: description: Subnet contains resolved Subnet selector values utilized for node launch @@ -692,6 +907,9 @@ spec: zone: description: The associated availability zone type: string + zoneID: + description: The associated availability zone ID + type: string required: - id - zone @@ -710,28 +928,31 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.18.0 creationTimestamp: null labels: addon.kops.k8s.io/name: karpenter.sh app.kubernetes.io/managed-by: kops k8s-addon: karpenter.sh - name: machines.karpenter.sh + name: nodeclaims.karpenter.sh spec: group: karpenter.sh names: categories: - karpenter - kind: Machine - listKind: MachineList - plural: machines - singular: machine + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + singular: nodeclaim scope: Cluster versions: - additionalPrinterColumns: - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type name: Type type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone name: Zone type: string @@ -744,184 +965,185 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - - jsonPath: .metadata.labels.karpenter\.sh/capacity-type - name: Capacity + - jsonPath: .status.imageID + name: ImageID + priority: 1 + type: string + - jsonPath: .status.providerID + name: ID priority: 1 type: string - - jsonPath: .metadata.labels.karpenter\.sh/provisioner-name - name: Provisioner + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool priority: 1 type: string - - jsonPath: .spec.machineTemplateRef.name - name: Template + - jsonPath: .spec.nodeClassRef.name + name: NodeClass priority: 1 type: string - name: v1alpha5 + - jsonPath: .status.conditions[?(@.type=="Drifted")].status + name: Drifted + priority: 1 + type: string + name: v1 schema: openAPIV3Schema: - description: Machine is the Schema for the Machines API + description: NodeClaim is the Schema for the NodeClaims API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: - description: MachineSpec describes the desired state of the Machine + description: NodeClaimSpec describes the desired state of the NodeClaim properties: - kubelet: - description: Kubelet are options passed to the kubelet when provisioning - nodes - properties: - clusterDNS: - description: clusterDNS is a list of IP addresses for the cluster - DNS server. Note that not all providers may use all addresses. - items: - type: string - type: array - containerRuntime: - description: ContainerRuntime is the container runtime to be used - with your worker nodes. - type: string - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for - containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - description: EvictionHard is the map of signal names to quantities - that define hard eviction thresholds - type: object - evictionMaxPodGracePeriod: - description: EvictionMaxPodGracePeriod is the maximum allowed - grace period (in seconds) to use when terminating pods in response - to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - description: EvictionSoft is the map of signal names to quantities - that define soft eviction thresholds - type: object - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names - to quantities that define grace periods for each eviction signal - type: object - imageGCHighThresholdPercent: - description: ImageGCHighThresholdPercent is the percent of disk - usage after which image garbage collection is always run. The - percent is calculated by dividing this field value by 100, so - this field must be between 0 and 100, inclusive. When specified, - the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: ImageGCLowThresholdPercent is the percent of disk - usage before which image garbage collection is never run. Lowest - disk usage to garbage collect to. The percent is calculated - by dividing this field value by 100, so the field value must - be between 0 and 100, inclusive. When specified, the value must - be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: KubeReserved contains resources reserved for Kubernetes - system components. - type: object - maxPods: - description: MaxPods is an override for the maximum number of - pods that can run on a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: PodsPerCore is an override for the number of pods - that can run on a worker node instance based on the number of - cpu cores. This value cannot exceed MaxPods, so, if MaxPods - is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: SystemReserved contains resources reserved for OS - system daemons and kernel memory. - type: object - type: object - machineTemplateRef: - description: MachineTemplateRef is a reference to an object that defines + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration properties: - apiVersion: + group: description: API version of the referent + pattern: ^[^/]*$ type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' kind: description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' name: description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' required: + - group + - kind - name type: object requirements: - description: Requirements are layered with Labels and applied to every - node. + description: Requirements are layered with GetLabels and applied to + every node. items: - description: A node selector requirement is a selector that contains - values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. properties: key: description: The label key that the selector applies to. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ type: string + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", + "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", + "topology.kubernetes.io/zone", "topology.kubernetes.io/region", + "node.kubernetes.io/instance-type", "kubernetes.io/arch", + "kubernetes.io/os", "node.kubernetes.io/windows-build"] + || self.find("^([^/]+)").endsWith("node.kubernetes.io") + || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") + || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] + || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/capacity-reservation-type", + "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", + "karpenter.k8s.aws/instance-encryption-in-transit-supported", + "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", + "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", + "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", + "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", + "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", + "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", + "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", + "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", + "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", + "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] + || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer operator: - description: Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and - Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt type: string values: - description: An array of string values. If the operator is In - or NotIn, the values array must be non-empty. If the operator - is Exists or DoesNotExist, the values array must be empty. - If the operator is Gt or Lt, the values array must have a - single element, which will be interpreted as an integer. This - array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ type: array + x-kubernetes-list-type: atomic required: - key - operator type: object + maxItems: 100 type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : + true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive + integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') + ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many + values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? + x.values.size() >= x.minValues : true)' resources: - description: Resources models the resource requirements for the Machine + description: Resources models the resource requirements for the NodeClaim to launch properties: requests: @@ -932,36 +1154,44 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: Requests describes the minimum required resources - for the Machine to launch + for the NodeClaim to launch type: object type: object startupTaints: - description: StartupTaints are taints that are applied to nodes upon - startup which are expected to be removed automatically within a - short period of time, typically by a DaemonSet that tolerates the - taint. These are commonly used by daemonsets to allow initialization - and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint - in order to have nodes provisioned for them. + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute type: string key: description: Required. The taint key to be applied to a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ type: string timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. format: date-time type: string value: description: The taint value corresponding to the taint key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ type: string required: - effect @@ -969,35 +1199,68 @@ spec: type: object type: array taints: - description: Taints will be applied to the machine's node. + description: Taints will be applied to the NodeClaim's node. items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute type: string key: description: Required. The taint key to be applied to a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ type: string timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. format: date-time type: string value: description: The taint value corresponding to the taint key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ type: string required: - effect - key type: object type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf status: - description: MachineStatus defines the observed state of Machine + description: NodeClaimStatus defines the observed state of NodeClaim properties: allocatable: additionalProperties: @@ -1007,7 +1270,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: Allocatable is the estimated allocatable capacity of - the machine + the node type: object capacity: additionalProperties: @@ -1016,49 +1279,677 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Capacity is the estimated full capacity of the machine + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional + helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the + node + type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + +--- + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: nodepools.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodePool + listKind: NodePoolList + plural: nodepools + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to Karpenter's + disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons for + disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + maxProperties: 100 + type: object + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", + "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", + "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", + "topology.kubernetes.io/zone", "topology.kubernetes.io/region", + "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] + || x.find("^([^/]+)").endsWith("node.kubernetes.io") || + x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") + || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") + || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] + || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.k8s.aws" is restricted + rule: self.all(x, x in ["karpenter.k8s.aws/capacity-reservation-type", + "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", + "karpenter.k8s.aws/instance-encryption-in-transit-supported", + "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", + "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", + "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", + "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", + "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", + "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", + "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", + "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", + "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", + "karpenter.k8s.aws/instance-accelerator-manufacturer", + "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that + defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' + required: + - group + - kind + - name + type: object + x-kubernetes-validations: + - message: nodeClassRef.group is immutable + rule: self.group == oldSelf.group + - message: nodeClassRef.kind is immutable + rule: self.kind == oldSelf.kind + requirements: + description: Requirements are layered with GetLabels and applied + to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies + to. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", + "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", + "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", + "topology.kubernetes.io/zone", "topology.kubernetes.io/region", + "node.kubernetes.io/instance-type", "kubernetes.io/arch", + "kubernetes.io/os", "node.kubernetes.io/windows-build"] + || self.find("^([^/]+)").endsWith("node.kubernetes.io") + || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") + || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") + || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] + || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/capacity-reservation-type", + "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", + "karpenter.k8s.aws/instance-encryption-in-transit-supported", + "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", + "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", + "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", + "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", + "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", + "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", + "karpenter.k8s.aws/instance-network-bandwidth", + "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", + "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", + "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", + "karpenter.k8s.aws/instance-accelerator-count"] + || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value + defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() + != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a + single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == + ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= + 0) : true)' + - message: requirements with 'minValues' must have at least + that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) + ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: Required. The taint key to be applied to + a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: Required. The taint key to be applied to + a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + required: + - spec type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: conditions: description: Conditions contains signals for health and readiness items: - description: 'Condition defines a readiness condition for a Knative - resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + description: Condition aliases the upstream type and adds additional + helper methods properties: lastTransitionTime: - description: LastTransitionTime is the last time the condition - transitioned from one status to another. We use VolatileTime - in place of metav1.Time to exclude this from creating equality.Semantic - differences (all other things held constant). + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time type: string message: - description: A human readable message indicating details about - the transition. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer reason: - description: The reason for the condition's last transition. - type: string - severity: - description: Severity with which to treat failures of this type - of condition. When this is not specified, it defaults to Error. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string status: - description: Status of the condition, one of True, False, Unknown. + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type of condition. + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: + - lastTransitionTime + - message + - reason - status - type type: object type: array - nodeName: - description: NodeName is the name of the corresponding node object - type: string - providerID: - description: ProviderID of the corresponding node object - type: string + nodeClassObservedGeneration: + description: |- + NodeClassObservedGeneration represents the observed nodeClass generation for referenced nodeClass. If this does not match + the actual NodeClass Generation, NodeRegistrationHealthy status condition on the NodePool will be reset + format: int64 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object type: object + required: + - spec type: object served: true storage: true @@ -1076,8 +1967,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter namespace: kube-system @@ -1091,6 +1982,7 @@ spec: --- apiVersion: v1 +automountServiceAccountToken: false kind: ServiceAccount metadata: creationTimestamp: null @@ -1099,106 +1991,14 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter namespace: kube-system --- -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: karpenter-cert - namespace: kube-system - ---- - -apiVersion: v1 -data: - loglevel.webhook: debug - zap-logger-config: | - { - "level": "debug", - "development": false, - "disableStacktrace": true, - "disableCaller": true, - "sampling": { - "initial": 100, - "thereafter": 100 - }, - "outputPaths": ["stdout"], - "errorOutputPaths": ["stderr"], - "encoding": "console", - "encoderConfig": { - "timeKey": "time", - "levelKey": "level", - "nameKey": "logger", - "callerKey": "caller", - "messageKey": "message", - "stacktraceKey": "stacktrace", - "levelEncoder": "capital", - "timeEncoder": "iso8601" - } - } -kind: ConfigMap -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: config-logging - namespace: kube-system - ---- - -apiVersion: v1 -data: - aws.assumeRoleARN: "" - aws.assumeRoleDuration: 15m - aws.clusterCABundle: "" - aws.clusterEndpoint: https://api.internal.minimal.example.com - aws.clusterName: minimal.example.com - aws.defaultInstanceProfile: "" - aws.enableENILimitedPodDensity: "true" - aws.enablePodENI: "false" - aws.interruptionQueueName: "" - aws.isolatedVPC: "false" - aws.vmMemoryOverheadPercent: "0.075" - batchIdleDuration: 1s - batchMaxDuration: 10s - featureGates.driftEnabled: "false" -kind: ConfigMap -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: karpenter-global-settings - namespace: kube-system - ---- - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -1208,8 +2008,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh rbac.authorization.k8s.io/aggregate-to-admin: "true" name: karpenter-admin @@ -1217,10 +2017,10 @@ rules: - apiGroups: - karpenter.sh resources: - - provisioners - - provisioners/status - - machines - - machines/status + - nodepools + - nodepools/status + - nodeclaims + - nodeclaims/status verbs: - get - list @@ -1231,7 +2031,7 @@ rules: - apiGroups: - karpenter.k8s.aws resources: - - awsnodetemplates + - ec2nodeclasses verbs: - get - list @@ -1251,18 +2051,18 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter-core rules: - apiGroups: - karpenter.sh resources: - - provisioners - - provisioners/status - - machines - - machines/status + - nodepools + - nodepools/status + - nodeclaims + - nodeclaims/status verbs: - get - list @@ -1285,6 +2085,7 @@ rules: resources: - storageclasses - csinodes + - volumeattachments verbs: - get - watch @@ -1300,18 +2101,17 @@ rules: - list - watch - apiGroups: - - admissionregistration.k8s.io + - policy resources: - - validatingwebhookconfigurations - - mutatingwebhookconfigurations + - poddisruptionbudgets verbs: - get - - watch - list + - watch - apiGroups: - - policy + - "" resources: - - poddisruptionbudgets + - events verbs: - get - list @@ -1319,8 +2119,8 @@ rules: - apiGroups: - karpenter.sh resources: - - machines - - machines/status + - nodeclaims + - nodeclaims/status verbs: - create - delete @@ -1329,8 +2129,8 @@ rules: - apiGroups: - karpenter.sh resources: - - provisioners - - provisioners/status + - nodepools + - nodepools/status verbs: - update - patch @@ -1346,9 +2146,9 @@ rules: resources: - nodes verbs: - - create - patch - delete + - update - apiGroups: - "" resources: @@ -1356,14 +2156,11 @@ rules: verbs: - create - apiGroups: - - admissionregistration.k8s.io - resourceNames: - - validation.webhook.karpenter.sh - - validation.webhook.config.karpenter.sh + - "" resources: - - validatingwebhookconfigurations + - pods verbs: - - update + - delete --- @@ -1376,40 +2173,24 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter rules: - apiGroups: - karpenter.k8s.aws resources: - - awsnodetemplates + - ec2nodeclasses verbs: - get - list - watch -- apiGroups: - - admissionregistration.k8s.io - resourceNames: - - validation.webhook.karpenter.k8s.aws - resources: - - validatingwebhookconfigurations - verbs: - - update -- apiGroups: - - admissionregistration.k8s.io - resourceNames: - - defaulting.webhook.karpenter.k8s.aws - resources: - - mutatingwebhookconfigurations - verbs: - - update - apiGroups: - karpenter.k8s.aws resources: - - awsnodetemplates - - awsnodetemplates/status + - ec2nodeclasses + - ec2nodeclasses/status verbs: - patch - update @@ -1425,8 +2206,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter-core roleRef: @@ -1449,8 +2230,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter roleRef: @@ -1473,8 +2254,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter namespace: kube-system @@ -1486,43 +2267,10 @@ rules: verbs: - get - watch -- apiGroups: - - "" - resources: - - configmaps - - namespaces - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - "" - resourceNames: - - karpenter-cert - resources: - - secrets - verbs: - - update -- apiGroups: - - "" - resourceNames: - - karpenter-global-settings - - config-logging - resources: - - configmaps - verbs: - - update - - patch - - delete - apiGroups: - coordination.k8s.io resourceNames: - karpenter-leader-election - - webhook.configmapwebhook.00-of-01 - - webhook.defaultingwebhook.00-of-01 - - webhook.validationwebhook.00-of-01 - - webhook.webhookcertificates.00-of-01 resources: - leases verbs: @@ -1534,12 +2282,6 @@ rules: - leases verbs: - create -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create --- @@ -1552,8 +2294,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter-dns namespace: kube-system @@ -1569,38 +2311,6 @@ rules: --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: karpenter-lease - namespace: kube-node-lease -rules: -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - delete - ---- - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -1610,8 +2320,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter namespace: kube-system @@ -1635,8 +2345,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter-dns namespace: kube-system @@ -1651,31 +2361,6 @@ subjects: --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: karpenter-lease - namespace: kube-node-lease -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: karpenter-lease -subjects: -- kind: ServiceAccount - name: karpenter - namespace: kube-system - ---- - apiVersion: v1 kind: Service metadata: @@ -1685,21 +2370,17 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter namespace: kube-system spec: ports: - name: http-metrics - port: 8000 + port: 8080 protocol: TCP targetPort: http-metrics - - name: https-webhook - port: 8443 - protocol: TCP - targetPort: https-webhook selector: app.kubernetes.io/instance: karpenter app.kubernetes.io/name: karpenter @@ -1716,8 +2397,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.6.2 + helm.sh/chart: karpenter-1.6.2 k8s-addon: karpenter.sh name: karpenter namespace: kube-system @@ -1726,7 +2407,8 @@ spec: revisionHistoryLimit: 10 selector: matchLabels: - karpenter: webhook + app.kubernetes.io/instance: karpenter + app.kubernetes.io/name: karpenter strategy: rollingUpdate: maxUnavailable: 1 @@ -1736,7 +2418,6 @@ spec: labels: app.kubernetes.io/instance: karpenter app.kubernetes.io/name: karpenter - karpenter: webhook kops.k8s.io/managed-by: kops spec: affinity: @@ -1768,16 +2449,21 @@ spec: app.kubernetes.io/instance: karpenter app.kubernetes.io/name: karpenter topologyKey: kubernetes.io/hostname + automountServiceAccountToken: true containers: - env: - name: KUBERNETES_MIN_VERSION value: 1.19.0-0 - name: KARPENTER_SERVICE value: karpenter - - name: WEBHOOK_PORT - value: "8443" + - name: LOG_LEVEL + value: debug + - name: LOG_OUTPUT_PATHS + value: stdout + - name: LOG_ERROR_OUTPUT_PATHS + value: stderr - name: METRICS_PORT - value: "8000" + value: "8080" - name: HEALTH_PROBE_PORT value: "8081" - name: SYSTEM_NAMESPACE @@ -1790,13 +2476,31 @@ spec: containerName: controller divisor: "0" resource: limits.memory + - name: FEATURE_GATES + value: ReservedCapacity=true,SpotToSpotConsolidation=false,NodeRepair=false + - name: BATCH_MAX_DURATION + value: 10s + - name: BATCH_IDLE_DURATION + value: 1s + - name: PREFERENCE_POLICY + value: Respect + - name: MIN_VALUES_POLICY + value: Strict + - name: CLUSTER_NAME + value: minimal.example.com + - name: CLUSTER_ENDPOINT + value: https://api.internal.minimal.example.com + - name: VM_MEMORY_OVERHEAD_PERCENT + value: "0.075" + - name: RESERVED_ENIS + value: "0" - name: AWS_REGION value: us-test-1 - name: AWS_ROLE_ARN value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com - name: AWS_WEB_IDENTITY_TOKEN_FILE value: /var/run/secrets/amazonaws.com/token - image: public.ecr.aws/karpenter/controller:v0.31.3 + image: public.ecr.aws/karpenter/controller:1.6.2 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -1806,15 +2510,12 @@ spec: timeoutSeconds: 30 name: controller ports: - - containerPort: 8000 + - containerPort: 8080 name: http-metrics protocol: TCP - containerPort: 8081 name: http protocol: TCP - - containerPort: 8443 - name: https-webhook - protocol: TCP readinessProbe: httpGet: path: /readyz @@ -1832,20 +2533,24 @@ spec: capabilities: drop: - ALL + privileged: false readOnlyRootFilesystem: true - runAsGroup: 65536 + runAsGroup: 65532 runAsNonRoot: true - runAsUser: 65536 - seccompProfile: - type: RuntimeDefault + runAsUser: 65532 volumeMounts: - mountPath: /var/run/secrets/amazonaws.com/ name: token-amazonaws-com readOnly: true dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux priorityClassName: system-cluster-critical + schedulerName: default-scheduler securityContext: - fsGroup: 10001 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault serviceAccountName: karpenter tolerations: - key: node-role.kubernetes.io/master @@ -1871,279 +2576,3 @@ spec: audience: amazonaws.com expirationSeconds: 86400 path: token - ---- - -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: defaulting.webhook.karpenter.k8s.aws -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - name: defaulting.webhook.karpenter.k8s.aws - rules: - - apiGroups: - - karpenter.k8s.aws - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - awsnodetemplates - - awsnodetemplates/status - scope: '*' - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - operations: - - CREATE - - UPDATE - resources: - - provisioners - - provisioners/status - sideEffects: None - ---- - -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: validation.webhook.karpenter.sh -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - name: validation.webhook.karpenter.sh - rules: - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - operations: - - CREATE - - UPDATE - resources: - - provisioners - - provisioners/status - sideEffects: None - ---- - -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: validation.webhook.config.karpenter.sh -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - name: validation.webhook.config.karpenter.sh - objectSelector: - matchLabels: - app.kubernetes.io/part-of: karpenter - sideEffects: None - ---- - -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: validation.webhook.karpenter.k8s.aws -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - name: validation.webhook.karpenter.k8s.aws - rules: - - apiGroups: - - karpenter.k8s.aws - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - awsnodetemplates - - awsnodetemplates/status - scope: '*' - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - operations: - - CREATE - - UPDATE - resources: - - provisioners - - provisioners/status - sideEffects: None - ---- - -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/managed-by: kops - k8s-addon: karpenter.sh - name: karpenter-nodes-default -spec: - launchTemplate: karpenter-nodes-default.minimal.example.com - subnetSelector: - kops.k8s.io/instance-group/karpenter-nodes-default: '*' - kubernetes.io/cluster/minimal.example.com: '*' - ---- - -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/managed-by: kops - k8s-addon: karpenter.sh - name: karpenter-nodes-default -spec: - consolidation: - enabled: true - kubeletConfiguration: - kubeReserved: - cpu: 500m - memory: 1G - maxPods: 50 - systemReserved: - cpu: 500m - memory: 1G - providerRef: - name: karpenter-nodes-default - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: - - spot - - on-demand - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: node.kubernetes.io/instance-type - operator: In - values: - - c5.large - startupTaints: - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - ---- - -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/managed-by: kops - k8s-addon: karpenter.sh - name: karpenter-nodes-single-machinetype -spec: - launchTemplate: karpenter-nodes-single-machinetype.minimal.example.com - subnetSelector: - kops.k8s.io/instance-group/karpenter-nodes-single-machinetype: '*' - kubernetes.io/cluster/minimal.example.com: '*' - ---- - -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/managed-by: kops - k8s-addon: karpenter.sh - name: karpenter-nodes-single-machinetype -spec: - consolidation: - enabled: true - providerRef: - name: karpenter-nodes-single-machinetype - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: - - spot - - on-demand - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: node.kubernetes.io/instance-type - operator: In - values: - - t2.medium - startupTaints: - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-node-termination-handler.aws-k8s-1.11_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-node-termination-handler.aws-k8s-1.11_content deleted file mode 100644 index da3a64de540e8..0000000000000 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-node-termination-handler.aws-k8s-1.11_content +++ /dev/null @@ -1,285 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: node-termination-handler.aws - app.kubernetes.io/instance: aws-node-termination-handler - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: aws-node-termination-handler - app.kubernetes.io/part-of: aws-node-termination-handler - app.kubernetes.io/version: v1.22.0 - k8s-addon: node-termination-handler.aws - k8s-app: aws-node-termination-handler - name: aws-node-termination-handler - namespace: kube-system - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: node-termination-handler.aws - app.kubernetes.io/instance: aws-node-termination-handler - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: aws-node-termination-handler - app.kubernetes.io/part-of: aws-node-termination-handler - app.kubernetes.io/version: v1.22.0 - k8s-addon: node-termination-handler.aws - name: aws-node-termination-handler -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - list - - get -- apiGroups: - - "" - resources: - - pods/eviction - verbs: - - create -- apiGroups: - - extensions - resources: - - daemonsets - verbs: - - get -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - get -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: node-termination-handler.aws - app.kubernetes.io/instance: aws-node-termination-handler - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: aws-node-termination-handler - app.kubernetes.io/part-of: aws-node-termination-handler - app.kubernetes.io/version: v1.22.0 - k8s-addon: node-termination-handler.aws - name: aws-node-termination-handler -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: aws-node-termination-handler -subjects: -- kind: ServiceAccount - name: aws-node-termination-handler - namespace: kube-system - ---- - -apiVersion: apps/v1 -kind: Deployment -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: node-termination-handler.aws - app.kubernetes.io/component: deployment - app.kubernetes.io/instance: aws-node-termination-handler - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: aws-node-termination-handler - app.kubernetes.io/part-of: aws-node-termination-handler - app.kubernetes.io/version: v1.22.0 - k8s-addon: node-termination-handler.aws - k8s-app: aws-node-termination-handler - name: aws-node-termination-handler - namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/instance: aws-node-termination-handler - app.kubernetes.io/name: aws-node-termination-handler - kubernetes.io/os: linux - template: - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/component: deployment - app.kubernetes.io/instance: aws-node-termination-handler - app.kubernetes.io/name: aws-node-termination-handler - k8s-app: aws-node-termination-handler - kops.k8s.io/managed-by: kops - kops.k8s.io/nth-mode: sqs - kubernetes.io/os: linux - spec: - containers: - - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ENABLE_PROBES_SERVER - value: "true" - - name: PROBES_SERVER_PORT - value: "8080" - - name: PROBES_SERVER_ENDPOINT - value: /healthz - - name: LOG_LEVEL - value: info - - name: JSON_LOGGING - value: "true" - - name: LOG_FORMAT_VERSION - value: "2" - - name: ENABLE_PROMETHEUS_SERVER - value: "false" - - name: PROMETHEUS_SERVER_PORT - value: "9092" - - name: CHECK_TAG_BEFORE_DRAINING - value: "true" - - name: MANAGED_TAG - value: aws-node-termination-handler/managed - - name: USE_PROVIDER_ID - value: "true" - - name: DRY_RUN - value: "false" - - name: CORDON_ONLY - value: "false" - - name: TAINT_NODE - value: "false" - - name: EXCLUDE_FROM_LOAD_BALANCERS - value: "true" - - name: DELETE_LOCAL_DATA - value: "true" - - name: IGNORE_DAEMON_SETS - value: "true" - - name: POD_TERMINATION_GRACE_PERIOD - value: "-1" - - name: NODE_TERMINATION_GRACE_PERIOD - value: "120" - - name: EMIT_KUBERNETES_EVENTS - value: "true" - - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS - value: "-1" - - name: ENABLE_SQS_TERMINATION_DRAINING - value: "true" - - name: QUEUE_URL - value: https://sqs.us-test-1.amazonaws.com/123456789012/minimal-example-com-nth - - name: DELETE_SQS_MSG_IF_NODE_NOT_FOUND - value: "false" - - name: WORKERS - value: "10" - - name: AWS_ROLE_ARN - value: arn:aws-test:iam::123456789012:role/aws-node-termination-handler.kube-system.sa.minimal.example.com - - name: AWS_WEB_IDENTITY_TOKEN_FILE - value: /var/run/secrets/amazonaws.com/token - image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.22.0 - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /healthz - port: 8080 - initialDelaySeconds: 5 - periodSeconds: 5 - name: aws-node-termination-handler - ports: - - containerPort: 8080 - name: liveness-probe - protocol: TCP - - containerPort: 9092 - name: metrics - protocol: TCP - resources: - requests: - cpu: 50m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - mountPath: /var/run/secrets/amazonaws.com/ - name: token-amazonaws-com - readOnly: true - priorityClassName: system-cluster-critical - securityContext: - fsGroup: 1000 - serviceAccountName: aws-node-termination-handler - topologySpreadConstraints: - - labelSelector: - matchLabels: - app.kubernetes.io/instance: aws-node-termination-handler - app.kubernetes.io/name: aws-node-termination-handler - kops.k8s.io/nth-mode: sqs - maxSkew: 1 - topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: ScheduleAnyway - - labelSelector: - matchLabels: - app.kubernetes.io/instance: aws-node-termination-handler - app.kubernetes.io/name: aws-node-termination-handler - kops.k8s.io/nth-mode: sqs - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - volumes: - - name: token-amazonaws-com - projected: - defaultMode: 420 - sources: - - serviceAccountToken: - audience: amazonaws.com - expirationSeconds: 86400 - path: token - ---- - -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: node-termination-handler.aws - app.kubernetes.io/instance: aws-node-termination-handler - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: aws-node-termination-handler - k8s-addon: node-termination-handler.aws - name: aws-node-termination-handler - namespace: kube-system -spec: - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/instance: aws-node-termination-handler - app.kubernetes.io/name: aws-node-termination-handler - kops.k8s.io/nth-mode: sqs diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupscript-karpenter-nodes-default_content similarity index 100% rename from tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data rename to tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupscript-karpenter-nodes-default_content diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupscript-karpenter-nodes-single-machinetype_content similarity index 100% rename from tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data rename to tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupscript-karpenter-nodes-single-machinetype_content diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index e727e859dfc46..228cb876739f8 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -4,8 +4,6 @@ locals { iam_openid_connect_provider_issuer = "discovery.example.com/minimal.example.com" kube-system-aws-cloud-controller-manager_role_arn = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.arn kube-system-aws-cloud-controller-manager_role_name = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.name - kube-system-aws-node-termination-handler_role_arn = aws_iam_role.aws-node-termination-handler-kube-system-sa-minimal-example-com.arn - kube-system-aws-node-termination-handler_role_name = aws_iam_role.aws-node-termination-handler-kube-system-sa-minimal-example-com.name kube-system-dns-controller_role_arn = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn kube-system-dns-controller_role_name = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name kube-system-ebs-csi-controller-sa_role_arn = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.arn @@ -50,14 +48,6 @@ output "kube-system-aws-cloud-controller-manager_role_name" { value = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.name } -output "kube-system-aws-node-termination-handler_role_arn" { - value = aws_iam_role.aws-node-termination-handler-kube-system-sa-minimal-example-com.arn -} - -output "kube-system-aws-node-termination-handler_role_name" { - value = aws_iam_role.aws-node-termination-handler-kube-system-sa-minimal-example-com.name -} - output "kube-system-dns-controller_role_arn" { value = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn } @@ -177,11 +167,6 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" propagate_at_launch = true value = "master-us-test-1a.masters.minimal.example.com" } - tag { - key = "aws-node-termination-handler/managed" - propagate_at_launch = true - value = "" - } tag { key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" propagate_at_launch = true @@ -242,11 +227,6 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { propagate_at_launch = true value = "nodes.minimal.example.com" } - tag { - key = "aws-node-termination-handler/managed" - propagate_at_launch = true - value = "" - } tag { key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" propagate_at_launch = true @@ -270,82 +250,6 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] } -resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" { - autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id - default_result = "CONTINUE" - heartbeat_timeout = 300 - lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" - name = "master-us-test-1a-NTHLifecycleHook" -} - -resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" { - autoscaling_group_name = aws_autoscaling_group.nodes-minimal-example-com.id - default_result = "CONTINUE" - heartbeat_timeout = 300 - lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" - name = "nodes-NTHLifecycleHook" -} - -resource "aws_cloudwatch_event_rule" "minimal-example-com-ASGLifecycle" { - event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-ASGLifecycle_event_pattern") - name = "minimal.example.com-ASGLifecycle" - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "minimal.example.com-ASGLifecycle" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } -} - -resource "aws_cloudwatch_event_rule" "minimal-example-com-InstanceScheduledChange" { - event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-InstanceScheduledChange_event_pattern") - name = "minimal.example.com-InstanceScheduledChange" - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "minimal.example.com-InstanceScheduledChange" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } -} - -resource "aws_cloudwatch_event_rule" "minimal-example-com-InstanceStateChange" { - event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-InstanceStateChange_event_pattern") - name = "minimal.example.com-InstanceStateChange" - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "minimal.example.com-InstanceStateChange" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } -} - -resource "aws_cloudwatch_event_rule" "minimal-example-com-SpotInterruption" { - event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-SpotInterruption_event_pattern") - name = "minimal.example.com-SpotInterruption" - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "minimal.example.com-SpotInterruption" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } -} - -resource "aws_cloudwatch_event_target" "minimal-example-com-ASGLifecycle-Target" { - arn = aws_sqs_queue.minimal-example-com-nth.arn - rule = aws_cloudwatch_event_rule.minimal-example-com-ASGLifecycle.id -} - -resource "aws_cloudwatch_event_target" "minimal-example-com-InstanceScheduledChange-Target" { - arn = aws_sqs_queue.minimal-example-com-nth.arn - rule = aws_cloudwatch_event_rule.minimal-example-com-InstanceScheduledChange.id -} - -resource "aws_cloudwatch_event_target" "minimal-example-com-InstanceStateChange-Target" { - arn = aws_sqs_queue.minimal-example-com-nth.arn - rule = aws_cloudwatch_event_rule.minimal-example-com-InstanceStateChange.id -} - -resource "aws_cloudwatch_event_target" "minimal-example-com-SpotInterruption-Target" { - arn = aws_sqs_queue.minimal-example-com-nth.arn - rule = aws_cloudwatch_event_rule.minimal-example-com-SpotInterruption.id -} - resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" { availability_zone = "us-test-1a" encrypted = false @@ -423,18 +327,6 @@ resource "aws_iam_role" "aws-cloud-controller-manager-kube-system-sa-minimal-exa } } -resource "aws_iam_role" "aws-node-termination-handler-kube-system-sa-minimal-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_aws-node-termination-handler.kube-system.sa.minimal.example.com_policy") - name = "aws-node-termination-handler.kube-system.sa.minimal.example.com" - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "aws-node-termination-handler.kube-system.sa.minimal.example.com" - "kubernetes.io/cluster/minimal.example.com" = "owned" - "service-account.kops.k8s.io/name" = "aws-node-termination-handler" - "service-account.kops.k8s.io/namespace" = "kube-system" - } -} - resource "aws_iam_role" "dns-controller-kube-system-sa-minimal-example-com" { assume_role_policy = file("${path.module}/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy") name = "dns-controller.kube-system.sa.minimal.example.com" @@ -497,12 +389,6 @@ resource "aws_iam_role_policy" "aws-cloud-controller-manager-kube-system-sa-mini role = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.name } -resource "aws_iam_role_policy" "aws-node-termination-handler-kube-system-sa-minimal-example-com" { - name = "aws-node-termination-handler.kube-system.sa.minimal.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_aws-node-termination-handler.kube-system.sa.minimal.example.com_policy") - role = aws_iam_role.aws-node-termination-handler-kube-system-sa-minimal-example-com.name -} - resource "aws_iam_role_policy" "dns-controller-kube-system-sa-minimal-example-com" { name = "dns-controller.kube-system.sa.minimal.example.com" policy = file("${path.module}/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy") @@ -552,156 +438,6 @@ resource "aws_key_pair" "kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663 } } -resource "aws_launch_template" "karpenter-nodes-default-minimal-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - encrypted = true - iops = 3000 - throughput = 125 - volume_size = 128 - volume_type = "gp3" - } - } - iam_instance_profile { - name = aws_iam_instance_profile.nodes-minimal-example-com.id - } - image_id = "ami-12345678" - key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - metadata_options { - http_endpoint = "enabled" - http_protocol_ipv6 = "disabled" - http_put_response_hop_limit = 1 - http_tokens = "required" - } - monitoring { - enabled = false - } - name = "karpenter-nodes-default.minimal.example.com" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - ipv6_address_count = 0 - security_groups = [aws_security_group.nodes-minimal-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes-default.minimal.example.com" - "aws-node-termination-handler/managed" = "" - "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-default" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes-default" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes-default.minimal.example.com" - "aws-node-termination-handler/managed" = "" - "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-default" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes-default" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } - } - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes-default.minimal.example.com" - "aws-node-termination-handler/managed" = "" - "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-default" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes-default" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } - user_data = filebase64("${path.module}/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data") -} - -resource "aws_launch_template" "karpenter-nodes-single-machinetype-minimal-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - encrypted = true - iops = 3000 - throughput = 125 - volume_size = 128 - volume_type = "gp3" - } - } - iam_instance_profile { - name = aws_iam_instance_profile.nodes-minimal-example-com.id - } - image_id = "ami-12345678" - key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - metadata_options { - http_endpoint = "enabled" - http_protocol_ipv6 = "disabled" - http_put_response_hop_limit = 1 - http_tokens = "required" - } - monitoring { - enabled = false - } - name = "karpenter-nodes-single-machinetype.minimal.example.com" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - ipv6_address_count = 0 - security_groups = [aws_security_group.nodes-minimal-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes-single-machinetype.minimal.example.com" - "aws-node-termination-handler/managed" = "" - "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-single-machinetype" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes-single-machinetype" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes-single-machinetype.minimal.example.com" - "aws-node-termination-handler/managed" = "" - "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-single-machinetype" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes-single-machinetype" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } - } - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes-single-machinetype.minimal.example.com" - "aws-node-termination-handler/managed" = "" - "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-single-machinetype" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes-single-machinetype" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } - user_data = filebase64("${path.module}/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data") -} - resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { block_device_mappings { device_name = "/dev/xvda" @@ -748,7 +484,6 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { tags = { "KubernetesCluster" = "minimal.example.com" "Name" = "master-us-test-1a.masters.minimal.example.com" - "aws-node-termination-handler/managed" = "" "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" @@ -763,7 +498,6 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { tags = { "KubernetesCluster" = "minimal.example.com" "Name" = "master-us-test-1a.masters.minimal.example.com" - "aws-node-termination-handler/managed" = "" "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" @@ -776,7 +510,6 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { tags = { "KubernetesCluster" = "minimal.example.com" "Name" = "master-us-test-1a.masters.minimal.example.com" - "aws-node-termination-handler/managed" = "" "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" @@ -830,7 +563,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" { tags = { "KubernetesCluster" = "minimal.example.com" "Name" = "nodes.minimal.example.com" - "aws-node-termination-handler/managed" = "" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "nodes" @@ -842,7 +574,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" { tags = { "KubernetesCluster" = "minimal.example.com" "Name" = "nodes.minimal.example.com" - "aws-node-termination-handler/managed" = "" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "nodes" @@ -852,7 +583,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" { tags = { "KubernetesCluster" = "minimal.example.com" "Name" = "nodes.minimal.example.com" - "aws-node-termination-handler/managed" = "" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "nodes" @@ -1032,14 +762,6 @@ resource "aws_s3_object" "minimal-example-com-addons-limit-range-addons-k8s-io" server_side_encryption = "AES256" } -resource "aws_s3_object" "minimal-example-com-addons-node-termination-handler-aws-k8s-1-11" { - bucket = "testingBucket" - content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-node-termination-handler.aws-k8s-1.11_content") - key = "clusters.example.com/minimal.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml" - provider = aws.files - server_side_encryption = "AES256" -} - resource "aws_s3_object" "minimal-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content") @@ -1080,6 +802,22 @@ resource "aws_s3_object" "nodeupconfig-nodes" { server_side_encryption = "AES256" } +resource "aws_s3_object" "nodeupscript-karpenter-nodes-default" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupscript-karpenter-nodes-default_content") + key = "clusters.example.com/minimal.example.com/igconfig/node/karpenter-nodes-default/nodeupscript.sh" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupscript-karpenter-nodes-single-machinetype" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupscript-karpenter-nodes-single-machinetype_content") + key = "clusters.example.com/minimal.example.com/igconfig/node/karpenter-nodes-single-machinetype/nodeupscript.sh" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_security_group" "masters-minimal-example-com" { description = "Security group for masters" name = "masters.minimal.example.com" @@ -1228,17 +966,6 @@ resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-udp-1 type = "ingress" } -resource "aws_sqs_queue" "minimal-example-com-nth" { - message_retention_seconds = 300 - name = "minimal-example-com-nth" - policy = file("${path.module}/data/aws_sqs_queue_minimal-example-com-nth_policy") - tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "minimal-example-com-nth" - "kubernetes.io/cluster/minimal.example.com" = "owned" - } -} - resource "aws_subnet" "us-test-1a-minimal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/upup/models/cloudup/resources/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml.template b/upup/models/cloudup/resources/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml.template index d22e1c58c53d1..b227ff101e1ec 100644 --- a/upup/models/cloudup/resources/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml.template +++ b/upup/models/cloudup/resources/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml.template @@ -726,6 +726,12 @@ spec: - effect: NoExecute operator: Exists tolerationSeconds: 300 + {{- if IsControlPlaneMode }} + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + {{- end }} {{ end }} securityContext: fsGroup: 1000 diff --git a/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.12.yaml.template index f97b406e17ec4..117ce77c2416f 100644 --- a/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.12.yaml.template @@ -138,7 +138,7 @@ spec: - key: "CriticalAddonsOnly" operator: "Exists" {{- end }} - {{- if KarpenterEnabled }} + {{- if IsControlPlaneMode }} - key: node-role.kubernetes.io/master operator: Exists - key: node-role.kubernetes.io/control-plane @@ -361,7 +361,9 @@ spec: - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}} - --logtostderr=true - --v=2 + {{- if not IsControlPlaneMode }} priorityClassName: system-cluster-critical + {{- end }} tolerations: {{- if KubeDNS.Tolerations }} {{ ToYAML .KubeDNS.Tolerations | indent 8 }} diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/helm-values.yaml b/upup/models/cloudup/resources/addons/karpenter.sh/helm-values.yaml new file mode 100644 index 0000000000000..68d60d9120776 --- /dev/null +++ b/upup/models/cloudup/resources/addons/karpenter.sh/helm-values.yaml @@ -0,0 +1,48 @@ +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: karpenter.sh/provisioner-name + operator: DoesNotExist + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: karpenter.sh/provisioner-name + operator: DoesNotExist + - key: node-role.kubernetes.io/master + operator: Exists +controller: + resources: + requests: + cpu: "{{ or .Karpenter.CPURequest \"500m\" }}" + memory: "{{ or .Karpenter.MemoryRequest \"1Gi\" }}" + limits: + memory: "{{ or .Karpenter.MemoryLimit \"1Gi\" }}" + env: + - name: AWS_REGION + value: "{{ Region }}" +logLevel: "{{ .Karpenter.LogLevel }}" +replicas: 1 +settings: + clusterEndpoint: "{{ printf \"https://{{ APIInternalName }}\" }}" + clusterName: "{{ printf \"{{ ClusterName }}\" }}" +tolerations: +- key: node-role.kubernetes.io/master + operator: Exists +- key: node-role.kubernetes.io/control-plane + operator: Exists +- key: CriticalAddonsOnly + operator: Exists +topologySpreadConstraints: +- maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index aa66e84eea670..0fdff48692c57 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -1,623 +1,656 @@ -# helm template karpenter oci://public.ecr.aws/karpenter/karpenter-crd \ -# --version v0.31.3 -# helm template karpenter oci://public.ecr.aws/karpenter/karpenter \ -# --version v0.31.3 \ +#helm template karpenter oci://public.ecr.aws/karpenter/karpenter \ +# --version 1.6.2 \ +# --include-crds \ # --namespace kube-system \ -# --set controller.resources.requests.cpu=500m \ -# --set controller.resources.requests.memory=1Gi \ -# --set controller.resources.limits.memory=1Gi +# --values helm-values.yaml +--- +# Source: karpenter/crds/karpenter.k8s.aws_ec2nodeclasses.yaml --- -# Source: karpenter-crd/templates/karpenter.sh_provisioners.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: provisioners.karpenter.sh + controller-gen.kubebuilder.io/version: v0.18.0 + name: ec2nodeclasses.karpenter.k8s.aws spec: - group: karpenter.sh + group: karpenter.k8s.aws names: categories: - karpenter - kind: Provisioner - listKind: ProvisionerList - plural: provisioners - singular: provisioner + kind: EC2NodeClass + listKind: EC2NodeClassList + plural: ec2nodeclasses + shortNames: + - ec2nc + - ec2ncs + singular: ec2nodeclass scope: Cluster versions: - additionalPrinterColumns: - - jsonPath: .spec.providerRef.name - name: Template + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready type: string - - jsonPath: .spec.weight - name: Weight + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.role + name: Role priority: 1 type: string - name: v1alpha5 + name: v1 schema: openAPIV3Schema: - description: Provisioner is the Schema for the Provisioners API + description: EC2NodeClass is the Schema for the EC2NodeClass API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: - description: ProvisionerSpec is the top level provisioner specification. - Provisioners launch nodes in response to pods that are unschedulable. - A single provisioner is capable of managing a diverse set of nodes. - Node properties are determined from a combination of provisioner and - pod scheduling constraints. + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. properties: - annotations: - additionalProperties: - type: string - description: Annotations are applied to every node. - type: object - consolidation: - description: Consolidation are the consolidation parameters - properties: - enabled: - description: Enabled enables consolidation if it has been set - type: boolean - type: object - kubeletConfiguration: - description: KubeletConfiguration are options passed to the kubelet - when provisioning nodes + amiFamily: + description: |- + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 + type: string + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + ssmParameter: + description: SSMParameter is the name (or ARN) of the SSM parameter containing the Image ID. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select amis. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', 'alias', 'ssmParameter'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias) || has(x.ssmParameter)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + The following are the supported values for each volume type: + + * gp3: 3,000-16,000 IOPS + + * io1: 100-64,000 IOPS + + * io2: 100-64,000 IOPS + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: Identifier (key ID, key alias, key ARN, or alias ARN) of the customer managed KMS key to use for EBS encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeInitializationRate: + description: |- + VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization, + in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume + initialization. Specifying a volume initialization rate ensures that the volume is initialized at a + predictable and consistent rate after creation. Only allowed if SnapshotID is set. + Valid Range: Minimum value of 100. Maximum value of 300. + format: int32 + maximum: 300 + minimum: 100 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + * gp2 and gp3: 1-16,384 + + * io1 and io2: 4-16,384 + + * st1 and sc1: 125-16,384 + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + - message: snapshotID must be set when volumeInitializationRate is set + rule: '!has(self.volumeInitializationRate) || (has(self.snapshotID) && self.snapshotID != '''')' + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + capacityReservationSelectorTerms: + description: |- + CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to + determine the set of eligible capacity reservations. + items: + properties: + id: + description: ID is the capacity reservation id in EC2 + pattern: ^cr-[0-9a-z]+$ + type: string + ownerID: + description: Owner is the owner id for the ami. + pattern: ^[0-9]{12}$ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select capacity reservations. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set along with tags in a capacity reservation selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))' + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. properties: clusterDNS: - description: clusterDNS is a list of IP addresses for the cluster - DNS server. Note that not all providers may use all addresses. + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. items: type: string type: array - containerRuntime: - description: ContainerRuntime is the container runtime to be used - with your worker nodes. - type: string cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for - containers that specify CPU limits. + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. type: boolean evictionHard: additionalProperties: type: string - description: EvictionHard is the map of signal names to quantities - that define hard eviction thresholds + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) evictionMaxPodGracePeriod: - description: EvictionMaxPodGracePeriod is the maximum allowed - grace period (in seconds) to use when terminating pods in response - to soft eviction thresholds being met. + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. format: int32 type: integer evictionSoft: additionalProperties: type: string - description: EvictionSoft is the map of signal names to quantities - that define soft eviction thresholds + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) evictionSoftGracePeriod: additionalProperties: type: string - description: EvictionSoftGracePeriod is the map of signal names - to quantities that define grace periods for each eviction signal + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) imageGCHighThresholdPercent: - description: ImageGCHighThresholdPercent is the percent of disk - usage after which image garbage collection is always run. The - percent is calculated by dividing this field value by 100, so - this field must be between 0 and 100, inclusive. When specified, - the value must be greater than ImageGCLowThresholdPercent. + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. format: int32 maximum: 100 minimum: 0 type: integer imageGCLowThresholdPercent: - description: ImageGCLowThresholdPercent is the percent of disk - usage before which image garbage collection is never run. Lowest - disk usage to garbage collect to. The percent is calculated - by dividing this field value by 100, so the field value must - be between 0 and 100, inclusive. When specified, the value must - be less than imageGCHighThresholdPercent + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent format: int32 maximum: 100 minimum: 0 type: integer kubeReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: KubeReserved contains resources reserved for Kubernetes - system components. + description: KubeReserved contains resources reserved for Kubernetes system components. type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) maxPods: - description: MaxPods is an override for the maximum number of - pods that can run on a worker node instance. + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. format: int32 minimum: 0 type: integer podsPerCore: - description: PodsPerCore is an override for the number of pods - that can run on a worker node instance based on the number of - cpu cores. This value cannot exceed MaxPods, so, if MaxPods - is a lower value, that value will be used. + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. format: int32 minimum: 0 type: integer systemReserved: additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: SystemReserved contains resources reserved for OS - system daemons and kernel memory. - type: object - type: object - labels: - additionalProperties: - type: string - description: Labels are layered with Requirements and applied to every - node. - type: object - limits: - description: Limits define a set of bounds for provisioning capacity. - properties: - resources: - additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Resources contains all the allocatable resources - that Karpenter supports for limiting. + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) type: object - provider: - description: Provider contains fields specific to your cloudprovider. - type: object - x-kubernetes-preserve-unknown-fields: true - providerRef: - description: ProviderRef is a reference to a dedicated CRD for the - chosen provider, that holds additional configuration options + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. properties: - apiVersion: - description: API version of the referent + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional type: string - required: - - name type: object - requirements: - description: Requirements are layered with Labels and applied to every - node. + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of security group selector terms. The terms are ORed. items: - description: A node selector requirement is a selector that contains - values, a key, and an operator that relates the key and values. + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: - key: - description: The label key that the selector applies to. + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ type: string - operator: - description: Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and - Lt. + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. type: string - values: - description: An array of string values. If the operator is In - or NotIn, the values array must be non-empty. If the operator - is Exists or DoesNotExist, the values array must be empty. - If the operator is Gt or Lt, the values array must have a - single element, which will be interpreted as an integer. This - array is replaced during a strategic merge patch. - items: + tags: + additionalProperties: type: string - type: array - required: - - key - - operator - type: object - type: array - startupTaints: - description: StartupTaints are taints that are applied to nodes upon - startup which are expected to be removed automatically within a - short period of time, typically by a DaemonSet that tolerates the - taint. These are commonly used by daemonsets to allow initialization - and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint - in order to have nodes provisioned for them. - items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. - properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key - type: object - type: array - taints: - description: Taints will be applied to every node launched by the - Provisioner. If specified, the provisioner will not provision nodes - for pods that do not have matching tolerations. Additional taints - will be created that match pod tolerations on a per-node basis. - items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. - properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key - type: object - type: array - ttlSecondsAfterEmpty: - description: "TTLSecondsAfterEmpty is the number of seconds the controller - will wait before attempting to delete a node, measured from when - the node is detected to be empty. A Node is considered to be empty - when it does not have pods scheduled to it, excluding daemonsets. - \n Termination due to no utilization is disabled if this field is - not set." - format: int64 - type: integer - ttlSecondsUntilExpired: - description: "TTLSecondsUntilExpired is the number of seconds the - controller will wait before terminating a node, measured from when - the node is created. This is useful to implement features like eventually - consistent node upgrade, memory leak protection, and disruption - testing. \n Termination due to expiration is disabled if this field - is not set." - format: int64 - type: integer - weight: - description: Weight is the priority given to the provisioner during - scheduling. A higher numerical weight indicates that this provisioner - will be ordered ahead of other provisioners with lower weights. - A provisioner with no weight will be treated as if it is a provisioner - with a weight of 0. - format: int32 - maximum: 100 - minimum: 1 - type: integer - type: object - status: - description: ProvisionerStatus defines the observed state of Provisioner - properties: - conditions: - description: Conditions is the set of conditions required for this - provisioner to scale its target, and indicates whether or not those - conditions are met. - items: - description: 'Condition defines a readiness condition for a Knative - resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' - properties: - lastTransitionTime: - description: LastTransitionTime is the last time the condition - transitioned from one status to another. We use VolatileTime - in place of metav1.Time to exclude this from creating equality.Semantic - differences (all other things held constant). - type: string - message: - description: A human readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - severity: - description: Severity with which to treat failures of this type - of condition. When this is not specified, it defaults to Error. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition. - type: string - required: - - status - - type + description: |- + Tags is a map of key/value tags used to select security groups. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') type: object + maxItems: 30 type: array - lastScaleTime: - description: LastScaleTime is the last time the Provisioner scaled - the number of nodes - format: date-time - type: string - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Resources is the list of resources that have been provisioned. - type: object - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -# Source: karpenter-crd/templates/karpenter.k8s.aws_awsnodetemplates.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: awsnodetemplates.karpenter.k8s.aws -spec: - group: karpenter.k8s.aws - names: - categories: - - karpenter - kind: AWSNodeTemplate - listKind: AWSNodeTemplateList - plural: awsnodetemplates - singular: awsnodetemplate - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: AWSNodeTemplate is the Schema for the AWSNodeTemplate API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: AWSNodeTemplateSpec is the top level specification for the - AWS Karpenter Provider. This will contain configuration necessary to - launch instances in AWS. - properties: - amiFamily: - description: AMIFamily is the AMI family that instances use. - type: string - amiSelector: - additionalProperties: - type: string - description: AMISelector discovers AMIs to be used by Amazon EC2 tags. - type: object - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - blockDeviceMappings: - description: BlockDeviceMappings to be applied to provisioned nodes. + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of subnet selector terms. The terms are ORed. items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: - deviceName: - description: The device name (for example, /dev/sdh or xvdh). + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ type: string - ebs: - description: EBS contains parameters used to automatically set - up EBS volumes when an instance is launched. - properties: - deleteOnTermination: - description: DeleteOnTermination indicates whether the EBS - volume is deleted on instance termination. - type: boolean - encrypted: - description: Encrypted indicates whether the EBS volume - is encrypted. Encrypted volumes can only be attached to - instances that support Amazon EBS encryption. If you are - creating a volume from a snapshot, you can't specify an - encryption value. - type: boolean - iops: - description: "IOPS is the number of I/O operations per second - (IOPS). For gp3, io1, and io2 volumes, this represents - the number of IOPS that are provisioned for the volume. - For gp2 volumes, this represents the baseline performance - of the volume and the rate at which the volume accumulates - I/O credits for bursting. \n The following are the supported - values for each volume type: \n * gp3: 3,000-16,000 IOPS - \n * io1: 100-64,000 IOPS \n * io2: 100-64,000 IOPS \n - For io1 and io2 volumes, we guarantee 64,000 IOPS only - for Instances built on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). - Other instance families guarantee performance up to 32,000 - IOPS. \n This parameter is supported for io1, io2, and - gp3 volumes only. This parameter is not supported for - gp2, st1, sc1, or standard volumes." - format: int64 - type: integer - kmsKeyID: - description: KMSKeyID (ARN) of the symmetric Key Management - Service (KMS) CMK used for encryption. - type: string - snapshotID: - description: SnapshotID is the ID of an EBS snapshot - type: string - throughput: - description: 'Throughput to provision for a gp3 volume, - with a maximum of 1,000 MiB/s. Valid Range: Minimum value - of 125. Maximum value of 1000.' - format: int64 - type: integer - volumeSize: - anyOf: - - type: integer - - type: string - description: "VolumeSize in GiBs. You must specify either - a snapshot ID or a volume size. The following are the - supported volumes sizes for each volume type: \n * gp2 - and gp3: 1-16,384 \n * io1 and io2: 4-16,384 \n * st1 - and sc1: 125-16,384 \n * standard: 1-1,024" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - volumeType: - description: VolumeType of the block device. For more information, - see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) - in the Amazon Elastic Compute Cloud User Guide. - type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') type: object + maxItems: 30 type: array - context: - description: Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html - type: string - detailedMonitoring: - description: DetailedMonitoring controls if detailed monitoring is - enabled for instances that are launched - type: boolean - instanceProfile: - description: InstanceProfile is the AWS identity that instances use. - type: string - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - launchTemplate: - description: 'LaunchTemplateName for the node. If not specified, a - launch template will be generated. NOTE: This field is for specifying - a custom launch template and is exposed in the Spec as `launchTemplate` - for backwards compatibility.' - type: string - metadataOptions: - description: "MetadataOptions for the generated launch template of - provisioned nodes. \n This specifies the exposure of the Instance - Metadata Service to provisioned EC2 nodes. For more information, - see Instance Metadata and User Data (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) - in the Amazon Elastic Compute Cloud User Guide. \n Refer to recommended, - security best practices (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) - for limiting exposure of Instance Metadata and User Data to pods. - If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - disabled, with httpPutResponseLimit of 2, and with httpTokens required." - properties: - httpEndpoint: - description: "HTTPEndpoint enables or disables the HTTP metadata - endpoint on provisioned nodes. If metadata options is non-nil, - but this parameter is not specified, the default state is \"enabled\". - \n If you specify a value of \"disabled\", instance metadata - will not be accessible on the node." - type: string - httpProtocolIPv6: - description: HTTPProtocolIPv6 enables or disables the IPv6 endpoint - for the instance metadata service on provisioned nodes. If metadata - options is non-nil, but this parameter is not specified, the - default state is "disabled". - type: string - httpPutResponseHopLimit: - description: HTTPPutResponseHopLimit is the desired HTTP PUT response - hop limit for instance metadata requests. The larger the number, - the further instance metadata requests can travel. Possible - values are integers from 1 to 64. If metadata options is non-nil, - but this parameter is not specified, the default value is 1. - format: int64 - type: integer - httpTokens: - description: "HTTPTokens determines the state of token usage for - instance metadata requests. If metadata options is non-nil, - but this parameter is not specified, the default state is \"optional\". - \n If the state is optional, one can choose to retrieve instance - metadata with or without a signed token header on the request. - If one retrieves the IAM role credentials without a token, the - version 1.0 role credentials are returned. If one retrieves - the IAM role credentials using a valid signed token, the version - 2.0 role credentials are returned. \n If the state is \"required\", - one must send a signed token header with any instance metadata - retrieval requests. In this state, retrieving the IAM role credentials - always returns the version 2.0 credentials; the version 1.0 - credentials are not available." - type: string - type: object - securityGroupSelector: - additionalProperties: - type: string - description: SecurityGroups specify the names of the security groups. - type: object - subnetSelector: - additionalProperties: - type: string - description: SubnetSelector discovers subnets by tags. A value of - "" is a wildcard. - type: object + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in a subnet selector term' + rule: '!self.all(x, has(x.id) && has(x.tags))' tags: additionalProperties: type: string - description: Tags to be applied on ec2 resources like instances and - launch templates. + description: Tags to be applied on ec2 resources like instances and launch templates. type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') userData: - description: UserData to be applied to the provisioned nodes. It must - be in the appropriate format based on the AMIFamily in use. Karpenter - will merge certain fields into this UserData to ensure nodes are - being provisioned with the correct configuration. + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms type: object + x-kubernetes-validations: + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) + - message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2'') : true)' + - message: if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2023'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2023'') : true)' + - message: if set, amiFamily must be 'Bottlerocket' or 'Custom' when using a Bottlerocket alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''bottlerocket'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Bottlerocket'') : true)' + - message: if set, amiFamily must be 'Windows2019' or 'Custom' when using a Windows2019 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2019'') : true)' + - message: if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2022'') : true)' + - message: must specify amiFamily if amiSelectorTerms does not contain an alias + rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)' status: - description: AWSNodeTemplateStatus contains the resolved state of the - AWSNodeTemplate + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass properties: amis: - description: AMI contains the current AMI values that are available - to the cluster under the AMI selectors. + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. items: - description: AMI contains resolved AMI selector values utilized - for node launch + description: AMI contains resolved AMI selector values utilized for node launch properties: + deprecated: + description: Deprecation status of the AMI + type: boolean id: description: ID of the AMI type: string @@ -625,32 +658,31 @@ spec: description: Name of the AMI type: string requirements: - description: Requirements of the AMI to be utilized on an instance - type + description: Requirements of the AMI to be utilized on an instance type items: - description: A node selector requirement is a selector that - contains values, a key, and an operator that relates the - key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set - of values. Valid operators are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator - is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. If the operator is Gt or Lt, the - values array must have a single element, which will - be interpreted as an integer. This array is replaced - during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator @@ -661,12 +693,127 @@ spec: - requirements type: object type: array + capacityReservations: + description: |- + CapacityReservations contains the current capacity reservation values that are available to this NodeClass under the + CapacityReservation selectors. + items: + properties: + availabilityZone: + description: The availability zone the capacity reservation is available in. + type: string + endTime: + description: |- + The time at which the capacity reservation expires. Once expired, the reserved capacity is released and Karpenter + will no longer be able to launch instances into that reservation. + format: date-time + type: string + id: + description: The id for the capacity reservation. + pattern: ^cr-[0-9a-z]+$ + type: string + instanceMatchCriteria: + description: Indicates the type of instance launches the capacity reservation accepts. + enum: + - open + - targeted + type: string + instanceType: + description: The instance type for the capacity reservation. + type: string + ownerID: + description: The ID of the AWS account that owns the capacity reservation. + pattern: ^[0-9]{12}$ + type: string + reservationType: + default: default + description: The type of capacity reservation. + enum: + - default + - capacity-block + type: string + state: + default: active + description: |- + The state of the capacity reservation. A capacity reservation is considered to be expiring if it is within the EC2 + reclaimation window. Only capacity-block reservations may be in this state. + enum: + - active + - expiring + type: string + required: + - availabilityZone + - id + - instanceMatchCriteria + - instanceType + - ownerID + type: object + type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile for the role + type: string securityGroups: - description: SecurityGroups contains the current Security Groups values - that are available to the cluster under the SecurityGroups selectors. + description: |- + SecurityGroups contains the current security group values that are available to the + cluster under the SecurityGroups selectors. items: - description: SecurityGroup contains resolved SecurityGroup selector - values utilized for node launch + description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch properties: id: description: ID of the security group @@ -679,11 +826,11 @@ spec: type: object type: array subnets: - description: Subnets contains the current Subnet values that are available - to the cluster under the subnet selectors. + description: |- + Subnets contains the current subnet values that are available to the + cluster under the subnet selectors. items: - description: Subnet contains resolved Subnet selector values utilized - for node launch + description: Subnet contains resolved Subnet selector values utilized for node launch properties: id: description: ID of the subnet @@ -691,6 +838,9 @@ spec: zone: description: The associated availability zone type: string + zoneID: + description: The associated availability zone ID + type: string required: - id - zone @@ -702,29 +852,34 @@ spec: storage: true subresources: status: {} + +--- +# Source: karpenter/crds/karpenter.sh_nodeclaims.yaml --- -# Source: karpenter-crd/templates/karpenter.sh_machines.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: machines.karpenter.sh + controller-gen.kubebuilder.io/version: v0.18.0 + name: nodeclaims.karpenter.sh spec: group: karpenter.sh names: categories: - karpenter - kind: Machine - listKind: MachineList - plural: machines - singular: machine + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + singular: nodeclaim scope: Cluster versions: - additionalPrinterColumns: - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type name: Type type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone name: Zone type: string @@ -737,185 +892,157 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - - jsonPath: .metadata.labels.karpenter\.sh/capacity-type - name: Capacity + - jsonPath: .status.imageID + name: ImageID priority: 1 type: string - - jsonPath: .metadata.labels.karpenter\.sh/provisioner-name - name: Provisioner + - jsonPath: .status.providerID + name: ID priority: 1 type: string - - jsonPath: .spec.machineTemplateRef.name - name: Template + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool priority: 1 type: string - name: v1alpha5 - schema: - openAPIV3Schema: - description: Machine is the Schema for the Machines API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: MachineSpec describes the desired state of the Machine - properties: - kubelet: - description: Kubelet are options passed to the kubelet when provisioning - nodes - properties: - clusterDNS: - description: clusterDNS is a list of IP addresses for the cluster - DNS server. Note that not all providers may use all addresses. - items: - type: string - type: array - containerRuntime: - description: ContainerRuntime is the container runtime to be used - with your worker nodes. - type: string - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for - containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - description: EvictionHard is the map of signal names to quantities - that define hard eviction thresholds - type: object - evictionMaxPodGracePeriod: - description: EvictionMaxPodGracePeriod is the maximum allowed - grace period (in seconds) to use when terminating pods in response - to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - description: EvictionSoft is the map of signal names to quantities - that define soft eviction thresholds - type: object - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names - to quantities that define grace periods for each eviction signal - type: object - imageGCHighThresholdPercent: - description: ImageGCHighThresholdPercent is the percent of disk - usage after which image garbage collection is always run. The - percent is calculated by dividing this field value by 100, so - this field must be between 0 and 100, inclusive. When specified, - the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: ImageGCLowThresholdPercent is the percent of disk - usage before which image garbage collection is never run. Lowest - disk usage to garbage collect to. The percent is calculated - by dividing this field value by 100, so the field value must - be between 0 and 100, inclusive. When specified, the value must - be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: KubeReserved contains resources reserved for Kubernetes - system components. - type: object - maxPods: - description: MaxPods is an override for the maximum number of - pods that can run on a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: PodsPerCore is an override for the number of pods - that can run on a worker node instance based on the number of - cpu cores. This value cannot exceed MaxPods, so, if MaxPods - is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: SystemReserved contains resources reserved for OS - system daemons and kernel memory. - type: object - type: object - machineTemplateRef: - description: MachineTemplateRef is a reference to an object that defines - provider specific configuration + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=="Drifted")].status + name: Drifted + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration properties: - apiVersion: + group: description: API version of the referent + pattern: ^[^/]*$ type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' kind: description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' name: description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' required: + - group + - kind - name type: object requirements: - description: Requirements are layered with Labels and applied to every - node. + description: Requirements are layered with GetLabels and applied to every node. items: - description: A node selector requirement is a selector that contains - values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. properties: key: description: The label key that the selector applies to. type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer operator: - description: Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and - Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt values: - description: An array of string values. If the operator is In - or NotIn, the values array must be non-empty. If the operator - is Exists or DoesNotExist, the values array must be empty. - If the operator is Gt or Lt, the values array must have a - single element, which will be interpreted as an integer. This - array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ required: - key - operator type: object + maxItems: 100 type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' resources: - description: Resources models the resource requirements for the Machine - to launch + description: Resources models the resource requirements for the NodeClaim to launch properties: requests: additionalProperties: @@ -924,73 +1051,113 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Requests describes the minimum required resources - for the Machine to launch + description: Requests describes the minimum required resources for the NodeClaim to launch type: object type: object startupTaints: - description: StartupTaints are taints that are applied to nodes upon - startup which are expected to be removed automatically within a - short period of time, typically by a DaemonSet that tolerates the - taint. These are commonly used by daemonsets to allow initialization - and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint - in order to have nodes provisioned for them. + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute key: description: Required. The taint key to be applied to a node. type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. format: date-time type: string value: description: The taint value corresponding to the taint key. type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ required: - effect - key type: object type: array taints: - description: Taints will be applied to the machine's node. + description: Taints will be applied to the NodeClaim's node. items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute key: description: Required. The taint key to be applied to a node. type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. format: date-time type: string value: description: The taint value corresponding to the taint key. type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ required: - effect - key type: object type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf status: - description: MachineStatus defines the observed state of Machine + description: NodeClaimStatus defines the observed state of NodeClaim properties: allocatable: additionalProperties: @@ -999,8 +1166,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Allocatable is the estimated allocatable capacity of - the machine + description: Allocatable is the estimated allocatable capacity of the node type: object capacity: additionalProperties: @@ -1009,42 +1175,71 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Capacity is the estimated full capacity of the machine + description: Capacity is the estimated full capacity of the node type: object conditions: description: Conditions contains signals for health and readiness items: - description: 'Condition defines a readiness condition for a Knative - resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + description: Condition aliases the upstream type and adds additional helper methods properties: lastTransitionTime: - description: LastTransitionTime is the last time the condition - transitioned from one status to another. We use VolatileTime - in place of metav1.Time to exclude this from creating equality.Semantic - differences (all other things held constant). + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time type: string message: - description: A human readable message indicating details about - the transition. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer reason: - description: The reason for the condition's last transition. - type: string - severity: - description: Severity with which to treat failures of this type - of condition. When this is not specified, it defaults to Error. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ type: string status: - description: Status of the condition, one of True, False, Unknown. + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type of condition. + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: + - lastTransitionTime - status - type type: object type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string nodeName: description: NodeName is the name of the corresponding node object type: string @@ -1052,11 +1247,542 @@ spec: description: ProviderID of the corresponding node object type: string type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + +--- +# Source: karpenter/crds/karpenter.sh_nodepools.yaml +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: nodepools.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodePool + listKind: NodePoolList + plural: nodepools + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.k8s.aws" is restricted + rule: self.all(x, x in ["karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' + required: + - group + - kind + - name + type: object + x-kubernetes-validations: + - message: nodeClassRef.group is immutable + rule: self.group == oldSelf.group + - message: nodeClassRef.kind is immutable + rule: self.kind == oldSelf.kind + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + nodeClassObservedGeneration: + description: |- + NodeClassObservedGeneration represents the observed nodeClass generation for referenced nodeClass. If this does not match + the actual NodeClass Generation, NodeRegistrationHealthy status condition on the NodePool will be reset + format: int64 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + required: + - spec type: object served: true storage: true subresources: status: {} + --- # Source: karpenter/templates/poddisruptionbudget.yaml apiVersion: policy/v1 @@ -1065,10 +1791,10 @@ metadata: name: karpenter namespace: kube-system labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm spec: maxUnavailable: 1 @@ -1084,97 +1810,12 @@ metadata: name: karpenter namespace: kube-system labels: - helm.sh/chart: karpenter-v0.31.3 - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" - app.kubernetes.io/managed-by: Helm ---- -# Source: karpenter/templates/secret-webhook-cert.yaml -apiVersion: v1 -kind: Secret -metadata: - name: karpenter-cert - namespace: kube-system - labels: - helm.sh/chart: karpenter-v0.31.3 - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" - app.kubernetes.io/managed-by: Helm -# data: {} # Injected by karpenter-webhook ---- -# Source: karpenter/templates/configmap-logging.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: config-logging - namespace: kube-system - labels: - helm.sh/chart: karpenter-v0.31.3 - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" - app.kubernetes.io/managed-by: Helm -data: - # https://github.com/uber-go/zap/blob/aa3e73ec0896f8b066ddf668597a02f89628ee50/config.go - zap-logger-config: | - { - "level": "{{ .Karpenter.LogLevel }}", - "development": false, - "disableStacktrace": true, - "disableCaller": true, - "sampling": { - "initial": 100, - "thereafter": 100 - }, - "outputPaths": ["stdout"], - "errorOutputPaths": ["stderr"], - "encoding": "{{ .Karpenter.LogEncoding }}", - "encoderConfig": { - "timeKey": "time", - "levelKey": "level", - "nameKey": "logger", - "callerKey": "caller", - "messageKey": "message", - "stacktraceKey": "stacktrace", - "levelEncoder": "capital", - "timeEncoder": "iso8601" - } - } - loglevel.webhook: "debug" ---- -# Source: karpenter/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: karpenter-global-settings - namespace: kube-system - labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm -data: - "aws.assumeRoleARN": "" - "aws.assumeRoleDuration": "15m" - "aws.clusterCABundle": "" - "aws.clusterEndpoint": "https://{{ APIInternalName }}" - "aws.clusterName": "{{ ClusterName }}" - "aws.defaultInstanceProfile": "" - {{ if not .Networking.AmazonVPC }} - "aws.enableENILimitedPodDensity": "true" - {{ else }} - "aws.enableENILimitedPodDensity": "false" - {{ end }} - "aws.enablePodENI": "false" - "aws.interruptionQueueName": "" - "aws.isolatedVPC": "false" - "aws.vmMemoryOverheadPercent": "0.075" - "batchIdleDuration": "1s" - "batchMaxDuration": "10s" - "featureGates.driftEnabled": "false" +automountServiceAccountToken: false --- # Source: karpenter/templates/aggregate-clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1183,18 +1824,18 @@ metadata: name: karpenter-admin labels: rbac.authorization.k8s.io/aggregate-to-admin: "true" - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm rules: - - apiGroups: ["karpenter.sh"] - resources: ["provisioners", "provisioners/status", "machines", "machines/status"] - verbs: ["get", "list", "watch", "create", "delete", "patch"] - - apiGroups: ["karpenter.k8s.aws"] - resources: ["awsnodetemplates"] - verbs: ["get", "list", "watch", "create", "delete", "patch"] +- apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] +- apiGroups: ["karpenter.k8s.aws"] + resources: ["ec2nodeclasses"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] --- # Source: karpenter/templates/clusterrole-core.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1202,51 +1843,50 @@ kind: ClusterRole metadata: name: karpenter-core labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm rules: - # Read - - apiGroups: ["karpenter.sh"] - resources: ["provisioners", "provisioners/status", "machines", "machines/status"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csinodes"] - verbs: ["get", "watch", "list"] - - apiGroups: ["apps"] - resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] - verbs: ["list", "watch"] - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] - verbs: ["get", "watch", "list"] - - apiGroups: [ "policy" ] - resources: [ "poddisruptionbudgets" ] - verbs: [ "get", "list", "watch" ] - # Write - - apiGroups: ["karpenter.sh"] - resources: ["machines", "machines/status"] - verbs: ["create", "delete", "update", "patch"] - - apiGroups: ["karpenter.sh"] - resources: ["provisioners", "provisioners/status"] - verbs: ["update", "patch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["create", "patch", "delete"] - - apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["validation.webhook.karpenter.sh", "validation.webhook.config.karpenter.sh"] +# Read +- apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] + verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes", "volumeattachments"] + verbs: ["get", "watch", "list"] +- apiGroups: ["apps"] + resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] + verbs: ["list", "watch"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["get", list, "watch"] +# Write +- apiGroups: ["karpenter.sh"] + resources: ["nodeclaims", "nodeclaims/status"] + verbs: ["create", "delete", "update", "patch"] +- apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status"] + verbs: ["update", "patch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["patch", "delete", "update"] +- apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["delete"] --- # Source: karpenter/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1254,28 +1894,20 @@ kind: ClusterRole metadata: name: karpenter labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm rules: - # Read - - apiGroups: ["karpenter.k8s.aws"] - resources: ["awsnodetemplates"] - verbs: ["get", "list", "watch"] - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["validation.webhook.karpenter.k8s.aws"] - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["defaulting.webhook.karpenter.k8s.aws"] - # Write - - apiGroups: ["karpenter.k8s.aws"] - resources: ["awsnodetemplates", "awsnodetemplates/status"] - verbs: ["patch", "update"] +# Read +- apiGroups: ["karpenter.k8s.aws"] + resources: ["ec2nodeclasses"] + verbs: ["get", "list", "watch"] +# Write +- apiGroups: ["karpenter.k8s.aws"] + resources: ["ec2nodeclasses", "ec2nodeclasses/status"] + verbs: ["patch", "update"] --- # Source: karpenter/templates/clusterrole-core.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1283,19 +1915,19 @@ kind: ClusterRoleBinding metadata: name: karpenter-core labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: karpenter-core subjects: - - kind: ServiceAccount - name: karpenter - namespace: kube-system +- kind: ServiceAccount + name: karpenter + namespace: kube-system --- # Source: karpenter/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1303,19 +1935,19 @@ kind: ClusterRoleBinding metadata: name: karpenter labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: karpenter subjects: - - kind: ServiceAccount - name: karpenter - namespace: kube-system +- kind: ServiceAccount + name: karpenter + namespace: kube-system --- # Source: karpenter/templates/role.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1324,47 +1956,27 @@ metadata: name: karpenter namespace: kube-system labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm rules: - # Read - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch"] - - apiGroups: [""] - resources: ["configmaps", "namespaces", "secrets"] - verbs: ["get", "list", "watch"] - # Write - - apiGroups: [""] - resources: ["secrets"] - verbs: ["update"] - resourceNames: ["karpenter-cert"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["update", "patch", "delete"] - resourceNames: - - karpenter-global-settings - - config-logging - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["patch", "update"] - resourceNames: - - "karpenter-leader-election" - - "webhook.configmapwebhook.00-of-01" - - "webhook.defaultingwebhook.00-of-01" - - "webhook.validationwebhook.00-of-01" - - "webhook.webhookcertificates.00-of-01" - # Cannot specify resourceNames on create - # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["create"] +# Read +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch"] +# Write +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["patch", "update"] + resourceNames: + - "karpenter-leader-election" +# Cannot specify resourceNames on create +# https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] --- # Source: karpenter/templates/role.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1373,39 +1985,17 @@ metadata: name: karpenter-dns namespace: kube-system labels: - helm.sh/chart: karpenter-v0.31.3 - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" - app.kubernetes.io/managed-by: Helm -rules: - # Read - - apiGroups: [""] - resources: ["services"] - resourceNames: ["kube-dns"] - verbs: ["get"] ---- -# Source: karpenter/templates/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: karpenter-lease - namespace: kube-node-lease - labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm rules: - # Read - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "list", "watch"] - # Write - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["delete"] +# Read +- apiGroups: [""] + resources: ["services"] + resourceNames: ["kube-dns"] + verbs: ["get"] --- # Source: karpenter/templates/rolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1414,19 +2004,19 @@ metadata: name: karpenter namespace: kube-system labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: karpenter subjects: - - kind: ServiceAccount - name: karpenter - namespace: kube-system +- kind: ServiceAccount + name: karpenter + namespace: kube-system --- # Source: karpenter/templates/rolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -1435,40 +2025,19 @@ metadata: name: karpenter-dns namespace: kube-system labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: karpenter-dns subjects: - - kind: ServiceAccount - name: karpenter - namespace: kube-system ---- -# Source: karpenter/templates/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: karpenter-lease - namespace: kube-node-lease - labels: - helm.sh/chart: karpenter-v0.31.3 - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" - app.kubernetes.io/managed-by: Helm -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: karpenter-lease -subjects: - - kind: ServiceAccount - name: karpenter - namespace: kube-system +- kind: ServiceAccount + name: karpenter + namespace: kube-system --- # Source: karpenter/templates/service.yaml apiVersion: v1 @@ -1477,22 +2046,18 @@ metadata: name: karpenter namespace: kube-system labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm spec: type: ClusterIP ports: - - name: http-metrics - port: 8000 - targetPort: http-metrics - protocol: TCP - - name: https-webhook - port: 8443 - targetPort: https-webhook - protocol: TCP + - name: http-metrics + port: 8080 + targetPort: http-metrics + protocol: TCP selector: app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter @@ -1504,10 +2069,10 @@ metadata: name: karpenter namespace: kube-system labels: - helm.sh/chart: karpenter-v0.31.3 + helm.sh/chart: karpenter-1.6.2 app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" + app.kubernetes.io/version: "1.6.2" app.kubernetes.io/managed-by: Helm spec: replicas: {{ ControlPlaneControllerReplicas false }} @@ -1517,15 +2082,21 @@ spec: maxUnavailable: 1 selector: matchLabels: - karpenter: webhook + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter template: metadata: labels: app.kubernetes.io/name: karpenter app.kubernetes.io/instance: karpenter - karpenter: webhook + annotations: spec: + automountServiceAccountToken: true serviceAccountName: karpenter + securityContext: + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault priorityClassName: "system-cluster-critical" {{ if not IsIPv6Only }} dnsPolicy: Default @@ -1533,73 +2104,93 @@ spec: # Must use ClusterFirst on IPv6 clusters in order to get DNS64 dnsPolicy: ClusterFirst {{ end }} + schedulerName: "default-scheduler" containers: - - name: controller - securityContext: - runAsUser: 65536 - runAsGroup: 65536 - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - image: {{ .Karpenter.Image }} - imagePullPolicy: IfNotPresent - env: - - name: KUBERNETES_MIN_VERSION - value: "1.19.0-0" - - name: KARPENTER_SERVICE - value: karpenter - - name: WEBHOOK_PORT - value: "8443" - - name: METRICS_PORT - value: "8000" - - name: HEALTH_PROBE_PORT - value: "8081" - - name: SYSTEM_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MEMORY_LIMIT - valueFrom: - resourceFieldRef: - containerName: controller - divisor: "0" - resource: limits.memory - - name: AWS_REGION - value: {{ Region }} - ports: - - name: http-metrics - containerPort: 8000 - protocol: TCP - - name: http - containerPort: 8081 - protocol: TCP - - name: https-webhook - containerPort: 8443 - protocol: TCP - livenessProbe: - initialDelaySeconds: 30 - timeoutSeconds: 30 - httpGet: - path: /healthz - port: http - readinessProbe: - initialDelaySeconds: 5 - timeoutSeconds: 30 - httpGet: - path: /readyz - port: http - resources: - limits: - memory: {{ or .Karpenter.MemoryLimit "1Gi" }} - requests: - cpu: {{ or .Karpenter.CPURequest "500m" }} - memory: {{ or .Karpenter.MemoryRequest "1Gi" }} - nodeSelector: null + - name: controller + securityContext: + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + image: "{{ .Karpenter.Image }}" + imagePullPolicy: IfNotPresent + env: + - name: KUBERNETES_MIN_VERSION + value: "1.19.0-0" + - name: KARPENTER_SERVICE + value: karpenter + - name: LOG_LEVEL + value: "{{ .Karpenter.LogLevel }}" + - name: LOG_OUTPUT_PATHS + value: "stdout" + - name: LOG_ERROR_OUTPUT_PATHS + value: "stderr" + - name: METRICS_PORT + value: "8080" + - name: HEALTH_PROBE_PORT + value: "8081" + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MEMORY_LIMIT + valueFrom: + resourceFieldRef: + containerName: controller + divisor: "0" + resource: limits.memory + - name: FEATURE_GATES + value: "ReservedCapacity=true,SpotToSpotConsolidation=false,NodeRepair=false" + - name: BATCH_MAX_DURATION + value: "10s" + - name: BATCH_IDLE_DURATION + value: "1s" + - name: PREFERENCE_POLICY + value: "Respect" + - name: MIN_VALUES_POLICY + value: "Strict" + - name: CLUSTER_NAME + value: "{{ ClusterName }}" + - name: CLUSTER_ENDPOINT + value: "https://{{ APIInternalName }}" + - name: VM_MEMORY_OVERHEAD_PERCENT + value: "0.075" + - name: RESERVED_ENIS + value: "0" + - name: AWS_REGION + value: '{{ Region }}' + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: http + containerPort: 8081 + protocol: TCP + livenessProbe: + initialDelaySeconds: 30 + timeoutSeconds: 30 + httpGet: + path: /healthz + port: http + readinessProbe: + initialDelaySeconds: 5 + timeoutSeconds: 30 + httpGet: + path: /readyz + port: http + resources: + limits: + memory: '{{ or .Karpenter.MemoryLimit "1Gi" }}' + requests: + cpu: '{{ or .Karpenter.CPURequest "500m" }}' + memory: '{{ or .Karpenter.MemoryRequest "1Gi" }}' + nodeSelector: + kubernetes.io/os: linux # The template below patches the .Values.affinity to add a default label selector where not specificed affinity: nodeAffinity: @@ -1632,243 +2223,17 @@ spec: topologyKey: kubernetes.io/hostname # The template below patches the .Values.topologySpreadConstraints to add a default label selector where not specificed topologySpreadConstraints: - - labelSelector: - matchLabels: - app.kubernetes.io/instance: karpenter - app.kubernetes.io/name: karpenter - maxSkew: 1 - topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + app.kubernetes.io/instance: karpenter + app.kubernetes.io/name: karpenter + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: CriticalAddonsOnly - operator: Exists ---- -# Source: karpenter/templates/webhooks.yaml -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: defaulting.webhook.karpenter.k8s.aws - labels: - helm.sh/chart: karpenter-v0.31.3 - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" - app.kubernetes.io/managed-by: Helm -webhooks: - - name: defaulting.webhook.karpenter.k8s.aws - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - sideEffects: None - rules: - - apiGroups: - - karpenter.k8s.aws - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - awsnodetemplates - - awsnodetemplates/status - scope: '*' - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - resources: - - provisioners - - provisioners/status - operations: - - CREATE - - UPDATE ---- -# Source: karpenter/templates/webhooks-core.yaml -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.karpenter.sh - labels: - helm.sh/chart: karpenter-v0.31.3 - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" - app.kubernetes.io/managed-by: Helm -webhooks: - - name: validation.webhook.karpenter.sh - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - sideEffects: None - rules: - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - resources: - - provisioners - - provisioners/status - operations: - - CREATE - - UPDATE ---- -# Source: karpenter/templates/webhooks-core.yaml -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.config.karpenter.sh - labels: - helm.sh/chart: karpenter-v0.31.3 - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" - app.kubernetes.io/managed-by: Helm -webhooks: - - name: validation.webhook.config.karpenter.sh - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - sideEffects: None - objectSelector: - matchLabels: - app.kubernetes.io/part-of: karpenter ---- -# Source: karpenter/templates/webhooks.yaml -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.karpenter.k8s.aws - labels: - helm.sh/chart: karpenter-v0.31.3 - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "0.31.3" - app.kubernetes.io/managed-by: Helm -webhooks: - - name: validation.webhook.karpenter.k8s.aws - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - sideEffects: None - rules: - - apiGroups: - - karpenter.k8s.aws - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - awsnodetemplates - - awsnodetemplates/status - scope: '*' - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - resources: - - provisioners - - provisioners/status - operations: - - CREATE - - UPDATE - -{{ range $name, $spec := GetNodeInstanceGroups }} -{{ if eq $spec.Manager "Karpenter" }} ---- -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: {{ $name }} -spec: - subnetSelector: - kops.k8s.io/instance-group/{{ $name }}: "*" - kubernetes.io/cluster/{{ ClusterName }}: "*" - launchTemplate: {{ $name }}.{{ ClusterName }} ---- -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: {{ $name }} -spec: - consolidation: - enabled: true -{{ with $spec.Kubelet }} -{{ if or .MaxPods .SystemReserved .KubeReserved }} - kubeletConfiguration: - {{ if .MaxPods }} - maxPods: {{ .MaxPods }} - {{ end }} - {{ if .SystemReserved }} - systemReserved: - {{ range $key, $val := .SystemReserved}} - {{ $key }}: "{{ $val }}" - {{ end }} - {{ end }} - {{ if .KubeReserved }} - kubeReserved: - {{ range $key, $val := .KubeReserved}} - {{ $key }}: "{{ $val }}" - {{ end }} - {{ end }} -{{ end }} -{{ end }} - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot", "on-demand"] - - key: kubernetes.io/arch - operator: In - values: ["{{ ArchitectureOfAMI $spec.Image }}"] - - key: "node.kubernetes.io/instance-type" - operator: In - values: - {{ range $type := KarpenterInstanceTypes $spec }} - - {{ $type }} - {{ end }} -{{ with $spec.Taints }} - taints: - {{ range $taintString := $spec.Taints }} - {{ $taint := ParseTaint $taintString }} - - key: {{ $taint.key }} - effect: {{ $taint.effect }} - {{ if $taint.value }} - value: "{{ $taint.value }}" - {{ end }} - {{ end }} -{{ end }} -{{ if $.ExternalCloudControllerManager }} - startupTaints: - - key: node.cloudprovider.kubernetes.io/uninitialized - effect: NoSchedule -{{ end }} -{{ with $spec.NodeLabels }} - labels: - {{ range $key, $value := . }} - {{ $key }}: "{{ $value }}" - {{ end }} -{{ end }} - providerRef: - name: {{ $name }} -{{ end }} -{{ end }} + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: CriticalAddonsOnly + operator: Exists diff --git a/upup/pkg/fi/cloudup/awsup/aws_cloud.go b/upup/pkg/fi/cloudup/awsup/aws_cloud.go index 1b8b98fe1ac2c..a73c8f5ccb319 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/aws_cloud.go @@ -757,32 +757,7 @@ func getKarpenterGroups(c AWSCloud, cluster *kops.Cluster, instancegroups []*kop func buildKarpenterGroup(c AWSCloud, cluster *kops.Cluster, ig *kops.InstanceGroup, nodes []v1.Node) (*cloudinstances.CloudInstanceGroup, error) { ctx := context.TODO() nodeMap := cloudinstances.GetNodeMap(nodes, cluster) - instances := make(map[string]*ec2types.Instance) - updatedInstances := make(map[string]*ec2types.Instance) clusterName := c.Tags()[TagClusterName] - var version string - - { - input := &ec2.DescribeLaunchTemplatesInput{ - Filters: []ec2types.Filter{ - NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name), - NewEC2Filter("tag:"+TagClusterName, clusterName), - }, - } - var list []ec2types.LaunchTemplate - paginator := ec2.NewDescribeLaunchTemplatesPaginator(c.EC2(), input) - for paginator.HasMorePages() { - page, err := paginator.NextPage(ctx) - if err != nil { - return nil, fmt.Errorf("error listing launch templates: %v", err) - } - list = append(list, page.LaunchTemplates...) - } - lt := list[0] - versionNumber := *lt.LatestVersionNumber - version = strconv.Itoa(int(versionNumber)) - - } karpenterGroup := &cloudinstances.CloudInstanceGroup{ InstanceGroup: ig, @@ -805,51 +780,12 @@ func buildKarpenterGroup(c AWSCloud, cluster *kops.Cluster, ig *kops.InstanceGro for _, r := range result.Reservations { for _, i := range r.Instances { id := aws.ToString(i.InstanceId) - instances[id] = &i + cloudInstance, _ := karpenterGroup.NewCloudInstance(aws.ToString(i.InstanceId), cloudinstances.CloudInstanceStatusUpToDate, nodeMap[id]) + addCloudInstanceData(cloudInstance, &i) } } } - klog.V(2).Infof("found %d karpenter instances", len(instances)) - - { - req := &ec2.DescribeInstancesInput{ - Filters: []ec2types.Filter{ - NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name), - NewEC2Filter("tag:"+TagClusterName, clusterName), - NewEC2Filter("instance-state-name", "pending", "running", "stopping", "stopped"), - NewEC2Filter("tag:aws:ec2launchtemplate:version", version), - }, - } - - result, err := c.EC2().DescribeInstances(ctx, req) - if err != nil { - return nil, err - } - - for _, r := range result.Reservations { - for _, i := range r.Instances { - id := aws.ToString(i.InstanceId) - updatedInstances[id] = &i - } - } - } - klog.V(2).Infof("found %d updated instances", len(updatedInstances)) - - { - for _, instance := range instances { - id := *instance.InstanceId - _, ready := updatedInstances[id] - var status string - if ready { - status = cloudinstances.CloudInstanceStatusUpToDate - } else { - status = cloudinstances.CloudInstanceStatusNeedsUpdate - } - cloudInstance, _ := karpenterGroup.NewCloudInstance(id, status, nodeMap[id]) - addCloudInstanceData(cloudInstance, instance) - } - } return karpenterGroup, nil } diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 32627fdde4573..d3e99b667b874 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -28,7 +28,6 @@ When defining a new function: package cloudup import ( - "context" "encoding/base64" "encoding/json" "fmt" @@ -41,10 +40,8 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" - "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" kopsroot "k8s.io/kops" @@ -396,12 +393,11 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS dest["ParseTaint"] = util.ParseTaint - dest["KarpenterEnabled"] = func() bool { + // IsControlPlaneMode signals that kOps is used to bootstrap the control plane and the nodes will be created by other means + // e.g. by Karpenter or Cluster API + dest["IsControlPlaneMode"] = func() bool { return cluster.Spec.Karpenter != nil && cluster.Spec.Karpenter.Enabled } - dest["KarpenterInstanceTypes"] = func(ig kops.InstanceGroupSpec) ([]string, error) { - return karpenterInstanceTypes(tf.cloud.(awsup.AWSCloud), ig) - } dest["PodIdentityWebhookConfigMapData"] = tf.podIdentityWebhookConfigMapData @@ -1014,116 +1010,6 @@ func (tf *TemplateFunctions) podIdentityWebhookConfigMapData() (string, error) { return fmt.Sprintf("%q", jsonBytes), err } -func karpenterInstanceTypes(cloud awsup.AWSCloud, ig kops.InstanceGroupSpec) ([]string, error) { - ctx := context.TODO() - var mixedInstancesPolicy *kops.MixedInstancesPolicySpec - - if ig.MachineType == "" && ig.MixedInstancesPolicy == nil { - // Karpenter thinks all clusters run VPC CNI and schedules thinking Node Capacity is constrainted by number of ENIs. - - // cpuMin is the reasonable lower limit for a Kubernetes Node - // Generally, it also avoids instances Karpenter thinks it can only schedule 4 Pods on. - cpuMin := resource.MustParse("2") - memoryMin := resource.MustParse(("2G")) - - mixedInstancesPolicy = &kops.MixedInstancesPolicySpec{ - InstanceRequirements: &kops.InstanceRequirementsSpec{ - CPU: &kops.MinMaxSpec{ - Min: &cpuMin, - }, - Memory: &kops.MinMaxSpec{ - Min: &memoryMin, - }, - }, - } - } - if ig.MixedInstancesPolicy != nil { - mixedInstancesPolicy = ig.MixedInstancesPolicy - } - - if mixedInstancesPolicy != nil { - if len(mixedInstancesPolicy.Instances) > 0 { - return mixedInstancesPolicy.Instances, nil - } - if mixedInstancesPolicy.InstanceRequirements != nil { - instanceRequirements := mixedInstancesPolicy.InstanceRequirements - ami, err := cloud.ResolveImage(ig.Image) - if err != nil { - return nil, err - } - arch := ami.Architecture - hv := ami.VirtualizationType - - ir := &ec2types.InstanceRequirementsRequest{ - VCpuCount: &ec2types.VCpuCountRangeRequest{}, - MemoryMiB: &ec2types.MemoryMiBRequest{}, - BurstablePerformance: ec2types.BurstablePerformanceIncluded, - InstanceGenerations: []ec2types.InstanceGeneration{ec2types.InstanceGenerationCurrent}, - } - cpu := instanceRequirements.CPU - if cpu != nil { - if cpu.Max != nil { - cpuMax, _ := instanceRequirements.CPU.Max.AsInt64() - ir.VCpuCount.Max = fi.PtrTo(int32(cpuMax)) - } - cpu := instanceRequirements.CPU - if cpu != nil { - if cpu.Max != nil { - cpuMax, _ := instanceRequirements.CPU.Max.AsInt64() - ir.VCpuCount.Max = fi.PtrTo(int32(cpuMax)) - } - if cpu.Min != nil { - cpuMin, _ := instanceRequirements.CPU.Min.AsInt64() - ir.VCpuCount.Min = fi.PtrTo(int32(cpuMin)) - } - } else { - ir.VCpuCount.Min = fi.PtrTo(int32(0)) - } - - memory := instanceRequirements.Memory - if memory != nil { - if memory.Max != nil { - memoryMax := instanceRequirements.Memory.Max.ScaledValue(resource.Mega) - ir.MemoryMiB.Max = fi.PtrTo(int32(memoryMax)) - } - if memory.Min != nil { - memoryMin := instanceRequirements.Memory.Min.ScaledValue(resource.Mega) - ir.MemoryMiB.Min = fi.PtrTo(int32(memoryMin)) - } - } else { - ir.MemoryMiB.Min = fi.PtrTo(int32(0)) - } - - ir.AcceleratorCount = &ec2types.AcceleratorCountRequest{ - Min: fi.PtrTo(int32(0)), - Max: fi.PtrTo(int32(0)), - } - - response, err := cloud.EC2().GetInstanceTypesFromInstanceRequirements(ctx, - &ec2.GetInstanceTypesFromInstanceRequirementsInput{ - ArchitectureTypes: []ec2types.ArchitectureType{ec2types.ArchitectureType(arch)}, - VirtualizationTypes: []ec2types.VirtualizationType{hv}, - InstanceRequirements: ir, - }, - ) - if err != nil { - return nil, err - } - types := []string{} - for _, it := range response.InstanceTypes { - types = append(types, *it.InstanceType) - } - if len(types) == 0 { - return nil, fmt.Errorf("no instances matched requirements") - } - return types, nil - } - } - } - - return []string{ig.MachineType}, nil -} - func (tf *TemplateFunctions) kopsFeatureEnabled(featureName string) (bool, error) { f, err := featureflag.Get(featureName) if err != nil { diff --git a/upup/pkg/fi/cloudup/template_functions_test.go b/upup/pkg/fi/cloudup/template_functions_test.go index d60b8d3519c6e..0f9d5f8ae96d1 100644 --- a/upup/pkg/fi/cloudup/template_functions_test.go +++ b/upup/pkg/fi/cloudup/template_functions_test.go @@ -21,13 +21,9 @@ import ( "reflect" "testing" - "github.com/aws/aws-sdk-go-v2/aws" - ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "k8s.io/kops/cloudmock/aws/mockec2" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/cloudup/awsup" ) func Test_TemplateFunctions_CloudControllerConfigArgv(t *testing.T) { @@ -277,29 +273,6 @@ func Test_TemplateFunctions_CloudControllerConfigArgv(t *testing.T) { } } -func Test_KarpenterInstanceTypes(t *testing.T) { - amiId := "ami-073c8c0760395aab8" - ec2Client := &mockec2.MockEC2{} - ec2Client.Images = append(ec2Client.Images, &ec2types.Image{ - CreationDate: aws.String("2016-10-21T20:07:19.000Z"), - ImageId: &amiId, - Name: aws.String("focal"), - OwnerId: aws.String(awsup.WellKnownAccountUbuntu), - RootDeviceName: aws.String("/dev/xvda"), - Architecture: ec2types.ArchitectureValuesX8664, - }) - ig := kops.InstanceGroupSpec{ - Image: amiId, - } - cloud := &awsup.MockAWSCloud{MockCloud: awsup.MockCloud{ - MockEC2: ec2Client, - }} - _, err := karpenterInstanceTypes(cloud, ig) - if err != nil { - t.Errorf("failed to fetch instance types: %v", err) - } -} - func TestKopsFeatureEnabled(t *testing.T) { tests := []struct { name string