From a62d23b0bc4ae92e00899abd14a465562650365d Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Tue, 22 Sep 2026 12:47:24 +0200 Subject: [PATCH 01/13] CKS: reconcile node capacity after live resize --- .../KubernetesClusterScaleWorker.java | 99 ++++++++- ...bernetesClusterNodeCapacityReconciler.java | 67 ++++++ ...etesClusterNodeCapacityReconcilerImpl.java | 190 ++++++++++++++++++ .../spring-kubernetes-service-context.xml | 1 + .../KubernetesClusterScaleWorkerTest.java | 33 +++ ...ClusterNodeCapacityReconcilerImplTest.java | 71 +++++++ 6 files changed, 455 insertions(+), 6 deletions(-) create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 003a852f3dbe..2df102531c66 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -28,6 +28,8 @@ import java.util.Set; import java.util.stream.Collectors; +import javax.inject.Inject; + import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.service.ServiceOfferingVO; import com.cloud.storage.VMTemplateVO; @@ -44,16 +46,21 @@ import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.exception.VirtualMachineMigrationException; +import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler.NodeAccess; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconcilerImpl; import com.cloud.network.IpAddress; import com.cloud.network.Network; import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.offering.ServiceOffering; import com.cloud.storage.LaunchPermissionVO; import com.cloud.uservm.UserVm; @@ -80,6 +87,9 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif private Boolean isAutoscalingEnabled; private long scaleTimeoutTime; + @Inject + protected KubernetesClusterNodeCapacityReconciler kubernetesClusterNodeCapacityReconciler; + protected KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); } @@ -378,15 +388,60 @@ private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, for (long i = 0; i < tobeScaledVMCount; i++) { KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i); UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); + if (userVM == null) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, unable to find cluster VM %s", + kubernetesCluster.getName(), vmMapVO.getVmId()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + ServiceOffering oldOffering = serviceOfferingDao.findById(userVM.getServiceOfferingId()); + boolean capacityChanged = KubernetesClusterNodeCapacityReconcilerImpl.capacityChanged(oldOffering, serviceOffering); + if (capacityChanged && KubernetesCluster.State.Running == originalState && ETCD == nodeType + && KubernetesCluster.ClusterType.CloudManaged == kubernetesCluster.getClusterType()) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s cannot live-resize dedicated etcd VM %s; " + + "etcd health reconciliation is not implemented", kubernetesCluster.getName(), userVM.getDisplayName()), + kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + boolean reconcileKubernetes = shouldReconcileNodeCapacity(vmMapVO, userVM, oldOffering, serviceOffering, nodeType); + NodeAccess nodeAccess = null; + NodeCapacitySnapshot before = null; boolean result = false; + boolean cksCordonCompleted = false; + boolean resizeSucceeded = serviceOffering.getId() == userVM.getServiceOfferingId(); try { - result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); - } catch (RuntimeException | ResourceUnavailableException | ManagementServerException | VirtualMachineMigrationException e) { + if (reconcileKubernetes) { + nodeAccess = resolveNodeAccess(userVM); + before = kubernetesClusterNodeCapacityReconciler.captureBefore(kubernetesCluster, userVM, nodeAccess); + kubernetesClusterNodeCapacityReconciler.cordonIfNeeded(kubernetesCluster, userVM, before, nodeAccess, scaleTimeoutTime); + cksCordonCompleted = !before.isUnschedulable() || before.isCloudStackResizeCordon(); + } + if (serviceOffering.getId() != userVM.getServiceOfferingId()) { + result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); + if (!result) { + logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s", kubernetesCluster.getName(), userVM.getDisplayName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + resizeSucceeded = true; + userVM = userVmDao.findById(userVM.getId()); + if (userVM == null || serviceOffering.getId() != userVM.getServiceOfferingId()) { + logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, VM %s did not reach target offering", kubernetesCluster.getName(), vmMapVO.getVmId()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } + if (reconcileKubernetes) { + kubernetesClusterNodeCapacityReconciler.verifyGuestResources(userVM, serviceOffering, before, nodeAccess, scaleTimeoutTime); + if (!kubernetesClusterNodeCapacityReconciler.isKubernetesResourcesCurrent(before, serviceOffering)) { + kubernetesClusterNodeCapacityReconciler.restartKubelet(userVM, nodeAccess, scaleTimeoutTime); + kubernetesClusterNodeCapacityReconciler.waitForKubernetesResources(kubernetesCluster, userVM, serviceOffering, before, nodeAccess, scaleTimeoutTime); + } + kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster, userVM, before, nodeAccess, scaleTimeoutTime); + } + } catch (Exception e) { + if (reconcileKubernetes && cksCordonCompleted && !resizeSucceeded) { + try { + kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster, userVM, before, nodeAccess, scaleTimeoutTime); + } catch (Exception cleanupException) { + logger.warn("Unable to restore schedulability for CKS node {} after a pre-resize failure", userVM.getUuid(), cleanupException); + } + } logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s due to %s", kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } - if (!result) { - logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s", kubernetesCluster.getName(), userVM.getDisplayName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } if (System.currentTimeMillis() > scaleTimeoutTime) { logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, scaling action timed out", kubernetesCluster.getName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } @@ -394,6 +449,38 @@ private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); } + private NodeAccess resolveNodeAccess(UserVm userVM) { + Pair controlAccess = getKubernetesClusterServerIpSshPort(null); + if (StringUtils.isBlank(controlAccess.first())) { + throw new CloudRuntimeException(String.format("Unable to resolve control-plane SSH for Kubernetes cluster %s", kubernetesCluster.getUuid())); + } + if (manager.isDirectAccess(network)) { + if (StringUtils.isBlank(userVM.getPrivateIpAddress())) { + throw new CloudRuntimeException(String.format("Unable to resolve private SSH address for VM %s", userVM.getUuid())); + } + return new NodeAccess(controlAccess.first(), controlAccess.second(), userVM.getPrivateIpAddress(), DEFAULT_SSH_PORT, + getControlNodeLoginUser(), getManagementServerSshPublicKeyFile()); + } + PortForwardingRuleVO sshRule = portForwardingRulesDao.listByVm(userVM.getId()).stream() + .filter(rule -> rule.getDestinationPortStart() == DEFAULT_SSH_PORT) + .filter(rule -> !FirewallRule.State.Revoke.equals(rule.getState())) + .findFirst().orElse(null); + if (sshRule == null) { + throw new CloudRuntimeException(String.format("Unable to resolve SSH port-forwarding rule for VM %s", userVM.getUuid())); + } + return new NodeAccess(controlAccess.first(), controlAccess.second(), controlAccess.first(), sshRule.getSourcePortStart(), + getControlNodeLoginUser(), getManagementServerSshPublicKeyFile()); + } + + protected boolean shouldReconcileNodeCapacity(KubernetesClusterVmMapVO vmMapVO, UserVm userVM, + ServiceOffering oldOffering, ServiceOffering targetOffering, + KubernetesClusterNodeType nodeType) { + return !vmMapVO.isExternalNode() + && KubernetesCluster.ClusterType.CloudManaged == kubernetesCluster.getClusterType() + && Hypervisor.HypervisorType.KVM == userVM.getHypervisorType() + && kubernetesClusterNodeCapacityReconciler.requiresKubeletRefresh(oldOffering, targetOffering, nodeType, originalState); + } + private void removeNodesFromCluster(List vmMaps) throws CloudRuntimeException { for (KubernetesClusterVmMapVO vmMapVO : vmMaps) { UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java new file mode 100644 index 000000000000..f52458774aca --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import java.io.File; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.offering.ServiceOffering; +import com.cloud.uservm.UserVm; + +/** Reconciles the guest and Kubernetes resource views after a live VM resize. */ +public interface KubernetesClusterNodeCapacityReconciler { + NodeCapacitySnapshot captureBefore(KubernetesCluster cluster, UserVm vm, NodeAccess access) throws Exception; + boolean requiresKubeletRefresh(ServiceOffering oldOffering, ServiceOffering newOffering, + KubernetesClusterNodeType nodeType, KubernetesCluster.State originalState); + void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception; + void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception; + boolean isKubernetesResourcesCurrent(NodeCapacitySnapshot snapshot, ServiceOffering target); + void restartKubelet(UserVm vm, NodeAccess access, long deadline) throws Exception; + NodeCapacitySnapshot waitForKubernetesResources(KubernetesCluster cluster, UserVm vm, ServiceOffering target, + NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception; + void restoreSchedulability(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception; + + final class NodeAccess { + private final String controlAddress; private final int controlPort; private final String nodeAddress; private final int nodePort; + private final String user; private final File sshKeyFile; + public NodeAccess(String controlAddress, int controlPort, String nodeAddress, int nodePort, String user, File sshKeyFile) { + this.controlAddress = controlAddress; this.controlPort = controlPort; this.nodeAddress = nodeAddress; this.nodePort = nodePort; + this.user = user; this.sshKeyFile = sshKeyFile; + } + public String getControlAddress() { return controlAddress; } public int getControlPort() { return controlPort; } + public String getNodeAddress() { return nodeAddress; } public int getNodePort() { return nodePort; } + public String getUser() { return user; } public File getSshKeyFile() { return sshKeyFile; } + } + + final class NodeCapacitySnapshot { + private final boolean unschedulable; private final boolean cloudStackResizeCordon; private final long guestOnlineCpuCount; + private final long guestMemoryKiB; private final long capacityCpuMillis; private final long capacityMemoryBytes; + private final long allocatableCpuMillis; private final long allocatableMemoryBytes; private final boolean ready; + public NodeCapacitySnapshot(boolean unschedulable, boolean cloudStackResizeCordon, long guestOnlineCpuCount, long guestMemoryKiB, + long capacityCpuMillis, long capacityMemoryBytes, long allocatableCpuMillis, long allocatableMemoryBytes, boolean ready) { + this.unschedulable = unschedulable; this.cloudStackResizeCordon = cloudStackResizeCordon; this.guestOnlineCpuCount = guestOnlineCpuCount; + this.guestMemoryKiB = guestMemoryKiB; this.capacityCpuMillis = capacityCpuMillis; this.capacityMemoryBytes = capacityMemoryBytes; + this.allocatableCpuMillis = allocatableCpuMillis; this.allocatableMemoryBytes = allocatableMemoryBytes; this.ready = ready; + } + public boolean isUnschedulable() { return unschedulable; } public boolean isCloudStackResizeCordon() { return cloudStackResizeCordon; } + public long getGuestOnlineCpuCount() { return guestOnlineCpuCount; } public long getGuestMemoryKiB() { return guestMemoryKiB; } + public long getCapacityCpuMillis() { return capacityCpuMillis; } public long getCapacityMemoryBytes() { return capacityMemoryBytes; } + public long getAllocatableCpuMillis() { return allocatableCpuMillis; } public long getAllocatableMemoryBytes() { return allocatableMemoryBytes; } + public boolean isReady() { return ready; } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java new file mode 100644 index 000000000000..a6785ec3fdfd --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.offering.ServiceOffering; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.commons.lang3.StringUtils; + +/** SSH-backed implementation for a rolling CKS live-resize reconciliation. */ +public class KubernetesClusterNodeCapacityReconcilerImpl implements KubernetesClusterNodeCapacityReconciler { + static final long MINIMUM_MEMORY_OVERHEAD_BYTES = 128L * 1024L * 1024L; + static final long KUBERNETES_MEMORY_REPORTING_TOLERANCE_BYTES = 16L * 1024L * 1024L; + private static final int COMMAND_TIMEOUT_MS = 30000; + private static final int POLL_INTERVAL_MS = 5000; + private static final Pattern NODE_NAME = Pattern.compile("[A-Za-z0-9][A-Za-z0-9.-]{0,252}"); + private static final Pattern MEMORY_QUANTITY = Pattern.compile("([0-9]+)(Ki|Mi|Gi|Ti|K|M|G|T)?"); + private static final String RESIZE_ANNOTATION = "cloudstack.apache.org/cks-live-resize"; + + @Override + public boolean requiresKubeletRefresh(ServiceOffering oldOffering, ServiceOffering newOffering, + KubernetesClusterNodeType nodeType, KubernetesCluster.State originalState) { + return KubernetesCluster.State.Running == originalState + && (KubernetesClusterNodeType.WORKER == nodeType || KubernetesClusterNodeType.CONTROL == nodeType) + && capacityChanged(oldOffering, newOffering); + } + + @Override + public NodeCapacitySnapshot captureBefore(KubernetesCluster cluster, UserVm vm, NodeAccess access) throws Exception { + return nodeSnapshot(cluster, vm, access, guestResources(access)); + } + + @Override + public void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception { + if (before.isUnschedulable() && !before.isCloudStackResizeCordon()) return; + String node = nodeName(vm); + executeControl(access, "sudo /opt/bin/kubectl annotate node " + node + " " + RESIZE_ANNOTATION + "=" + cluster.getUuid() + " --overwrite"); + executeControl(access, "sudo /opt/bin/kubectl cordon " + node); + while (System.currentTimeMillis() < deadline) { + if (nodeSnapshot(cluster, vm, access, null).isUnschedulable()) return; + sleep(); + } + throw failure("CORDON", vm, "Kubernetes node did not become unschedulable"); + } + + @Override + public void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception { + while (System.currentTimeMillis() < deadline) { + if (guestMatchesTarget(guestResources(access), target)) return; + sleep(); + } + throw failure("GUEST_VERIFY", vm, "guest CPU or memory did not reach the target offering"); + } + + @Override + public boolean isKubernetesResourcesCurrent(NodeCapacitySnapshot snapshot, ServiceOffering target) { + GuestResources guest = new GuestResources(snapshot.getGuestOnlineCpuCount(), snapshot.getGuestMemoryKiB()); + return guestMatchesTarget(guest, target) && kubernetesMatchesGuest(snapshot, snapshot); + } + + @Override + public void restartKubelet(UserVm vm, NodeAccess access, long deadline) throws Exception { + executeNode(access, "sudo systemctl restart kubelet"); + while (System.currentTimeMillis() < deadline) { + if ("active".equals(executeNode(access, "sudo systemctl is-active kubelet").trim())) return; + sleep(); + } + throw failure("KUBELET_RESTART", vm, "kubelet did not become active"); + } + + @Override + public NodeCapacitySnapshot waitForKubernetesResources(KubernetesCluster cluster, UserVm vm, ServiceOffering target, + NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception { + while (System.currentTimeMillis() < deadline) { + GuestResources guest = guestResources(access); + NodeCapacitySnapshot observed = nodeSnapshot(cluster, vm, access, guest); + if (guestMatchesTarget(guest, target) && kubernetesMatchesGuest(observed, before)) return observed; + sleep(); + } + throw failure("CAPACITY_VERIFY", vm, "Kubernetes capacity did not match the resized guest"); + } + + @Override + public void restoreSchedulability(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception { + if (before.isUnschedulable() && !before.isCloudStackResizeCordon()) return; + String node = nodeName(vm); + executeControl(access, "sudo /opt/bin/kubectl uncordon " + node); + executeControl(access, "sudo /opt/bin/kubectl annotate node " + node + " " + RESIZE_ANNOTATION + "-"); + while (System.currentTimeMillis() < deadline) { + if (!nodeSnapshot(cluster, vm, access, null).isUnschedulable()) return; + sleep(); + } + throw failure("RESTORE_SCHEDULABILITY", vm, "Kubernetes node remained unschedulable"); + } + + public static boolean capacityChanged(ServiceOffering oldOffering, ServiceOffering targetOffering) { + return oldOffering != null && targetOffering != null + && (!Objects.equals(oldOffering.getCpu(), targetOffering.getCpu()) + || !Objects.equals(oldOffering.getRamSize(), targetOffering.getRamSize())); + } + + static long parseCpuMillis(String value) { return value.endsWith("m") ? Long.parseLong(value.substring(0, value.length() - 1)) : Long.parseLong(value) * 1000L; } + static long parseMemoryBytes(String value) { + Matcher matcher = MEMORY_QUANTITY.matcher(value); + if (!matcher.matches()) throw new IllegalArgumentException("Unsupported Kubernetes memory quantity"); + long number = Long.parseLong(matcher.group(1)); String unit = matcher.group(2); + if (unit == null) return number; + switch (unit) { + case "Ki": return number * 1024L; case "Mi": return number * 1024L * 1024L; case "Gi": return number * 1024L * 1024L * 1024L; + case "Ti": return number * 1024L * 1024L * 1024L * 1024L; case "K": return number * 1000L; case "M": return number * 1000L * 1000L; + case "G": return number * 1000L * 1000L * 1000L; case "T": return number * 1000L * 1000L * 1000L * 1000L; + default: throw new IllegalArgumentException("Unsupported Kubernetes memory quantity"); + } + } + + private NodeCapacitySnapshot nodeSnapshot(KubernetesCluster cluster, UserVm vm, NodeAccess access, GuestResources guest) throws Exception { + JsonObject node = new JsonParser().parse(executeControl(access, "sudo /opt/bin/kubectl get node " + nodeName(vm) + " -o json")).getAsJsonObject(); + JsonObject annotations = node.getAsJsonObject("metadata").has("annotations") ? node.getAsJsonObject("metadata").getAsJsonObject("annotations") : null; + boolean cksCordon = annotations != null && annotations.has(RESIZE_ANNOTATION) && cluster.getUuid().equals(annotations.get(RESIZE_ANNOTATION).getAsString()); + JsonObject spec = node.has("spec") ? node.getAsJsonObject("spec") : new JsonObject(); JsonObject status = node.getAsJsonObject("status"); + JsonObject capacity = status.getAsJsonObject("capacity"); JsonObject allocatable = status.getAsJsonObject("allocatable"); + return new NodeCapacitySnapshot(spec.has("unschedulable") && spec.get("unschedulable").getAsBoolean(), cksCordon, + guest == null ? 0 : guest.cpu, guest == null ? 0 : guest.memoryKiB, parseCpuMillis(capacity.get("cpu").getAsString()), + parseMemoryBytes(capacity.get("memory").getAsString()), parseCpuMillis(allocatable.get("cpu").getAsString()), + parseMemoryBytes(allocatable.get("memory").getAsString()), isReady(status)); + } + + private boolean isReady(JsonObject status) { + JsonArray conditions = status.getAsJsonArray("conditions"); + for (JsonElement condition : conditions) { JsonObject item = condition.getAsJsonObject(); if ("Ready".equals(item.get("type").getAsString()) && "True".equals(item.get("status").getAsString())) return true; } + return false; + } + + private GuestResources guestResources(NodeAccess access) throws Exception { + String[] values = executeNode(access, "getconf _NPROCESSORS_ONLN; awk '/^MemTotal:/ {print $2}' /proc/meminfo").trim().split("\\s+"); + if (values.length != 2) throw new CloudRuntimeException("Unable to read guest CPU and memory"); + return new GuestResources(Long.parseLong(values[0]), Long.parseLong(values[1])); + } + + private boolean guestMatchesTarget(GuestResources observed, ServiceOffering target) { + long targetMemory = target.getRamSize() * 1024L * 1024L; long observedMemory = observed.memoryKiB * 1024L; + long overhead = Math.max(targetMemory / 50L, MINIMUM_MEMORY_OVERHEAD_BYTES); + return observed.cpu == target.getCpu() && observedMemory <= targetMemory && targetMemory - observedMemory <= overhead; + } + + private boolean kubernetesMatchesGuest(NodeCapacitySnapshot observed, NodeCapacitySnapshot before) { + return observed.isReady() && observed.getCapacityCpuMillis() == observed.getGuestOnlineCpuCount() * 1000L + && Math.abs(observed.getCapacityMemoryBytes() - observed.getGuestMemoryKiB() * 1024L) <= KUBERNETES_MEMORY_REPORTING_TOLERANCE_BYTES + && (observed.getGuestOnlineCpuCount() <= before.getGuestOnlineCpuCount() || observed.getAllocatableCpuMillis() > before.getAllocatableCpuMillis()) + && (observed.getGuestMemoryKiB() <= before.getGuestMemoryKiB() || observed.getAllocatableMemoryBytes() > before.getAllocatableMemoryBytes()); + } + + private String executeControl(NodeAccess access, String command) throws Exception { return execute(access.getControlAddress(), access.getControlPort(), access, command); } + private String executeNode(NodeAccess access, String command) throws Exception { return execute(access.getNodeAddress(), access.getNodePort(), access, command); } + private String execute(String address, int port, NodeAccess access, String command) throws Exception { + Pair result = SshHelper.sshExecute(address, port, access.getUser(), access.getSshKeyFile(), null, command, 10000, 10000, COMMAND_TIMEOUT_MS); + if (Boolean.TRUE.equals(result.first())) return StringUtils.defaultString(result.second()); + throw new CloudRuntimeException("CKS live-resize command failed"); + } + private String nodeName(UserVm vm) { String name = StringUtils.lowerCase(vm.getHostName()); if (!NODE_NAME.matcher(name).matches()) throw new CloudRuntimeException("Invalid Kubernetes node name for live resize"); return name; } + private CloudRuntimeException failure(String phase, UserVm vm, String message) { return new CloudRuntimeException("CKS live resize " + phase + " failed for VM " + vm.getUuid() + ": " + message); } + private void sleep() throws InterruptedException { Thread.sleep(POLL_INTERVAL_MS); } + private static class GuestResources { private final long cpu; private final long memoryKiB; GuestResources(long cpu, long memoryKiB) { this.cpu = cpu; this.memoryKiB = memoryKiB; } } +} diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml index 053366786292..9fe19200d6e7 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml @@ -34,6 +34,7 @@ + diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index c9299bdbaa67..ffd9a28288cc 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -20,6 +20,8 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler; +import com.cloud.hypervisor.Hypervisor; import com.cloud.offering.ServiceOffering; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; @@ -40,6 +42,7 @@ import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterScaleWorkerTest { @@ -54,6 +57,8 @@ public class KubernetesClusterScaleWorkerTest { private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @Mock private UserVmDao userVmDao; + @Mock + private KubernetesClusterNodeCapacityReconciler kubernetesClusterNodeCapacityReconciler; private KubernetesClusterScaleWorker worker; @@ -187,4 +192,32 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { Assert.assertTrue(toRemove.isEmpty()); } + + @Test + public void testShouldReconcileNodeCapacityOnlyForManagedKvmNodes() { + KubernetesCluster runningManagedCluster = Mockito.mock(KubernetesCluster.class); + Mockito.when(runningManagedCluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(runningManagedCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + KubernetesClusterScaleWorker scaleWorker = new KubernetesClusterScaleWorker(runningManagedCluster, + new java.util.HashMap<>(), 1L, null, false, null, null, clusterManager); + scaleWorker.kubernetesClusterNodeCapacityReconciler = kubernetesClusterNodeCapacityReconciler; + + KubernetesClusterVmMapVO managedNode = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(managedNode.isExternalNode()).thenReturn(false); + UserVmVO kvmNode = Mockito.mock(UserVmVO.class); + Mockito.when(kvmNode.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + ServiceOffering oldOffering = Mockito.mock(ServiceOffering.class); + ServiceOffering targetOffering = Mockito.mock(ServiceOffering.class); + Mockito.when(kubernetesClusterNodeCapacityReconciler.requiresKubeletRefresh(oldOffering, targetOffering, WORKER, + KubernetesCluster.State.Running)).thenReturn(true); + + Assert.assertTrue(scaleWorker.shouldReconcileNodeCapacity(managedNode, kvmNode, oldOffering, targetOffering, WORKER)); + + Mockito.when(managedNode.isExternalNode()).thenReturn(true); + Assert.assertFalse(scaleWorker.shouldReconcileNodeCapacity(managedNode, kvmNode, oldOffering, targetOffering, WORKER)); + + Mockito.when(managedNode.isExternalNode()).thenReturn(false); + Mockito.when(kvmNode.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.XenServer); + Assert.assertFalse(scaleWorker.shouldReconcileNodeCapacity(managedNode, kvmNode, oldOffering, targetOffering, WORKER)); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java new file mode 100644 index 000000000000..729803c7187e --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.offering.ServiceOffering; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class KubernetesClusterNodeCapacityReconcilerImplTest { + private final KubernetesClusterNodeCapacityReconciler reconciler = new KubernetesClusterNodeCapacityReconcilerImpl(); + + @Test + public void testParseKubernetesQuantities() { + Assert.assertEquals(3900L, KubernetesClusterNodeCapacityReconcilerImpl.parseCpuMillis("3900m")); + Assert.assertEquals(4000L, KubernetesClusterNodeCapacityReconcilerImpl.parseCpuMillis("4")); + Assert.assertEquals(4037048L * 1024L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("4037048Ki")); + Assert.assertEquals(4L * 1024L * 1024L * 1024L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("4Gi")); + } + + @Test + public void testOnlyRunningKubernetesNodesWithChangedCpuOrRamRequireRefresh() { + ServiceOffering oldOffering = offering(2, 2048); + ServiceOffering cpuOffering = offering(4, 2048); + ServiceOffering ramOffering = offering(2, 4096); + ServiceOffering capOnlyOffering = offering(2, 2048); + + Assert.assertTrue(reconciler.requiresKubeletRefresh(oldOffering, cpuOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Running)); + Assert.assertTrue(reconciler.requiresKubeletRefresh(oldOffering, ramOffering, KubernetesClusterNodeType.CONTROL, KubernetesCluster.State.Running)); + Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, capOnlyOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Running)); + Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, cpuOffering, KubernetesClusterNodeType.ETCD, KubernetesCluster.State.Running)); + Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, cpuOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Stopped)); + } + + @Test + public void testCurrentKubernetesResourcesDoNotRequireASecondKubeletRestart() { + long memoryKiB = 4L * 1024L * 1024L; + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot current = + new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, 4, memoryKiB, + 4000, memoryKiB * 1024L, 3900, memoryKiB * 1024L - 128L * 1024L * 1024L, true); + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot stale = + new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, 4, memoryKiB, + 2000, memoryKiB * 1024L, 1900, memoryKiB * 1024L - 128L * 1024L * 1024L, true); + + Assert.assertTrue(reconciler.isKubernetesResourcesCurrent(current, offering(4, 4096))); + Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(stale, offering(4, 4096))); + } + + private ServiceOffering offering(int cpu, int memory) { + ServiceOffering offering = Mockito.mock(ServiceOffering.class); + Mockito.when(offering.getCpu()).thenReturn(cpu); + Mockito.when(offering.getRamSize()).thenReturn(memory); + return offering; + } +} From 95a51822705c1eec650a2a58033f60e2f0038ebe Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Tue, 22 Sep 2026 13:32:09 +0200 Subject: [PATCH 02/13] CKS: cover node capacity reconciliation ownership --- ...etesClusterNodeCapacityReconcilerImpl.java | 4 +- ...ClusterNodeCapacityReconcilerImplTest.java | 82 +++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java index a6785ec3fdfd..68576ce12c9e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java @@ -176,8 +176,8 @@ private boolean kubernetesMatchesGuest(NodeCapacitySnapshot observed, NodeCapaci && (observed.getGuestMemoryKiB() <= before.getGuestMemoryKiB() || observed.getAllocatableMemoryBytes() > before.getAllocatableMemoryBytes()); } - private String executeControl(NodeAccess access, String command) throws Exception { return execute(access.getControlAddress(), access.getControlPort(), access, command); } - private String executeNode(NodeAccess access, String command) throws Exception { return execute(access.getNodeAddress(), access.getNodePort(), access, command); } + protected String executeControl(NodeAccess access, String command) throws Exception { return execute(access.getControlAddress(), access.getControlPort(), access, command); } + protected String executeNode(NodeAccess access, String command) throws Exception { return execute(access.getNodeAddress(), access.getNodePort(), access, command); } private String execute(String address, int port, NodeAccess access, String command) throws Exception { Pair result = SshHelper.sshExecute(address, port, access.getUser(), access.getSshKeyFile(), null, command, 10000, 10000, COMMAND_TIMEOUT_MS); if (Boolean.TRUE.equals(result.first())) return StringUtils.defaultString(result.second()); diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java index 729803c7187e..3707c8014bff 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java @@ -16,9 +16,13 @@ // under the License. package com.cloud.kubernetes.cluster.utils; +import java.util.ArrayList; +import java.util.List; + import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.offering.ServiceOffering; +import com.cloud.uservm.UserVm; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -62,10 +66,88 @@ public void testCurrentKubernetesResourcesDoNotRequireASecondKubeletRestart() { Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(stale, offering(4, 4096))); } + @Test + public void testCaptureBeforeReadsStructuredNodeState() throws Exception { + FakeReconciler fakeReconciler = new FakeReconciler(nodeJson(false)); + KubernetesCluster cluster = cluster("cluster-uuid"); + UserVm vm = vm("worker-1"); + + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot snapshot = fakeReconciler.captureBefore(cluster, vm, access()); + + Assert.assertFalse(snapshot.isUnschedulable()); + Assert.assertTrue(snapshot.isReady()); + Assert.assertEquals(2L, snapshot.getGuestOnlineCpuCount()); + Assert.assertEquals(2097152L, snapshot.getGuestMemoryKiB()); + Assert.assertEquals(3900L, snapshot.getCapacityCpuMillis()); + Assert.assertEquals(4L * 1024L * 1024L * 1024L, snapshot.getCapacityMemoryBytes()); + } + + @Test + public void testCordonAndRestoreOnlyNodesOwnedByCks() throws Exception { + KubernetesCluster cluster = cluster("cluster-uuid"); + UserVm vm = vm("worker-1"); + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot previouslyCordon = + new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(true, false, 0, 0, 0, 0, 0, 0, true); + FakeReconciler operatorCordon = new FakeReconciler(nodeJson(true)); + + operatorCordon.cordonIfNeeded(cluster, vm, previouslyCordon, access(), System.currentTimeMillis() + 1000L); + operatorCordon.restoreSchedulability(cluster, vm, previouslyCordon, access(), System.currentTimeMillis() + 1000L); + Assert.assertTrue(operatorCordon.controlCommands.isEmpty()); + + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot cksCordon = + new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(true, true, 0, 0, 0, 0, 0, 0, true); + FakeReconciler cleanup = new FakeReconciler(nodeJson(false)); + cleanup.restoreSchedulability(cluster, vm, cksCordon, access(), System.currentTimeMillis() + 1000L); + + Assert.assertEquals("sudo /opt/bin/kubectl uncordon worker-1", cleanup.controlCommands.get(0)); + Assert.assertEquals("sudo /opt/bin/kubectl annotate node worker-1 cloudstack.apache.org/cks-live-resize-", cleanup.controlCommands.get(1)); + } + private ServiceOffering offering(int cpu, int memory) { ServiceOffering offering = Mockito.mock(ServiceOffering.class); Mockito.when(offering.getCpu()).thenReturn(cpu); Mockito.when(offering.getRamSize()).thenReturn(memory); return offering; } + + private KubernetesCluster cluster(String uuid) { + KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); + Mockito.when(cluster.getUuid()).thenReturn(uuid); + return cluster; + } + + private UserVm vm(String hostname) { + UserVm vm = Mockito.mock(UserVm.class); + Mockito.when(vm.getHostName()).thenReturn(hostname); + Mockito.when(vm.getUuid()).thenReturn("vm-uuid"); + return vm; + } + + private KubernetesClusterNodeCapacityReconciler.NodeAccess access() { + return new KubernetesClusterNodeCapacityReconciler.NodeAccess("control", 22, "node", 22, "root", null); + } + + private String nodeJson(boolean unschedulable) { + return String.format("{\"metadata\":{\"annotations\":{}},\"spec\":{\"unschedulable\":%s},\"status\":{\"capacity\":{\"cpu\":\"3900m\",\"memory\":\"4Gi\"},\"allocatable\":{\"cpu\":\"3700m\",\"memory\":\"3900Mi\"},\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}", unschedulable); + } + + private static class FakeReconciler extends KubernetesClusterNodeCapacityReconcilerImpl { + private final String nodeJson; + private final List controlCommands = new ArrayList<>(); + + FakeReconciler(String nodeJson) { + this.nodeJson = nodeJson; + } + + @Override + protected String executeControl(KubernetesClusterNodeCapacityReconciler.NodeAccess access, String command) { + controlCommands.add(command); + return command.contains(" get node ") ? nodeJson : ""; + } + + @Override + protected String executeNode(KubernetesClusterNodeCapacityReconciler.NodeAccess access, String command) { + return "2\n2097152\n"; + } + } } From f57838e2374b107a54d2289e04930d6671f035e7 Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Tue, 22 Sep 2026 13:37:16 +0200 Subject: [PATCH 03/13] CKS: test live resize reconciliation guards --- .../KubernetesClusterScaleWorkerTest.java | 3 +++ ...etesClusterNodeCapacityReconcilerImplTest.java | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index ffd9a28288cc..2c16ea755731 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -219,5 +219,8 @@ public void testShouldReconcileNodeCapacityOnlyForManagedKvmNodes() { Mockito.when(managedNode.isExternalNode()).thenReturn(false); Mockito.when(kvmNode.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.XenServer); Assert.assertFalse(scaleWorker.shouldReconcileNodeCapacity(managedNode, kvmNode, oldOffering, targetOffering, WORKER)); + + Mockito.when(runningManagedCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.ExternalManaged); + Assert.assertFalse(scaleWorker.shouldReconcileNodeCapacity(managedNode, kvmNode, oldOffering, targetOffering, WORKER)); } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java index 3707c8014bff..eee5b154888b 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java @@ -103,6 +103,21 @@ public void testCordonAndRestoreOnlyNodesOwnedByCks() throws Exception { Assert.assertEquals("sudo /opt/bin/kubectl annotate node worker-1 cloudstack.apache.org/cks-live-resize-", cleanup.controlCommands.get(1)); } + @Test + public void testCordonAddsOwnershipAnnotationBeforeCordoning() throws Exception { + KubernetesCluster cluster = cluster("cluster-uuid"); + UserVm vm = vm("worker-1"); + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot schedulable = + new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, 0, 0, 0, 0, 0, 0, true); + FakeReconciler fakeReconciler = new FakeReconciler(nodeJson(true)); + + fakeReconciler.cordonIfNeeded(cluster, vm, schedulable, access(), System.currentTimeMillis() + 1000L); + + Assert.assertEquals("sudo /opt/bin/kubectl annotate node worker-1 cloudstack.apache.org/cks-live-resize=cluster-uuid --overwrite", + fakeReconciler.controlCommands.get(0)); + Assert.assertEquals("sudo /opt/bin/kubectl cordon worker-1", fakeReconciler.controlCommands.get(1)); + } + private ServiceOffering offering(int cpu, int memory) { ServiceOffering offering = Mockito.mock(ServiceOffering.class); Mockito.when(offering.getCpu()).thenReturn(cpu); From a40ea44fd3a81f43eef36763c1f1bf13a54033c0 Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Tue, 22 Sep 2026 13:39:38 +0200 Subject: [PATCH 04/13] CKS: improve live resize failure diagnostics --- .../actionworkers/KubernetesClusterScaleWorker.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 2df102531c66..a7a80dd40b65 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -440,7 +440,14 @@ private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, logger.warn("Unable to restore schedulability for CKS node {} after a pre-resize failure", userVM.getUuid(), cleanupException); } } - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s due to %s", kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + String recovery = resizeSucceeded + ? "The node remains cordoned; check kubelet on the VM and retry the CKS scale operation." + : "The VM was not resized; CKS attempted to restore its temporary cordon."; + String message = String.format("Scaling Kubernetes cluster %s (UUID %s) failed for %s node VM %s (UUID %s), " + + "offering %s to %s: %s %s", kubernetesCluster.getName(), kubernetesCluster.getUuid(), nodeType, + userVM.getDisplayName(), userVM.getUuid(), oldOffering == null ? null : oldOffering.getId(), + serviceOffering.getId(), e.getMessage(), recovery); + logTransitStateAndThrow(Level.ERROR, message, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } if (System.currentTimeMillis() > scaleTimeoutTime) { logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, scaling action timed out", kubernetesCluster.getName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); From 780d621dbe1e2b57ffa6118a548766b783724d4b Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Tue, 22 Sep 2026 13:42:01 +0200 Subject: [PATCH 05/13] CKS: resolve live resize target public IP --- .../cluster/actionworkers/KubernetesClusterScaleWorker.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index a7a80dd40b65..797e82a7bbc3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -475,7 +475,11 @@ private NodeAccess resolveNodeAccess(UserVm userVM) { if (sshRule == null) { throw new CloudRuntimeException(String.format("Unable to resolve SSH port-forwarding rule for VM %s", userVM.getUuid())); } - return new NodeAccess(controlAccess.first(), controlAccess.second(), controlAccess.first(), sshRule.getSourcePortStart(), + IpAddress targetPublicIp = network.getVpcId() == null ? getNetworkSourceNatIp(network) : getVpcTierKubernetesPublicIp(network); + if (targetPublicIp == null) { + throw new CloudRuntimeException(String.format("Unable to resolve target-node public IP for VM %s", userVM.getUuid())); + } + return new NodeAccess(controlAccess.first(), controlAccess.second(), targetPublicIp.getAddress().addr(), sshRule.getSourcePortStart(), getControlNodeLoginUser(), getManagementServerSshPublicKeyFile()); } From 4216946a4741310b381cc283a5adc3b8432383bc Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Tue, 22 Sep 2026 17:08:33 +0200 Subject: [PATCH 06/13] CKS: register live resize capacity reconciler --- .../KubernetesClusterNodeCapacityReconcilerImpl.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java index 68576ce12c9e..2c8d9efc4caf 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java @@ -32,8 +32,15 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.apache.commons.lang3.StringUtils; - -/** SSH-backed implementation for a rolling CKS live-resize reconciliation. */ +import org.springframework.stereotype.Component; + +/** + * SSH-backed implementation for a rolling CKS live-resize reconciliation. + * + * Component scanning makes this implementation available to the scale worker, which obtains it + * through {@code ComponentContext}. + */ +@Component public class KubernetesClusterNodeCapacityReconcilerImpl implements KubernetesClusterNodeCapacityReconciler { static final long MINIMUM_MEMORY_OVERHEAD_BYTES = 128L * 1024L * 1024L; static final long KUBERNETES_MEMORY_REPORTING_TOLERANCE_BYTES = 16L * 1024L * 1024L; From 7e5a95a7593f381b1b684e905f417f95465e8210 Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Tue, 20 Oct 2026 07:06:35 +0200 Subject: [PATCH 07/13] CKS: retry capacity reconciliation after live resize --- .../KubernetesClusterScaleWorker.java | 27 +++++++++++++------ ...etesClusterNodeCapacityReconcilerImpl.java | 16 +++++------ .../spring-kubernetes-service-context.xml | 2 -- .../KubernetesClusterScaleWorkerTest.java | 26 ++++++++++++++++++ ...ClusterNodeCapacityReconcilerImplTest.java | 17 +++++++++++- 5 files changed, 67 insertions(+), 21 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 797e82a7bbc3..780f361dbdf0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -28,8 +28,6 @@ import java.util.Set; import java.util.stream.Collectors; -import javax.inject.Inject; - import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.service.ServiceOfferingVO; import com.cloud.storage.VMTemplateVO; @@ -87,8 +85,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif private Boolean isAutoscalingEnabled; private long scaleTimeoutTime; - @Inject - protected KubernetesClusterNodeCapacityReconciler kubernetesClusterNodeCapacityReconciler; + protected KubernetesClusterNodeCapacityReconciler kubernetesClusterNodeCapacityReconciler = new KubernetesClusterNodeCapacityReconcilerImpl(); protected KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); @@ -401,26 +398,30 @@ private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } boolean reconcileKubernetes = shouldReconcileNodeCapacity(vmMapVO, userVM, oldOffering, serviceOffering, nodeType); + boolean reconciliationRetryCandidate = isReconciliationRetryCandidate(vmMapVO, userVM, serviceOffering, nodeType); NodeAccess nodeAccess = null; NodeCapacitySnapshot before = null; boolean result = false; boolean cksCordonCompleted = false; - boolean resizeSucceeded = serviceOffering.getId() == userVM.getServiceOfferingId(); + boolean resizeSucceeded = Objects.equals(serviceOffering.getId(), userVM.getServiceOfferingId()); try { - if (reconcileKubernetes) { + if (reconcileKubernetes || reconciliationRetryCandidate) { nodeAccess = resolveNodeAccess(userVM); before = kubernetesClusterNodeCapacityReconciler.captureBefore(kubernetesCluster, userVM, nodeAccess); + reconcileKubernetes = reconcileKubernetes || before.isCloudStackResizeCordon(); + } + if (reconcileKubernetes) { kubernetesClusterNodeCapacityReconciler.cordonIfNeeded(kubernetesCluster, userVM, before, nodeAccess, scaleTimeoutTime); cksCordonCompleted = !before.isUnschedulable() || before.isCloudStackResizeCordon(); } - if (serviceOffering.getId() != userVM.getServiceOfferingId()) { + if (!Objects.equals(serviceOffering.getId(), userVM.getServiceOfferingId())) { result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); if (!result) { logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s", kubernetesCluster.getName(), userVM.getDisplayName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } resizeSucceeded = true; userVM = userVmDao.findById(userVM.getId()); - if (userVM == null || serviceOffering.getId() != userVM.getServiceOfferingId()) { + if (userVM == null || !Objects.equals(serviceOffering.getId(), userVM.getServiceOfferingId())) { logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, VM %s did not reach target offering", kubernetesCluster.getName(), vmMapVO.getVmId()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } } @@ -492,6 +493,16 @@ protected boolean shouldReconcileNodeCapacity(KubernetesClusterVmMapVO vmMapVO, && kubernetesClusterNodeCapacityReconciler.requiresKubeletRefresh(oldOffering, targetOffering, nodeType, originalState); } + protected boolean isReconciliationRetryCandidate(KubernetesClusterVmMapVO vmMapVO, UserVm userVM, + ServiceOffering targetOffering, KubernetesClusterNodeType nodeType) { + return !vmMapVO.isExternalNode() + && KubernetesCluster.ClusterType.CloudManaged == kubernetesCluster.getClusterType() + && Hypervisor.HypervisorType.KVM == userVM.getHypervisorType() + && (WORKER == nodeType || CONTROL == nodeType) + && KubernetesCluster.State.Running == originalState + && Objects.equals(targetOffering.getId(), userVM.getServiceOfferingId()); + } + private void removeNodesFromCluster(List vmMaps) throws CloudRuntimeException { for (KubernetesClusterVmMapVO vmMapVO : vmMaps) { UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java index 2c8d9efc4caf..2502490238d5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java @@ -32,17 +32,13 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -/** - * SSH-backed implementation for a rolling CKS live-resize reconciliation. - * - * Component scanning makes this implementation available to the scale worker, which obtains it - * through {@code ComponentContext}. - */ -@Component + +/** SSH-backed implementation for a rolling CKS live-resize reconciliation. */ public class KubernetesClusterNodeCapacityReconcilerImpl implements KubernetesClusterNodeCapacityReconciler { - static final long MINIMUM_MEMORY_OVERHEAD_BYTES = 128L * 1024L * 1024L; + // KVM QA observes 4,568,788 KiB in a 4,608 MiB guest after memory hotplug + // (about 146 MiB below the offering), so keep a small margin above that + // platform overhead while still rejecting materially undersized guests. + static final long MINIMUM_MEMORY_OVERHEAD_BYTES = 160L * 1024L * 1024L; static final long KUBERNETES_MEMORY_REPORTING_TOLERANCE_BYTES = 16L * 1024L * 1024L; private static final int COMMAND_TIMEOUT_MS = 30000; private static final int POLL_INTERVAL_MS = 5000; diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml index 9fe19200d6e7..fedc57071c5d 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml @@ -34,8 +34,6 @@ - - diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index 2c16ea755731..8a35c06f0c3f 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -21,6 +21,7 @@ import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconcilerImpl; import com.cloud.hypervisor.Hypervisor; import com.cloud.offering.ServiceOffering; import com.cloud.service.ServiceOfferingVO; @@ -193,6 +194,11 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { Assert.assertTrue(toRemove.isEmpty()); } + @Test + public void testUsesLiveResizeReconcilerWithoutComponentContextRegistration() { + Assert.assertTrue(worker.kubernetesClusterNodeCapacityReconciler instanceof KubernetesClusterNodeCapacityReconcilerImpl); + } + @Test public void testShouldReconcileNodeCapacityOnlyForManagedKvmNodes() { KubernetesCluster runningManagedCluster = Mockito.mock(KubernetesCluster.class); @@ -223,4 +229,24 @@ public void testShouldReconcileNodeCapacityOnlyForManagedKvmNodes() { Mockito.when(runningManagedCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.ExternalManaged); Assert.assertFalse(scaleWorker.shouldReconcileNodeCapacity(managedNode, kvmNode, oldOffering, targetOffering, WORKER)); } + + @Test + public void testCksOwnedCordonCanRetryReconciliationAfterVmReachedTargetOffering() { + KubernetesCluster runningManagedCluster = Mockito.mock(KubernetesCluster.class); + Mockito.when(runningManagedCluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(runningManagedCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + KubernetesClusterScaleWorker scaleWorker = new KubernetesClusterScaleWorker(runningManagedCluster, + new java.util.HashMap<>(), 1L, null, false, null, null, clusterManager); + + KubernetesClusterVmMapVO managedNode = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(managedNode.isExternalNode()).thenReturn(false); + UserVmVO kvmNode = Mockito.mock(UserVmVO.class); + Mockito.when(kvmNode.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + Mockito.when(kvmNode.getServiceOfferingId()).thenReturn(Long.parseLong("1024")); + ServiceOffering targetOffering = Mockito.mock(ServiceOffering.class); + Mockito.when(targetOffering.getId()).thenReturn(Long.parseLong("1024")); + + Assert.assertTrue(scaleWorker.isReconciliationRetryCandidate(managedNode, kvmNode, targetOffering, WORKER)); + Assert.assertFalse(scaleWorker.isReconciliationRetryCandidate(managedNode, kvmNode, targetOffering, DEFAULT)); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java index eee5b154888b..f54d9e566e59 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java @@ -48,6 +48,8 @@ public void testOnlyRunningKubernetesNodesWithChangedCpuOrRamRequireRefresh() { Assert.assertTrue(reconciler.requiresKubeletRefresh(oldOffering, cpuOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Running)); Assert.assertTrue(reconciler.requiresKubeletRefresh(oldOffering, ramOffering, KubernetesClusterNodeType.CONTROL, KubernetesCluster.State.Running)); Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, capOnlyOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Running)); + Assert.assertFalse(reconciler.requiresKubeletRefresh(null, cpuOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Running)); + Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, null, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Running)); Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, cpuOffering, KubernetesClusterNodeType.ETCD, KubernetesCluster.State.Running)); Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, cpuOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Stopped)); } @@ -118,6 +120,18 @@ public void testCordonAddsOwnershipAnnotationBeforeCordoning() throws Exception Assert.assertEquals("sudo /opt/bin/kubectl cordon worker-1", fakeReconciler.controlCommands.get(1)); } + @Test + public void testGuestVerificationAllowsObservedKvmHotplugOverhead() throws Exception { + FakeReconciler fakeReconciler = new FakeReconciler(nodeJson(false)); + fakeReconciler.nodeResources = "4\n4568788\n"; + + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot before = + new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, 2, 2097152L, 2000L, 2L * 1024L * 1024L * 1024L, + 1900L, 1900L * 1024L * 1024L, true); + + fakeReconciler.verifyGuestResources(vm("worker-1"), offering(4, 4608), before, access(), System.currentTimeMillis() + 1000L); + } + private ServiceOffering offering(int cpu, int memory) { ServiceOffering offering = Mockito.mock(ServiceOffering.class); Mockito.when(offering.getCpu()).thenReturn(cpu); @@ -149,6 +163,7 @@ private String nodeJson(boolean unschedulable) { private static class FakeReconciler extends KubernetesClusterNodeCapacityReconcilerImpl { private final String nodeJson; private final List controlCommands = new ArrayList<>(); + private String nodeResources = "2\n2097152\n"; FakeReconciler(String nodeJson) { this.nodeJson = nodeJson; @@ -162,7 +177,7 @@ protected String executeControl(KubernetesClusterNodeCapacityReconciler.NodeAcce @Override protected String executeNode(KubernetesClusterNodeCapacityReconciler.NodeAccess access, String command) { - return "2\n2097152\n"; + return nodeResources; } } } From 5480119108462433a2b611b7d25d67e2e92f627c Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Tue, 20 Oct 2026 14:59:56 +0200 Subject: [PATCH 08/13] CKS: online guest resources after live resize --- .../actionworkers/KubernetesClusterScaleWorker.java | 1 + .../KubernetesClusterNodeCapacityReconciler.java | 1 + .../KubernetesClusterNodeCapacityReconcilerImpl.java | 9 +++++++++ ...ernetesClusterNodeCapacityReconcilerImplTest.java | 12 ++++++++++++ 4 files changed, 23 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 780f361dbdf0..2ce1332d5c31 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -426,6 +426,7 @@ private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, } } if (reconcileKubernetes) { + kubernetesClusterNodeCapacityReconciler.onlineGuestResources(nodeAccess); kubernetesClusterNodeCapacityReconciler.verifyGuestResources(userVM, serviceOffering, before, nodeAccess, scaleTimeoutTime); if (!kubernetesClusterNodeCapacityReconciler.isKubernetesResourcesCurrent(before, serviceOffering)) { kubernetesClusterNodeCapacityReconciler.restartKubelet(userVM, nodeAccess, scaleTimeoutTime); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java index f52458774aca..3fbabca6e322 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java @@ -29,6 +29,7 @@ public interface KubernetesClusterNodeCapacityReconciler { boolean requiresKubeletRefresh(ServiceOffering oldOffering, ServiceOffering newOffering, KubernetesClusterNodeType nodeType, KubernetesCluster.State originalState); void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception; + void onlineGuestResources(NodeAccess access) throws Exception; void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception; boolean isKubernetesResourcesCurrent(NodeCapacitySnapshot snapshot, ServiceOffering target); void restartKubelet(UserVm vm, NodeAccess access, long deadline) throws Exception; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java index 2502490238d5..bec9b91a83e9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java @@ -45,6 +45,10 @@ public class KubernetesClusterNodeCapacityReconcilerImpl implements KubernetesCl private static final Pattern NODE_NAME = Pattern.compile("[A-Za-z0-9][A-Za-z0-9.-]{0,252}"); private static final Pattern MEMORY_QUANTITY = Pattern.compile("([0-9]+)(Ki|Mi|Gi|Ti|K|M|G|T)?"); private static final String RESIZE_ANNOTATION = "cloudstack.apache.org/cks-live-resize"; + static final String ONLINE_HOTPLUGGED_RESOURCES = "sudo sh -c 'for cpu in /sys/devices/system/cpu/cpu[0-9]*; do " + + "[ -e \"$cpu/online\" ] || continue; echo 1 > \"$cpu/online\"; done; " + + "for memory in /sys/devices/system/memory/memory[0-9]*; do " + + "[ -e \"$memory/online\" ] || continue; echo online > \"$memory/online\"; done'"; @Override public boolean requiresKubeletRefresh(ServiceOffering oldOffering, ServiceOffering newOffering, @@ -72,6 +76,11 @@ public void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySna throw failure("CORDON", vm, "Kubernetes node did not become unschedulable"); } + @Override + public void onlineGuestResources(NodeAccess access) throws Exception { + executeNode(access, ONLINE_HOTPLUGGED_RESOURCES); + } + @Override public void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception { while (System.currentTimeMillis() < deadline) { diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java index f54d9e566e59..7611f582f13b 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java @@ -132,6 +132,16 @@ public void testGuestVerificationAllowsObservedKvmHotplugOverhead() throws Excep fakeReconciler.verifyGuestResources(vm("worker-1"), offering(4, 4608), before, access(), System.currentTimeMillis() + 1000L); } + @Test + public void testOnlineGuestResourcesActivatesHotpluggedCpuAndMemory() throws Exception { + FakeReconciler fakeReconciler = new FakeReconciler(nodeJson(false)); + + fakeReconciler.onlineGuestResources(access()); + + Assert.assertEquals(KubernetesClusterNodeCapacityReconcilerImpl.ONLINE_HOTPLUGGED_RESOURCES, + fakeReconciler.nodeCommands.get(0)); + } + private ServiceOffering offering(int cpu, int memory) { ServiceOffering offering = Mockito.mock(ServiceOffering.class); Mockito.when(offering.getCpu()).thenReturn(cpu); @@ -163,6 +173,7 @@ private String nodeJson(boolean unschedulable) { private static class FakeReconciler extends KubernetesClusterNodeCapacityReconcilerImpl { private final String nodeJson; private final List controlCommands = new ArrayList<>(); + private final List nodeCommands = new ArrayList<>(); private String nodeResources = "2\n2097152\n"; FakeReconciler(String nodeJson) { @@ -177,6 +188,7 @@ protected String executeControl(KubernetesClusterNodeCapacityReconciler.NodeAcce @Override protected String executeNode(KubernetesClusterNodeCapacityReconciler.NodeAccess access, String command) { + nodeCommands.add(command); return nodeResources; } } From 2a404aa1ffcccc2cc4b1e131562856c1fc7b754c Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Tue, 20 Oct 2026 15:36:50 +0200 Subject: [PATCH 09/13] CKS: recompute cluster capacity from node type offerings --- .../KubernetesClusterScaleWorker.java | 59 +++++++-------- .../KubernetesClusterScaleWorkerTest.java | 73 ++++++++++++++----- 2 files changed, 81 insertions(+), 51 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 2ce1332d5c31..4d8852611ca6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -29,7 +29,6 @@ import java.util.stream.Collectors; import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; -import com.cloud.service.ServiceOfferingVO; import com.cloud.storage.VMTemplateVO; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -219,7 +218,7 @@ private KubernetesClusterVO updateKubernetesClusterEntryForNodeType(final Long n final ServiceOffering newServiceOffering, final boolean updateNodeOffering, boolean updateClusterOffering) throws CloudRuntimeException { final ServiceOffering serviceOffering = newServiceOffering == null ? - serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()) : newServiceOffering; + serviceOfferingDao.findById(getConfiguredOfferingIdForNodeType(nodeType)) : newServiceOffering; final Long serviceOfferingId = newServiceOffering == null ? null : serviceOffering.getId(); Pair clusterCountAndCapacity = calculateNewClusterCountAndCapacity(newWorkerSize, nodeType, serviceOffering); @@ -236,32 +235,36 @@ private KubernetesClusterVO updateKubernetesClusterEntryForNodeType(final Long n } protected Pair calculateNewClusterCountAndCapacity(Long newWorkerSize, KubernetesClusterNodeType nodeType, ServiceOffering serviceOffering) { - long cores; - long memory; - long totalClusterSize = newWorkerSize == null ? kubernetesCluster.getTotalNodeCount() : (newWorkerSize + kubernetesCluster.getControlNodeCount() + kubernetesCluster.getEtcdNodeCount()); - - if (nodeType == DEFAULT) { - cores = serviceOffering.getCpu() * totalClusterSize; - memory = serviceOffering.getRamSize() * totalClusterSize; - } else { - long nodeCount = getNodeCountForType(nodeType, kubernetesCluster); - Long existingOfferingId = getExistingOfferingIdForNodeType(nodeType, kubernetesCluster); - if (existingOfferingId == null) { - existingOfferingId = serviceOffering.getId(); - } - ServiceOfferingVO previousOffering = serviceOfferingDao.findById(existingOfferingId); - Pair previousNodesCapacity = calculateNodesCapacity(previousOffering, nodeCount); - if (WORKER == nodeType) { - nodeCount = newWorkerSize == null ? kubernetesCluster.getNodeCount() : newWorkerSize; + long cores = 0; + long memory = 0; + // VM offerings have already changed here. Recompute the total from the + // configured node types instead of subtracting a supposedly old VM offering. + for (KubernetesClusterNodeType type : List.of(CONTROL, WORKER, ETCD)) { + long nodeCount = WORKER == type && newWorkerSize != null ? newWorkerSize : getNodeCountForType(type, kubernetesCluster); + if (nodeCount == 0) { + continue; } - Pair newNodesCapacity = calculateNodesCapacity(serviceOffering, nodeCount); - Pair newClusterCapacity = calculateClusterNewCapacity(kubernetesCluster, previousNodesCapacity, newNodesCapacity); - cores = newClusterCapacity.first(); - memory = newClusterCapacity.second(); + ServiceOffering offering = nodeType == DEFAULT || nodeType == type ? serviceOffering : + serviceOfferingDao.findByIdIncludingRemoved(getConfiguredOfferingIdForNodeType(type)); + Pair capacity = calculateNodesCapacity(offering, nodeCount); + cores += capacity.first(); + memory += capacity.second(); } return new Pair<>(cores, memory); } + private Long getConfiguredOfferingIdForNodeType(KubernetesClusterNodeType nodeType) { + Long offeringId = null; + if (WORKER == nodeType) { + offeringId = kubernetesCluster.getWorkerNodeServiceOfferingId(); + } else if (CONTROL == nodeType) { + offeringId = kubernetesCluster.getControlNodeServiceOfferingId(); + } else if (ETCD == nodeType) { + offeringId = kubernetesCluster.getEtcdNodeServiceOfferingId(); + } + return offeringId == null ? kubernetesCluster.getServiceOfferingId() : offeringId; + } + private long getNodeCountForType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) { if (WORKER == nodeType) { return kubernetesCluster.getNodeCount(); @@ -273,16 +276,6 @@ private long getNodeCountForType(KubernetesClusterNodeType nodeType, KubernetesC return kubernetesCluster.getTotalNodeCount(); } - protected Pair calculateClusterNewCapacity(KubernetesCluster kubernetesCluster, - Pair previousNodeTypeCapacity, - Pair newNodeTypeCapacity) { - long previousCores = kubernetesCluster.getCores(); - long previousMemory = kubernetesCluster.getMemory(); - long newCores = previousCores - previousNodeTypeCapacity.first() + newNodeTypeCapacity.first(); - long newMemory = previousMemory - previousNodeTypeCapacity.second() + newNodeTypeCapacity.second(); - return new Pair<>(newCores, newMemory); - } - protected Pair calculateNodesCapacity(ServiceOffering offering, long nodeCount) { return new Pair<>(offering.getCpu() * nodeCount, offering.getRamSize() * nodeCount); } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index 8a35c06f0c3f..170035d71355 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -98,21 +98,13 @@ public void testCalculateNewClusterCountAndCapacityAllNodesScaleSize() { @Test public void testCalculateNewClusterCountAndCapacityNodeTypeScaleControlOffering() { long controlNodes = 2L; - long kubernetesClusterId = 10L; - Mockito.when(kubernetesCluster.getId()).thenReturn(kubernetesClusterId); Mockito.when(kubernetesCluster.getControlNodeCount()).thenReturn(controlNodes); - - ServiceOfferingVO existingOffering = Mockito.mock(ServiceOfferingVO.class); - int existingCores = 2; - int existingMemory = 2048; - Mockito.when(existingOffering.getCpu()).thenReturn(existingCores); - Mockito.when(existingOffering.getRamSize()).thenReturn(existingMemory); + Mockito.when(kubernetesCluster.getNodeCount()).thenReturn(4L); + Mockito.when(kubernetesCluster.getWorkerNodeServiceOfferingId()).thenReturn(defaultOfferingId); int remainingClusterCpu = 8; int remainingClusterMemory = 12288; - Mockito.when(kubernetesCluster.getCores()).thenReturn(remainingClusterCpu + (controlNodes * existingCores)); - Mockito.when(kubernetesCluster.getMemory()).thenReturn(remainingClusterMemory + (controlNodes * existingMemory)); - - Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(existingOffering); + ServiceOfferingVO workerOffering = offering(2, 3072); + Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(defaultOfferingId)).thenReturn(workerOffering); ServiceOfferingVO newOffering = Mockito.mock(ServiceOfferingVO.class); int newCores = 4; @@ -120,12 +112,6 @@ public void testCalculateNewClusterCountAndCapacityNodeTypeScaleControlOffering( Mockito.when(newOffering.getCpu()).thenReturn(newCores); Mockito.when(newOffering.getRamSize()).thenReturn(newMemory); - KubernetesClusterVmMapVO controlNodeVM1 = Mockito.mock(KubernetesClusterVmMapVO.class); - Mockito.when(controlNodeVM1.getVmId()).thenReturn(10L); - UserVmVO userVmVO = Mockito.mock(UserVmVO.class); - Mockito.when(userVmVO.getServiceOfferingId()).thenReturn(defaultOfferingId); - Mockito.when(userVmDao.findById(10L)).thenReturn(userVmVO); - Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesClusterId, CONTROL)).thenReturn(List.of(controlNodeVM1)); Pair newClusterCapacity = worker.calculateNewClusterCountAndCapacity(null, CONTROL, newOffering); long expectedCores = remainingClusterCpu + (controlNodes * newCores); @@ -134,6 +120,57 @@ public void testCalculateNewClusterCountAndCapacityNodeTypeScaleControlOffering( Assert.assertEquals(expectedMemory, newClusterCapacity.second().longValue()); } + @Test + public void testWorkerResizeRecomputesStaleTotalsWithoutReadingResizedVmOfferings() { + Mockito.when(kubernetesCluster.getControlNodeCount()).thenReturn(1L); + Mockito.when(kubernetesCluster.getNodeCount()).thenReturn(1L); + Mockito.when(kubernetesCluster.getControlNodeServiceOfferingId()).thenReturn(2L); + Mockito.doReturn(offering(3, 3072)).when(serviceOfferingDao).findByIdIncludingRemoved(2L); + + Pair capacity = worker.calculateNewClusterCountAndCapacity(null, WORKER, offering(3, 3072)); + + Assert.assertEquals(Long.valueOf(6), capacity.first()); + Assert.assertEquals(Long.valueOf(6144), capacity.second()); + Mockito.verify(kubernetesCluster, Mockito.never()).getCores(); + Mockito.verify(kubernetesCluster, Mockito.never()).getMemory(); + Mockito.verifyNoInteractions(userVmDao, kubernetesClusterVmMapDao); + } + + @Test + public void testWorkerCountChangePreservesMixedControlAndEtcdOfferings() { + Mockito.when(kubernetesCluster.getControlNodeCount()).thenReturn(1L); + Mockito.when(kubernetesCluster.getEtcdNodeCount()).thenReturn(3L); + Mockito.when(kubernetesCluster.getControlNodeServiceOfferingId()).thenReturn(2L); + Mockito.when(kubernetesCluster.getEtcdNodeServiceOfferingId()).thenReturn(3L); + Mockito.doReturn(offering(4, 4096)).when(serviceOfferingDao).findByIdIncludingRemoved(2L); + Mockito.doReturn(offering(2, 2048)).when(serviceOfferingDao).findByIdIncludingRemoved(3L); + + Pair capacity = worker.calculateNewClusterCountAndCapacity(2L, WORKER, offering(3, 3072)); + + Assert.assertEquals(Long.valueOf(16), capacity.first()); + Assert.assertEquals(Long.valueOf(16384), capacity.second()); + } + + @Test + public void testCapacityFallsBackToClusterOfferingForLegacyNodeTypes() { + Mockito.when(kubernetesCluster.getControlNodeCount()).thenReturn(1L); + Mockito.when(kubernetesCluster.getControlNodeServiceOfferingId()).thenReturn(null); + Mockito.when(kubernetesCluster.getServiceOfferingId()).thenReturn(defaultOfferingId); + Mockito.doReturn(offering(2, 2048)).when(serviceOfferingDao).findByIdIncludingRemoved(defaultOfferingId); + + Pair capacity = worker.calculateNewClusterCountAndCapacity(1L, WORKER, offering(3, 3072)); + + Assert.assertEquals(Long.valueOf(5), capacity.first()); + Assert.assertEquals(Long.valueOf(5120), capacity.second()); + } + + private ServiceOfferingVO offering(int cpu, int memory) { + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(offering.getCpu()).thenReturn(cpu); + Mockito.when(offering.getRamSize()).thenReturn(memory); + return offering; + } + @Test public void testGetWorkerNodesToRemoveForDownsize_singleRemoval() { From 978b29173ed70c695b50bce9ea7ecdf7436ae070 Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Tue, 20 Oct 2026 20:16:48 +0200 Subject: [PATCH 10/13] CKS: preserve node offerings across scale and retry paths --- .../KubernetesClusterScaleWorker.java | 20 +++--- .../KubernetesClusterScaleWorkerTest.java | 65 +++++++++++++++++++ 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 4d8852611ca6..0b53c89b6183 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -338,7 +338,7 @@ private void validateKubernetesClusterScaleSizeParameters(KubernetesClusterNodeT } // Check capacity and transition state final long newVmRequiredCount = clusterSize - originalClusterSize; - final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + final ServiceOffering clusterServiceOffering = getServiceOfferingForNodeTypeOnCluster(nodeType, kubernetesCluster); if (clusterServiceOffering == null) { logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling failed for Kubernetes cluster : %s, cluster service offering not found", kubernetesCluster.getName())); } @@ -361,7 +361,7 @@ private void validateKubernetesClusterScaleSizeParameters(KubernetesClusterNodeT } } - private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, ServiceOffering serviceOffering, + protected void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, ServiceOffering serviceOffering, boolean updateNodeOffering, boolean updateClusterOffering) throws CloudRuntimeException { validateKubernetesClusterScaleOfferingParameters(); List scalingStates = List.of(KubernetesCluster.State.Scaling, KubernetesCluster.State.ScalingStoppedCluster); @@ -708,6 +708,9 @@ public boolean scaleCluster() throws CloudRuntimeException { for (KubernetesClusterNodeType nodeType : Arrays.asList(CONTROL, ETCD, WORKER)) { boolean isWorkerNode = WORKER == nodeType; final long newVMRequired = (!isWorkerNode || clusterSize == null) ? 0 : clusterSize - originalClusterSize; + if (nodeType == ETCD && kubernetesCluster.getEtcdNodeCount() == 0) { + continue; + } if (!scaleClusterDefaultOffering && !serviceOfferingNodeTypeMap.containsKey(nodeType.name()) && newVMRequired == 0) { continue; } @@ -777,15 +780,8 @@ protected boolean isServiceOfferingScalingNeededForNodeType(ServiceOffering exis } protected Long getExistingOfferingIdForNodeType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) { - List clusterVms = kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(), nodeType); - if (CollectionUtils.isEmpty(clusterVms)) { - return kubernetesCluster.getServiceOfferingId(); - } - KubernetesClusterVmMapVO clusterVm = clusterVms.get(0); - UserVmVO clusterUserVm = userVmDao.findById(clusterVm.getVmId()); - if (clusterUserVm == null) { - return kubernetesCluster.getServiceOfferingId(); - } - return clusterUserVm.getServiceOfferingId(); + // Cluster metadata is committed only after every selected VM succeeds. + // A partially resized first VM must not make a retry skip the remaining nodes. + return getConfiguredOfferingIdForNodeType(nodeType); } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index 170035d71355..ac4ece5a268a 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -43,6 +43,7 @@ import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; @RunWith(MockitoJUnitRunner.class) @@ -171,6 +172,70 @@ private ServiceOfferingVO offering(int cpu, int memory) { return offering; } + @Test + public void testExistingOfferingUsesCommittedMetadataRatherThanPartiallyResizedFirstVm() { + Mockito.when(kubernetesCluster.getWorkerNodeServiceOfferingId()).thenReturn(17L); + Assert.assertEquals(Long.valueOf(17), worker.getExistingOfferingIdForNodeType(WORKER, kubernetesCluster)); + Mockito.verifyNoInteractions(userVmDao, kubernetesClusterVmMapDao); + } + + @Test + public void testWorkerPlanningUsesNodeOfferingWithoutLegacyGlobalOffering() { + Mockito.when(kubernetesCluster.getWorkerNodeServiceOfferingId()).thenReturn(17L); + ServiceOfferingVO workerOffering = offering(3, 3072); + Mockito.when(serviceOfferingDao.findById(17L)).thenReturn(workerOffering); + Assert.assertSame(workerOffering, worker.getServiceOfferingForNodeTypeOnCluster(WORKER, kubernetesCluster)); + } + + @Test + public void testGlobalOfferingChangeSkipsAbsentDedicatedEtcdNodes() { + Mockito.when(kubernetesCluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(kubernetesCluster.getServiceOfferingId()).thenReturn(null); + Mockito.when(kubernetesCluster.getControlNodeServiceOfferingId()).thenReturn(11L); + Mockito.when(kubernetesCluster.getWorkerNodeServiceOfferingId()).thenReturn(12L); + ServiceOfferingVO controlOffering = Mockito.mock(ServiceOfferingVO.class); + ServiceOfferingVO workerOffering = Mockito.mock(ServiceOfferingVO.class); + ServiceOfferingVO target = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(controlOffering.getId()).thenReturn(11L); + Mockito.when(workerOffering.getId()).thenReturn(12L); + Mockito.when(target.getId()).thenReturn(13L); + Mockito.when(serviceOfferingDao.findById(11L)).thenReturn(controlOffering); + Mockito.when(serviceOfferingDao.findById(12L)).thenReturn(workerOffering); + KubernetesClusterScaleWorker scaleWorker = Mockito.spy(new KubernetesClusterScaleWorker(kubernetesCluster, + java.util.Map.of(DEFAULT.name(), target), null, null, null, null, null, clusterManager)); + scaleWorker.serviceOfferingDao = serviceOfferingDao; + Mockito.doNothing().when(scaleWorker).init(); + Mockito.doNothing().when(scaleWorker).scaleKubernetesClusterOffering(Mockito.any(), Mockito.any(), Mockito.anyBoolean(), Mockito.anyBoolean()); + Mockito.doReturn(true).when(scaleWorker).stateTransitTo(Mockito.anyLong(), Mockito.any()); + + Assert.assertTrue(scaleWorker.scaleCluster()); + + Mockito.verify(scaleWorker).scaleKubernetesClusterOffering(CONTROL, target, true, false); + Mockito.verify(scaleWorker).scaleKubernetesClusterOffering(WORKER, target, true, true); + Mockito.verify(scaleWorker, Mockito.never()).scaleKubernetesClusterOffering(Mockito.eq(ETCD), Mockito.any(), Mockito.anyBoolean(), Mockito.anyBoolean()); + } + + @Test + public void testRetryUsesUncommittedWorkerOfferingToEnterResizeLoop() { + Mockito.when(kubernetesCluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(kubernetesCluster.getWorkerNodeServiceOfferingId()).thenReturn(11L); + ServiceOfferingVO old = Mockito.mock(ServiceOfferingVO.class); + ServiceOfferingVO target = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(old.getId()).thenReturn(11L); + Mockito.when(target.getId()).thenReturn(12L); + Mockito.when(serviceOfferingDao.findById(11L)).thenReturn(old); + KubernetesClusterScaleWorker scaleWorker = Mockito.spy(new KubernetesClusterScaleWorker(kubernetesCluster, + java.util.Map.of(WORKER.name(), target), null, null, null, null, null, clusterManager)); + scaleWorker.serviceOfferingDao = serviceOfferingDao; + Mockito.doNothing().when(scaleWorker).init(); + Mockito.doNothing().when(scaleWorker).scaleKubernetesClusterOffering(Mockito.any(), Mockito.any(), Mockito.anyBoolean(), Mockito.anyBoolean()); + Mockito.doReturn(true).when(scaleWorker).stateTransitTo(Mockito.anyLong(), Mockito.any()); + + Assert.assertTrue(scaleWorker.scaleCluster()); + + Mockito.verify(scaleWorker).scaleKubernetesClusterOffering(WORKER, target, true, false); + } + @Test public void testGetWorkerNodesToRemoveForDownsize_singleRemoval() { From f92e52d736b9491309f6be3baac250de18c198c9 Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Wed, 21 Oct 2026 17:31:12 +0200 Subject: [PATCH 11/13] test(cks): cover live resize capacity reconciliation failure paths --- ...ClusterNodeCapacityReconcilerImplTest.java | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java index 7611f582f13b..0e37dd8a5839 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java @@ -23,6 +23,7 @@ import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.offering.ServiceOffering; import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -142,6 +143,175 @@ public void testOnlineGuestResourcesActivatesHotpluggedCpuAndMemory() throws Exc fakeReconciler.nodeCommands.get(0)); } + @Test + public void testParseMemoryDecimalBinaryAndUnitlessQuantities() { + Assert.assertEquals(123L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("123")); + Assert.assertEquals(2L * 1024L * 1024L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2Mi")); + Assert.assertEquals(2L * 1024L * 1024L * 1024L * 1024L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2Ti")); + Assert.assertEquals(2000L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2K")); + Assert.assertEquals(2000000L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2M")); + Assert.assertEquals(2000000000L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2G")); + Assert.assertEquals(2000000000000L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2T")); + Assert.assertThrows(IllegalArgumentException.class, () -> KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("4Gi; reboot")); + } + + @Test + public void testRestartKubeletWaitsForActiveService() throws Exception { + FakeReconciler fake = new FakeReconciler(nodeJson(false)); + fake.nodeResources = " active\n"; + + fake.restartKubelet(vm("worker-1"), access(), System.currentTimeMillis() + 1000L); + + Assert.assertEquals(List.of("sudo systemctl restart kubelet", "sudo systemctl is-active kubelet"), fake.nodeCommands); + } + + @Test + public void testExpiredDeadlinesReportTheFailedPhaseAndVm() { + FakeReconciler fake = new FakeReconciler(nodeJson(false)); + KubernetesCluster cluster = cluster("cluster-uuid"); + UserVm vm = vm("worker-1"); + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot before = snapshot(2, 2048, true); + long expired = System.currentTimeMillis() - 1L; + + assertPhaseFailure("CORDON", () -> fake.cordonIfNeeded(cluster, vm, before, access(), expired)); + assertPhaseFailure("GUEST_VERIFY", () -> fake.verifyGuestResources(vm, offering(4, 4096), before, access(), expired)); + assertPhaseFailure("KUBELET_RESTART", () -> fake.restartKubelet(vm, access(), expired)); + assertPhaseFailure("CAPACITY_VERIFY", () -> fake.waitForKubernetesResources(cluster, vm, offering(4, 4096), before, access(), expired)); + assertPhaseFailure("RESTORE_SCHEDULABILITY", () -> fake.restoreSchedulability(cluster, vm, before, access(), expired)); + } + + @Test + public void testWaitForKubernetesResourcesRequiresIncreasedAllocatableCpuAndMemory() throws Exception { + FakeReconciler fake = new FakeReconciler(nodeJson(false).replace("3900m", "4")); + fake.nodeResources = "4\n4194304\n"; + + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot observed = fake.waitForKubernetesResources( + cluster("cluster-uuid"), vm("worker-1"), offering(4, 4096), snapshot(2, 2048, true), + access(), System.currentTimeMillis() + 1000L); + + Assert.assertTrue(observed.isReady()); + Assert.assertEquals(4000L, observed.getCapacityCpuMillis()); + Assert.assertEquals(3700L, observed.getAllocatableCpuMillis()); + Assert.assertEquals(3900L * 1024L * 1024L, observed.getAllocatableMemoryBytes()); + } + + @Test(timeout = 15000L) + public void testWaitRejectsUnchangedAllocatableCpuAfterCpuIncrease() { + FakeReconciler fake = new FakeReconciler(nodeJson(false).replace("3900m", "4").replace("3700m", "1900m")); + fake.nodeResources = "4\n4194304\n"; + + assertPhaseFailure("CAPACITY_VERIFY", () -> fake.waitForKubernetesResources(cluster("cluster-uuid"), vm("worker-1"), + offering(4, 4096), snapshot(2, 2048, true), access(), System.currentTimeMillis() + 1000L)); + + Assert.assertFalse(fake.controlCommands.isEmpty()); + Assert.assertTrue(fake.controlCommands.stream().noneMatch(command -> command.contains("uncordon"))); + } + + @Test(timeout = 15000L) + public void testWaitRejectsStaleAllocatableMemoryAfterMemoryIncrease() { + FakeReconciler fake = new FakeReconciler(nodeJson(false).replace("3900m", "4").replace("3900Mi", "1900Mi")); + fake.nodeResources = "4\n4194304\n"; + + assertPhaseFailure("CAPACITY_VERIFY", () -> fake.waitForKubernetesResources(cluster("cluster-uuid"), vm("worker-1"), + offering(4, 4096), snapshot(2, 2048, true), access(), System.currentTimeMillis() + 1000L)); + + Assert.assertFalse(fake.controlCommands.isEmpty()); + } + + @Test(timeout = 15000L) + public void testRestartKubeletDoesNotTreatInactiveServiceAsSuccess() { + FakeReconciler fake = new FakeReconciler(nodeJson(false)); + fake.nodeResources = "inactive\n"; + + assertPhaseFailure("KUBELET_RESTART", () -> fake.restartKubelet(vm("worker-1"), access(), System.currentTimeMillis() + 1000L)); + + Assert.assertEquals(List.of("sudo systemctl restart kubelet", "sudo systemctl is-active kubelet"), fake.nodeCommands); + } + + @Test + public void testResourceVerificationRejectsStaleNotReadyOrOutOfToleranceNodes() { + ServiceOffering target = offering(4, 4096); + Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(snapshot(4, 4096, false), target)); + Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(snapshot(2, 4096, true), target)); + Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(snapshot(4, 4097, true), target)); + Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(snapshot(4, 3800, true), target)); + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot staleMemory = + new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, 4, 4194304L, + 4000L, 2L * 1024L * 1024L * 1024L, 3700L, 1900L * 1024L * 1024L, true); + Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(staleMemory, target)); + } + + @Test + public void testCaptureRecognizesOnlyThisClustersCordonOwnership() throws Exception { + String owned = nodeJson(true).replace("\"annotations\":{}", + "\"annotations\":{\"cloudstack.apache.org/cks-live-resize\":\"cluster-uuid\"}"); + FakeReconciler fake = new FakeReconciler(owned); + Assert.assertTrue(fake.captureBefore(cluster("cluster-uuid"), vm("worker-1"), access()).isCloudStackResizeCordon()); + Assert.assertFalse(fake.captureBefore(cluster("another-cluster"), vm("worker-1"), access()).isCloudStackResizeCordon()); + } + + @Test + public void testCaptureHandlesMissingOptionalFieldsAndNotReadyNode() throws Exception { + String minimal = nodeJson(false).replace("\"annotations\":{}", "\"name\":\"worker-1\"") + .replace("\"spec\":{\"unschedulable\":false},", "") + .replace("\"status\":\"True\"", "\"status\":\"False\""); + FakeReconciler fake = new FakeReconciler(minimal); + KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot observed = + fake.captureBefore(cluster("cluster-uuid"), vm("worker-1"), access()); + Assert.assertFalse(observed.isReady()); + Assert.assertFalse(observed.isUnschedulable()); + Assert.assertFalse(observed.isCloudStackResizeCordon()); + } + + @Test + public void testInvalidNodeNameIsRejectedBeforeControlCommand() { + FakeReconciler fake = new FakeReconciler(nodeJson(false)); + Assert.assertThrows(CloudRuntimeException.class, + () -> fake.captureBefore(cluster("cluster-uuid"), vm("worker; reboot"), access())); + Assert.assertTrue(fake.controlCommands.isEmpty()); + } + + @Test + public void testNodeNameIsLowercaseInKubectlCommand() throws Exception { + FakeReconciler fake = new FakeReconciler(nodeJson(false)); + fake.captureBefore(cluster("cluster-uuid"), vm("Worker-1"), access()); + Assert.assertEquals("sudo /opt/bin/kubectl get node worker-1 -o json", fake.controlCommands.get(0)); + } + + @Test + public void testMalformedGuestResourceOutputStopsBeforeKubectl() { + FakeReconciler fake = new FakeReconciler(nodeJson(false)); + fake.nodeResources = "4\n"; + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, + () -> fake.captureBefore(cluster("cluster-uuid"), vm("worker-1"), access())); + Assert.assertEquals("Unable to read guest CPU and memory", error.getMessage()); + Assert.assertTrue(fake.controlCommands.isEmpty()); + } + + @Test + public void testNodeAccessPreservesSeparateControlAndWorkerEndpoints() { + java.io.File key = new java.io.File("test-only-key"); + KubernetesClusterNodeCapacityReconciler.NodeAccess access = + new KubernetesClusterNodeCapacityReconciler.NodeAccess("control", 2222, "worker", 2223, "cloud", key); + Assert.assertEquals("control", access.getControlAddress()); + Assert.assertEquals(2222, access.getControlPort()); + Assert.assertEquals("worker", access.getNodeAddress()); + Assert.assertEquals(2223, access.getNodePort()); + Assert.assertEquals("cloud", access.getUser()); + Assert.assertSame(key, access.getSshKeyFile()); + } + + private KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot snapshot(int cpu, int memoryMiB, boolean ready) { + long bytes = memoryMiB * 1024L * 1024L; + return new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, cpu, memoryMiB * 1024L, + cpu * 1000L, bytes, cpu * 1000L - 100L, bytes - 128L * 1024L * 1024L, ready); + } + + private void assertPhaseFailure(String phase, org.junit.function.ThrowingRunnable action) { + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, action); + Assert.assertTrue(error.getMessage(), error.getMessage().contains(phase + " failed for VM vm-uuid")); + } + private ServiceOffering offering(int cpu, int memory) { ServiceOffering offering = Mockito.mock(ServiceOffering.class); Mockito.when(offering.getCpu()).thenReturn(cpu); From a4021f52ef3fa1aa62d72612b199d05174867adc Mon Sep 17 00:00:00 2001 From: andrijapanicsb Date: Mon, 26 Oct 2026 20:44:25 +0100 Subject: [PATCH 12/13] CKS: simplify live resize orchestration and narrow failure handling --- .../KubernetesClusterScaleWorker.java | 173 +++++---- ...bernetesClusterNodeCapacityReconciler.java | 14 +- ...etesClusterNodeCapacityReconcilerImpl.java | 64 +++- .../NodeCapacityReconciliationException.java | 32 ++ ...bernetesClusterNodeOfferingResizeTest.java | 327 ++++++++++++++++++ ...ClusterNodeCapacityReconcilerImplTest.java | 86 +++++ 6 files changed, 608 insertions(+), 88 deletions(-) create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeCapacityReconciliationException.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizeTest.java diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 0b53c89b6183..6a91a347eb28 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -43,6 +43,7 @@ import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.VirtualMachineMigrationException; import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; @@ -376,79 +377,123 @@ protected void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType List vmList = kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(), nodeType); final long tobeScaledVMCount = Math.min(vmList.size(), size); for (long i = 0; i < tobeScaledVMCount; i++) { - KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i); - UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); - if (userVM == null) { + new NodeOfferingResize(vmList.get((int) i), nodeType, serviceOffering).execute(); + if (System.currentTimeMillis() > scaleTimeoutTime) { + logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, scaling action timed out", kubernetesCluster.getName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } + kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); + } + + private final class NodeOfferingResize { + private final KubernetesClusterVmMapVO vmMap; + private final KubernetesClusterNodeType nodeType; + private final ServiceOffering targetOffering; + private final ServiceOffering oldOffering; + private UserVmVO vm; + private NodeAccess access; + private NodeCapacitySnapshot before; + private boolean reconcileKubernetes; + private boolean cksCordonCompleted; + private boolean resizeSucceeded; + + private NodeOfferingResize(KubernetesClusterVmMapVO vmMap, KubernetesClusterNodeType nodeType, ServiceOffering targetOffering) { + this.vmMap = vmMap; + this.nodeType = nodeType; + this.targetOffering = targetOffering; + vm = userVmDao.findById(vmMap.getVmId()); + if (vm == null) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, unable to find cluster VM %s", - kubernetesCluster.getName(), vmMapVO.getVmId()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + kubernetesCluster.getName(), vmMap.getVmId()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + oldOffering = serviceOfferingDao.findById(vm.getServiceOfferingId()); + resizeSucceeded = Objects.equals(targetOffering.getId(), vm.getServiceOfferingId()); + } + + private void execute() { + validateLiveResize(); + try { + prepareReconciliation(); + resizeVm(); + reconcileCapacity(); + } catch (ResourceUnavailableException | ManagementServerException | VirtualMachineMigrationException | CloudRuntimeException e) { + restoreAfterFailedResize(); + reportFailure(e); } - ServiceOffering oldOffering = serviceOfferingDao.findById(userVM.getServiceOfferingId()); - boolean capacityChanged = KubernetesClusterNodeCapacityReconcilerImpl.capacityChanged(oldOffering, serviceOffering); - if (capacityChanged && KubernetesCluster.State.Running == originalState && ETCD == nodeType + } + + private void validateLiveResize() { + if (KubernetesClusterNodeCapacityReconcilerImpl.capacityChanged(oldOffering, targetOffering) + && KubernetesCluster.State.Running == originalState && ETCD == nodeType && KubernetesCluster.ClusterType.CloudManaged == kubernetesCluster.getClusterType()) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s cannot live-resize dedicated etcd VM %s; " + - "etcd health reconciliation is not implemented", kubernetesCluster.getName(), userVM.getDisplayName()), + "etcd health reconciliation is not implemented", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - boolean reconcileKubernetes = shouldReconcileNodeCapacity(vmMapVO, userVM, oldOffering, serviceOffering, nodeType); - boolean reconciliationRetryCandidate = isReconciliationRetryCandidate(vmMapVO, userVM, serviceOffering, nodeType); - NodeAccess nodeAccess = null; - NodeCapacitySnapshot before = null; - boolean result = false; - boolean cksCordonCompleted = false; - boolean resizeSucceeded = Objects.equals(serviceOffering.getId(), userVM.getServiceOfferingId()); - try { - if (reconcileKubernetes || reconciliationRetryCandidate) { - nodeAccess = resolveNodeAccess(userVM); - before = kubernetesClusterNodeCapacityReconciler.captureBefore(kubernetesCluster, userVM, nodeAccess); - reconcileKubernetes = reconcileKubernetes || before.isCloudStackResizeCordon(); - } - if (reconcileKubernetes) { - kubernetesClusterNodeCapacityReconciler.cordonIfNeeded(kubernetesCluster, userVM, before, nodeAccess, scaleTimeoutTime); - cksCordonCompleted = !before.isUnschedulable() || before.isCloudStackResizeCordon(); - } - if (!Objects.equals(serviceOffering.getId(), userVM.getServiceOfferingId())) { - result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); - if (!result) { - logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s", kubernetesCluster.getName(), userVM.getDisplayName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - resizeSucceeded = true; - userVM = userVmDao.findById(userVM.getId()); - if (userVM == null || !Objects.equals(serviceOffering.getId(), userVM.getServiceOfferingId())) { - logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, VM %s did not reach target offering", kubernetesCluster.getName(), vmMapVO.getVmId()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - } - if (reconcileKubernetes) { - kubernetesClusterNodeCapacityReconciler.onlineGuestResources(nodeAccess); - kubernetesClusterNodeCapacityReconciler.verifyGuestResources(userVM, serviceOffering, before, nodeAccess, scaleTimeoutTime); - if (!kubernetesClusterNodeCapacityReconciler.isKubernetesResourcesCurrent(before, serviceOffering)) { - kubernetesClusterNodeCapacityReconciler.restartKubelet(userVM, nodeAccess, scaleTimeoutTime); - kubernetesClusterNodeCapacityReconciler.waitForKubernetesResources(kubernetesCluster, userVM, serviceOffering, before, nodeAccess, scaleTimeoutTime); - } - kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster, userVM, before, nodeAccess, scaleTimeoutTime); - } - } catch (Exception e) { - if (reconcileKubernetes && cksCordonCompleted && !resizeSucceeded) { - try { - kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster, userVM, before, nodeAccess, scaleTimeoutTime); - } catch (Exception cleanupException) { - logger.warn("Unable to restore schedulability for CKS node {} after a pre-resize failure", userVM.getUuid(), cleanupException); - } - } - String recovery = resizeSucceeded - ? "The node remains cordoned; check kubelet on the VM and retry the CKS scale operation." - : "The VM was not resized; CKS attempted to restore its temporary cordon."; - String message = String.format("Scaling Kubernetes cluster %s (UUID %s) failed for %s node VM %s (UUID %s), " + - "offering %s to %s: %s %s", kubernetesCluster.getName(), kubernetesCluster.getUuid(), nodeType, - userVM.getDisplayName(), userVM.getUuid(), oldOffering == null ? null : oldOffering.getId(), - serviceOffering.getId(), e.getMessage(), recovery); - logTransitStateAndThrow(Level.ERROR, message, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } + + private void prepareReconciliation() { + reconcileKubernetes = shouldReconcileNodeCapacity(vmMap, vm, oldOffering, targetOffering, nodeType); + if (reconcileKubernetes || isReconciliationRetryCandidate(vmMap, vm, targetOffering, nodeType)) { + access = resolveNodeAccess(vm); + before = kubernetesClusterNodeCapacityReconciler.captureBefore(kubernetesCluster, vm, access); + reconcileKubernetes = reconcileKubernetes || before.isCloudStackResizeCordon(); } - if (System.currentTimeMillis() > scaleTimeoutTime) { - logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, scaling action timed out", kubernetesCluster.getName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + if (reconcileKubernetes) { + kubernetesClusterNodeCapacityReconciler.cordonIfNeeded(kubernetesCluster, vm, before, access, scaleTimeoutTime); + cksCordonCompleted = !before.isUnschedulable() || before.isCloudStackResizeCordon(); } } - kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); + + private void resizeVm() throws ResourceUnavailableException, ManagementServerException, VirtualMachineMigrationException { + if (resizeSucceeded) { + return; + } + if (!userVmManager.upgradeVirtualMachine(vm.getId(), targetOffering.getId(), new HashMap<>())) { + throw new CloudRuntimeException("Unable to scale cluster VM " + vm.getDisplayName()); + } + resizeSucceeded = true; + UserVmVO resizedVm = userVmDao.findById(vm.getId()); + if (resizedVm == null || !Objects.equals(targetOffering.getId(), resizedVm.getServiceOfferingId())) { + throw new CloudRuntimeException("VM " + vm.getUuid() + " did not reach target offering"); + } + vm = resizedVm; + } + + private void reconcileCapacity() { + if (!reconcileKubernetes) { + return; + } + kubernetesClusterNodeCapacityReconciler.onlineGuestResources(access); + kubernetesClusterNodeCapacityReconciler.verifyGuestResources(vm, targetOffering, before, access, scaleTimeoutTime); + if (!kubernetesClusterNodeCapacityReconciler.isKubernetesResourcesCurrent(before, targetOffering)) { + kubernetesClusterNodeCapacityReconciler.restartKubelet(vm, access, scaleTimeoutTime); + kubernetesClusterNodeCapacityReconciler.waitForKubernetesResources(kubernetesCluster, vm, targetOffering, before, access, scaleTimeoutTime); + } + kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster, vm, before, access, scaleTimeoutTime); + } + + private void restoreAfterFailedResize() { + if (!reconcileKubernetes || !cksCordonCompleted || resizeSucceeded) { + return; + } + try { + kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster, vm, before, access, scaleTimeoutTime); + } catch (CloudRuntimeException cleanupException) { + logger.warn("Unable to restore schedulability for CKS node {} after a pre-resize failure", vm.getUuid(), cleanupException); + } + } + + private void reportFailure(Exception cause) { + String recovery = resizeSucceeded + ? "The node remains cordoned; check kubelet on the VM and retry the CKS scale operation." + : "The VM was not resized; CKS attempted to restore its temporary cordon."; + String message = String.format("Scaling Kubernetes cluster %s (UUID %s) failed for %s node VM %s (UUID %s), " + + "offering %s to %s: %s %s", kubernetesCluster.getName(), kubernetesCluster.getUuid(), nodeType, + vm.getDisplayName(), vm.getUuid(), oldOffering == null ? null : oldOffering.getId(), + targetOffering.getId(), cause.getMessage(), recovery); + logTransitStateAndThrow(Level.ERROR, message, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, cause); + } } private NodeAccess resolveNodeAccess(UserVm userVM) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java index 3fbabca6e322..18e7a2c17742 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java @@ -25,17 +25,17 @@ /** Reconciles the guest and Kubernetes resource views after a live VM resize. */ public interface KubernetesClusterNodeCapacityReconciler { - NodeCapacitySnapshot captureBefore(KubernetesCluster cluster, UserVm vm, NodeAccess access) throws Exception; + NodeCapacitySnapshot captureBefore(KubernetesCluster cluster, UserVm vm, NodeAccess access); boolean requiresKubeletRefresh(ServiceOffering oldOffering, ServiceOffering newOffering, KubernetesClusterNodeType nodeType, KubernetesCluster.State originalState); - void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception; - void onlineGuestResources(NodeAccess access) throws Exception; - void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception; + void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline); + void onlineGuestResources(NodeAccess access); + void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline); boolean isKubernetesResourcesCurrent(NodeCapacitySnapshot snapshot, ServiceOffering target); - void restartKubelet(UserVm vm, NodeAccess access, long deadline) throws Exception; + void restartKubelet(UserVm vm, NodeAccess access, long deadline); NodeCapacitySnapshot waitForKubernetesResources(KubernetesCluster cluster, UserVm vm, ServiceOffering target, - NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception; - void restoreSchedulability(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception; + NodeCapacitySnapshot before, NodeAccess access, long deadline); + void restoreSchedulability(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline); final class NodeAccess { private final String controlAddress; private final int controlPort; private final String nodeAddress; private final int nodePort; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java index bec9b91a83e9..ad7cf71099b6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java @@ -59,12 +59,12 @@ public boolean requiresKubeletRefresh(ServiceOffering oldOffering, ServiceOfferi } @Override - public NodeCapacitySnapshot captureBefore(KubernetesCluster cluster, UserVm vm, NodeAccess access) throws Exception { + public NodeCapacitySnapshot captureBefore(KubernetesCluster cluster, UserVm vm, NodeAccess access) { return nodeSnapshot(cluster, vm, access, guestResources(access)); } @Override - public void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception { + public void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) { if (before.isUnschedulable() && !before.isCloudStackResizeCordon()) return; String node = nodeName(vm); executeControl(access, "sudo /opt/bin/kubectl annotate node " + node + " " + RESIZE_ANNOTATION + "=" + cluster.getUuid() + " --overwrite"); @@ -77,12 +77,12 @@ public void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySna } @Override - public void onlineGuestResources(NodeAccess access) throws Exception { + public void onlineGuestResources(NodeAccess access) { executeNode(access, ONLINE_HOTPLUGGED_RESOURCES); } @Override - public void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception { + public void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline) { while (System.currentTimeMillis() < deadline) { if (guestMatchesTarget(guestResources(access), target)) return; sleep(); @@ -97,7 +97,7 @@ public boolean isKubernetesResourcesCurrent(NodeCapacitySnapshot snapshot, Servi } @Override - public void restartKubelet(UserVm vm, NodeAccess access, long deadline) throws Exception { + public void restartKubelet(UserVm vm, NodeAccess access, long deadline) { executeNode(access, "sudo systemctl restart kubelet"); while (System.currentTimeMillis() < deadline) { if ("active".equals(executeNode(access, "sudo systemctl is-active kubelet").trim())) return; @@ -108,7 +108,7 @@ public void restartKubelet(UserVm vm, NodeAccess access, long deadline) throws E @Override public NodeCapacitySnapshot waitForKubernetesResources(KubernetesCluster cluster, UserVm vm, ServiceOffering target, - NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception { + NodeCapacitySnapshot before, NodeAccess access, long deadline) { while (System.currentTimeMillis() < deadline) { GuestResources guest = guestResources(access); NodeCapacitySnapshot observed = nodeSnapshot(cluster, vm, access, guest); @@ -119,7 +119,7 @@ public NodeCapacitySnapshot waitForKubernetesResources(KubernetesCluster cluster } @Override - public void restoreSchedulability(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception { + public void restoreSchedulability(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) { if (before.isUnschedulable() && !before.isCloudStackResizeCordon()) return; String node = nodeName(vm); executeControl(access, "sudo /opt/bin/kubectl uncordon " + node); @@ -151,7 +151,7 @@ static long parseMemoryBytes(String value) { } } - private NodeCapacitySnapshot nodeSnapshot(KubernetesCluster cluster, UserVm vm, NodeAccess access, GuestResources guest) throws Exception { + private NodeCapacitySnapshot nodeSnapshot(KubernetesCluster cluster, UserVm vm, NodeAccess access, GuestResources guest) { JsonObject node = new JsonParser().parse(executeControl(access, "sudo /opt/bin/kubectl get node " + nodeName(vm) + " -o json")).getAsJsonObject(); JsonObject annotations = node.getAsJsonObject("metadata").has("annotations") ? node.getAsJsonObject("metadata").getAsJsonObject("annotations") : null; boolean cksCordon = annotations != null && annotations.has(RESIZE_ANNOTATION) && cluster.getUuid().equals(annotations.get(RESIZE_ANNOTATION).getAsString()); @@ -169,7 +169,7 @@ private boolean isReady(JsonObject status) { return false; } - private GuestResources guestResources(NodeAccess access) throws Exception { + private GuestResources guestResources(NodeAccess access) { String[] values = executeNode(access, "getconf _NPROCESSORS_ONLN; awk '/^MemTotal:/ {print $2}' /proc/meminfo").trim().split("\\s+"); if (values.length != 2) throw new CloudRuntimeException("Unable to read guest CPU and memory"); return new GuestResources(Long.parseLong(values[0]), Long.parseLong(values[1])); @@ -188,15 +188,45 @@ private boolean kubernetesMatchesGuest(NodeCapacitySnapshot observed, NodeCapaci && (observed.getGuestMemoryKiB() <= before.getGuestMemoryKiB() || observed.getAllocatableMemoryBytes() > before.getAllocatableMemoryBytes()); } - protected String executeControl(NodeAccess access, String command) throws Exception { return execute(access.getControlAddress(), access.getControlPort(), access, command); } - protected String executeNode(NodeAccess access, String command) throws Exception { return execute(access.getNodeAddress(), access.getNodePort(), access, command); } - private String execute(String address, int port, NodeAccess access, String command) throws Exception { - Pair result = SshHelper.sshExecute(address, port, access.getUser(), access.getSshKeyFile(), null, command, 10000, 10000, COMMAND_TIMEOUT_MS); - if (Boolean.TRUE.equals(result.first())) return StringUtils.defaultString(result.second()); - throw new CloudRuntimeException("CKS live-resize command failed"); + protected String executeControl(NodeAccess access, String command) { + return execute(access.getControlAddress(), access.getControlPort(), access, command); + } + + protected String executeNode(NodeAccess access, String command) { + return execute(access.getNodeAddress(), access.getNodePort(), access, command); + } + + private String execute(String address, int port, NodeAccess access, String command) { + Pair result; + try { + result = SshHelper.sshExecute(address, port, access.getUser(), access.getSshKeyFile(), null, command, 10000, 10000, COMMAND_TIMEOUT_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new NodeCapacityReconciliationException("CKS live-resize command interrupted", e); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + // SshHelper declares Exception, including authentication failures. + // Translate only at this legacy boundary, not around the resize workflow. + throw new NodeCapacityReconciliationException("CKS live-resize SSH command failed", e); + } + if (!Boolean.TRUE.equals(result.first())) { + throw new NodeCapacityReconciliationException("CKS live-resize command failed"); + } + return StringUtils.defaultString(result.second()); } private String nodeName(UserVm vm) { String name = StringUtils.lowerCase(vm.getHostName()); if (!NODE_NAME.matcher(name).matches()) throw new CloudRuntimeException("Invalid Kubernetes node name for live resize"); return name; } - private CloudRuntimeException failure(String phase, UserVm vm, String message) { return new CloudRuntimeException("CKS live resize " + phase + " failed for VM " + vm.getUuid() + ": " + message); } - private void sleep() throws InterruptedException { Thread.sleep(POLL_INTERVAL_MS); } + private NodeCapacityReconciliationException failure(String phase, UserVm vm, String message) { + return new NodeCapacityReconciliationException("CKS live resize " + phase + " failed for VM " + vm.getUuid() + ": " + message); + } + + private void sleep() { + try { + Thread.sleep(POLL_INTERVAL_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new NodeCapacityReconciliationException("CKS live-resize resource polling interrupted", e); + } + } private static class GuestResources { private final long cpu; private final long memoryKiB; GuestResources(long cpu, long memoryKiB) { this.cpu = cpu; this.memoryKiB = memoryKiB; } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeCapacityReconciliationException.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeCapacityReconciliationException.java new file mode 100644 index 000000000000..e9ede51eef6b --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeCapacityReconciliationException.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import com.cloud.utils.exception.CloudRuntimeException; + +/** An operational failure while refreshing a node's guest or Kubernetes resources. */ +public class NodeCapacityReconciliationException extends CloudRuntimeException { + private static final long serialVersionUID = 1L; + + public NodeCapacityReconciliationException(String message) { + super(message); + } + + public NodeCapacityReconciliationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizeTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizeTest.java new file mode 100644 index 000000000000..0abcce43d45e --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizeTest.java @@ -0,0 +1,327 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.HashMap; +import java.util.List; + +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.VirtualMachineMigrationException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot; +import com.cloud.kubernetes.cluster.utils.NodeCapacityReconciliationException; +import com.cloud.network.Network; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.UserVmManager; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.dao.UserVmDao; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; + +public class KubernetesClusterNodeOfferingResizeTest { + private final KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); + private final KubernetesClusterManagerImpl manager = Mockito.mock(KubernetesClusterManagerImpl.class); + private final KubernetesClusterVmMapDao vmMapDao = Mockito.mock(KubernetesClusterVmMapDao.class); + private final UserVmDao vmDao = Mockito.mock(UserVmDao.class); + private final UserVmManager vmManager = Mockito.mock(UserVmManager.class); + private final ServiceOfferingDao offeringDao = Mockito.mock(ServiceOfferingDao.class); + private final KubernetesClusterNodeCapacityReconciler reconciler = Mockito.mock(KubernetesClusterNodeCapacityReconciler.class); + private final ServiceOfferingVO oldOffering = offering(10L, 2, 2048); + private final ServiceOfferingVO target = offering(20L, 4, 4096); + private final UserVmVO first = vm(1L, 10L); + private final UserVmVO second = vm(2L, 10L); + private final UserVmVO resizedFirst = vm(1L, 20L); + private final UserVmVO resizedSecond = vm(2L, 20L); + private KubernetesClusterScaleWorker worker; + + @Before + public void setUp() throws Exception { + Mockito.when(cluster.getId()).thenReturn(100L); + Mockito.when(cluster.getName()).thenReturn("test-cluster"); + Mockito.when(cluster.getUuid()).thenReturn("cluster-uuid"); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(cluster.getNodeCount()).thenReturn(2L); + Mockito.when(cluster.getTotalNodeCount()).thenReturn(2L); + worker = Mockito.spy(new KubernetesClusterScaleWorker(cluster, new HashMap<>(), null, null, null, null, null, manager)); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Scaling); + worker.userVmDao = vmDao; + worker.userVmManager = vmManager; + worker.serviceOfferingDao = offeringDao; + worker.kubernetesClusterVmMapDao = vmMapDao; + worker.kubernetesClusterNodeCapacityReconciler = reconciler; + ReflectionTestUtils.setField(worker, "scaleTimeoutTime", System.currentTimeMillis() + 60000L); + ReflectionTestUtils.setField(worker, "network", Mockito.mock(Network.class)); + Mockito.when(manager.isDirectAccess(any())).thenReturn(true); + Mockito.doReturn(new Pair<>("control", 22)).when(worker).getKubernetesClusterServerIpSshPort(null); + Mockito.doReturn("cloud").when(worker).getControlNodeLoginUser(); + Mockito.doReturn(null).when(worker).getManagementServerSshPublicKeyFile(); + Mockito.doReturn(true).when(worker).stateTransitTo(anyLong(), any()); + Mockito.doNothing().when(worker).deleteTemplateLaunchPermission(); + Mockito.doReturn(Mockito.mock(KubernetesClusterVO.class)).when(worker).updateKubernetesClusterEntry( + any(), any(), any(), any(), any(), any(), any(), eq(WORKER), anyBoolean(), anyBoolean()); + List maps = List.of(vmMap(1L), vmMap(2L)); + Mockito.when(vmMapDao.listByClusterId(100L)).thenReturn(maps); + Mockito.when(vmMapDao.listByClusterIdAndVmType(100L, WORKER)).thenReturn(maps); + Mockito.when(offeringDao.findById(10L)).thenReturn(oldOffering); + Mockito.when(offeringDao.findById(20L)).thenReturn(target); + Mockito.when(vmDao.findById(1L)).thenReturn(first); + Mockito.when(vmDao.findById(2L)).thenReturn(second); + Mockito.when(vmManager.upgradeVirtualMachine(eq(1L), eq(20L), any())).thenAnswer(invocation -> { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedFirst); + return true; + }); + Mockito.when(vmManager.upgradeVirtualMachine(eq(2L), eq(20L), any())).thenAnswer(invocation -> { + Mockito.when(vmDao.findById(2L)).thenReturn(resizedSecond); + return true; + }); + Mockito.when(reconciler.requiresKubeletRefresh(oldOffering, target, WORKER, KubernetesCluster.State.Running)).thenReturn(true); + Mockito.when(reconciler.captureBefore(eq(cluster), any(), any())).thenReturn(snapshot(false, false)); + } + + @Test + public void testNodesCompleteReconciliationSequentially() throws Exception { + scale(); + + InOrder order = Mockito.inOrder(reconciler, vmManager); + order.verify(reconciler).cordonIfNeeded(eq(cluster), eq(first), any(), any(), anyLong()); + order.verify(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + order.verify(reconciler).verifyGuestResources(eq(resizedFirst), eq(target), any(), any(), anyLong()); + order.verify(reconciler).restartKubelet(eq(resizedFirst), any(), anyLong()); + order.verify(reconciler).waitForKubernetesResources(eq(cluster), eq(resizedFirst), eq(target), any(), any(), anyLong()); + order.verify(reconciler).restoreSchedulability(eq(cluster), eq(resizedFirst), any(), any(), anyLong()); + order.verify(reconciler).cordonIfNeeded(eq(cluster), eq(second), any(), any(), anyLong()); + order.verify(vmManager).upgradeVirtualMachine(eq(2L), eq(20L), any()); + order.verify(reconciler).restoreSchedulability(eq(cluster), eq(resizedSecond), any(), any(), anyLong()); + } + + @Test + public void testResourceUnavailableRestoresTemporaryCordon() throws Exception { + assertPreResizeFailure(new ResourceUnavailableException("host unavailable", UserVmVO.class, 1L)); + } + + @Test + public void testManagementFailureRestoresTemporaryCordon() throws Exception { + assertPreResizeFailure(new ManagementServerException("upgrade failed")); + } + + @Test + public void testMigrationFailureRestoresTemporaryCordon() throws Exception { + assertPreResizeFailure(new VirtualMachineMigrationException("migration failed")); + } + + @Test + public void testRejectedVmResizeRestoresTemporaryCordon() throws Exception { + Mockito.doReturn(false).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); + Assert.assertTrue(error.getMessage().contains("The VM was not resized")); + Mockito.verify(reconciler).restoreSchedulability(eq(cluster), eq(first), any(), any(), anyLong()); + assertSecondNodeUntouched(); + } + + @Test + public void testFailedCleanupPreservesOriginalResizeFailure() throws Exception { + ManagementServerException failure = new ManagementServerException("original failure"); + Mockito.doThrow(new NodeCapacityReconciliationException("cleanup failed")).when(reconciler) + .restoreSchedulability(eq(cluster), eq(first), any(), any(), anyLong()); + assertPreResizeFailure(failure); + } + + @Test + public void testOperatorCordonIsNotRestoredAfterRejectedResize() throws Exception { + Mockito.when(reconciler.captureBefore(eq(cluster), eq(first), any())).thenReturn(snapshot(true, false)); + Mockito.doReturn(false).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + Assert.assertThrows(CloudRuntimeException.class, this::scale); + Mockito.verify(reconciler, Mockito.never()).restoreSchedulability(any(), any(), any(), any(), anyLong()); + assertSecondNodeUntouched(); + } + + @Test + public void testPostResizeFailureLeavesNodeCordonedAndSecondNodeUntouched() { + NodeCapacityReconciliationException failure = new NodeCapacityReconciliationException("guest verification failed"); + Mockito.doThrow(failure).when(reconciler).verifyGuestResources(eq(resizedFirst), eq(target), any(), any(), anyLong()); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); + + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(error.getMessage().contains("The node remains cordoned")); + Mockito.verify(reconciler, Mockito.never()).restoreSchedulability(any(), any(), any(), any(), anyLong()); + assertSecondNodeUntouched(); + } + + @Test + public void testSnapshotFailureStopsBeforeCordonOrVmResize() { + NodeCapacityReconciliationException failure = new NodeCapacityReconciliationException("SSH unavailable"); + Mockito.when(reconciler.captureBefore(eq(cluster), eq(first), any())).thenThrow(failure); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); + + Assert.assertSame(failure, error.getCause()); + Mockito.verifyNoInteractions(vmManager); + Mockito.verify(reconciler, Mockito.never()).cordonIfNeeded(any(), any(), any(), any(), anyLong()); + Mockito.verify(reconciler, Mockito.never()).restoreSchedulability(any(), any(), any(), any(), anyLong()); + assertSecondNodeUntouched(); + } + + @Test + public void testCordonFailureStopsBeforeVmResize() { + Mockito.doThrow(new NodeCapacityReconciliationException("cordon failed")).when(reconciler) + .cordonIfNeeded(eq(cluster), eq(first), any(), any(), anyLong()); + + Assert.assertThrows(CloudRuntimeException.class, this::scale); + + Mockito.verifyNoInteractions(vmManager); + Mockito.verify(reconciler, Mockito.never()).restoreSchedulability(any(), any(), any(), any(), anyLong()); + assertSecondNodeUntouched(); + } + + @Test + public void testRetryReconcilesAlreadyResizedNodeBeforeResizingNext() throws Exception { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedFirst); + Mockito.when(reconciler.captureBefore(eq(cluster), eq(resizedFirst), any())).thenReturn(snapshot(true, true)); + + scale(); + + Mockito.verify(vmManager, Mockito.never()).upgradeVirtualMachine(eq(1L), anyLong(), any()); + InOrder order = Mockito.inOrder(reconciler, vmManager); + order.verify(reconciler).restartKubelet(eq(resizedFirst), any(), anyLong()); + order.verify(reconciler).restoreSchedulability(eq(cluster), eq(resizedFirst), any(), any(), anyLong()); + order.verify(vmManager).upgradeVirtualMachine(eq(2L), eq(20L), any()); + } + + @Test + public void testCurrentResourcesDoNotRestartKubelet() { + Mockito.when(reconciler.isKubernetesResourcesCurrent(any(), eq(target))).thenReturn(true); + scale(); + Mockito.verify(reconciler, Mockito.never()).restartKubelet(any(), any(), anyLong()); + Mockito.verify(reconciler).restoreSchedulability(eq(cluster), eq(resizedFirst), any(), any(), anyLong()); + } + + @Test + public void testStoppedClusterResizesWithoutGuestOrKubeletOperations() throws Exception { + ReflectionTestUtils.setField(worker, "originalState", KubernetesCluster.State.Stopped); + scale(); + Mockito.verify(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + Mockito.verify(vmManager).upgradeVirtualMachine(eq(2L), eq(20L), any()); + Mockito.verify(reconciler, Mockito.never()).captureBefore(any(), any(), any()); + Mockito.verify(reconciler, Mockito.never()).restartKubelet(any(), any(), anyLong()); + } + + @Test + public void testCreatedClusterUpdatesMetadataWithoutTouchingVms() { + ReflectionTestUtils.setField(worker, "originalState", KubernetesCluster.State.Created); + scale(); + Mockito.verifyNoInteractions(vmDao, vmManager, reconciler); + Mockito.verify(worker).updateKubernetesClusterEntry(eq(8L), eq(8192L), any(), eq(20L), + any(), any(), any(), eq(WORKER), eq(true), eq(false)); + } + + @Test + public void testMissingVmAfterSuccessfulResizeDoesNotHideFailureWithNullPointer() throws Exception { + Mockito.doAnswer(invocation -> { + Mockito.when(vmDao.findById(1L)).thenReturn(null); + return true; + }).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); + + Assert.assertTrue(error.getMessage().contains("VM vm-1 did not reach target offering")); + Mockito.verify(reconciler, Mockito.never()).restoreSchedulability(any(), any(), any(), any(), anyLong()); + assertSecondNodeUntouched(); + } + + @Test + public void testUnexpectedProgrammingExceptionIsNotWrappedAsOperationalFailure() throws Exception { + NullPointerException failure = new NullPointerException("programming error"); + Mockito.doThrow(failure).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + Assert.assertSame(failure, Assert.assertThrows(NullPointerException.class, this::scale)); + + Mockito.verify(worker, Mockito.never()).stateTransitTo(100L, KubernetesCluster.Event.OperationFailed); + assertSecondNodeUntouched(); + } + + private void scale() { + worker.scaleKubernetesClusterOffering(WORKER, target, true, false); + } + + private void assertPreResizeFailure(Exception failure) throws Exception { + Mockito.doThrow(failure).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); + Assert.assertSame(failure, error.getCause()); + Mockito.verify(reconciler).restoreSchedulability(eq(cluster), eq(first), any(), any(), anyLong()); + Mockito.verify(worker).stateTransitTo(100L, KubernetesCluster.Event.OperationFailed); + assertSecondNodeUntouched(); + } + + private void assertSecondNodeUntouched() { + Mockito.verify(vmDao, Mockito.never()).findById(2L); + Mockito.verify(reconciler, Mockito.never()).captureBefore(eq(cluster), eq(second), any()); + Mockito.verify(worker, Mockito.never()).updateKubernetesClusterEntry( + any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), anyBoolean()); + } + + private ServiceOfferingVO offering(long id, int cpu, int memory) { + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(offering.getId()).thenReturn(id); + Mockito.when(offering.getCpu()).thenReturn(cpu); + Mockito.when(offering.getRamSize()).thenReturn(memory); + return offering; + } + + private UserVmVO vm(long id, long offeringId) { + UserVmVO vm = Mockito.mock(UserVmVO.class); + Mockito.when(vm.getId()).thenReturn(id); + Mockito.when(vm.getUuid()).thenReturn("vm-" + id); + Mockito.when(vm.getDisplayName()).thenReturn("worker-" + id); + Mockito.when(vm.getPrivateIpAddress()).thenReturn("node-" + id); + Mockito.when(vm.getServiceOfferingId()).thenReturn(offeringId); + Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.KVM); + return vm; + } + + private KubernetesClusterVmMapVO vmMap(long id) { + KubernetesClusterVmMapVO map = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(map.getVmId()).thenReturn(id); + return map; + } + + private NodeCapacitySnapshot snapshot(boolean unschedulable, boolean cksCordon) { + return new NodeCapacitySnapshot(unschedulable, cksCordon, 2L, 2097152L, + 2000L, 2147483648L, 1900L, 2013265920L, true); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java index 0e37dd8a5839..8d2876d8f35e 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java @@ -23,9 +23,12 @@ import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.offering.ServiceOffering; import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; import org.junit.Assert; import org.junit.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; public class KubernetesClusterNodeCapacityReconcilerImplTest { @@ -301,6 +304,89 @@ public void testNodeAccessPreservesSeparateControlAndWorkerEndpoints() { Assert.assertSame(key, access.getSshKeyFile()); } + @Test + public void testSshSuccessReturnsOutput() { + KubernetesClusterNodeCapacityReconcilerImpl real = new KubernetesClusterNodeCapacityReconcilerImpl(); + KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); + try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { + ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) + .thenReturn(new Pair<>(true, "output")); + Assert.assertEquals("output", real.executeNode(access, "command")); + } + } + + @Test + public void testRejectedSshCommandHasSpecificReconciliationFailure() { + KubernetesClusterNodeCapacityReconcilerImpl real = new KubernetesClusterNodeCapacityReconcilerImpl(); + KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); + try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { + ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) + .thenReturn(new Pair<>(false, "remote error")); + Assert.assertThrows(NodeCapacityReconciliationException.class, () -> real.executeNode(access, "command")); + } + } + + @Test + public void testLegacySshExceptionIsTranslatedAtTransportBoundary() { + KubernetesClusterNodeCapacityReconcilerImpl real = new KubernetesClusterNodeCapacityReconcilerImpl(); + KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); + Exception failure = new Exception("SSH authentication failed"); + try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { + ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) + .thenThrow(failure); + NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, + () -> real.executeNode(access, "command")); + Assert.assertSame(failure, error.getCause()); + } + } + + @Test + public void testSshInterruptionPreservesInterruptFlag() { + KubernetesClusterNodeCapacityReconcilerImpl real = new KubernetesClusterNodeCapacityReconcilerImpl(); + KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); + InterruptedException failure = new InterruptedException("cancelled"); + try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { + ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) + .thenThrow(failure); + NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, + () -> real.executeNode(access, "command")); + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + @Test + public void testSshProgrammingErrorsAreNotTranslated() { + KubernetesClusterNodeCapacityReconcilerImpl real = new KubernetesClusterNodeCapacityReconcilerImpl(); + KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); + NullPointerException failure = new NullPointerException("programming error"); + try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { + ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) + .thenThrow(failure); + Assert.assertSame(failure, Assert.assertThrows(NullPointerException.class, () -> real.executeNode(access, "command"))); + } + } + + @Test + public void testPollingInterruptionPreservesInterruptFlag() { + FakeReconciler fake = new FakeReconciler(nodeJson(false)); + UserVm vm = vm("worker-1"); + KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); + long deadline = System.currentTimeMillis() + 10000L; + fake.nodeResources = "inactive"; + Thread.currentThread().interrupt(); + try { + NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, + () -> fake.restartKubelet(vm, access, deadline)); + Assert.assertTrue(error.getCause() instanceof InterruptedException); + Assert.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + private KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot snapshot(int cpu, int memoryMiB, boolean ready) { long bytes = memoryMiB * 1024L * 1024L; return new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, cpu, memoryMiB * 1024L, From 306cc2e02f7bb6c3dae6f6b72a39dc0a5e4df0e5 Mon Sep 17 00:00:00 2001 From: Andrija Panic Date: Tue, 27 Oct 2026 23:21:18 +0100 Subject: [PATCH 13/13] CKS: rework live resize reconciliation after review - Move the per-node resize into KubernetesClusterNodeOfferingResizer and make the reconciler a plain class with small top-level value classes. - Mark a node as pending and cordon it in one conditional kubectl patch, and finish or release it with one patch too. Conflicts, rejected patches and lost replies are settled by reading the node again, a limited number of times. - Keep the pending marker after any failure. Remove the CKS cordon only when the VM did not reach the target offering, so a retry of the same scale finishes the work. A VM already on the target offering is skipped after one control-plane read unless it still has unfinished work. - Refuse the resize before changing anything when the guest already has more CPUs or memory than the target offering. - Check guest memory by its online memory blocks instead of MemTotal. MemTotal leaves out what the kernel keeps for itself, which grows with the memory the VM booted with, so the old margin rejected healthy nodes of about 5 to 20 GiB, for example on a CPU-only change. - Restart kubelet only when the node is not up to date or kubelet is not active, and check that allocatable is between zero and capacity instead of comparing with an earlier snapshot. - Catch RuntimeException at the per-node boundary again, as on main, so an unexpected error still cleans up and marks the scale as failed. - Resize dedicated etcd VMs the same way as on main. - Reach a node through the IP of its own SSH port-forwarding rule, which fixes Netris VPC tiers. - Share the per-type offering lookup, include removed offerings for the current offering and the cluster totals, and check the offerings of all affected node types before any VM is resized. - Check the scale timeout before each node instead of after it. - Revert an unrelated whitespace change in the Spring context. --- .../KubernetesClusterActionWorker.java | 10 +- .../KubernetesClusterNodeOfferingResizer.java | 284 +++++++ .../KubernetesClusterScaleWorker.java | 251 ++---- .../cluster/utils/GuestResources.java | 63 ++ ...bernetesClusterNodeCapacityReconciler.java | 502 ++++++++++- ...etesClusterNodeCapacityReconcilerImpl.java | 232 ------ .../cluster/utils/KubernetesClusterUtil.java | 24 +- .../kubernetes/cluster/utils/NodeAccess.java | 67 ++ .../cluster/utils/NodeCapacitySnapshot.java | 106 +++ .../cluster/utils/NodeResources.java | 36 + .../spring-kubernetes-service-context.xml | 1 + ...bernetesClusterNodeOfferingResizeTest.java | 327 -------- ...ernetesClusterNodeOfferingResizerTest.java | 563 +++++++++++++ ...ernetesClusterScaleWorkerOfferingTest.java | 253 ++++++ .../KubernetesClusterScaleWorkerTest.java | 72 +- ...ClusterNodeCapacityReconcilerImplTest.java | 451 ---------- ...etesClusterNodeCapacityReconcilerTest.java | 781 ++++++++++++++++++ .../utils/KubernetesClusterUtilTest.java | 18 + 18 files changed, 2756 insertions(+), 1285 deletions(-) create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizer.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/GuestResources.java delete mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeAccess.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeCapacitySnapshot.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeResources.java delete mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizeTest.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizerTest.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerOfferingTest.java delete mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerTest.java diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 07c9f12a50e2..2b937da95a85 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -976,6 +976,14 @@ public void setKeys(String[] keys) { protected ServiceOffering getServiceOfferingForNodeTypeOnCluster(KubernetesClusterNodeType nodeType, KubernetesCluster cluster) { + return serviceOfferingDao.findById(getServiceOfferingIdForNodeTypeOnCluster(nodeType, cluster)); + } + + /** + * The offering ID set for a node type on the cluster, falling back to the cluster offering. + */ + protected Long getServiceOfferingIdForNodeTypeOnCluster(KubernetesClusterNodeType nodeType, + KubernetesCluster cluster) { Long offeringId = null; Long defaultOfferingId = cluster.getServiceOfferingId(); Long controlOfferingId = cluster.getControlNodeServiceOfferingId(); @@ -994,7 +1002,7 @@ protected ServiceOffering getServiceOfferingForNodeTypeOnCluster(KubernetesClust logger.error(msg); throw new CloudRuntimeException(msg); } - return serviceOfferingDao.findById(offeringId); + return offeringId; } protected boolean isDefaultTemplateUsed() { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizer.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizer.java new file mode 100644 index 000000000000..d9b5c7812239 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizer.java @@ -0,0 +1,284 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.HashMap; +import java.util.function.Function; + +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.VirtualMachineMigrationException; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.kubernetes.cluster.utils.GuestResources; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler; +import com.cloud.kubernetes.cluster.utils.NodeAccess; +import com.cloud.kubernetes.cluster.utils.NodeCapacitySnapshot; +import com.cloud.kubernetes.cluster.utils.NodeCapacityReconciliationException; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.UserVmManager; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.dao.UserVmDao; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + +/** + * Moves the VMs of a CKS cluster to a new service offering one at a time. For control and worker nodes of a + * running CloudManaged KVM cluster it also makes Kubernetes report the new CPU and memory before the next + * VM is changed. Other VMs, including dedicated etcd VMs, get the plain VM resize. + */ +public class KubernetesClusterNodeOfferingResizer { + protected Logger logger = LogManager.getLogger(getClass()); + + private final KubernetesCluster kubernetesCluster; + private final boolean clusterRunning; + private final UserVmDao userVmDao; + private final ServiceOfferingDao serviceOfferingDao; + private final UserVmManager userVmManager; + private final KubernetesClusterNodeCapacityReconciler reconciler; + private final Function nodeAccessResolver; + + public KubernetesClusterNodeOfferingResizer(KubernetesCluster kubernetesCluster, + boolean clusterRunning, + UserVmDao userVmDao, + ServiceOfferingDao serviceOfferingDao, + UserVmManager userVmManager, + KubernetesClusterNodeCapacityReconciler reconciler, + Function nodeAccessResolver) { + this.kubernetesCluster = kubernetesCluster; + this.clusterRunning = clusterRunning; + this.userVmDao = userVmDao; + this.serviceOfferingDao = serviceOfferingDao; + this.userVmManager = userVmManager; + this.reconciler = reconciler; + this.nodeAccessResolver = nodeAccessResolver; + } + + /** + * Moves one VM to the target offering. A failure is thrown as a CloudRuntimeException that keeps the + * original failure as its cause and says what state the VM and its Kubernetes node were left in. + */ + public void resize(KubernetesClusterVmMapVO vmMap, KubernetesClusterNodeType nodeType, ServiceOffering targetOffering, long deadline) { + UserVmVO vm = userVmDao.findById(vmMap.getVmId()); + if (vm == null) { + throw new CloudRuntimeException(String.format("unable to find the VM with ID %d", vmMap.getVmId())); + } + String context = String.format("unable to scale %s node VM %s (%s) to offering %s", nodeType, vm.getDisplayName(), vm.getUuid(), + targetOffering.getName()); + boolean onTarget = isOnTarget(vm, targetOffering); + String skipReason = getReconciliationSkipReason(vmMap, vm, nodeType); + if (skipReason != null) { + logSkip(vm, skipReason); + if (!onTarget) { + resizeWithoutReconciliation(vm, targetOffering, context); + } + return; + } + if (onTarget) { + finishPendingReconciliation(vm, targetOffering, deadline, context, "it is already on the target offering"); + return; + } + ServiceOffering currentOffering = serviceOfferingDao.findByIdIncludingRemoved(vm.getServiceOfferingId()); + if (currentOffering == null) { + throw new CloudRuntimeException(context + ": the current offering of the VM was not found. The VM was not resized."); + } + if (!KubernetesClusterNodeCapacityReconciler.capacityChanged(currentOffering, targetOffering)) { + resizeWithoutReconciliation(vm, targetOffering, context); + finishPendingReconciliation(vm, targetOffering, deadline, context, "the offering does not change its CPU or RAM"); + return; + } + resizeAndReconcile(vm, targetOffering, deadline, context); + } + + private String getReconciliationSkipReason(KubernetesClusterVmMapVO vmMap, UserVm vm, KubernetesClusterNodeType nodeType) { + if (vmMap.isExternalNode()) { + return "it is an external node"; + } + if (KubernetesCluster.ClusterType.CloudManaged != kubernetesCluster.getClusterType()) { + return "the cluster is not CloudManaged"; + } + if (Hypervisor.HypervisorType.KVM != vm.getHypervisorType()) { + return "it is not a KVM VM"; + } + if (!clusterRunning) { + return "the cluster was not Running"; + } + if (WORKER != nodeType && CONTROL != nodeType) { + return "only control and worker VMs are Kubernetes nodes"; + } + return null; + } + + private void logSkip(UserVm vm, String reason) { + logger.info("Not reconciling the Kubernetes capacity of VM {} in Kubernetes cluster {}: {}", vm, kubernetesCluster, reason); + } + + private boolean isOnTarget(UserVm vm, ServiceOffering targetOffering) { + return targetOffering.getId() == vm.getServiceOfferingId(); + } + + private void resizeWithoutReconciliation(UserVm vm, ServiceOffering targetOffering, String context) { + boolean resized; + try { + resized = userVmManager.upgradeVirtualMachine(vm.getId(), targetOffering.getId(), new HashMap<>()); + } catch (ResourceUnavailableException | ManagementServerException | VirtualMachineMigrationException | RuntimeException e) { + throw new CloudRuntimeException(String.format("%s: %s", context, describe(e)), e); + } + if (!resized) { + throw new CloudRuntimeException(context); + } + } + + /** + * The VM is on the target offering and its CPU and RAM need no change; finish its node only if an earlier + * resize left it unverified. Only the Node is read when there is nothing to finish. + */ + private void finishPendingReconciliation(UserVm vm, ServiceOffering targetOffering, long deadline, String context, String skipReason) { + try { + NodeAccess access = nodeAccessResolver.apply(vm); + NodeCapacitySnapshot node = reconciler.readNode(vm, access); + String clusterUuid = kubernetesCluster.getUuid(); + if (node.hasForeignMarker(clusterUuid)) { + throw new NodeCapacityReconciliationException("the Kubernetes node carries live resize markers of another cluster"); + } + if (!node.isPendingFor(clusterUuid) && !node.hasOwnershipMarker(clusterUuid)) { + logSkip(vm, skipReason); + return; + } + logger.info("Finishing the unverified Kubernetes capacity reconciliation of VM {} in Kubernetes cluster {}", vm, kubernetesCluster); + reconcile(vm, targetOffering, access, node.getUid(), deadline); + } catch (RuntimeException e) { + throw new CloudRuntimeException(String.format("%s: %s. The VM is already on the target offering; retry the same scale operation " + + "to finish reconciling its Kubernetes node.", context, describe(e)), e); + } + } + + private void resizeAndReconcile(UserVmVO vm, ServiceOffering targetOffering, long deadline, String context) { + NodeAccess access; + NodeCapacitySnapshot initial; + try { + access = nodeAccessResolver.apply(vm); + initial = reconciler.readNode(vm, access); + if (initial.hasForeignMarker(kubernetesCluster.getUuid())) { + throw new NodeCapacityReconciliationException("the Kubernetes node carries live resize markers of another cluster"); + } + GuestResources guest = reconciler.readGuest(vm, access); + if (KubernetesClusterNodeCapacityReconciler.isGuestAboveTarget(guest, targetOffering)) { + throw new CloudRuntimeException(String.format("the VM already has %d CPUs and %d MiB of memory online, more than the requested " + + "offering; use an offering at least as large as the VM's current resources", guest.getOnlineCpus(), + guest.getOnlineMemoryBytes() / ByteScaleUtils.MiB)); + } + } catch (RuntimeException e) { + throw new CloudRuntimeException(String.format("%s: %s. The VM was not resized and its Kubernetes node was not changed.", + context, describe(e)), e); + } + NodeCapacitySnapshot started; + try { + started = reconciler.startResize(kubernetesCluster, vm, access, initial.getUid()); + } catch (RuntimeException e) { + throw new CloudRuntimeException(String.format("%s: %s. The VM was not resized; its Kubernetes node may already be marked for " + + "the resize and cordoned. Retry the same scale operation.", context, describe(e)), e); + } + try { + UserVmVO resizedVm = upgrade(vm, targetOffering); + reconcile(resizedVm, targetOffering, access, initial.getUid(), deadline); + } catch (ResourceUnavailableException | ManagementServerException | VirtualMachineMigrationException | RuntimeException e) { + String recovery = recoverAfterFailure(vm, targetOffering, access, initial, started, e); + throw new CloudRuntimeException(String.format("%s: %s. %s", context, describe(e), recovery), e); + } + } + + private UserVmVO upgrade(UserVm vm, ServiceOffering targetOffering) + throws ResourceUnavailableException, ManagementServerException, VirtualMachineMigrationException { + if (!userVmManager.upgradeVirtualMachine(vm.getId(), targetOffering.getId(), new HashMap<>())) { + throw new CloudRuntimeException("the VM could not be scaled"); + } + UserVmVO resizedVm = userVmDao.findById(vm.getId()); + if (resizedVm == null || !isOnTarget(resizedVm, targetOffering)) { + throw new CloudRuntimeException("the VM did not reach the target offering"); + } + return resizedVm; + } + + /** + * After a failed or uncertain resize the pending marker always stays, so a retry finds the node. The CKS + * cordon stays only when the VM is on the target offering; otherwise the next scale resizes the VM again + * (KVM only adds what is missing) and the cordon would just keep the node unschedulable until then. + */ + private String recoverAfterFailure(UserVm vm, ServiceOffering targetOffering, NodeAccess access, + NodeCapacitySnapshot initial, NodeCapacitySnapshot started, Exception failure) { + String retry = " Retry the same scale operation to finish it."; + if (Thread.currentThread().isInterrupted()) { + return "The operation was interrupted and the Kubernetes node was left as it is." + retry; + } + UserVmVO currentVm = null; + try { + currentVm = userVmDao.findById(vm.getId()); + } catch (RuntimeException e) { + failure.addSuppressed(e); + } + if (currentVm == null) { + return "The VM could not be read again, so its Kubernetes node was left as it is." + retry; + } + String clusterUuid = kubernetesCluster.getUuid(); + if (isOnTarget(currentVm, targetOffering)) { + String cordon = started.isCordonOwnedBy(clusterUuid) ? " and cordoned" : ""; + return String.format("The VM was resized, but its Kubernetes node is not reconciled yet and stays marked for reconciliation%s.%s", + cordon, retry); + } + String cordon = ""; + // Release only a cordon this attempt added: the node was schedulable before it started. + if (started.isCordonOwnedBy(clusterUuid) && !initial.isUnschedulable()) { + try { + reconciler.releaseCordon(kubernetesCluster, vm, access, initial.getUid()); + cordon = " CKS removed its temporary cordon from the Kubernetes node."; + } catch (RuntimeException e) { + failure.addSuppressed(e); + cordon = String.format(" CKS could not remove its temporary cordon from the Kubernetes node: %s.", describe(e)); + } + } + return "The VM may have been partly resized on its host." + cordon + retry; + } + + private void reconcile(UserVm vm, ServiceOffering targetOffering, NodeAccess access, String nodeUid, long deadline) { + reconciler.onlineGuestResources(access); + GuestResources guest = reconciler.waitForGuestResources(vm, targetOffering, access, deadline); + if (!reconciler.isKubernetesNodeAlreadyResized(reconciler.readNode(vm, access), guest) || !reconciler.isKubeletActive(access)) { + reconciler.restartKubelet(vm, access, deadline); + reconciler.waitForKubernetesResources(vm, guest, access, deadline); + } + NodeCapacitySnapshot node = reconciler.finishResize(kubernetesCluster, vm, access, nodeUid); + logger.info("Kubernetes node of VM {} in Kubernetes cluster {} reports {}m CPU and {} bytes of memory after the resize and is {}", + vm, kubernetesCluster, node.getCapacity().getCpuMillis(), node.getCapacity().getMemoryBytes(), + node.isUnschedulable() ? "unschedulable" : "schedulable"); + } + + private static String describe(Exception e) { + return StringUtils.defaultIfEmpty(e.getMessage(), e.getClass().getName()); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 6a91a347eb28..c484fe550909 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -19,9 +19,8 @@ import java.io.File; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; +import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -43,8 +42,6 @@ import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.exception.VirtualMachineMigrationException; -import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterService; @@ -52,9 +49,7 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler; -import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler.NodeAccess; -import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot; -import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconcilerImpl; +import com.cloud.kubernetes.cluster.utils.NodeAccess; import com.cloud.network.IpAddress; import com.cloud.network.Network; import com.cloud.network.rules.FirewallRule; @@ -64,6 +59,7 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.NetUtils; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.UserVmVO; import org.apache.logging.log4j.Level; @@ -75,6 +71,9 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModifierActionWorker { + // Control nodes are scaled first and workers last. + private static final List SCALING_ORDER = List.of(CONTROL, ETCD, WORKER); + private Map serviceOfferingNodeTypeMap; private Long clusterSize; private List nodeIds; @@ -85,7 +84,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif private Boolean isAutoscalingEnabled; private long scaleTimeoutTime; - protected KubernetesClusterNodeCapacityReconciler kubernetesClusterNodeCapacityReconciler = new KubernetesClusterNodeCapacityReconcilerImpl(); + protected KubernetesClusterNodeCapacityReconciler kubernetesClusterNodeCapacityReconciler = new KubernetesClusterNodeCapacityReconciler(); protected KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); @@ -219,7 +218,7 @@ private KubernetesClusterVO updateKubernetesClusterEntryForNodeType(final Long n final ServiceOffering newServiceOffering, final boolean updateNodeOffering, boolean updateClusterOffering) throws CloudRuntimeException { final ServiceOffering serviceOffering = newServiceOffering == null ? - serviceOfferingDao.findById(getConfiguredOfferingIdForNodeType(nodeType)) : newServiceOffering; + getOfferingIncludingRemovedForNodeType(nodeType) : newServiceOffering; final Long serviceOfferingId = newServiceOffering == null ? null : serviceOffering.getId(); Pair clusterCountAndCapacity = calculateNewClusterCountAndCapacity(newWorkerSize, nodeType, serviceOffering); @@ -240,13 +239,12 @@ protected Pair calculateNewClusterCountAndCapacity(Long newWorkerSiz long memory = 0; // VM offerings have already changed here. Recompute the total from the // configured node types instead of subtracting a supposedly old VM offering. - for (KubernetesClusterNodeType type : List.of(CONTROL, WORKER, ETCD)) { + for (KubernetesClusterNodeType type : CLUSTER_NODES_TYPES_LIST) { long nodeCount = WORKER == type && newWorkerSize != null ? newWorkerSize : getNodeCountForType(type, kubernetesCluster); if (nodeCount == 0) { continue; } - ServiceOffering offering = nodeType == DEFAULT || nodeType == type ? serviceOffering : - serviceOfferingDao.findByIdIncludingRemoved(getConfiguredOfferingIdForNodeType(type)); + ServiceOffering offering = nodeType == DEFAULT || nodeType == type ? serviceOffering : getOfferingIncludingRemovedForNodeType(type); Pair capacity = calculateNodesCapacity(offering, nodeCount); cores += capacity.first(); memory += capacity.second(); @@ -254,16 +252,18 @@ protected Pair calculateNewClusterCountAndCapacity(Long newWorkerSiz return new Pair<>(cores, memory); } - private Long getConfiguredOfferingIdForNodeType(KubernetesClusterNodeType nodeType) { - Long offeringId = null; - if (WORKER == nodeType) { - offeringId = kubernetesCluster.getWorkerNodeServiceOfferingId(); - } else if (CONTROL == nodeType) { - offeringId = kubernetesCluster.getControlNodeServiceOfferingId(); - } else if (ETCD == nodeType) { - offeringId = kubernetesCluster.getEtcdNodeServiceOfferingId(); + /** + * The offering the existing nodes of a type run with. It may have been deleted since, which must not + * break the cluster's CPU and memory totals. + */ + private ServiceOffering getOfferingIncludingRemovedForNodeType(KubernetesClusterNodeType nodeType) { + Long offeringId = getServiceOfferingIdForNodeTypeOnCluster(nodeType, kubernetesCluster); + ServiceOffering offering = serviceOfferingDao.findByIdIncludingRemoved(offeringId); + if (offering == null) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to find the service offering of its %s nodes", + kubernetesCluster.getName(), nodeType), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - return offeringId == null ? kubernetesCluster.getServiceOfferingId() : offeringId; + return offering; } private long getNodeCountForType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) { @@ -375,128 +375,35 @@ protected void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType } final long size = getNodeCountForType(nodeType, kubernetesCluster); List vmList = kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(), nodeType); - final long tobeScaledVMCount = Math.min(vmList.size(), size); - for (long i = 0; i < tobeScaledVMCount; i++) { - new NodeOfferingResize(vmList.get((int) i), nodeType, serviceOffering).execute(); + final int tobeScaledVMCount = (int) Math.min(vmList.size(), size); + KubernetesClusterNodeOfferingResizer nodeResizer = createNodeOfferingResizer(); + for (int i = 0; i < tobeScaledVMCount; i++) { if (System.currentTimeMillis() > scaleTimeoutTime) { - logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, scaling action timed out", kubernetesCluster.getName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - } - kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); - } - - private final class NodeOfferingResize { - private final KubernetesClusterVmMapVO vmMap; - private final KubernetesClusterNodeType nodeType; - private final ServiceOffering targetOffering; - private final ServiceOffering oldOffering; - private UserVmVO vm; - private NodeAccess access; - private NodeCapacitySnapshot before; - private boolean reconcileKubernetes; - private boolean cksCordonCompleted; - private boolean resizeSucceeded; - - private NodeOfferingResize(KubernetesClusterVmMapVO vmMap, KubernetesClusterNodeType nodeType, ServiceOffering targetOffering) { - this.vmMap = vmMap; - this.nodeType = nodeType; - this.targetOffering = targetOffering; - vm = userVmDao.findById(vmMap.getVmId()); - if (vm == null) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, unable to find cluster VM %s", - kubernetesCluster.getName(), vmMap.getVmId()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, scaling action timed out after %d of %d %s VMs were scaled", + kubernetesCluster.getName(), i, tobeScaledVMCount, nodeType), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - oldOffering = serviceOfferingDao.findById(vm.getServiceOfferingId()); - resizeSucceeded = Objects.equals(targetOffering.getId(), vm.getServiceOfferingId()); - } - - private void execute() { - validateLiveResize(); try { - prepareReconciliation(); - resizeVm(); - reconcileCapacity(); - } catch (ResourceUnavailableException | ManagementServerException | VirtualMachineMigrationException | CloudRuntimeException e) { - restoreAfterFailedResize(); - reportFailure(e); - } - } - - private void validateLiveResize() { - if (KubernetesClusterNodeCapacityReconcilerImpl.capacityChanged(oldOffering, targetOffering) - && KubernetesCluster.State.Running == originalState && ETCD == nodeType - && KubernetesCluster.ClusterType.CloudManaged == kubernetesCluster.getClusterType()) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s cannot live-resize dedicated etcd VM %s; " + - "etcd health reconciliation is not implemented", kubernetesCluster.getName(), vm.getDisplayName()), - kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - } - - private void prepareReconciliation() { - reconcileKubernetes = shouldReconcileNodeCapacity(vmMap, vm, oldOffering, targetOffering, nodeType); - if (reconcileKubernetes || isReconciliationRetryCandidate(vmMap, vm, targetOffering, nodeType)) { - access = resolveNodeAccess(vm); - before = kubernetesClusterNodeCapacityReconciler.captureBefore(kubernetesCluster, vm, access); - reconcileKubernetes = reconcileKubernetes || before.isCloudStackResizeCordon(); - } - if (reconcileKubernetes) { - kubernetesClusterNodeCapacityReconciler.cordonIfNeeded(kubernetesCluster, vm, before, access, scaleTimeoutTime); - cksCordonCompleted = !before.isUnschedulable() || before.isCloudStackResizeCordon(); - } - } - - private void resizeVm() throws ResourceUnavailableException, ManagementServerException, VirtualMachineMigrationException { - if (resizeSucceeded) { - return; - } - if (!userVmManager.upgradeVirtualMachine(vm.getId(), targetOffering.getId(), new HashMap<>())) { - throw new CloudRuntimeException("Unable to scale cluster VM " + vm.getDisplayName()); - } - resizeSucceeded = true; - UserVmVO resizedVm = userVmDao.findById(vm.getId()); - if (resizedVm == null || !Objects.equals(targetOffering.getId(), resizedVm.getServiceOfferingId())) { - throw new CloudRuntimeException("VM " + vm.getUuid() + " did not reach target offering"); - } - vm = resizedVm; - } - - private void reconcileCapacity() { - if (!reconcileKubernetes) { - return; - } - kubernetesClusterNodeCapacityReconciler.onlineGuestResources(access); - kubernetesClusterNodeCapacityReconciler.verifyGuestResources(vm, targetOffering, before, access, scaleTimeoutTime); - if (!kubernetesClusterNodeCapacityReconciler.isKubernetesResourcesCurrent(before, targetOffering)) { - kubernetesClusterNodeCapacityReconciler.restartKubelet(vm, access, scaleTimeoutTime); - kubernetesClusterNodeCapacityReconciler.waitForKubernetesResources(kubernetesCluster, vm, targetOffering, before, access, scaleTimeoutTime); - } - kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster, vm, before, access, scaleTimeoutTime); - } - - private void restoreAfterFailedResize() { - if (!reconcileKubernetes || !cksCordonCompleted || resizeSucceeded) { - return; - } - try { - kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster, vm, before, access, scaleTimeoutTime); - } catch (CloudRuntimeException cleanupException) { - logger.warn("Unable to restore schedulability for CKS node {} after a pre-resize failure", vm.getUuid(), cleanupException); + nodeResizer.resize(vmList.get(i), nodeType, serviceOffering, scaleTimeoutTime); + } catch (RuntimeException e) { + String reason = StringUtils.defaultIfEmpty(e.getMessage(), e.getClass().getName()); + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, %s", kubernetesCluster.getName(), reason), + kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } } + kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); + } - private void reportFailure(Exception cause) { - String recovery = resizeSucceeded - ? "The node remains cordoned; check kubelet on the VM and retry the CKS scale operation." - : "The VM was not resized; CKS attempted to restore its temporary cordon."; - String message = String.format("Scaling Kubernetes cluster %s (UUID %s) failed for %s node VM %s (UUID %s), " + - "offering %s to %s: %s %s", kubernetesCluster.getName(), kubernetesCluster.getUuid(), nodeType, - vm.getDisplayName(), vm.getUuid(), oldOffering == null ? null : oldOffering.getId(), - targetOffering.getId(), cause.getMessage(), recovery); - logTransitStateAndThrow(Level.ERROR, message, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, cause); - } + protected KubernetesClusterNodeOfferingResizer createNodeOfferingResizer() { + return new KubernetesClusterNodeOfferingResizer(kubernetesCluster, + KubernetesCluster.State.Running == originalState, + userVmDao, + serviceOfferingDao, + userVmManager, + kubernetesClusterNodeCapacityReconciler, + this::resolveNodeAccess); } - private NodeAccess resolveNodeAccess(UserVm userVM) { + protected NodeAccess resolveNodeAccess(UserVm userVM) { Pair controlAccess = getKubernetesClusterServerIpSshPort(null); if (StringUtils.isBlank(controlAccess.first())) { throw new CloudRuntimeException(String.format("Unable to resolve control-plane SSH for Kubernetes cluster %s", kubernetesCluster.getUuid())); @@ -508,40 +415,25 @@ private NodeAccess resolveNodeAccess(UserVm userVM) { return new NodeAccess(controlAccess.first(), controlAccess.second(), userVM.getPrivateIpAddress(), DEFAULT_SSH_PORT, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile()); } + // The node is reached through its own SSH port-forwarding rule. Use that rule's public IP: on Netris + // VPC tiers it is the NAT IP, not the load balancer IP of the cluster. PortForwardingRuleVO sshRule = portForwardingRulesDao.listByVm(userVM.getId()).stream() .filter(rule -> rule.getDestinationPortStart() == DEFAULT_SSH_PORT) - .filter(rule -> !FirewallRule.State.Revoke.equals(rule.getState())) + .filter(rule -> NetUtils.TCP_PROTO.equalsIgnoreCase(rule.getProtocol())) + .filter(rule -> rule.getNetworkId() == network.getId()) + .filter(rule -> FirewallRule.State.Revoke != rule.getState()) .findFirst().orElse(null); if (sshRule == null) { throw new CloudRuntimeException(String.format("Unable to resolve SSH port-forwarding rule for VM %s", userVM.getUuid())); } - IpAddress targetPublicIp = network.getVpcId() == null ? getNetworkSourceNatIp(network) : getVpcTierKubernetesPublicIp(network); - if (targetPublicIp == null) { - throw new CloudRuntimeException(String.format("Unable to resolve target-node public IP for VM %s", userVM.getUuid())); + IpAddress sshIp = sshRule.getSourceIpAddressId() == null ? null : ipAddressDao.findById(sshRule.getSourceIpAddressId()); + if (sshIp == null) { + throw new CloudRuntimeException(String.format("Unable to resolve the public IP of the SSH port-forwarding rule for VM %s", userVM.getUuid())); } - return new NodeAccess(controlAccess.first(), controlAccess.second(), targetPublicIp.getAddress().addr(), sshRule.getSourcePortStart(), + return new NodeAccess(controlAccess.first(), controlAccess.second(), sshIp.getAddress().addr(), sshRule.getSourcePortStart(), getControlNodeLoginUser(), getManagementServerSshPublicKeyFile()); } - protected boolean shouldReconcileNodeCapacity(KubernetesClusterVmMapVO vmMapVO, UserVm userVM, - ServiceOffering oldOffering, ServiceOffering targetOffering, - KubernetesClusterNodeType nodeType) { - return !vmMapVO.isExternalNode() - && KubernetesCluster.ClusterType.CloudManaged == kubernetesCluster.getClusterType() - && Hypervisor.HypervisorType.KVM == userVM.getHypervisorType() - && kubernetesClusterNodeCapacityReconciler.requiresKubeletRefresh(oldOffering, targetOffering, nodeType, originalState); - } - - protected boolean isReconciliationRetryCandidate(KubernetesClusterVmMapVO vmMapVO, UserVm userVM, - ServiceOffering targetOffering, KubernetesClusterNodeType nodeType) { - return !vmMapVO.isExternalNode() - && KubernetesCluster.ClusterType.CloudManaged == kubernetesCluster.getClusterType() - && Hypervisor.HypervisorType.KVM == userVM.getHypervisorType() - && (WORKER == nodeType || CONTROL == nodeType) - && KubernetesCluster.State.Running == originalState - && Objects.equals(targetOffering.getId(), userVM.getServiceOfferingId()); - } - private void removeNodesFromCluster(List vmMaps) throws CloudRuntimeException { for (KubernetesClusterVmMapVO vmMapVO : vmMaps) { UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); @@ -749,18 +641,15 @@ public boolean scaleCluster() throws CloudRuntimeException { final boolean autoscalingChanged = isAutoscalingChanged(); ServiceOffering defaultServiceOffering = serviceOfferingNodeTypeMap.getOrDefault(DEFAULT.name(), null); + Map existingOfferings = getExistingOfferingsOfScaledNodeTypes(scaleClusterDefaultOffering, originalClusterSize); - for (KubernetesClusterNodeType nodeType : Arrays.asList(CONTROL, ETCD, WORKER)) { - boolean isWorkerNode = WORKER == nodeType; - final long newVMRequired = (!isWorkerNode || clusterSize == null) ? 0 : clusterSize - originalClusterSize; - if (nodeType == ETCD && kubernetesCluster.getEtcdNodeCount() == 0) { + for (KubernetesClusterNodeType nodeType : SCALING_ORDER) { + ServiceOffering existingServiceOffering = existingOfferings.get(nodeType); + if (existingServiceOffering == null) { continue; } - if (!scaleClusterDefaultOffering && !serviceOfferingNodeTypeMap.containsKey(nodeType.name()) && newVMRequired == 0) { - continue; - } - - ServiceOffering existingServiceOffering = getExistingServiceOfferingForNodeType(nodeType, kubernetesCluster); + boolean isWorkerNode = WORKER == nodeType; + final long newVMRequired = (!isWorkerNode || clusterSize == null) ? 0 : clusterSize - originalClusterSize; ServiceOffering scalingServiceOffering = serviceOfferingNodeTypeMap.getOrDefault(nodeType.name(), defaultServiceOffering); boolean isNodeOfferingScalingNeeded = isServiceOfferingScalingNeededForNodeType(existingServiceOffering, scalingServiceOffering); boolean updateNodeOffering = serviceOfferingNodeTypeMap.containsKey(nodeType.name()) || isNodeOfferingScalingNeeded; @@ -794,12 +683,28 @@ public boolean scaleCluster() throws CloudRuntimeException { return true; } + /** + * Looks up the current offering of every node type this request changes before any VM is touched, so a + * missing offering stops the scale before, not after, the VMs of another node type were resized. + */ + private Map getExistingOfferingsOfScaledNodeTypes(boolean scaleClusterDefaultOffering, + long originalClusterSize) { + Map existingOfferings = new EnumMap<>(KubernetesClusterNodeType.class); + for (KubernetesClusterNodeType nodeType : SCALING_ORDER) { + boolean workerCountChanges = WORKER == nodeType && clusterSize != null && clusterSize != originalClusterSize; + if (nodeType == ETCD && kubernetesCluster.getEtcdNodeCount() == 0) { + continue; + } + if (!scaleClusterDefaultOffering && !serviceOfferingNodeTypeMap.containsKey(nodeType.name()) && !workerCountChanges) { + continue; + } + existingOfferings.put(nodeType, getExistingServiceOfferingForNodeType(nodeType, kubernetesCluster)); + } + return existingOfferings; + } + private ServiceOffering getExistingServiceOfferingForNodeType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) { Long existingOfferingId = getExistingOfferingIdForNodeType(nodeType, kubernetesCluster); - if (existingOfferingId == null) { - logAndThrow(Level.ERROR, String.format("The Kubernetes cluster %s does not have a service offering set for node type %s", - kubernetesCluster.getName(), nodeType.name())); - } ServiceOffering existingOffering = serviceOfferingDao.findById(existingOfferingId); if (existingOffering == null) { logAndThrow(Level.ERROR, String.format("Cannot find service offering with ID %s set on the Kubernetes cluster %s node type %s", @@ -827,6 +732,6 @@ protected boolean isServiceOfferingScalingNeededForNodeType(ServiceOffering exis protected Long getExistingOfferingIdForNodeType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) { // Cluster metadata is committed only after every selected VM succeeds. // A partially resized first VM must not make a retry skip the remaining nodes. - return getConfiguredOfferingIdForNodeType(nodeType); + return getServiceOfferingIdForNodeTypeOnCluster(nodeType, kubernetesCluster); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/GuestResources.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/GuestResources.java new file mode 100644 index 000000000000..52cc8a01e6dd --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/GuestResources.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; + +/** CPU and memory as seen inside a node VM. */ +public final class GuestResources { + private final long onlineCpus; + private final long memoryKiB; + private final long onlineMemoryBytes; + private final long memoryBlockBytes; + + /** + * @param onlineCpus CPUs online in the guest + * @param memoryKiB MemTotal from /proc/meminfo, which kubelet reports as the node's memory capacity + * @param onlineMemoryBytes size of the online memory blocks under /sys/devices/system/memory + * @param memoryBlockBytes size of one of those memory blocks + */ + public GuestResources(long onlineCpus, + long memoryKiB, + long onlineMemoryBytes, + long memoryBlockBytes) { + this.onlineCpus = onlineCpus; + this.memoryKiB = memoryKiB; + this.onlineMemoryBytes = onlineMemoryBytes; + this.memoryBlockBytes = memoryBlockBytes; + } + + public long getOnlineCpus() { + return onlineCpus; + } + + public long getMemoryKiB() { + return memoryKiB; + } + + public long getMemoryBytes() { + return memoryKiB * ByteScaleUtils.KiB; + } + + public long getOnlineMemoryBytes() { + return onlineMemoryBytes; + } + + public long getMemoryBlockBytes() { + return memoryBlockBytes; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java index 18e7a2c17742..e38ca3702456 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java @@ -16,53 +16,469 @@ // under the License. package com.cloud.kubernetes.cluster.utils; -import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.cloud.kubernetes.cluster.KubernetesCluster; -import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.offering.ServiceOffering; import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.ssh.SshHelper; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonParser; + +/** + * Brings the guest and Kubernetes views of a CKS node's CPU and memory up to date after a live VM resize, + * using kubectl on the control plane and commands on the node itself. + */ +public class KubernetesClusterNodeCapacityReconciler { + protected Logger logger = LogManager.getLogger(getClass()); + + static final long KUBERNETES_MEMORY_REPORTING_TOLERANCE_BYTES = 16L * ByteScaleUtils.MiB; + static final int MAX_NODE_UPDATE_ATTEMPTS = 3; + static final long POLL_INTERVAL_MS = 5000L; + static final String GET_NODE_COMMAND = "sudo /opt/bin/kubectl get node %s -o json --request-timeout=20s"; + // CPUs online, MemTotal in KiB, memory block size in hex bytes, and the number of online memory blocks. + static final String GUEST_RESOURCES_COMMAND = "getconf _NPROCESSORS_ONLN; awk '/^MemTotal:/ {print $2}' /proc/meminfo; " + + "cat /sys/devices/system/memory/block_size_bytes; cat /sys/devices/system/memory/memory*/state | grep -cx online"; + static final String RESTART_KUBELET_COMMAND = "sudo systemctl restart kubelet"; + // is-active exits non-zero for every state except active, so print the state instead of failing. + static final String KUBELET_STATE_COMMAND = "systemctl is-active kubelet 2>/dev/null || true"; + static final String ONLINE_HOTPLUGGED_RESOURCES = "sudo sh -c 'for cpu in /sys/devices/system/cpu/cpu[0-9]*; do " + + "[ -e \"$cpu/online\" ] || continue; echo 1 > \"$cpu/online\"; done; " + + "for memory in /sys/devices/system/memory/memory[0-9]*; do " + + "[ -e \"$memory/online\" ] || continue; echo online > \"$memory/online\"; done'"; + private static final int CONNECT_TIMEOUT_MS = 10000; + private static final int COMMAND_TIMEOUT_MS = 30000; + private static final Pattern NODE_NAME = Pattern.compile("[a-z0-9][a-z0-9.-]{0,252}"); + private static final Pattern PATCH_VALUE = Pattern.compile("[A-Za-z0-9-]{1,64}"); + private static final Pattern GUEST_VALUE = Pattern.compile("[1-9][0-9]{0,17}"); + private static final Pattern GUEST_HEX_VALUE = Pattern.compile("[0-9a-fA-F]{1,15}"); + private static final Pattern CPU_QUANTITY = Pattern.compile("([0-9]{1,18})(m)?"); + private static final Pattern MEMORY_QUANTITY = Pattern.compile("([0-9]{1,18})(Ki|Mi|Gi|Ti|K|M|G|T)?"); + + /** Whether the target offering changes the CPU count or the memory of the current one. */ + public static boolean capacityChanged(ServiceOffering currentOffering, ServiceOffering targetOffering) { + return !Objects.equals(currentOffering.getCpu(), targetOffering.getCpu()) + || !Objects.equals(currentOffering.getRamSize(), targetOffering.getRamSize()); + } + + /** Whether the guest has more CPUs online, or at least one memory block more, than the target offering provides. */ + public static boolean isGuestAboveTarget(GuestResources guest, ServiceOffering target) { + long targetMemory = target.getRamSize() * ByteScaleUtils.MiB; + return guest.getOnlineCpus() > target.getCpu() || guest.getOnlineMemoryBytes() - targetMemory >= guest.getMemoryBlockBytes(); + } + + /** + * Whether the guest has exactly the target CPUs online and its online memory blocks add up to the target + * memory. MemTotal is not compared with the offering: it leaves out the memory the kernel keeps for + * itself, which grows with the memory the VM booted with. The memory blocks add up to the RAM of the VM, + * rounded to a whole block, so a memory step smaller than one block (128 MiB on most VMs) is not seen. + */ + static boolean guestMatchesTarget(GuestResources guest, ServiceOffering target) { + long targetMemory = target.getRamSize() * ByteScaleUtils.MiB; + return guest.getOnlineCpus() == target.getCpu() + && Math.abs(guest.getOnlineMemoryBytes() - targetMemory) < guest.getMemoryBlockBytes(); + } + + /** + * Whether the Kubernetes node already reports the guest's current CPUs and memory. Kubelet computes + * allocatable as capacity minus its reservations in the same status update, so a Ready node whose + * capacity matches the guest and whose allocatable lies between zero and its capacity is up to date. + */ + public boolean isKubernetesNodeAlreadyResized(NodeCapacitySnapshot node, GuestResources guest) { + NodeResources capacity = node.getCapacity(); + NodeResources allocatable = node.getAllocatable(); + return node.isReady() + && capacity.getCpuMillis() == guest.getOnlineCpus() * 1000L + && Math.abs(capacity.getMemoryBytes() - guest.getMemoryBytes()) <= KUBERNETES_MEMORY_REPORTING_TOLERANCE_BYTES + && allocatable.getCpuMillis() >= 0 && allocatable.getCpuMillis() <= capacity.getCpuMillis() + && allocatable.getMemoryBytes() >= 0 && allocatable.getMemoryBytes() <= capacity.getMemoryBytes(); + } + + /** Reads the Kubernetes Node of a VM through the control plane. */ + public NodeCapacitySnapshot readNode(UserVm vm, NodeAccess access) { + String output = executeControl(access, String.format(GET_NODE_COMMAND, nodeName(vm))); + try { + return parseNode(output); + } catch (JsonParseException | IllegalArgumentException | ArithmeticException e) { + throw failure("READ_NODE", vm, "unexpected kubectl output: " + e.getMessage(), e); + } + } + + /** Reads the CPUs online, MemTotal and the online memory blocks inside the VM. */ + public GuestResources readGuest(UserVm vm, NodeAccess access) { + String[] values = executeNode(access, GUEST_RESOURCES_COMMAND).trim().split("\\s+"); + if (values.length != 4 || !GUEST_VALUE.matcher(values[0]).matches() || !GUEST_VALUE.matcher(values[1]).matches() + || !GUEST_HEX_VALUE.matcher(values[2]).matches() || !GUEST_VALUE.matcher(values[3]).matches()) { + throw failure("READ_GUEST", vm, "unexpected guest CPU and memory output"); + } + long memoryBlockBytes = Long.parseLong(values[2], 16); + if (memoryBlockBytes == 0) { + throw failure("READ_GUEST", vm, "unexpected guest memory block size"); + } + try { + return new GuestResources(Long.parseLong(values[0]), + Long.parseLong(values[1]), + Math.multiplyExact(Long.parseLong(values[3]), memoryBlockBytes), + memoryBlockBytes); + } catch (ArithmeticException e) { + throw failure("READ_GUEST", vm, "unexpected guest memory size", e); + } + } + + public void onlineGuestResources(NodeAccess access) { + executeNode(access, ONLINE_HOTPLUGGED_RESOURCES); + } + + public boolean isKubeletActive(NodeAccess access) { + return "active".equals(executeNode(access, KUBELET_STATE_COMMAND).trim()); + } + + /** + * Marks the node as having unverified resize work and, unless it is already cordoned, cordons it and + * records that CKS owns the cordon. An operator's cordon is left as it is. + */ + public NodeCapacitySnapshot startResize(KubernetesCluster cluster, UserVm vm, NodeAccess access, String nodeUid) { + String owner = patchValue(cluster.getUuid(), vm); + return updateNode("START", vm, access, nodeUid, node -> { + if (node.hasForeignMarker(owner)) { + throw failure("START", vm, "the Kubernetes node carries live resize markers of another cluster"); + } + if (node.isPendingFor(owner) && node.isUnschedulable()) { + return null; + } + JsonObject annotations = new JsonObject(); + annotations.addProperty(NodeCapacitySnapshot.PENDING_ANNOTATION, owner); + if (node.isUnschedulable()) { + return nodePatch(vm, node, annotations, null); + } + annotations.addProperty(NodeCapacitySnapshot.CORDON_ANNOTATION, owner); + return nodePatch(vm, node, annotations, true); + }); + } + + /** Removes this cluster's markers and, only if CKS owned the cordon, makes the node schedulable again. */ + public NodeCapacitySnapshot finishResize(KubernetesCluster cluster, UserVm vm, NodeAccess access, String nodeUid) { + String owner = cluster.getUuid(); + return updateNode("FINISH", vm, access, nodeUid, node -> { + boolean pending = node.isPendingFor(owner); + boolean ownershipMarker = node.hasOwnershipMarker(owner); + if (!pending && !ownershipMarker) { + return null; + } + JsonObject annotations = new JsonObject(); + if (pending) { + annotations.add(NodeCapacitySnapshot.PENDING_ANNOTATION, JsonNull.INSTANCE); + } + if (ownershipMarker) { + annotations.add(NodeCapacitySnapshot.CORDON_ANNOTATION, JsonNull.INSTANCE); + } + return nodePatch(vm, node, annotations, node.isCordonOwnedBy(owner) ? Boolean.FALSE : null); + }); + } + + /** Removes a CKS cordon and its ownership marker but keeps the pending marker. */ + public NodeCapacitySnapshot releaseCordon(KubernetesCluster cluster, UserVm vm, NodeAccess access, String nodeUid) { + String owner = cluster.getUuid(); + return updateNode("RELEASE_CORDON", vm, access, nodeUid, node -> { + if (!node.hasOwnershipMarker(owner)) { + return null; + } + JsonObject annotations = new JsonObject(); + annotations.add(NodeCapacitySnapshot.CORDON_ANNOTATION, JsonNull.INSTANCE); + return nodePatch(vm, node, annotations, node.isCordonOwnedBy(owner) ? Boolean.FALSE : null); + }); + } + + /** Waits until the guest has the target resources online; fails at once if it has more than the target. */ + public GuestResources waitForGuestResources(UserVm vm, ServiceOffering target, NodeAccess access, long deadline) { + return poll("GUEST_VERIFY", vm, deadline, "guest CPU or memory did not reach the target offering", () -> { + GuestResources guest = readGuest(vm, access); + if (isGuestAboveTarget(guest, target)) { + throw failure("GUEST_VERIFY", vm, String.format("the guest has %d CPUs and %d MiB of memory online, more than the target offering", + guest.getOnlineCpus(), guest.getOnlineMemoryBytes() / ByteScaleUtils.MiB)); + } + return guestMatchesTarget(guest, target) ? guest : null; + }); + } + + /** Restarts kubelet so it reads the new machine resources, and waits until it is active again. */ + public void restartKubelet(UserVm vm, NodeAccess access, long deadline) { + if (currentTimeMillis() >= deadline) { + throw failure("KUBELET_RESTART", vm, "the scale timeout was reached before kubelet was restarted"); + } + executeNode(access, RESTART_KUBELET_COMMAND); + poll("KUBELET_RESTART", vm, deadline, "kubelet did not become active", () -> isKubeletActive(access) ? Boolean.TRUE : null); + } + + public NodeCapacitySnapshot waitForKubernetesResources(UserVm vm, GuestResources guest, NodeAccess access, long deadline) { + return poll("CAPACITY_VERIFY", vm, deadline, "Kubernetes node capacity did not match the resized guest", () -> { + NodeCapacitySnapshot node = readNode(vm, access); + return isKubernetesNodeAlreadyResized(node, guest) ? node : null; + }); + } + + static NodeCapacitySnapshot parseNode(String output) { + JsonObject node = requireObject(JsonParser.parseString(output), "node"); + JsonObject metadata = requireObject(node.get("metadata"), "metadata"); + JsonObject status = requireObject(node.get("status"), "status"); + JsonElement spec = node.get("spec"); + boolean unschedulable = spec != null && spec.isJsonObject() && isTrue(spec.getAsJsonObject().get("unschedulable")); + return new NodeCapacitySnapshot(requireString(metadata.get("uid"), "metadata.uid"), + requireString(metadata.get("resourceVersion"), "metadata.resourceVersion"), + unschedulable, + readAnnotations(metadata.get("annotations")), + isReady(status.get("conditions")), + readResources(status.get("capacity"), "status.capacity"), + readResources(status.get("allocatable"), "status.allocatable")); + } + + static long parseCpuMillis(String value) { + Matcher matcher = CPU_QUANTITY.matcher(value); + if (!matcher.matches()) { + throw new IllegalArgumentException("unsupported Kubernetes CPU quantity " + value); + } + long number = Long.parseLong(matcher.group(1)); + return matcher.group(2) == null ? Math.multiplyExact(number, 1000L) : number; + } + + static long parseMemoryBytes(String value) { + Matcher matcher = MEMORY_QUANTITY.matcher(value); + if (!matcher.matches()) { + throw new IllegalArgumentException("unsupported Kubernetes memory quantity " + value); + } + return Math.multiplyExact(Long.parseLong(matcher.group(1)), memoryUnitBytes(matcher.group(2))); + } + + private static long memoryUnitBytes(String unit) { + if (unit == null) { + return 1L; + } + switch (unit) { + case "Ki": + return ByteScaleUtils.KiB; + case "Mi": + return ByteScaleUtils.MiB; + case "Gi": + return ByteScaleUtils.GiB; + case "Ti": + return ByteScaleUtils.GiB * ByteScaleUtils.KiB; + case "K": + return 1000L; + case "M": + return 1000L * 1000L; + case "G": + return 1000L * 1000L * 1000L; + case "T": + return 1000L * 1000L * 1000L * 1000L; + default: + throw new IllegalArgumentException("unsupported Kubernetes memory unit " + unit); + } + } + + private static JsonObject requireObject(JsonElement element, String name) { + if (element == null || !element.isJsonObject()) { + throw new IllegalArgumentException("missing " + name); + } + return element.getAsJsonObject(); + } + + private static String requireString(JsonElement element, String name) { + if (!isString(element)) { + throw new IllegalArgumentException("missing " + name); + } + return element.getAsString(); + } + + private static boolean isString(JsonElement element) { + return element != null && element.isJsonPrimitive() && element.getAsJsonPrimitive().isString(); + } + + private static boolean isTrue(JsonElement element) { + return element != null && element.isJsonPrimitive() && element.getAsJsonPrimitive().isBoolean() && element.getAsBoolean(); + } + + private static Map readAnnotations(JsonElement element) { + Map annotations = new HashMap<>(); + if (element == null || !element.isJsonObject()) { + return annotations; + } + for (Map.Entry annotation : element.getAsJsonObject().entrySet()) { + if (isString(annotation.getValue())) { + annotations.put(annotation.getKey(), annotation.getValue().getAsString()); + } + } + return annotations; + } + + private static boolean isReady(JsonElement conditions) { + if (conditions == null || !conditions.isJsonArray()) { + return false; + } + for (JsonElement condition : conditions.getAsJsonArray()) { + if (!condition.isJsonObject()) { + continue; + } + JsonObject item = condition.getAsJsonObject(); + if (isString(item.get("type")) && "Ready".equals(item.get("type").getAsString()) + && isString(item.get("status")) && "True".equals(item.get("status").getAsString())) { + return true; + } + } + return false; + } + + private static NodeResources readResources(JsonElement element, String name) { + JsonObject resources = requireObject(element, name); + return new NodeResources(parseCpuMillis(requireString(resources.get("cpu"), name + ".cpu")), + parseMemoryBytes(requireString(resources.get("memory"), name + ".memory"))); + } + + /** + * Applies the patch that patchFor returns for a fresh read of the node, until patchFor finds nothing + * left to change. Every patch carries the resourceVersion it was based on, so the API server rejects it + * if the node changed in the meantime. After a conflict, a rejected update or a lost reply the next + * read shows what was applied. + */ + private NodeCapacitySnapshot updateNode(String phase, UserVm vm, NodeAccess access, String nodeUid, + Function patchFor) { + String nodeName = nodeName(vm); + for (int attempt = 0; attempt <= MAX_NODE_UPDATE_ATTEMPTS; attempt++) { + NodeCapacitySnapshot node = readNode(vm, access); + if (!node.getUid().equals(nodeUid)) { + throw failure(phase, vm, "the Kubernetes node was replaced during the resize"); + } + String patch = patchFor.apply(node); + if (patch == null) { + return node; + } + if (attempt < MAX_NODE_UPDATE_ATTEMPTS) { + applyPatch(nodeName, patch, access); + } + } + throw failure(phase, vm, String.format("the Kubernetes node could not be updated in %d attempts", MAX_NODE_UPDATE_ATTEMPTS)); + } + + private void applyPatch(String nodeName, String patch, NodeAccess access) { + try { + executeControl(access, KubernetesClusterUtil.getKubernetesNodePatchCommand(nodeName, patch)); + } catch (NodeCapacityReconciliationException e) { + if (Thread.currentThread().isInterrupted()) { + throw e; + } + logger.debug("Updating Kubernetes node {} did not succeed, reading it again: {}", nodeName, e.getMessage()); + } + } + + private String nodePatch(UserVm vm, NodeCapacitySnapshot node, JsonObject annotations, Boolean unschedulable) { + JsonObject metadata = new JsonObject(); + metadata.addProperty("resourceVersion", patchValue(node.getResourceVersion(), vm)); + metadata.add("annotations", annotations); + JsonObject patch = new JsonObject(); + patch.add("metadata", metadata); + if (unschedulable != null) { + JsonObject spec = new JsonObject(); + spec.addProperty("unschedulable", unschedulable); + patch.add("spec", spec); + } + return patch.toString(); + } + + /** Values go into a single-quoted shell argument, so allow only characters that need no quoting. */ + private String patchValue(String value, UserVm vm) { + if (value == null || !PATCH_VALUE.matcher(value).matches()) { + throw failure("NODE_UPDATE", vm, "unexpected value in a Kubernetes node update: " + value); + } + return value; + } + + /** Observes once, then keeps observing while time remains; at most one observation follows the deadline. */ + private T poll(String phase, UserVm vm, long deadline, String timeoutMessage, Supplier observation) { + while (true) { + T result = observation.get(); + if (result != null) { + return result; + } + long remaining = deadline - currentTimeMillis(); + if (remaining <= 0) { + throw failure(phase, vm, timeoutMessage); + } + sleep(Math.min(POLL_INTERVAL_MS, remaining)); + } + } + + protected long currentTimeMillis() { + return System.currentTimeMillis(); + } + + protected void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new NodeCapacityReconciliationException("CKS live resize polling was interrupted", e); + } + } + + protected String executeControl(NodeAccess access, String command) { + return execute(access.getControlAddress(), access.getControlPort(), access, command); + } + + protected String executeNode(NodeAccess access, String command) { + return execute(access.getNodeAddress(), access.getNodePort(), access, command); + } + + private String execute(String address, int port, NodeAccess access, String command) { + Pair result; + try { + result = SshHelper.sshExecute(address, port, access.getUser(), access.getSshKeyFile(), null, command, + CONNECT_TIMEOUT_MS, CONNECT_TIMEOUT_MS, COMMAND_TIMEOUT_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new NodeCapacityReconciliationException(String.format("Command '%s' on %s was interrupted", command, address), e); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + // SshHelper declares Exception, including connection and authentication failures. + throw new NodeCapacityReconciliationException(String.format("Unable to run command '%s' on %s: %s", command, address, e.getMessage()), e); + } + if (!Boolean.TRUE.equals(result.first())) { + throw new NodeCapacityReconciliationException(String.format("Command '%s' failed on %s: %s", + command, address, StringUtils.abbreviate(StringUtils.trimToEmpty(result.second()), 300))); + } + return StringUtils.defaultString(result.second()); + } + + private String nodeName(UserVm vm) { + String name = KubernetesClusterUtil.getKubernetesNodeName(vm); + if (StringUtils.isBlank(name)) { + throw failure("NODE_NAME", vm, "the VM has no host name"); + } + if (!NODE_NAME.matcher(name).matches()) { + throw failure("NODE_NAME", vm, "the VM host name is not a valid Kubernetes node name"); + } + return name; + } + + private NodeCapacityReconciliationException failure(String phase, UserVm vm, String message) { + return failure(phase, vm, message, null); + } -/** Reconciles the guest and Kubernetes resource views after a live VM resize. */ -public interface KubernetesClusterNodeCapacityReconciler { - NodeCapacitySnapshot captureBefore(KubernetesCluster cluster, UserVm vm, NodeAccess access); - boolean requiresKubeletRefresh(ServiceOffering oldOffering, ServiceOffering newOffering, - KubernetesClusterNodeType nodeType, KubernetesCluster.State originalState); - void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline); - void onlineGuestResources(NodeAccess access); - void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline); - boolean isKubernetesResourcesCurrent(NodeCapacitySnapshot snapshot, ServiceOffering target); - void restartKubelet(UserVm vm, NodeAccess access, long deadline); - NodeCapacitySnapshot waitForKubernetesResources(KubernetesCluster cluster, UserVm vm, ServiceOffering target, - NodeCapacitySnapshot before, NodeAccess access, long deadline); - void restoreSchedulability(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline); - - final class NodeAccess { - private final String controlAddress; private final int controlPort; private final String nodeAddress; private final int nodePort; - private final String user; private final File sshKeyFile; - public NodeAccess(String controlAddress, int controlPort, String nodeAddress, int nodePort, String user, File sshKeyFile) { - this.controlAddress = controlAddress; this.controlPort = controlPort; this.nodeAddress = nodeAddress; this.nodePort = nodePort; - this.user = user; this.sshKeyFile = sshKeyFile; - } - public String getControlAddress() { return controlAddress; } public int getControlPort() { return controlPort; } - public String getNodeAddress() { return nodeAddress; } public int getNodePort() { return nodePort; } - public String getUser() { return user; } public File getSshKeyFile() { return sshKeyFile; } - } - - final class NodeCapacitySnapshot { - private final boolean unschedulable; private final boolean cloudStackResizeCordon; private final long guestOnlineCpuCount; - private final long guestMemoryKiB; private final long capacityCpuMillis; private final long capacityMemoryBytes; - private final long allocatableCpuMillis; private final long allocatableMemoryBytes; private final boolean ready; - public NodeCapacitySnapshot(boolean unschedulable, boolean cloudStackResizeCordon, long guestOnlineCpuCount, long guestMemoryKiB, - long capacityCpuMillis, long capacityMemoryBytes, long allocatableCpuMillis, long allocatableMemoryBytes, boolean ready) { - this.unschedulable = unschedulable; this.cloudStackResizeCordon = cloudStackResizeCordon; this.guestOnlineCpuCount = guestOnlineCpuCount; - this.guestMemoryKiB = guestMemoryKiB; this.capacityCpuMillis = capacityCpuMillis; this.capacityMemoryBytes = capacityMemoryBytes; - this.allocatableCpuMillis = allocatableCpuMillis; this.allocatableMemoryBytes = allocatableMemoryBytes; this.ready = ready; - } - public boolean isUnschedulable() { return unschedulable; } public boolean isCloudStackResizeCordon() { return cloudStackResizeCordon; } - public long getGuestOnlineCpuCount() { return guestOnlineCpuCount; } public long getGuestMemoryKiB() { return guestMemoryKiB; } - public long getCapacityCpuMillis() { return capacityCpuMillis; } public long getCapacityMemoryBytes() { return capacityMemoryBytes; } - public long getAllocatableCpuMillis() { return allocatableCpuMillis; } public long getAllocatableMemoryBytes() { return allocatableMemoryBytes; } - public boolean isReady() { return ready; } + private NodeCapacityReconciliationException failure(String phase, UserVm vm, String message, Exception cause) { + return new NodeCapacityReconciliationException(String.format("CKS live resize %s failed for VM %s: %s", phase, vm.getUuid(), message), cause); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java deleted file mode 100644 index ad7cf71099b6..000000000000 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImpl.java +++ /dev/null @@ -1,232 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.kubernetes.cluster.utils; - -import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.cloud.kubernetes.cluster.KubernetesCluster; -import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; -import com.cloud.offering.ServiceOffering; -import com.cloud.uservm.UserVm; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.ssh.SshHelper; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import org.apache.commons.lang3.StringUtils; - -/** SSH-backed implementation for a rolling CKS live-resize reconciliation. */ -public class KubernetesClusterNodeCapacityReconcilerImpl implements KubernetesClusterNodeCapacityReconciler { - // KVM QA observes 4,568,788 KiB in a 4,608 MiB guest after memory hotplug - // (about 146 MiB below the offering), so keep a small margin above that - // platform overhead while still rejecting materially undersized guests. - static final long MINIMUM_MEMORY_OVERHEAD_BYTES = 160L * 1024L * 1024L; - static final long KUBERNETES_MEMORY_REPORTING_TOLERANCE_BYTES = 16L * 1024L * 1024L; - private static final int COMMAND_TIMEOUT_MS = 30000; - private static final int POLL_INTERVAL_MS = 5000; - private static final Pattern NODE_NAME = Pattern.compile("[A-Za-z0-9][A-Za-z0-9.-]{0,252}"); - private static final Pattern MEMORY_QUANTITY = Pattern.compile("([0-9]+)(Ki|Mi|Gi|Ti|K|M|G|T)?"); - private static final String RESIZE_ANNOTATION = "cloudstack.apache.org/cks-live-resize"; - static final String ONLINE_HOTPLUGGED_RESOURCES = "sudo sh -c 'for cpu in /sys/devices/system/cpu/cpu[0-9]*; do " - + "[ -e \"$cpu/online\" ] || continue; echo 1 > \"$cpu/online\"; done; " - + "for memory in /sys/devices/system/memory/memory[0-9]*; do " - + "[ -e \"$memory/online\" ] || continue; echo online > \"$memory/online\"; done'"; - - @Override - public boolean requiresKubeletRefresh(ServiceOffering oldOffering, ServiceOffering newOffering, - KubernetesClusterNodeType nodeType, KubernetesCluster.State originalState) { - return KubernetesCluster.State.Running == originalState - && (KubernetesClusterNodeType.WORKER == nodeType || KubernetesClusterNodeType.CONTROL == nodeType) - && capacityChanged(oldOffering, newOffering); - } - - @Override - public NodeCapacitySnapshot captureBefore(KubernetesCluster cluster, UserVm vm, NodeAccess access) { - return nodeSnapshot(cluster, vm, access, guestResources(access)); - } - - @Override - public void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) { - if (before.isUnschedulable() && !before.isCloudStackResizeCordon()) return; - String node = nodeName(vm); - executeControl(access, "sudo /opt/bin/kubectl annotate node " + node + " " + RESIZE_ANNOTATION + "=" + cluster.getUuid() + " --overwrite"); - executeControl(access, "sudo /opt/bin/kubectl cordon " + node); - while (System.currentTimeMillis() < deadline) { - if (nodeSnapshot(cluster, vm, access, null).isUnschedulable()) return; - sleep(); - } - throw failure("CORDON", vm, "Kubernetes node did not become unschedulable"); - } - - @Override - public void onlineGuestResources(NodeAccess access) { - executeNode(access, ONLINE_HOTPLUGGED_RESOURCES); - } - - @Override - public void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline) { - while (System.currentTimeMillis() < deadline) { - if (guestMatchesTarget(guestResources(access), target)) return; - sleep(); - } - throw failure("GUEST_VERIFY", vm, "guest CPU or memory did not reach the target offering"); - } - - @Override - public boolean isKubernetesResourcesCurrent(NodeCapacitySnapshot snapshot, ServiceOffering target) { - GuestResources guest = new GuestResources(snapshot.getGuestOnlineCpuCount(), snapshot.getGuestMemoryKiB()); - return guestMatchesTarget(guest, target) && kubernetesMatchesGuest(snapshot, snapshot); - } - - @Override - public void restartKubelet(UserVm vm, NodeAccess access, long deadline) { - executeNode(access, "sudo systemctl restart kubelet"); - while (System.currentTimeMillis() < deadline) { - if ("active".equals(executeNode(access, "sudo systemctl is-active kubelet").trim())) return; - sleep(); - } - throw failure("KUBELET_RESTART", vm, "kubelet did not become active"); - } - - @Override - public NodeCapacitySnapshot waitForKubernetesResources(KubernetesCluster cluster, UserVm vm, ServiceOffering target, - NodeCapacitySnapshot before, NodeAccess access, long deadline) { - while (System.currentTimeMillis() < deadline) { - GuestResources guest = guestResources(access); - NodeCapacitySnapshot observed = nodeSnapshot(cluster, vm, access, guest); - if (guestMatchesTarget(guest, target) && kubernetesMatchesGuest(observed, before)) return observed; - sleep(); - } - throw failure("CAPACITY_VERIFY", vm, "Kubernetes capacity did not match the resized guest"); - } - - @Override - public void restoreSchedulability(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) { - if (before.isUnschedulable() && !before.isCloudStackResizeCordon()) return; - String node = nodeName(vm); - executeControl(access, "sudo /opt/bin/kubectl uncordon " + node); - executeControl(access, "sudo /opt/bin/kubectl annotate node " + node + " " + RESIZE_ANNOTATION + "-"); - while (System.currentTimeMillis() < deadline) { - if (!nodeSnapshot(cluster, vm, access, null).isUnschedulable()) return; - sleep(); - } - throw failure("RESTORE_SCHEDULABILITY", vm, "Kubernetes node remained unschedulable"); - } - - public static boolean capacityChanged(ServiceOffering oldOffering, ServiceOffering targetOffering) { - return oldOffering != null && targetOffering != null - && (!Objects.equals(oldOffering.getCpu(), targetOffering.getCpu()) - || !Objects.equals(oldOffering.getRamSize(), targetOffering.getRamSize())); - } - - static long parseCpuMillis(String value) { return value.endsWith("m") ? Long.parseLong(value.substring(0, value.length() - 1)) : Long.parseLong(value) * 1000L; } - static long parseMemoryBytes(String value) { - Matcher matcher = MEMORY_QUANTITY.matcher(value); - if (!matcher.matches()) throw new IllegalArgumentException("Unsupported Kubernetes memory quantity"); - long number = Long.parseLong(matcher.group(1)); String unit = matcher.group(2); - if (unit == null) return number; - switch (unit) { - case "Ki": return number * 1024L; case "Mi": return number * 1024L * 1024L; case "Gi": return number * 1024L * 1024L * 1024L; - case "Ti": return number * 1024L * 1024L * 1024L * 1024L; case "K": return number * 1000L; case "M": return number * 1000L * 1000L; - case "G": return number * 1000L * 1000L * 1000L; case "T": return number * 1000L * 1000L * 1000L * 1000L; - default: throw new IllegalArgumentException("Unsupported Kubernetes memory quantity"); - } - } - - private NodeCapacitySnapshot nodeSnapshot(KubernetesCluster cluster, UserVm vm, NodeAccess access, GuestResources guest) { - JsonObject node = new JsonParser().parse(executeControl(access, "sudo /opt/bin/kubectl get node " + nodeName(vm) + " -o json")).getAsJsonObject(); - JsonObject annotations = node.getAsJsonObject("metadata").has("annotations") ? node.getAsJsonObject("metadata").getAsJsonObject("annotations") : null; - boolean cksCordon = annotations != null && annotations.has(RESIZE_ANNOTATION) && cluster.getUuid().equals(annotations.get(RESIZE_ANNOTATION).getAsString()); - JsonObject spec = node.has("spec") ? node.getAsJsonObject("spec") : new JsonObject(); JsonObject status = node.getAsJsonObject("status"); - JsonObject capacity = status.getAsJsonObject("capacity"); JsonObject allocatable = status.getAsJsonObject("allocatable"); - return new NodeCapacitySnapshot(spec.has("unschedulable") && spec.get("unschedulable").getAsBoolean(), cksCordon, - guest == null ? 0 : guest.cpu, guest == null ? 0 : guest.memoryKiB, parseCpuMillis(capacity.get("cpu").getAsString()), - parseMemoryBytes(capacity.get("memory").getAsString()), parseCpuMillis(allocatable.get("cpu").getAsString()), - parseMemoryBytes(allocatable.get("memory").getAsString()), isReady(status)); - } - - private boolean isReady(JsonObject status) { - JsonArray conditions = status.getAsJsonArray("conditions"); - for (JsonElement condition : conditions) { JsonObject item = condition.getAsJsonObject(); if ("Ready".equals(item.get("type").getAsString()) && "True".equals(item.get("status").getAsString())) return true; } - return false; - } - - private GuestResources guestResources(NodeAccess access) { - String[] values = executeNode(access, "getconf _NPROCESSORS_ONLN; awk '/^MemTotal:/ {print $2}' /proc/meminfo").trim().split("\\s+"); - if (values.length != 2) throw new CloudRuntimeException("Unable to read guest CPU and memory"); - return new GuestResources(Long.parseLong(values[0]), Long.parseLong(values[1])); - } - - private boolean guestMatchesTarget(GuestResources observed, ServiceOffering target) { - long targetMemory = target.getRamSize() * 1024L * 1024L; long observedMemory = observed.memoryKiB * 1024L; - long overhead = Math.max(targetMemory / 50L, MINIMUM_MEMORY_OVERHEAD_BYTES); - return observed.cpu == target.getCpu() && observedMemory <= targetMemory && targetMemory - observedMemory <= overhead; - } - - private boolean kubernetesMatchesGuest(NodeCapacitySnapshot observed, NodeCapacitySnapshot before) { - return observed.isReady() && observed.getCapacityCpuMillis() == observed.getGuestOnlineCpuCount() * 1000L - && Math.abs(observed.getCapacityMemoryBytes() - observed.getGuestMemoryKiB() * 1024L) <= KUBERNETES_MEMORY_REPORTING_TOLERANCE_BYTES - && (observed.getGuestOnlineCpuCount() <= before.getGuestOnlineCpuCount() || observed.getAllocatableCpuMillis() > before.getAllocatableCpuMillis()) - && (observed.getGuestMemoryKiB() <= before.getGuestMemoryKiB() || observed.getAllocatableMemoryBytes() > before.getAllocatableMemoryBytes()); - } - - protected String executeControl(NodeAccess access, String command) { - return execute(access.getControlAddress(), access.getControlPort(), access, command); - } - - protected String executeNode(NodeAccess access, String command) { - return execute(access.getNodeAddress(), access.getNodePort(), access, command); - } - - private String execute(String address, int port, NodeAccess access, String command) { - Pair result; - try { - result = SshHelper.sshExecute(address, port, access.getUser(), access.getSshKeyFile(), null, command, 10000, 10000, COMMAND_TIMEOUT_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new NodeCapacityReconciliationException("CKS live-resize command interrupted", e); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - // SshHelper declares Exception, including authentication failures. - // Translate only at this legacy boundary, not around the resize workflow. - throw new NodeCapacityReconciliationException("CKS live-resize SSH command failed", e); - } - if (!Boolean.TRUE.equals(result.first())) { - throw new NodeCapacityReconciliationException("CKS live-resize command failed"); - } - return StringUtils.defaultString(result.second()); - } - private String nodeName(UserVm vm) { String name = StringUtils.lowerCase(vm.getHostName()); if (!NODE_NAME.matcher(name).matches()) throw new CloudRuntimeException("Invalid Kubernetes node name for live resize"); return name; } - private NodeCapacityReconciliationException failure(String phase, UserVm vm, String message) { - return new NodeCapacityReconciliationException("CKS live resize " + phase + " failed for VM " + vm.getUuid() + ": " + message); - } - - private void sleep() { - try { - Thread.sleep(POLL_INTERVAL_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new NodeCapacityReconciliationException("CKS live-resize resource polling interrupted", e); - } - } - private static class GuestResources { private final long cpu; private final long memoryKiB; GuestResources(long cpu, long memoryKiB) { this.cpu = cpu; this.memoryKiB = memoryKiB; } } -} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 023fc9ec7fad..e04debd592c8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -50,6 +50,25 @@ public class KubernetesClusterUtil { public static final String CLUSTER_NODE_READY_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'"; public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\") print $5}'"; + public static final String CLUSTER_NODE_PATCH_COMMAND = "sudo /opt/bin/kubectl patch node %s --type merge --request-timeout=20s -p '%s'"; + + /** + * The Kubernetes node name of a CKS cluster VM, which is its host name in lowercase. + */ + public static String getKubernetesNodeName(final UserVm userVm) { + return StringUtils.lowerCase(userVm.getHostName()); + } + + /** + * Command that applies a JSON merge patch to a node. Annotations and spec.unschedulable in one patch + * change in a single API update, so a failure can't leave the node with only some of them changed. + */ + public static String getKubernetesNodePatchCommand(final String nodeName, final String mergePatch) { + if (StringUtils.contains(mergePatch, '\'')) { + throw new IllegalArgumentException("A node patch must not contain single quotes"); + } + return String.format(CLUSTER_NODE_PATCH_COMMAND, nodeName, mergePatch); + } public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port, String user, File sshKeyFile, String nodeName) throws Exception { @@ -108,10 +127,7 @@ public static boolean uncordonKubernetesClusterNode(final KubernetesCluster kube final String user, final File sshKeyFile, final UserVm userVm, final long timeoutTime, final int waitDuration) { - String hostName = userVm.getHostName(); - if (StringUtils.isNotEmpty(hostName)) { - hostName = hostName.toLowerCase(); - } + String hostName = getKubernetesNodeName(userVm); while (System.currentTimeMillis() < timeoutTime) { Pair result = null; try { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeAccess.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeAccess.java new file mode 100644 index 000000000000..0f25f408bfed --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeAccess.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import java.io.File; + +/** SSH endpoints for a CKS node: the control plane runs kubectl, the node itself runs guest commands. */ +public final class NodeAccess { + private final String controlAddress; + private final int controlPort; + private final String nodeAddress; + private final int nodePort; + private final String user; + private final File sshKeyFile; + + public NodeAccess(String controlAddress, + int controlPort, + String nodeAddress, + int nodePort, + String user, + File sshKeyFile) { + this.controlAddress = controlAddress; + this.controlPort = controlPort; + this.nodeAddress = nodeAddress; + this.nodePort = nodePort; + this.user = user; + this.sshKeyFile = sshKeyFile; + } + + public String getControlAddress() { + return controlAddress; + } + + public int getControlPort() { + return controlPort; + } + + public String getNodeAddress() { + return nodeAddress; + } + + public int getNodePort() { + return nodePort; + } + + public String getUser() { + return user; + } + + public File getSshKeyFile() { + return sshKeyFile; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeCapacitySnapshot.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeCapacitySnapshot.java new file mode 100644 index 000000000000..2f4b148299b2 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeCapacitySnapshot.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + +/** The parts of a Kubernetes Node object that a live resize reads. */ +public final class NodeCapacitySnapshot { + /** Set while a node has resize work that has not been verified yet; the value is the cluster UUID. */ + public static final String PENDING_ANNOTATION = "cloudstack.apache.org/cks-live-resize-pending"; + /** Set when CKS, not an operator, cordoned the node for a resize; the value is the cluster UUID. */ + public static final String CORDON_ANNOTATION = "cloudstack.apache.org/cks-live-resize"; + + private final String uid; + private final String resourceVersion; + private final boolean unschedulable; + private final Map annotations; + private final boolean ready; + private final NodeResources capacity; + private final NodeResources allocatable; + + public NodeCapacitySnapshot(String uid, + String resourceVersion, + boolean unschedulable, + Map annotations, + boolean ready, + NodeResources capacity, + NodeResources allocatable) { + this.uid = uid; + this.resourceVersion = resourceVersion; + this.unschedulable = unschedulable; + this.annotations = Collections.unmodifiableMap(new HashMap<>(annotations)); + this.ready = ready; + this.capacity = capacity; + this.allocatable = allocatable; + } + + public String getUid() { + return uid; + } + + public String getResourceVersion() { + return resourceVersion; + } + + public boolean isUnschedulable() { + return unschedulable; + } + + public boolean isReady() { + return ready; + } + + public NodeResources getCapacity() { + return capacity; + } + + public NodeResources getAllocatable() { + return allocatable; + } + + public boolean isPendingFor(String clusterUuid) { + return clusterUuid.equals(annotations.get(PENDING_ANNOTATION)); + } + + /** Whether CKS cordoned the node: it is unschedulable and carries this cluster's ownership marker. */ + public boolean isCordonOwnedBy(String clusterUuid) { + return unschedulable && hasOwnershipMarker(clusterUuid); + } + + /** + * Whether this cluster's ownership marker is set. kubectl uncordon keeps annotations, so the marker can + * be left on a node somebody has uncordoned since; it then no longer means that CKS owns a cordon. + */ + public boolean hasOwnershipMarker(String clusterUuid) { + return clusterUuid.equals(annotations.get(CORDON_ANNOTATION)); + } + + /** Whether a resize marker carries another cluster's UUID; such markers are never changed. */ + public boolean hasForeignMarker(String clusterUuid) { + return isForeign(annotations.get(PENDING_ANNOTATION), clusterUuid) + || isForeign(annotations.get(CORDON_ANNOTATION), clusterUuid); + } + + private static boolean isForeign(String owner, String clusterUuid) { + return StringUtils.isNotEmpty(owner) && !owner.equals(clusterUuid); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeResources.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeResources.java new file mode 100644 index 000000000000..14e5f39dc6df --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/NodeResources.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +/** CPU and memory quantities from a Kubernetes Node status, such as its capacity or allocatable. */ +public final class NodeResources { + private final long cpuMillis; + private final long memoryBytes; + + public NodeResources(long cpuMillis, long memoryBytes) { + this.cpuMillis = cpuMillis; + this.memoryBytes = memoryBytes; + } + + public long getCpuMillis() { + return cpuMillis; + } + + public long getMemoryBytes() { + return memoryBytes; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml index fedc57071c5d..053366786292 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml @@ -34,6 +34,7 @@ + diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizeTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizeTest.java deleted file mode 100644 index 0abcce43d45e..000000000000 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizeTest.java +++ /dev/null @@ -1,327 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.kubernetes.cluster.actionworkers; - -import java.util.HashMap; -import java.util.List; - -import com.cloud.exception.ManagementServerException; -import com.cloud.exception.ResourceUnavailableException; -import com.cloud.exception.VirtualMachineMigrationException; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.kubernetes.cluster.KubernetesCluster; -import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; -import com.cloud.kubernetes.cluster.KubernetesClusterVO; -import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; -import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; -import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler; -import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot; -import com.cloud.kubernetes.cluster.utils.NodeCapacityReconciliationException; -import com.cloud.network.Network; -import com.cloud.service.ServiceOfferingVO; -import com.cloud.service.dao.ServiceOfferingDao; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.vm.UserVmManager; -import com.cloud.vm.UserVmVO; -import com.cloud.vm.dao.UserVmDao; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.mockito.InOrder; -import org.mockito.Mockito; -import org.springframework.test.util.ReflectionTestUtils; - -import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; - -public class KubernetesClusterNodeOfferingResizeTest { - private final KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); - private final KubernetesClusterManagerImpl manager = Mockito.mock(KubernetesClusterManagerImpl.class); - private final KubernetesClusterVmMapDao vmMapDao = Mockito.mock(KubernetesClusterVmMapDao.class); - private final UserVmDao vmDao = Mockito.mock(UserVmDao.class); - private final UserVmManager vmManager = Mockito.mock(UserVmManager.class); - private final ServiceOfferingDao offeringDao = Mockito.mock(ServiceOfferingDao.class); - private final KubernetesClusterNodeCapacityReconciler reconciler = Mockito.mock(KubernetesClusterNodeCapacityReconciler.class); - private final ServiceOfferingVO oldOffering = offering(10L, 2, 2048); - private final ServiceOfferingVO target = offering(20L, 4, 4096); - private final UserVmVO first = vm(1L, 10L); - private final UserVmVO second = vm(2L, 10L); - private final UserVmVO resizedFirst = vm(1L, 20L); - private final UserVmVO resizedSecond = vm(2L, 20L); - private KubernetesClusterScaleWorker worker; - - @Before - public void setUp() throws Exception { - Mockito.when(cluster.getId()).thenReturn(100L); - Mockito.when(cluster.getName()).thenReturn("test-cluster"); - Mockito.when(cluster.getUuid()).thenReturn("cluster-uuid"); - Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); - Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); - Mockito.when(cluster.getNodeCount()).thenReturn(2L); - Mockito.when(cluster.getTotalNodeCount()).thenReturn(2L); - worker = Mockito.spy(new KubernetesClusterScaleWorker(cluster, new HashMap<>(), null, null, null, null, null, manager)); - Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Scaling); - worker.userVmDao = vmDao; - worker.userVmManager = vmManager; - worker.serviceOfferingDao = offeringDao; - worker.kubernetesClusterVmMapDao = vmMapDao; - worker.kubernetesClusterNodeCapacityReconciler = reconciler; - ReflectionTestUtils.setField(worker, "scaleTimeoutTime", System.currentTimeMillis() + 60000L); - ReflectionTestUtils.setField(worker, "network", Mockito.mock(Network.class)); - Mockito.when(manager.isDirectAccess(any())).thenReturn(true); - Mockito.doReturn(new Pair<>("control", 22)).when(worker).getKubernetesClusterServerIpSshPort(null); - Mockito.doReturn("cloud").when(worker).getControlNodeLoginUser(); - Mockito.doReturn(null).when(worker).getManagementServerSshPublicKeyFile(); - Mockito.doReturn(true).when(worker).stateTransitTo(anyLong(), any()); - Mockito.doNothing().when(worker).deleteTemplateLaunchPermission(); - Mockito.doReturn(Mockito.mock(KubernetesClusterVO.class)).when(worker).updateKubernetesClusterEntry( - any(), any(), any(), any(), any(), any(), any(), eq(WORKER), anyBoolean(), anyBoolean()); - List maps = List.of(vmMap(1L), vmMap(2L)); - Mockito.when(vmMapDao.listByClusterId(100L)).thenReturn(maps); - Mockito.when(vmMapDao.listByClusterIdAndVmType(100L, WORKER)).thenReturn(maps); - Mockito.when(offeringDao.findById(10L)).thenReturn(oldOffering); - Mockito.when(offeringDao.findById(20L)).thenReturn(target); - Mockito.when(vmDao.findById(1L)).thenReturn(first); - Mockito.when(vmDao.findById(2L)).thenReturn(second); - Mockito.when(vmManager.upgradeVirtualMachine(eq(1L), eq(20L), any())).thenAnswer(invocation -> { - Mockito.when(vmDao.findById(1L)).thenReturn(resizedFirst); - return true; - }); - Mockito.when(vmManager.upgradeVirtualMachine(eq(2L), eq(20L), any())).thenAnswer(invocation -> { - Mockito.when(vmDao.findById(2L)).thenReturn(resizedSecond); - return true; - }); - Mockito.when(reconciler.requiresKubeletRefresh(oldOffering, target, WORKER, KubernetesCluster.State.Running)).thenReturn(true); - Mockito.when(reconciler.captureBefore(eq(cluster), any(), any())).thenReturn(snapshot(false, false)); - } - - @Test - public void testNodesCompleteReconciliationSequentially() throws Exception { - scale(); - - InOrder order = Mockito.inOrder(reconciler, vmManager); - order.verify(reconciler).cordonIfNeeded(eq(cluster), eq(first), any(), any(), anyLong()); - order.verify(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); - order.verify(reconciler).verifyGuestResources(eq(resizedFirst), eq(target), any(), any(), anyLong()); - order.verify(reconciler).restartKubelet(eq(resizedFirst), any(), anyLong()); - order.verify(reconciler).waitForKubernetesResources(eq(cluster), eq(resizedFirst), eq(target), any(), any(), anyLong()); - order.verify(reconciler).restoreSchedulability(eq(cluster), eq(resizedFirst), any(), any(), anyLong()); - order.verify(reconciler).cordonIfNeeded(eq(cluster), eq(second), any(), any(), anyLong()); - order.verify(vmManager).upgradeVirtualMachine(eq(2L), eq(20L), any()); - order.verify(reconciler).restoreSchedulability(eq(cluster), eq(resizedSecond), any(), any(), anyLong()); - } - - @Test - public void testResourceUnavailableRestoresTemporaryCordon() throws Exception { - assertPreResizeFailure(new ResourceUnavailableException("host unavailable", UserVmVO.class, 1L)); - } - - @Test - public void testManagementFailureRestoresTemporaryCordon() throws Exception { - assertPreResizeFailure(new ManagementServerException("upgrade failed")); - } - - @Test - public void testMigrationFailureRestoresTemporaryCordon() throws Exception { - assertPreResizeFailure(new VirtualMachineMigrationException("migration failed")); - } - - @Test - public void testRejectedVmResizeRestoresTemporaryCordon() throws Exception { - Mockito.doReturn(false).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); - CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); - Assert.assertTrue(error.getMessage().contains("The VM was not resized")); - Mockito.verify(reconciler).restoreSchedulability(eq(cluster), eq(first), any(), any(), anyLong()); - assertSecondNodeUntouched(); - } - - @Test - public void testFailedCleanupPreservesOriginalResizeFailure() throws Exception { - ManagementServerException failure = new ManagementServerException("original failure"); - Mockito.doThrow(new NodeCapacityReconciliationException("cleanup failed")).when(reconciler) - .restoreSchedulability(eq(cluster), eq(first), any(), any(), anyLong()); - assertPreResizeFailure(failure); - } - - @Test - public void testOperatorCordonIsNotRestoredAfterRejectedResize() throws Exception { - Mockito.when(reconciler.captureBefore(eq(cluster), eq(first), any())).thenReturn(snapshot(true, false)); - Mockito.doReturn(false).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); - Assert.assertThrows(CloudRuntimeException.class, this::scale); - Mockito.verify(reconciler, Mockito.never()).restoreSchedulability(any(), any(), any(), any(), anyLong()); - assertSecondNodeUntouched(); - } - - @Test - public void testPostResizeFailureLeavesNodeCordonedAndSecondNodeUntouched() { - NodeCapacityReconciliationException failure = new NodeCapacityReconciliationException("guest verification failed"); - Mockito.doThrow(failure).when(reconciler).verifyGuestResources(eq(resizedFirst), eq(target), any(), any(), anyLong()); - - CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); - - Assert.assertSame(failure, error.getCause()); - Assert.assertTrue(error.getMessage().contains("The node remains cordoned")); - Mockito.verify(reconciler, Mockito.never()).restoreSchedulability(any(), any(), any(), any(), anyLong()); - assertSecondNodeUntouched(); - } - - @Test - public void testSnapshotFailureStopsBeforeCordonOrVmResize() { - NodeCapacityReconciliationException failure = new NodeCapacityReconciliationException("SSH unavailable"); - Mockito.when(reconciler.captureBefore(eq(cluster), eq(first), any())).thenThrow(failure); - - CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); - - Assert.assertSame(failure, error.getCause()); - Mockito.verifyNoInteractions(vmManager); - Mockito.verify(reconciler, Mockito.never()).cordonIfNeeded(any(), any(), any(), any(), anyLong()); - Mockito.verify(reconciler, Mockito.never()).restoreSchedulability(any(), any(), any(), any(), anyLong()); - assertSecondNodeUntouched(); - } - - @Test - public void testCordonFailureStopsBeforeVmResize() { - Mockito.doThrow(new NodeCapacityReconciliationException("cordon failed")).when(reconciler) - .cordonIfNeeded(eq(cluster), eq(first), any(), any(), anyLong()); - - Assert.assertThrows(CloudRuntimeException.class, this::scale); - - Mockito.verifyNoInteractions(vmManager); - Mockito.verify(reconciler, Mockito.never()).restoreSchedulability(any(), any(), any(), any(), anyLong()); - assertSecondNodeUntouched(); - } - - @Test - public void testRetryReconcilesAlreadyResizedNodeBeforeResizingNext() throws Exception { - Mockito.when(vmDao.findById(1L)).thenReturn(resizedFirst); - Mockito.when(reconciler.captureBefore(eq(cluster), eq(resizedFirst), any())).thenReturn(snapshot(true, true)); - - scale(); - - Mockito.verify(vmManager, Mockito.never()).upgradeVirtualMachine(eq(1L), anyLong(), any()); - InOrder order = Mockito.inOrder(reconciler, vmManager); - order.verify(reconciler).restartKubelet(eq(resizedFirst), any(), anyLong()); - order.verify(reconciler).restoreSchedulability(eq(cluster), eq(resizedFirst), any(), any(), anyLong()); - order.verify(vmManager).upgradeVirtualMachine(eq(2L), eq(20L), any()); - } - - @Test - public void testCurrentResourcesDoNotRestartKubelet() { - Mockito.when(reconciler.isKubernetesResourcesCurrent(any(), eq(target))).thenReturn(true); - scale(); - Mockito.verify(reconciler, Mockito.never()).restartKubelet(any(), any(), anyLong()); - Mockito.verify(reconciler).restoreSchedulability(eq(cluster), eq(resizedFirst), any(), any(), anyLong()); - } - - @Test - public void testStoppedClusterResizesWithoutGuestOrKubeletOperations() throws Exception { - ReflectionTestUtils.setField(worker, "originalState", KubernetesCluster.State.Stopped); - scale(); - Mockito.verify(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); - Mockito.verify(vmManager).upgradeVirtualMachine(eq(2L), eq(20L), any()); - Mockito.verify(reconciler, Mockito.never()).captureBefore(any(), any(), any()); - Mockito.verify(reconciler, Mockito.never()).restartKubelet(any(), any(), anyLong()); - } - - @Test - public void testCreatedClusterUpdatesMetadataWithoutTouchingVms() { - ReflectionTestUtils.setField(worker, "originalState", KubernetesCluster.State.Created); - scale(); - Mockito.verifyNoInteractions(vmDao, vmManager, reconciler); - Mockito.verify(worker).updateKubernetesClusterEntry(eq(8L), eq(8192L), any(), eq(20L), - any(), any(), any(), eq(WORKER), eq(true), eq(false)); - } - - @Test - public void testMissingVmAfterSuccessfulResizeDoesNotHideFailureWithNullPointer() throws Exception { - Mockito.doAnswer(invocation -> { - Mockito.when(vmDao.findById(1L)).thenReturn(null); - return true; - }).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); - - CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); - - Assert.assertTrue(error.getMessage().contains("VM vm-1 did not reach target offering")); - Mockito.verify(reconciler, Mockito.never()).restoreSchedulability(any(), any(), any(), any(), anyLong()); - assertSecondNodeUntouched(); - } - - @Test - public void testUnexpectedProgrammingExceptionIsNotWrappedAsOperationalFailure() throws Exception { - NullPointerException failure = new NullPointerException("programming error"); - Mockito.doThrow(failure).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); - - Assert.assertSame(failure, Assert.assertThrows(NullPointerException.class, this::scale)); - - Mockito.verify(worker, Mockito.never()).stateTransitTo(100L, KubernetesCluster.Event.OperationFailed); - assertSecondNodeUntouched(); - } - - private void scale() { - worker.scaleKubernetesClusterOffering(WORKER, target, true, false); - } - - private void assertPreResizeFailure(Exception failure) throws Exception { - Mockito.doThrow(failure).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); - CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); - Assert.assertSame(failure, error.getCause()); - Mockito.verify(reconciler).restoreSchedulability(eq(cluster), eq(first), any(), any(), anyLong()); - Mockito.verify(worker).stateTransitTo(100L, KubernetesCluster.Event.OperationFailed); - assertSecondNodeUntouched(); - } - - private void assertSecondNodeUntouched() { - Mockito.verify(vmDao, Mockito.never()).findById(2L); - Mockito.verify(reconciler, Mockito.never()).captureBefore(eq(cluster), eq(second), any()); - Mockito.verify(worker, Mockito.never()).updateKubernetesClusterEntry( - any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), anyBoolean()); - } - - private ServiceOfferingVO offering(long id, int cpu, int memory) { - ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); - Mockito.when(offering.getId()).thenReturn(id); - Mockito.when(offering.getCpu()).thenReturn(cpu); - Mockito.when(offering.getRamSize()).thenReturn(memory); - return offering; - } - - private UserVmVO vm(long id, long offeringId) { - UserVmVO vm = Mockito.mock(UserVmVO.class); - Mockito.when(vm.getId()).thenReturn(id); - Mockito.when(vm.getUuid()).thenReturn("vm-" + id); - Mockito.when(vm.getDisplayName()).thenReturn("worker-" + id); - Mockito.when(vm.getPrivateIpAddress()).thenReturn("node-" + id); - Mockito.when(vm.getServiceOfferingId()).thenReturn(offeringId); - Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.KVM); - return vm; - } - - private KubernetesClusterVmMapVO vmMap(long id) { - KubernetesClusterVmMapVO map = Mockito.mock(KubernetesClusterVmMapVO.class); - Mockito.when(map.getVmId()).thenReturn(id); - return map; - } - - private NodeCapacitySnapshot snapshot(boolean unschedulable, boolean cksCordon) { - return new NodeCapacitySnapshot(unschedulable, cksCordon, 2L, 2097152L, - 2000L, 2147483648L, 1900L, 2013265920L, true); - } -} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizerTest.java new file mode 100644 index 000000000000..e160304e96c8 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeOfferingResizerTest.java @@ -0,0 +1,563 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.VirtualMachineMigrationException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.kubernetes.cluster.utils.GuestResources; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler; +import com.cloud.kubernetes.cluster.utils.NodeAccess; +import com.cloud.kubernetes.cluster.utils.NodeCapacityReconciliationException; +import com.cloud.kubernetes.cluster.utils.NodeCapacitySnapshot; +import com.cloud.kubernetes.cluster.utils.NodeResources; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.UserVmManager; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.dao.UserVmDao; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; + +public class KubernetesClusterNodeOfferingResizerTest { + private static final String CLUSTER_UUID = "cluster-uuid"; + private static final long DEADLINE = 600000L; + private static final long MIB = 1024L * 1024L; + + private final KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); + private final UserVmDao vmDao = Mockito.mock(UserVmDao.class); + private final ServiceOfferingDao offeringDao = Mockito.mock(ServiceOfferingDao.class); + private final UserVmManager vmManager = Mockito.mock(UserVmManager.class); + private final KubernetesClusterNodeCapacityReconciler reconciler = Mockito.mock(KubernetesClusterNodeCapacityReconciler.class); + private final NodeAccess access = new NodeAccess("control", 22, "node", 2222, "cloud", null); + private final ServiceOfferingVO current = offering(10L, 2, 2048); + private final ServiceOfferingVO target = offering(20L, 4, 4096); + private final UserVmVO vm = vm(10L); + private final UserVmVO resizedVm = vm(20L); + private final KubernetesClusterVmMapVO vmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + private final GuestResources resizedGuest = guest(4, 4026532L, 4096); + private boolean clusterRunning = true; + private int accessResolutions; + + @Before + public void setUp() throws Exception { + Mockito.when(cluster.getUuid()).thenReturn(CLUSTER_UUID); + Mockito.when(cluster.getName()).thenReturn("test-cluster"); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(vmMap.getVmId()).thenReturn(1L); + Mockito.when(vmDao.findById(1L)).thenReturn(vm); + Mockito.when(offeringDao.findByIdIncludingRemoved(10L)).thenReturn(current); + Mockito.when(vmManager.upgradeVirtualMachine(eq(1L), eq(20L), any())).thenAnswer(invocation -> { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedVm); + return true; + }); + Mockito.when(reconciler.readNode(any(), eq(access))).thenReturn(node(false, null, null)); + Mockito.when(reconciler.readGuest(vm, access)).thenReturn(guest(2, 2029380L, 2048)); + Mockito.when(reconciler.startResize(cluster, vm, access, "uid-1")).thenReturn(node(true, CLUSTER_UUID, CLUSTER_UUID)); + Mockito.when(reconciler.waitForGuestResources(any(), eq(target), eq(access), eq(DEADLINE))).thenReturn(resizedGuest); + Mockito.when(reconciler.isKubeletActive(access)).thenReturn(true); + Mockito.when(reconciler.finishResize(eq(cluster), any(), eq(access), eq("uid-1"))).thenReturn(node(false, null, null)); + } + + @Test + public void testResizesAndReconcilesAWorkerNode() throws Exception { + resize(WORKER); + + InOrder order = Mockito.inOrder(reconciler, vmManager); + order.verify(reconciler).readNode(vm, access); + order.verify(reconciler).readGuest(vm, access); + order.verify(reconciler).startResize(cluster, vm, access, "uid-1"); + order.verify(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + order.verify(reconciler).onlineGuestResources(access); + order.verify(reconciler).waitForGuestResources(resizedVm, target, access, DEADLINE); + order.verify(reconciler).restartKubelet(resizedVm, access, DEADLINE); + order.verify(reconciler).waitForKubernetesResources(resizedVm, resizedGuest, access, DEADLINE); + order.verify(reconciler).finishResize(cluster, resizedVm, access, "uid-1"); + Mockito.verify(reconciler, Mockito.never()).releaseCordon(any(), any(), any(), any()); + Mockito.verify(offeringDao, Mockito.never()).findById(anyLong()); + } + + @Test + public void testUpToDateNodeWithActiveKubeletIsNotRestarted() { + Mockito.when(reconciler.isKubernetesNodeAlreadyResized(any(), eq(resizedGuest))).thenReturn(true); + + resize(WORKER); + + Mockito.verify(reconciler, Mockito.never()).restartKubelet(any(), any(), anyLong()); + Mockito.verify(reconciler).finishResize(cluster, resizedVm, access, "uid-1"); + } + + @Test + public void testInactiveKubeletIsRestartedEvenIfTheNodeLooksUpToDate() { + Mockito.when(reconciler.isKubernetesNodeAlreadyResized(any(), eq(resizedGuest))).thenReturn(true); + Mockito.when(reconciler.isKubeletActive(access)).thenReturn(false); + + resize(WORKER); + + Mockito.verify(reconciler).restartKubelet(resizedVm, access, DEADLINE); + Mockito.verify(reconciler).finishResize(cluster, resizedVm, access, "uid-1"); + } + + @Test + public void testDedicatedEtcdVmGetsThePlainResize() throws Exception { + assertPlainResize(ETCD); + } + + @Test + public void testExternalNodeGetsThePlainResize() throws Exception { + Mockito.when(vmMap.isExternalNode()).thenReturn(true); + assertPlainResize(WORKER); + } + + @Test + public void testExternallyManagedClusterGetsThePlainResize() throws Exception { + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.ExternalManaged); + assertPlainResize(WORKER); + } + + @Test + public void testNonKvmVmGetsThePlainResize() throws Exception { + Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.VMware); + assertPlainResize(WORKER); + } + + @Test + public void testClusterThatWasNotRunningGetsThePlainResize() throws Exception { + clusterRunning = false; + assertPlainResize(WORKER); + } + + @Test + public void testOfferingWithTheSameCpuAndRamGetsThePlainResizeAndOneNodeRead() throws Exception { + ServiceOfferingVO sameCapacity = offering(10L, 4, 4096); + Mockito.when(offeringDao.findByIdIncludingRemoved(10L)).thenReturn(sameCapacity); + + resize(WORKER); + + Mockito.verify(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + Mockito.verify(reconciler).readNode(vm, access); + Mockito.verifyNoMoreInteractions(reconciler); + } + + @Test + public void testOfferingWithTheSameCpuAndRamFinishesWorkLeftByAnEarlierResize() throws Exception { + ServiceOfferingVO sameCapacity = offering(10L, 4, 4096); + Mockito.when(offeringDao.findByIdIncludingRemoved(10L)).thenReturn(sameCapacity); + Mockito.when(reconciler.readNode(vm, access)).thenReturn(node(true, CLUSTER_UUID, CLUSTER_UUID)); + + resize(WORKER); + + InOrder order = Mockito.inOrder(vmManager, reconciler); + order.verify(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + order.verify(reconciler).waitForGuestResources(vm, target, access, DEADLINE); + order.verify(reconciler).finishResize(cluster, vm, access, "uid-1"); + Mockito.verify(reconciler, Mockito.never()).startResize(any(), any(), any(), any()); + } + + @Test + public void testPlainResizeFailureKeepsTheCause() throws Exception { + ResourceUnavailableException failure = new ResourceUnavailableException("host unavailable", UserVmVO.class, 1L); + Mockito.doThrow(failure).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(ETCD)); + + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("unable to scale ETCD node VM worker-1")); + } + + @Test + public void testPlainResizeRejectedByTheVmApiFails() throws Exception { + Mockito.doReturn(false).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + Assert.assertThrows(CloudRuntimeException.class, () -> resize(ETCD)); + } + + @Test + public void testVmOnTargetWithoutMarkersIsSkippedWithoutGuestAccess() throws Exception { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedVm); + + resize(WORKER); + + Mockito.verify(reconciler).readNode(resizedVm, access); + Mockito.verifyNoMoreInteractions(reconciler); + Mockito.verifyNoInteractions(vmManager); + } + + @Test + public void testVmOnTargetWithPendingWorkIsFinishedBeforeAnythingElse() throws Exception { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedVm); + Mockito.when(reconciler.readNode(resizedVm, access)).thenReturn(node(true, CLUSTER_UUID, null)); + + resize(WORKER); + + Mockito.verify(reconciler).onlineGuestResources(access); + Mockito.verify(reconciler).waitForGuestResources(resizedVm, target, access, DEADLINE); + Mockito.verify(reconciler).finishResize(cluster, resizedVm, access, "uid-1"); + Mockito.verify(reconciler, Mockito.never()).startResize(any(), any(), any(), any()); + Mockito.verifyNoInteractions(vmManager); + } + + @Test + public void testLeftoverOwnershipMarkerOnASchedulableNodeIsCleanedUp() { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedVm); + Mockito.when(reconciler.readNode(resizedVm, access)).thenReturn(node(false, null, CLUSTER_UUID)); + + resize(WORKER); + + Mockito.verify(reconciler).finishResize(cluster, resizedVm, access, "uid-1"); + } + + @Test + public void testOwnershipMarkerWithoutPendingMarkerIsStillFinished() { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedVm); + Mockito.when(reconciler.readNode(resizedVm, access)).thenReturn(node(true, null, CLUSTER_UUID)); + + resize(WORKER); + + Mockito.verify(reconciler).finishResize(cluster, resizedVm, access, "uid-1"); + } + + @Test + public void testUnreadableNodeOfAVmOnTargetFailsInsteadOfBeingSkipped() { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedVm); + NodeCapacityReconciliationException failure = new NodeCapacityReconciliationException("control plane unreachable"); + Mockito.when(reconciler.readNode(resizedVm, access)).thenThrow(failure); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("retry the same scale operation")); + Mockito.verify(reconciler, Mockito.never()).readGuest(any(), any()); + } + + @Test + public void testMarkersOfAnotherClusterAreNotTreatedAsDone() { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedVm); + Mockito.when(reconciler.readNode(resizedVm, access)).thenReturn(node(true, "another-cluster", null)); + + Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Mockito.verify(reconciler, Mockito.never()).finishResize(any(), any(), any(), any()); + } + + @Test + public void testVmOnTargetThatIsNotAKubernetesNodeIsLeftAlone() { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedVm); + + resize(ETCD); + + Mockito.verifyNoInteractions(reconciler, vmManager); + } + + @Test + public void testGuestAboveTheTargetIsRejectedBeforeAnyChange() { + Mockito.when(reconciler.readGuest(vm, access)).thenReturn(guest(4, 8152860L, 8192)); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertTrue(error.getMessage(), error.getMessage().contains("more than the requested offering")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("The VM was not resized and its Kubernetes node was not changed")); + Mockito.verify(reconciler, Mockito.never()).startResize(any(), any(), any(), any()); + Mockito.verifyNoInteractions(vmManager); + } + + @Test + public void testGuestReadFailureStopsBeforeAnyChange() { + NodeCapacityReconciliationException failure = new NodeCapacityReconciliationException("SSH unavailable"); + Mockito.when(reconciler.readGuest(vm, access)).thenThrow(failure); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertSame(failure, error.getCause()); + Mockito.verify(reconciler, Mockito.never()).startResize(any(), any(), any(), any()); + Mockito.verifyNoInteractions(vmManager); + } + + @Test + public void testStartFailureStopsBeforeTheVmResize() { + NodeCapacityReconciliationException failure = new NodeCapacityReconciliationException("patch failed"); + Mockito.when(reconciler.startResize(cluster, vm, access, "uid-1")).thenThrow(failure); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("may already be marked")); + Mockito.verifyNoInteractions(vmManager); + Mockito.verify(reconciler, Mockito.never()).releaseCordon(any(), any(), any(), any()); + } + + @Test + public void testResourceUnavailableReleasesTheCksCordonAndKeepsPendingWork() throws Exception { + assertCordonReleasedAfter(new ResourceUnavailableException("host unavailable", UserVmVO.class, 1L)); + } + + @Test + public void testManagementFailureReleasesTheCksCordonAndKeepsPendingWork() throws Exception { + assertCordonReleasedAfter(new ManagementServerException("upgrade failed")); + } + + @Test + public void testMigrationFailureReleasesTheCksCordonAndKeepsPendingWork() throws Exception { + assertCordonReleasedAfter(new VirtualMachineMigrationException("migration failed")); + } + + @Test + public void testUnexpectedErrorReleasesTheCksCordonAndKeepsItsCause() throws Exception { + assertCordonReleasedAfter(new NullPointerException("programming error")); + } + + @Test + public void testRejectedVmResizeReleasesTheCksCordon() throws Exception { + Mockito.doReturn(false).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertTrue(error.getMessage(), error.getMessage().contains("may have been partly resized")); + Mockito.verify(reconciler).releaseCordon(cluster, vm, access, "uid-1"); + } + + @Test + public void testFailureAfterTheVmReachedTheTargetKeepsPendingWorkAndCordon() throws Exception { + ManagementServerException failure = new ManagementServerException("agent answer lost"); + Mockito.doAnswer(invocation -> { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedVm); + throw failure; + }).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("stays marked for reconciliation and cordoned")); + Mockito.verify(reconciler, Mockito.never()).releaseCordon(any(), any(), any(), any()); + Mockito.verify(reconciler, Mockito.never()).finishResize(any(), any(), any(), any()); + } + + @Test + public void testReconciliationFailureAfterTheResizeKeepsPendingWorkAndCordon() { + NodeCapacityReconciliationException failure = new NodeCapacityReconciliationException("guest verification failed"); + Mockito.when(reconciler.waitForGuestResources(resizedVm, target, access, DEADLINE)).thenThrow(failure); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("stays marked for reconciliation and cordoned")); + Mockito.verify(reconciler, Mockito.never()).releaseCordon(any(), any(), any(), any()); + Mockito.verify(reconciler, Mockito.never()).finishResize(any(), any(), any(), any()); + } + + @Test + public void testOperatorCordonIsNeverReleased() throws Exception { + Mockito.when(reconciler.readNode(vm, access)).thenReturn(node(true, null, null)); + Mockito.when(reconciler.startResize(cluster, vm, access, "uid-1")).thenReturn(node(true, CLUSTER_UUID, null)); + Mockito.doReturn(false).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Mockito.verify(reconciler, Mockito.never()).releaseCordon(any(), any(), any(), any()); + } + + @Test + public void testCordonOfAnEarlierAttemptIsKeptAfterAFailedRetry() throws Exception { + Mockito.when(reconciler.readNode(vm, access)).thenReturn(node(true, CLUSTER_UUID, CLUSTER_UUID)); + Mockito.doReturn(false).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Mockito.verify(reconciler, Mockito.never()).releaseCordon(any(), any(), any(), any()); + } + + @Test + public void testLeftoverOwnershipMarkerDoesNotKeepThisAttemptsCordon() throws Exception { + Mockito.when(reconciler.readNode(vm, access)).thenReturn(node(false, null, CLUSTER_UUID)); + Mockito.doReturn(false).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Mockito.verify(reconciler).releaseCordon(cluster, vm, access, "uid-1"); + } + + @Test + public void testUnreadableVmAfterAFailureLeavesTheNodeAsItIs() throws Exception { + ManagementServerException failure = new ManagementServerException("upgrade failed"); + CloudRuntimeException databaseFailure = new CloudRuntimeException("database unavailable"); + Mockito.when(vmDao.findById(1L)).thenReturn(vm).thenThrow(databaseFailure); + Mockito.doThrow(failure).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("could not be read again")); + Assert.assertSame(databaseFailure, failure.getSuppressed()[0]); + Mockito.verify(reconciler, Mockito.never()).releaseCordon(any(), any(), any(), any()); + } + + @Test + public void testFailedCordonReleaseKeepsTheOriginalFailure() throws Exception { + ManagementServerException failure = new ManagementServerException("upgrade failed"); + NodeCapacityReconciliationException releaseFailure = new NodeCapacityReconciliationException("control plane unreachable"); + Mockito.doThrow(failure).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + Mockito.when(reconciler.releaseCordon(cluster, vm, access, "uid-1")).thenThrow(releaseFailure); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertSame(failure, error.getCause()); + Assert.assertSame(releaseFailure, failure.getSuppressed()[0]); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("could not remove its temporary cordon")); + } + + @Test + public void testInterruptedResizeLeavesTheNodeAsItIs() throws Exception { + Mockito.doAnswer(invocation -> { + Thread.currentThread().interrupt(); + throw new ManagementServerException("interrupted"); + }).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + try { + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertTrue(error.getMessage(), error.getMessage().contains("The operation was interrupted")); + Mockito.verify(reconciler, Mockito.never()).releaseCordon(any(), any(), any(), any()); + } finally { + Thread.interrupted(); + } + } + + @Test + public void testRetryOfTheSameScaleConvergesAndReconciles() throws Exception { + Mockito.doThrow(new ManagementServerException("agent timeout")).doAnswer(invocation -> { + Mockito.when(vmDao.findById(1L)).thenReturn(resizedVm); + return true; + }).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + Mockito.verify(reconciler).releaseCordon(cluster, vm, access, "uid-1"); + Mockito.when(reconciler.readNode(vm, access)).thenReturn(node(false, CLUSTER_UUID, null)); + + resize(WORKER); + + Mockito.verify(vmManager, Mockito.times(2)).upgradeVirtualMachine(eq(1L), eq(20L), any()); + Mockito.verify(reconciler).finishResize(cluster, resizedVm, access, "uid-1"); + } + + @Test + public void testMissingCurrentOfferingFailsBeforeAnyChange() { + Mockito.when(offeringDao.findByIdIncludingRemoved(10L)).thenReturn(null); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertTrue(error.getMessage(), error.getMessage().contains("current offering of the VM was not found")); + Mockito.verifyNoInteractions(reconciler, vmManager); + } + + @Test + public void testMissingVmFails() { + Mockito.when(vmDao.findById(1L)).thenReturn(null); + + Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Mockito.verifyNoInteractions(reconciler, vmManager); + } + + private void resize(KubernetesClusterNodeType nodeType) { + new KubernetesClusterNodeOfferingResizer(cluster, + clusterRunning, + vmDao, + offeringDao, + vmManager, + reconciler, + userVm -> { + accessResolutions++; + return access; + }).resize(vmMap, nodeType, target, DEADLINE); + } + + private void assertPlainResize(KubernetesClusterNodeType nodeType) throws Exception { + resize(nodeType); + + Mockito.verify(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + Mockito.verifyNoInteractions(reconciler); + Assert.assertEquals(0, accessResolutions); + } + + private void assertCordonReleasedAfter(Exception failure) throws Exception { + Mockito.doThrow(failure).when(vmManager).upgradeVirtualMachine(eq(1L), eq(20L), any()); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, () -> resize(WORKER)); + + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("may have been partly resized")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("removed its temporary cordon")); + Mockito.verify(reconciler).releaseCordon(cluster, vm, access, "uid-1"); + Mockito.verify(reconciler, Mockito.never()).finishResize(any(), any(), any(), any()); + } + + private static NodeCapacitySnapshot node(boolean unschedulable, String pendingOwner, String cordonOwner) { + Map annotations = new HashMap<>(); + if (pendingOwner != null) { + annotations.put(NodeCapacitySnapshot.PENDING_ANNOTATION, pendingOwner); + } + if (cordonOwner != null) { + annotations.put(NodeCapacitySnapshot.CORDON_ANNOTATION, cordonOwner); + } + return new NodeCapacitySnapshot("uid-1", + "7", + unschedulable, + annotations, + true, + new NodeResources(4000L, 4L * 1024L * 1024L * 1024L), + new NodeResources(3900L, 3L * 1024L * 1024L * 1024L)); + } + + private static GuestResources guest(long cpus, long memTotalKiB, long onlineMemoryMiB) { + return new GuestResources(cpus, memTotalKiB, onlineMemoryMiB * MIB, 128L * MIB); + } + + private static ServiceOfferingVO offering(long id, int cpu, int memory) { + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(offering.getId()).thenReturn(id); + Mockito.when(offering.getCpu()).thenReturn(cpu); + Mockito.when(offering.getRamSize()).thenReturn(memory); + Mockito.when(offering.getName()).thenReturn("offering-" + id); + return offering; + } + + private static UserVmVO vm(long offeringId) { + UserVmVO vm = Mockito.mock(UserVmVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getUuid()).thenReturn("vm-uuid"); + Mockito.when(vm.getDisplayName()).thenReturn("worker-1"); + Mockito.when(vm.getServiceOfferingId()).thenReturn(offeringId); + Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.KVM); + return vm; + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerOfferingTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerOfferingTest.java new file mode 100644 index 000000000000..7f4262f48ba1 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerOfferingTest.java @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.HashMap; +import java.util.List; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.cluster.utils.NodeAccess; +import com.cloud.network.Network; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Ip; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; + +public class KubernetesClusterScaleWorkerOfferingTest { + private final KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); + private final KubernetesClusterManagerImpl manager = Mockito.mock(KubernetesClusterManagerImpl.class); + private final KubernetesClusterVmMapDao vmMapDao = Mockito.mock(KubernetesClusterVmMapDao.class); + private final KubernetesClusterNodeOfferingResizer resizer = Mockito.mock(KubernetesClusterNodeOfferingResizer.class); + private final PortForwardingRulesDao portForwardingRulesDao = Mockito.mock(PortForwardingRulesDao.class); + private final IPAddressDao ipAddressDao = Mockito.mock(IPAddressDao.class); + private final Network network = Mockito.mock(Network.class); + private final ServiceOfferingVO target = Mockito.mock(ServiceOfferingVO.class); + private final KubernetesClusterVmMapVO firstNode = Mockito.mock(KubernetesClusterVmMapVO.class); + private final KubernetesClusterVmMapVO secondNode = Mockito.mock(KubernetesClusterVmMapVO.class); + private final UserVm vm = Mockito.mock(UserVm.class); + private KubernetesClusterScaleWorker worker; + + @Before + public void setUp() { + Mockito.when(cluster.getId()).thenReturn(100L); + Mockito.when(cluster.getName()).thenReturn("test-cluster"); + Mockito.when(cluster.getUuid()).thenReturn("cluster-uuid"); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(cluster.getNodeCount()).thenReturn(2L); + Mockito.when(cluster.getTotalNodeCount()).thenReturn(2L); + worker = Mockito.spy(new KubernetesClusterScaleWorker(cluster, new HashMap<>(), null, null, null, null, null, manager)); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Scaling); + worker.kubernetesClusterVmMapDao = vmMapDao; + worker.portForwardingRulesDao = portForwardingRulesDao; + worker.ipAddressDao = ipAddressDao; + ReflectionTestUtils.setField(worker, "network", network); + ReflectionTestUtils.setField(worker, "scaleTimeoutTime", System.currentTimeMillis() + 60000L); + Mockito.doReturn(resizer).when(worker).createNodeOfferingResizer(); + Mockito.doReturn(true).when(worker).stateTransitTo(anyLong(), any()); + Mockito.doNothing().when(worker).deleteTemplateLaunchPermission(); + Mockito.doReturn(Mockito.mock(KubernetesClusterVO.class)).when(worker).updateKubernetesClusterEntry( + any(), any(), any(), any(), any(), any(), any(), eq(WORKER), anyBoolean(), anyBoolean()); + Mockito.doReturn(new Pair<>("control", 2222)).when(worker).getKubernetesClusterServerIpSshPort(null); + Mockito.doReturn("cloud").when(worker).getControlNodeLoginUser(); + Mockito.doReturn(null).when(worker).getManagementServerSshPublicKeyFile(); + List nodes = List.of(firstNode, secondNode); + Mockito.when(vmMapDao.listByClusterId(100L)).thenReturn(nodes); + Mockito.when(vmMapDao.listByClusterIdAndVmType(100L, WORKER)).thenReturn(nodes); + Mockito.when(target.getId()).thenReturn(20L); + Mockito.when(target.getCpu()).thenReturn(4); + Mockito.when(target.getRamSize()).thenReturn(4096); + Mockito.when(network.getId()).thenReturn(200L); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getUuid()).thenReturn("vm-uuid"); + Mockito.when(vm.getPrivateIpAddress()).thenReturn("10.1.1.11"); + } + + @Test + public void testNodesAreResizedOneAfterAnotherBeforeTheClusterIsUpdated() { + scale(); + + InOrder order = Mockito.inOrder(resizer, worker); + order.verify(resizer).resize(eq(firstNode), eq(WORKER), eq(target), anyLong()); + order.verify(resizer).resize(eq(secondNode), eq(WORKER), eq(target), anyLong()); + order.verify(worker).updateKubernetesClusterEntry(eq(8L), eq(8192L), any(), eq(20L), any(), any(), any(), eq(WORKER), eq(true), eq(false)); + Mockito.verify(worker, Mockito.never()).stateTransitTo(100L, KubernetesCluster.Event.OperationFailed); + } + + @Test + public void testNodeFailureMarksTheScaleFailedAndStopsBeforeTheNextNode() { + CloudRuntimeException failure = new CloudRuntimeException("unable to scale WORKER node VM worker-1"); + Mockito.doThrow(failure).when(resizer).resize(eq(firstNode), eq(WORKER), eq(target), anyLong()); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); + + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("unable to scale WORKER node VM worker-1")); + Mockito.verify(worker).stateTransitTo(100L, KubernetesCluster.Event.OperationFailed); + Mockito.verify(resizer, Mockito.never()).resize(eq(secondNode), any(), any(), anyLong()); + assertClusterNotUpdated(); + } + + @Test + public void testUnexpectedErrorFromANodeAlsoMarksTheScaleFailed() { + NullPointerException failure = new NullPointerException(); + Mockito.doThrow(failure).when(resizer).resize(eq(firstNode), eq(WORKER), eq(target), anyLong()); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); + + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("NullPointerException")); + Mockito.verify(worker).stateTransitTo(100L, KubernetesCluster.Event.OperationFailed); + assertClusterNotUpdated(); + } + + @Test + public void testExpiredTimeoutStopsBeforeTheNextNode() { + Mockito.doAnswer(invocation -> { + ReflectionTestUtils.setField(worker, "scaleTimeoutTime", System.currentTimeMillis() - 1000L); + return null; + }).when(resizer).resize(eq(firstNode), eq(WORKER), eq(target), anyLong()); + + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, this::scale); + + Assert.assertTrue(error.getMessage(), error.getMessage().contains("timed out after 1 of 2 WORKER VMs were scaled")); + Mockito.verify(resizer, Mockito.never()).resize(eq(secondNode), any(), any(), anyLong()); + Mockito.verify(worker).stateTransitTo(100L, KubernetesCluster.Event.OperationFailed); + assertClusterNotUpdated(); + } + + @Test + public void testTimeoutReachedWhileTheLastNodeFinishesIsNotAFailure() { + Mockito.doAnswer(invocation -> { + ReflectionTestUtils.setField(worker, "scaleTimeoutTime", System.currentTimeMillis() - 1000L); + return null; + }).when(resizer).resize(eq(secondNode), eq(WORKER), eq(target), anyLong()); + + scale(); + + Mockito.verify(worker, Mockito.never()).stateTransitTo(100L, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker).updateKubernetesClusterEntry(any(), any(), any(), eq(20L), any(), any(), any(), eq(WORKER), eq(true), eq(false)); + } + + @Test + public void testCreatedClusterUpdatesMetadataWithoutTouchingVms() { + ReflectionTestUtils.setField(worker, "originalState", KubernetesCluster.State.Created); + + scale(); + + Mockito.verifyNoInteractions(resizer); + Mockito.verify(worker).updateKubernetesClusterEntry(eq(8L), eq(8192L), any(), eq(20L), any(), any(), any(), eq(WORKER), eq(true), eq(false)); + } + + @Test + public void testDirectAccessUsesThePrivateAddressOfTheNode() { + Mockito.when(manager.isDirectAccess(network)).thenReturn(true); + + NodeAccess access = worker.resolveNodeAccess(vm); + + Assert.assertEquals("control", access.getControlAddress()); + Assert.assertEquals(2222, access.getControlPort()); + Assert.assertEquals("10.1.1.11", access.getNodeAddress()); + Assert.assertEquals(22, access.getNodePort()); + Mockito.verifyNoInteractions(portForwardingRulesDao); + } + + @Test + public void testNodeAccessUsesThePublicIpAndPortOfTheSshRule() { + PortForwardingRuleVO udpRule = sshRule("udp", FirewallRule.State.Active, 4L, 2222); + PortForwardingRuleVO revokedRule = sshRule("tcp", FirewallRule.State.Revoke, 4L, 2222); + PortForwardingRuleVO natRule = sshRule("tcp", FirewallRule.State.Active, 5L, 2223); + Mockito.when(portForwardingRulesDao.listByVm(1L)).thenReturn(List.of(udpRule, revokedRule, natRule)); + IPAddressVO natIp = Mockito.mock(IPAddressVO.class); + Mockito.when(natIp.getAddress()).thenReturn(new Ip("203.0.113.5")); + Mockito.when(ipAddressDao.findById(5L)).thenReturn(natIp); + + NodeAccess access = worker.resolveNodeAccess(vm); + + Assert.assertEquals("control", access.getControlAddress()); + Assert.assertEquals("203.0.113.5", access.getNodeAddress()); + Assert.assertEquals(2223, access.getNodePort()); + // On Netris VPC tiers the cluster's public IP is the load balancer IP, not the NAT IP of the SSH rules. + Mockito.verify(worker, Mockito.never()).getVpcTierKubernetesPublicIp(any()); + Mockito.verify(worker, Mockito.never()).getNetworkSourceNatIp(any()); + } + + @Test + public void testMissingSshRuleFails() { + PortForwardingRuleVO revokedRule = sshRule("tcp", FirewallRule.State.Revoke, 5L, 2223); + Mockito.when(portForwardingRulesDao.listByVm(1L)).thenReturn(List.of(revokedRule)); + + Assert.assertThrows(CloudRuntimeException.class, () -> worker.resolveNodeAccess(vm)); + } + + @Test + public void testMissingIpOfTheSshRuleFails() { + PortForwardingRuleVO ruleWithoutIp = sshRule("tcp", FirewallRule.State.Active, 5L, 2223); + Mockito.when(portForwardingRulesDao.listByVm(1L)).thenReturn(List.of(ruleWithoutIp)); + + Assert.assertThrows(CloudRuntimeException.class, () -> worker.resolveNodeAccess(vm)); + } + + @Test + public void testMissingControlPlaneAddressFails() { + Mockito.doReturn(new Pair<>(null, 2222)).when(worker).getKubernetesClusterServerIpSshPort(null); + + Assert.assertThrows(CloudRuntimeException.class, () -> worker.resolveNodeAccess(vm)); + } + + private void scale() { + worker.scaleKubernetesClusterOffering(WORKER, target, true, false); + } + + private void assertClusterNotUpdated() { + Mockito.verify(worker, Mockito.never()).updateKubernetesClusterEntry( + any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), anyBoolean()); + } + + private PortForwardingRuleVO sshRule(String protocol, FirewallRule.State state, long sourceIpId, int sourcePort) { + PortForwardingRuleVO rule = Mockito.mock(PortForwardingRuleVO.class); + Mockito.when(rule.getDestinationPortStart()).thenReturn(22); + Mockito.when(rule.getProtocol()).thenReturn(protocol); + Mockito.when(rule.getNetworkId()).thenReturn(200L); + Mockito.when(rule.getState()).thenReturn(state); + Mockito.when(rule.getSourceIpAddressId()).thenReturn(sourceIpId); + Mockito.when(rule.getSourcePortStart()).thenReturn(sourcePort); + return rule; + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index ac4ece5a268a..997fb6df8c1a 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -20,14 +20,11 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; -import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler; -import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconcilerImpl; -import com.cloud.hypervisor.Hypervisor; import com.cloud.offering.ServiceOffering; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.utils.Pair; -import com.cloud.vm.UserVmVO; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.dao.UserVmDao; import org.junit.Assert; import org.junit.Before; @@ -59,8 +56,6 @@ public class KubernetesClusterScaleWorkerTest { private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @Mock private UserVmDao userVmDao; - @Mock - private KubernetesClusterNodeCapacityReconciler kubernetesClusterNodeCapacityReconciler; private KubernetesClusterScaleWorker worker; @@ -298,57 +293,26 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { @Test public void testUsesLiveResizeReconcilerWithoutComponentContextRegistration() { - Assert.assertTrue(worker.kubernetesClusterNodeCapacityReconciler instanceof KubernetesClusterNodeCapacityReconcilerImpl); + Assert.assertNotNull(worker.kubernetesClusterNodeCapacityReconciler); } @Test - public void testShouldReconcileNodeCapacityOnlyForManagedKvmNodes() { - KubernetesCluster runningManagedCluster = Mockito.mock(KubernetesCluster.class); - Mockito.when(runningManagedCluster.getState()).thenReturn(KubernetesCluster.State.Running); - Mockito.when(runningManagedCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); - KubernetesClusterScaleWorker scaleWorker = new KubernetesClusterScaleWorker(runningManagedCluster, - new java.util.HashMap<>(), 1L, null, false, null, null, clusterManager); - scaleWorker.kubernetesClusterNodeCapacityReconciler = kubernetesClusterNodeCapacityReconciler; - - KubernetesClusterVmMapVO managedNode = Mockito.mock(KubernetesClusterVmMapVO.class); - Mockito.when(managedNode.isExternalNode()).thenReturn(false); - UserVmVO kvmNode = Mockito.mock(UserVmVO.class); - Mockito.when(kvmNode.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); - ServiceOffering oldOffering = Mockito.mock(ServiceOffering.class); - ServiceOffering targetOffering = Mockito.mock(ServiceOffering.class); - Mockito.when(kubernetesClusterNodeCapacityReconciler.requiresKubeletRefresh(oldOffering, targetOffering, WORKER, - KubernetesCluster.State.Running)).thenReturn(true); - - Assert.assertTrue(scaleWorker.shouldReconcileNodeCapacity(managedNode, kvmNode, oldOffering, targetOffering, WORKER)); - - Mockito.when(managedNode.isExternalNode()).thenReturn(true); - Assert.assertFalse(scaleWorker.shouldReconcileNodeCapacity(managedNode, kvmNode, oldOffering, targetOffering, WORKER)); - - Mockito.when(managedNode.isExternalNode()).thenReturn(false); - Mockito.when(kvmNode.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.XenServer); - Assert.assertFalse(scaleWorker.shouldReconcileNodeCapacity(managedNode, kvmNode, oldOffering, targetOffering, WORKER)); - - Mockito.when(runningManagedCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.ExternalManaged); - Assert.assertFalse(scaleWorker.shouldReconcileNodeCapacity(managedNode, kvmNode, oldOffering, targetOffering, WORKER)); - } + public void testMissingOfferingOfALaterNodeTypeFailsBeforeAnyVmIsResized() { + Mockito.when(kubernetesCluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(kubernetesCluster.getControlNodeServiceOfferingId()).thenReturn(11L); + Mockito.when(kubernetesCluster.getWorkerNodeServiceOfferingId()).thenReturn(12L); + ServiceOfferingVO controlOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(11L)).thenReturn(controlOffering); + ServiceOfferingVO target = Mockito.mock(ServiceOfferingVO.class); + KubernetesClusterScaleWorker scaleWorker = Mockito.spy(new KubernetesClusterScaleWorker(kubernetesCluster, + java.util.Map.of(CONTROL.name(), target, WORKER.name(), target), null, null, null, null, null, clusterManager)); + scaleWorker.serviceOfferingDao = serviceOfferingDao; + Mockito.doNothing().when(scaleWorker).init(); - @Test - public void testCksOwnedCordonCanRetryReconciliationAfterVmReachedTargetOffering() { - KubernetesCluster runningManagedCluster = Mockito.mock(KubernetesCluster.class); - Mockito.when(runningManagedCluster.getState()).thenReturn(KubernetesCluster.State.Running); - Mockito.when(runningManagedCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); - KubernetesClusterScaleWorker scaleWorker = new KubernetesClusterScaleWorker(runningManagedCluster, - new java.util.HashMap<>(), 1L, null, false, null, null, clusterManager); - - KubernetesClusterVmMapVO managedNode = Mockito.mock(KubernetesClusterVmMapVO.class); - Mockito.when(managedNode.isExternalNode()).thenReturn(false); - UserVmVO kvmNode = Mockito.mock(UserVmVO.class); - Mockito.when(kvmNode.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); - Mockito.when(kvmNode.getServiceOfferingId()).thenReturn(Long.parseLong("1024")); - ServiceOffering targetOffering = Mockito.mock(ServiceOffering.class); - Mockito.when(targetOffering.getId()).thenReturn(Long.parseLong("1024")); - - Assert.assertTrue(scaleWorker.isReconciliationRetryCandidate(managedNode, kvmNode, targetOffering, WORKER)); - Assert.assertFalse(scaleWorker.isReconciliationRetryCandidate(managedNode, kvmNode, targetOffering, DEFAULT)); + CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, scaleWorker::scaleCluster); + + Assert.assertTrue(error.getMessage(), error.getMessage().contains("ID 12") && error.getMessage().contains("WORKER")); + + Mockito.verify(scaleWorker, Mockito.never()).scaleKubernetesClusterOffering(Mockito.any(), Mockito.any(), Mockito.anyBoolean(), Mockito.anyBoolean()); } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java deleted file mode 100644 index 8d2876d8f35e..000000000000 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerImplTest.java +++ /dev/null @@ -1,451 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.kubernetes.cluster.utils; - -import java.util.ArrayList; -import java.util.List; - -import com.cloud.kubernetes.cluster.KubernetesCluster; -import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; -import com.cloud.offering.ServiceOffering; -import com.cloud.uservm.UserVm; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.ssh.SshHelper; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; - -public class KubernetesClusterNodeCapacityReconcilerImplTest { - private final KubernetesClusterNodeCapacityReconciler reconciler = new KubernetesClusterNodeCapacityReconcilerImpl(); - - @Test - public void testParseKubernetesQuantities() { - Assert.assertEquals(3900L, KubernetesClusterNodeCapacityReconcilerImpl.parseCpuMillis("3900m")); - Assert.assertEquals(4000L, KubernetesClusterNodeCapacityReconcilerImpl.parseCpuMillis("4")); - Assert.assertEquals(4037048L * 1024L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("4037048Ki")); - Assert.assertEquals(4L * 1024L * 1024L * 1024L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("4Gi")); - } - - @Test - public void testOnlyRunningKubernetesNodesWithChangedCpuOrRamRequireRefresh() { - ServiceOffering oldOffering = offering(2, 2048); - ServiceOffering cpuOffering = offering(4, 2048); - ServiceOffering ramOffering = offering(2, 4096); - ServiceOffering capOnlyOffering = offering(2, 2048); - - Assert.assertTrue(reconciler.requiresKubeletRefresh(oldOffering, cpuOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Running)); - Assert.assertTrue(reconciler.requiresKubeletRefresh(oldOffering, ramOffering, KubernetesClusterNodeType.CONTROL, KubernetesCluster.State.Running)); - Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, capOnlyOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Running)); - Assert.assertFalse(reconciler.requiresKubeletRefresh(null, cpuOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Running)); - Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, null, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Running)); - Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, cpuOffering, KubernetesClusterNodeType.ETCD, KubernetesCluster.State.Running)); - Assert.assertFalse(reconciler.requiresKubeletRefresh(oldOffering, cpuOffering, KubernetesClusterNodeType.WORKER, KubernetesCluster.State.Stopped)); - } - - @Test - public void testCurrentKubernetesResourcesDoNotRequireASecondKubeletRestart() { - long memoryKiB = 4L * 1024L * 1024L; - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot current = - new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, 4, memoryKiB, - 4000, memoryKiB * 1024L, 3900, memoryKiB * 1024L - 128L * 1024L * 1024L, true); - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot stale = - new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, 4, memoryKiB, - 2000, memoryKiB * 1024L, 1900, memoryKiB * 1024L - 128L * 1024L * 1024L, true); - - Assert.assertTrue(reconciler.isKubernetesResourcesCurrent(current, offering(4, 4096))); - Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(stale, offering(4, 4096))); - } - - @Test - public void testCaptureBeforeReadsStructuredNodeState() throws Exception { - FakeReconciler fakeReconciler = new FakeReconciler(nodeJson(false)); - KubernetesCluster cluster = cluster("cluster-uuid"); - UserVm vm = vm("worker-1"); - - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot snapshot = fakeReconciler.captureBefore(cluster, vm, access()); - - Assert.assertFalse(snapshot.isUnschedulable()); - Assert.assertTrue(snapshot.isReady()); - Assert.assertEquals(2L, snapshot.getGuestOnlineCpuCount()); - Assert.assertEquals(2097152L, snapshot.getGuestMemoryKiB()); - Assert.assertEquals(3900L, snapshot.getCapacityCpuMillis()); - Assert.assertEquals(4L * 1024L * 1024L * 1024L, snapshot.getCapacityMemoryBytes()); - } - - @Test - public void testCordonAndRestoreOnlyNodesOwnedByCks() throws Exception { - KubernetesCluster cluster = cluster("cluster-uuid"); - UserVm vm = vm("worker-1"); - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot previouslyCordon = - new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(true, false, 0, 0, 0, 0, 0, 0, true); - FakeReconciler operatorCordon = new FakeReconciler(nodeJson(true)); - - operatorCordon.cordonIfNeeded(cluster, vm, previouslyCordon, access(), System.currentTimeMillis() + 1000L); - operatorCordon.restoreSchedulability(cluster, vm, previouslyCordon, access(), System.currentTimeMillis() + 1000L); - Assert.assertTrue(operatorCordon.controlCommands.isEmpty()); - - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot cksCordon = - new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(true, true, 0, 0, 0, 0, 0, 0, true); - FakeReconciler cleanup = new FakeReconciler(nodeJson(false)); - cleanup.restoreSchedulability(cluster, vm, cksCordon, access(), System.currentTimeMillis() + 1000L); - - Assert.assertEquals("sudo /opt/bin/kubectl uncordon worker-1", cleanup.controlCommands.get(0)); - Assert.assertEquals("sudo /opt/bin/kubectl annotate node worker-1 cloudstack.apache.org/cks-live-resize-", cleanup.controlCommands.get(1)); - } - - @Test - public void testCordonAddsOwnershipAnnotationBeforeCordoning() throws Exception { - KubernetesCluster cluster = cluster("cluster-uuid"); - UserVm vm = vm("worker-1"); - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot schedulable = - new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, 0, 0, 0, 0, 0, 0, true); - FakeReconciler fakeReconciler = new FakeReconciler(nodeJson(true)); - - fakeReconciler.cordonIfNeeded(cluster, vm, schedulable, access(), System.currentTimeMillis() + 1000L); - - Assert.assertEquals("sudo /opt/bin/kubectl annotate node worker-1 cloudstack.apache.org/cks-live-resize=cluster-uuid --overwrite", - fakeReconciler.controlCommands.get(0)); - Assert.assertEquals("sudo /opt/bin/kubectl cordon worker-1", fakeReconciler.controlCommands.get(1)); - } - - @Test - public void testGuestVerificationAllowsObservedKvmHotplugOverhead() throws Exception { - FakeReconciler fakeReconciler = new FakeReconciler(nodeJson(false)); - fakeReconciler.nodeResources = "4\n4568788\n"; - - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot before = - new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, 2, 2097152L, 2000L, 2L * 1024L * 1024L * 1024L, - 1900L, 1900L * 1024L * 1024L, true); - - fakeReconciler.verifyGuestResources(vm("worker-1"), offering(4, 4608), before, access(), System.currentTimeMillis() + 1000L); - } - - @Test - public void testOnlineGuestResourcesActivatesHotpluggedCpuAndMemory() throws Exception { - FakeReconciler fakeReconciler = new FakeReconciler(nodeJson(false)); - - fakeReconciler.onlineGuestResources(access()); - - Assert.assertEquals(KubernetesClusterNodeCapacityReconcilerImpl.ONLINE_HOTPLUGGED_RESOURCES, - fakeReconciler.nodeCommands.get(0)); - } - - @Test - public void testParseMemoryDecimalBinaryAndUnitlessQuantities() { - Assert.assertEquals(123L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("123")); - Assert.assertEquals(2L * 1024L * 1024L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2Mi")); - Assert.assertEquals(2L * 1024L * 1024L * 1024L * 1024L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2Ti")); - Assert.assertEquals(2000L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2K")); - Assert.assertEquals(2000000L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2M")); - Assert.assertEquals(2000000000L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2G")); - Assert.assertEquals(2000000000000L, KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("2T")); - Assert.assertThrows(IllegalArgumentException.class, () -> KubernetesClusterNodeCapacityReconcilerImpl.parseMemoryBytes("4Gi; reboot")); - } - - @Test - public void testRestartKubeletWaitsForActiveService() throws Exception { - FakeReconciler fake = new FakeReconciler(nodeJson(false)); - fake.nodeResources = " active\n"; - - fake.restartKubelet(vm("worker-1"), access(), System.currentTimeMillis() + 1000L); - - Assert.assertEquals(List.of("sudo systemctl restart kubelet", "sudo systemctl is-active kubelet"), fake.nodeCommands); - } - - @Test - public void testExpiredDeadlinesReportTheFailedPhaseAndVm() { - FakeReconciler fake = new FakeReconciler(nodeJson(false)); - KubernetesCluster cluster = cluster("cluster-uuid"); - UserVm vm = vm("worker-1"); - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot before = snapshot(2, 2048, true); - long expired = System.currentTimeMillis() - 1L; - - assertPhaseFailure("CORDON", () -> fake.cordonIfNeeded(cluster, vm, before, access(), expired)); - assertPhaseFailure("GUEST_VERIFY", () -> fake.verifyGuestResources(vm, offering(4, 4096), before, access(), expired)); - assertPhaseFailure("KUBELET_RESTART", () -> fake.restartKubelet(vm, access(), expired)); - assertPhaseFailure("CAPACITY_VERIFY", () -> fake.waitForKubernetesResources(cluster, vm, offering(4, 4096), before, access(), expired)); - assertPhaseFailure("RESTORE_SCHEDULABILITY", () -> fake.restoreSchedulability(cluster, vm, before, access(), expired)); - } - - @Test - public void testWaitForKubernetesResourcesRequiresIncreasedAllocatableCpuAndMemory() throws Exception { - FakeReconciler fake = new FakeReconciler(nodeJson(false).replace("3900m", "4")); - fake.nodeResources = "4\n4194304\n"; - - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot observed = fake.waitForKubernetesResources( - cluster("cluster-uuid"), vm("worker-1"), offering(4, 4096), snapshot(2, 2048, true), - access(), System.currentTimeMillis() + 1000L); - - Assert.assertTrue(observed.isReady()); - Assert.assertEquals(4000L, observed.getCapacityCpuMillis()); - Assert.assertEquals(3700L, observed.getAllocatableCpuMillis()); - Assert.assertEquals(3900L * 1024L * 1024L, observed.getAllocatableMemoryBytes()); - } - - @Test(timeout = 15000L) - public void testWaitRejectsUnchangedAllocatableCpuAfterCpuIncrease() { - FakeReconciler fake = new FakeReconciler(nodeJson(false).replace("3900m", "4").replace("3700m", "1900m")); - fake.nodeResources = "4\n4194304\n"; - - assertPhaseFailure("CAPACITY_VERIFY", () -> fake.waitForKubernetesResources(cluster("cluster-uuid"), vm("worker-1"), - offering(4, 4096), snapshot(2, 2048, true), access(), System.currentTimeMillis() + 1000L)); - - Assert.assertFalse(fake.controlCommands.isEmpty()); - Assert.assertTrue(fake.controlCommands.stream().noneMatch(command -> command.contains("uncordon"))); - } - - @Test(timeout = 15000L) - public void testWaitRejectsStaleAllocatableMemoryAfterMemoryIncrease() { - FakeReconciler fake = new FakeReconciler(nodeJson(false).replace("3900m", "4").replace("3900Mi", "1900Mi")); - fake.nodeResources = "4\n4194304\n"; - - assertPhaseFailure("CAPACITY_VERIFY", () -> fake.waitForKubernetesResources(cluster("cluster-uuid"), vm("worker-1"), - offering(4, 4096), snapshot(2, 2048, true), access(), System.currentTimeMillis() + 1000L)); - - Assert.assertFalse(fake.controlCommands.isEmpty()); - } - - @Test(timeout = 15000L) - public void testRestartKubeletDoesNotTreatInactiveServiceAsSuccess() { - FakeReconciler fake = new FakeReconciler(nodeJson(false)); - fake.nodeResources = "inactive\n"; - - assertPhaseFailure("KUBELET_RESTART", () -> fake.restartKubelet(vm("worker-1"), access(), System.currentTimeMillis() + 1000L)); - - Assert.assertEquals(List.of("sudo systemctl restart kubelet", "sudo systemctl is-active kubelet"), fake.nodeCommands); - } - - @Test - public void testResourceVerificationRejectsStaleNotReadyOrOutOfToleranceNodes() { - ServiceOffering target = offering(4, 4096); - Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(snapshot(4, 4096, false), target)); - Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(snapshot(2, 4096, true), target)); - Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(snapshot(4, 4097, true), target)); - Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(snapshot(4, 3800, true), target)); - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot staleMemory = - new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, 4, 4194304L, - 4000L, 2L * 1024L * 1024L * 1024L, 3700L, 1900L * 1024L * 1024L, true); - Assert.assertFalse(reconciler.isKubernetesResourcesCurrent(staleMemory, target)); - } - - @Test - public void testCaptureRecognizesOnlyThisClustersCordonOwnership() throws Exception { - String owned = nodeJson(true).replace("\"annotations\":{}", - "\"annotations\":{\"cloudstack.apache.org/cks-live-resize\":\"cluster-uuid\"}"); - FakeReconciler fake = new FakeReconciler(owned); - Assert.assertTrue(fake.captureBefore(cluster("cluster-uuid"), vm("worker-1"), access()).isCloudStackResizeCordon()); - Assert.assertFalse(fake.captureBefore(cluster("another-cluster"), vm("worker-1"), access()).isCloudStackResizeCordon()); - } - - @Test - public void testCaptureHandlesMissingOptionalFieldsAndNotReadyNode() throws Exception { - String minimal = nodeJson(false).replace("\"annotations\":{}", "\"name\":\"worker-1\"") - .replace("\"spec\":{\"unschedulable\":false},", "") - .replace("\"status\":\"True\"", "\"status\":\"False\""); - FakeReconciler fake = new FakeReconciler(minimal); - KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot observed = - fake.captureBefore(cluster("cluster-uuid"), vm("worker-1"), access()); - Assert.assertFalse(observed.isReady()); - Assert.assertFalse(observed.isUnschedulable()); - Assert.assertFalse(observed.isCloudStackResizeCordon()); - } - - @Test - public void testInvalidNodeNameIsRejectedBeforeControlCommand() { - FakeReconciler fake = new FakeReconciler(nodeJson(false)); - Assert.assertThrows(CloudRuntimeException.class, - () -> fake.captureBefore(cluster("cluster-uuid"), vm("worker; reboot"), access())); - Assert.assertTrue(fake.controlCommands.isEmpty()); - } - - @Test - public void testNodeNameIsLowercaseInKubectlCommand() throws Exception { - FakeReconciler fake = new FakeReconciler(nodeJson(false)); - fake.captureBefore(cluster("cluster-uuid"), vm("Worker-1"), access()); - Assert.assertEquals("sudo /opt/bin/kubectl get node worker-1 -o json", fake.controlCommands.get(0)); - } - - @Test - public void testMalformedGuestResourceOutputStopsBeforeKubectl() { - FakeReconciler fake = new FakeReconciler(nodeJson(false)); - fake.nodeResources = "4\n"; - CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, - () -> fake.captureBefore(cluster("cluster-uuid"), vm("worker-1"), access())); - Assert.assertEquals("Unable to read guest CPU and memory", error.getMessage()); - Assert.assertTrue(fake.controlCommands.isEmpty()); - } - - @Test - public void testNodeAccessPreservesSeparateControlAndWorkerEndpoints() { - java.io.File key = new java.io.File("test-only-key"); - KubernetesClusterNodeCapacityReconciler.NodeAccess access = - new KubernetesClusterNodeCapacityReconciler.NodeAccess("control", 2222, "worker", 2223, "cloud", key); - Assert.assertEquals("control", access.getControlAddress()); - Assert.assertEquals(2222, access.getControlPort()); - Assert.assertEquals("worker", access.getNodeAddress()); - Assert.assertEquals(2223, access.getNodePort()); - Assert.assertEquals("cloud", access.getUser()); - Assert.assertSame(key, access.getSshKeyFile()); - } - - @Test - public void testSshSuccessReturnsOutput() { - KubernetesClusterNodeCapacityReconcilerImpl real = new KubernetesClusterNodeCapacityReconcilerImpl(); - KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); - try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { - ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) - .thenReturn(new Pair<>(true, "output")); - Assert.assertEquals("output", real.executeNode(access, "command")); - } - } - - @Test - public void testRejectedSshCommandHasSpecificReconciliationFailure() { - KubernetesClusterNodeCapacityReconcilerImpl real = new KubernetesClusterNodeCapacityReconcilerImpl(); - KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); - try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { - ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) - .thenReturn(new Pair<>(false, "remote error")); - Assert.assertThrows(NodeCapacityReconciliationException.class, () -> real.executeNode(access, "command")); - } - } - - @Test - public void testLegacySshExceptionIsTranslatedAtTransportBoundary() { - KubernetesClusterNodeCapacityReconcilerImpl real = new KubernetesClusterNodeCapacityReconcilerImpl(); - KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); - Exception failure = new Exception("SSH authentication failed"); - try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { - ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) - .thenThrow(failure); - NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, - () -> real.executeNode(access, "command")); - Assert.assertSame(failure, error.getCause()); - } - } - - @Test - public void testSshInterruptionPreservesInterruptFlag() { - KubernetesClusterNodeCapacityReconcilerImpl real = new KubernetesClusterNodeCapacityReconcilerImpl(); - KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); - InterruptedException failure = new InterruptedException("cancelled"); - try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { - ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) - .thenThrow(failure); - NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, - () -> real.executeNode(access, "command")); - Assert.assertSame(failure, error.getCause()); - Assert.assertTrue(Thread.currentThread().isInterrupted()); - } finally { - Thread.interrupted(); - } - } - - @Test - public void testSshProgrammingErrorsAreNotTranslated() { - KubernetesClusterNodeCapacityReconcilerImpl real = new KubernetesClusterNodeCapacityReconcilerImpl(); - KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); - NullPointerException failure = new NullPointerException("programming error"); - try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { - ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) - .thenThrow(failure); - Assert.assertSame(failure, Assert.assertThrows(NullPointerException.class, () -> real.executeNode(access, "command"))); - } - } - - @Test - public void testPollingInterruptionPreservesInterruptFlag() { - FakeReconciler fake = new FakeReconciler(nodeJson(false)); - UserVm vm = vm("worker-1"); - KubernetesClusterNodeCapacityReconciler.NodeAccess access = access(); - long deadline = System.currentTimeMillis() + 10000L; - fake.nodeResources = "inactive"; - Thread.currentThread().interrupt(); - try { - NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, - () -> fake.restartKubelet(vm, access, deadline)); - Assert.assertTrue(error.getCause() instanceof InterruptedException); - Assert.assertTrue(Thread.currentThread().isInterrupted()); - } finally { - Thread.interrupted(); - } - } - - private KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot snapshot(int cpu, int memoryMiB, boolean ready) { - long bytes = memoryMiB * 1024L * 1024L; - return new KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot(false, false, cpu, memoryMiB * 1024L, - cpu * 1000L, bytes, cpu * 1000L - 100L, bytes - 128L * 1024L * 1024L, ready); - } - - private void assertPhaseFailure(String phase, org.junit.function.ThrowingRunnable action) { - CloudRuntimeException error = Assert.assertThrows(CloudRuntimeException.class, action); - Assert.assertTrue(error.getMessage(), error.getMessage().contains(phase + " failed for VM vm-uuid")); - } - - private ServiceOffering offering(int cpu, int memory) { - ServiceOffering offering = Mockito.mock(ServiceOffering.class); - Mockito.when(offering.getCpu()).thenReturn(cpu); - Mockito.when(offering.getRamSize()).thenReturn(memory); - return offering; - } - - private KubernetesCluster cluster(String uuid) { - KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); - Mockito.when(cluster.getUuid()).thenReturn(uuid); - return cluster; - } - - private UserVm vm(String hostname) { - UserVm vm = Mockito.mock(UserVm.class); - Mockito.when(vm.getHostName()).thenReturn(hostname); - Mockito.when(vm.getUuid()).thenReturn("vm-uuid"); - return vm; - } - - private KubernetesClusterNodeCapacityReconciler.NodeAccess access() { - return new KubernetesClusterNodeCapacityReconciler.NodeAccess("control", 22, "node", 22, "root", null); - } - - private String nodeJson(boolean unschedulable) { - return String.format("{\"metadata\":{\"annotations\":{}},\"spec\":{\"unschedulable\":%s},\"status\":{\"capacity\":{\"cpu\":\"3900m\",\"memory\":\"4Gi\"},\"allocatable\":{\"cpu\":\"3700m\",\"memory\":\"3900Mi\"},\"conditions\":[{\"type\":\"Ready\",\"status\":\"True\"}]}}", unschedulable); - } - - private static class FakeReconciler extends KubernetesClusterNodeCapacityReconcilerImpl { - private final String nodeJson; - private final List controlCommands = new ArrayList<>(); - private final List nodeCommands = new ArrayList<>(); - private String nodeResources = "2\n2097152\n"; - - FakeReconciler(String nodeJson) { - this.nodeJson = nodeJson; - } - - @Override - protected String executeControl(KubernetesClusterNodeCapacityReconciler.NodeAccess access, String command) { - controlCommands.add(command); - return command.contains(" get node ") ? nodeJson : ""; - } - - @Override - protected String executeNode(KubernetesClusterNodeCapacityReconciler.NodeAccess access, String command) { - nodeCommands.add(command); - return nodeResources; - } - } -} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerTest.java new file mode 100644 index 000000000000..e3ad8ca710d7 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconcilerTest.java @@ -0,0 +1,781 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import java.io.File; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.function.ThrowingRunnable; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.offering.ServiceOffering; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.ssh.SshHelper; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +public class KubernetesClusterNodeCapacityReconcilerTest { + private static final String CLUSTER_UUID = "cluster-uuid"; + private static final String PENDING = NodeCapacitySnapshot.PENDING_ANNOTATION; + private static final String CORDON = NodeCapacitySnapshot.CORDON_ANNOTATION; + private static final long MIB = 1024L * 1024L; + private static final long GIB = 1024L * MIB; + + private final KubernetesCluster cluster = cluster(); + private final UserVm vm = vm("worker-1"); + private final NodeAccess access = new NodeAccess("control", 22, "node", 22, "root", null); + private final KubernetesClusterNodeCapacityReconciler reconciler = new KubernetesClusterNodeCapacityReconciler(); + + @Test + public void testParseCpuAndMemoryQuantities() { + Assert.assertEquals(3900L, KubernetesClusterNodeCapacityReconciler.parseCpuMillis("3900m")); + Assert.assertEquals(4000L, KubernetesClusterNodeCapacityReconciler.parseCpuMillis("4")); + Assert.assertEquals(123L, KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("123")); + Assert.assertEquals(4037048L * 1024L, KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("4037048Ki")); + Assert.assertEquals(2L * 1024L * 1024L, KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("2Mi")); + Assert.assertEquals(4L * GIB, KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("4Gi")); + Assert.assertEquals(2L * 1024L * GIB, KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("2Ti")); + Assert.assertEquals(2000L, KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("2K")); + Assert.assertEquals(2000000L, KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("2M")); + Assert.assertEquals(2000000000L, KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("2G")); + Assert.assertEquals(2000000000000L, KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("2T")); + } + + @Test + public void testQuantityParsingRejectsMalformedAndOverflowingValues() { + Assert.assertThrows(IllegalArgumentException.class, () -> KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("4Gi; reboot")); + Assert.assertThrows(IllegalArgumentException.class, () -> KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("1.5Gi")); + Assert.assertThrows(IllegalArgumentException.class, () -> KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("-1")); + Assert.assertThrows(IllegalArgumentException.class, () -> KubernetesClusterNodeCapacityReconciler.parseCpuMillis("1e3")); + Assert.assertThrows(ArithmeticException.class, () -> KubernetesClusterNodeCapacityReconciler.parseMemoryBytes("99999999999999999Ti")); + Assert.assertThrows(ArithmeticException.class, () -> KubernetesClusterNodeCapacityReconciler.parseCpuMillis("99999999999999999")); + } + + @Test + public void testParseNodeReadsIdentityMarkersSchedulingAndResources() { + FakeNode node = new FakeNode(); + node.unschedulable = true; + node.annotations.put(PENDING, CLUSTER_UUID); + node.annotations.put(CORDON, CLUSTER_UUID); + + NodeCapacitySnapshot snapshot = KubernetesClusterNodeCapacityReconciler.parseNode(node.toJson()); + + Assert.assertEquals("uid-1", snapshot.getUid()); + Assert.assertEquals("7", snapshot.getResourceVersion()); + Assert.assertTrue(snapshot.isUnschedulable()); + Assert.assertTrue(snapshot.isReady()); + Assert.assertTrue(snapshot.isPendingFor(CLUSTER_UUID)); + Assert.assertTrue(snapshot.isCordonOwnedBy(CLUSTER_UUID)); + Assert.assertFalse(snapshot.hasForeignMarker(CLUSTER_UUID)); + Assert.assertTrue(snapshot.hasForeignMarker("another-cluster")); + Assert.assertEquals(2000L, snapshot.getCapacity().getCpuMillis()); + Assert.assertEquals(2L * GIB, snapshot.getCapacity().getMemoryBytes()); + Assert.assertEquals(1900L, snapshot.getAllocatable().getCpuMillis()); + Assert.assertEquals(1900L * 1024L * 1024L, snapshot.getAllocatable().getMemoryBytes()); + } + + @Test + public void testParseNodeAcceptsMissingOptionalFields() { + JsonObject node = JsonParser.parseString(new FakeNode().toJson()).getAsJsonObject(); + node.remove("spec"); + node.getAsJsonObject("metadata").remove("annotations"); + node.getAsJsonObject("status").remove("conditions"); + + NodeCapacitySnapshot snapshot = KubernetesClusterNodeCapacityReconciler.parseNode(node.toString()); + + Assert.assertFalse(snapshot.isUnschedulable()); + Assert.assertFalse(snapshot.isReady()); + Assert.assertFalse(snapshot.isPendingFor(CLUSTER_UUID)); + Assert.assertFalse(snapshot.hasForeignMarker(CLUSTER_UUID)); + } + + @Test + public void testParseNodeRejectsMissingRequiredFields() { + JsonObject withoutUid = JsonParser.parseString(new FakeNode().toJson()).getAsJsonObject(); + withoutUid.getAsJsonObject("metadata").remove("uid"); + JsonObject withoutMemory = JsonParser.parseString(new FakeNode().toJson()).getAsJsonObject(); + withoutMemory.getAsJsonObject("status").getAsJsonObject("capacity").remove("memory"); + + String withoutUidJson = withoutUid.toString(); + String withoutMemoryJson = withoutMemory.toString(); + + Assert.assertThrows(IllegalArgumentException.class, () -> KubernetesClusterNodeCapacityReconciler.parseNode(withoutUidJson)); + Assert.assertThrows(IllegalArgumentException.class, () -> KubernetesClusterNodeCapacityReconciler.parseNode(withoutMemoryJson)); + Assert.assertThrows(IllegalArgumentException.class, () -> KubernetesClusterNodeCapacityReconciler.parseNode("[]")); + } + + @Test + public void testMalformedKubectlOutputFailsWithPhaseAndVm() { + FakeReconciler fake = new FakeReconciler(); + fake.nodeOutput = "Error from server: not json"; + + assertPhaseFailure("READ_NODE", () -> fake.readNode(vm, access)); + } + + @Test + public void testNodeNameIsLowercaseInKubectlCommand() { + FakeReconciler fake = new FakeReconciler(); + + fake.readNode(vm("Worker-1"), access); + + Assert.assertEquals("sudo /opt/bin/kubectl get node worker-1 -o json --request-timeout=20s", fake.controlCommands.get(0)); + } + + @Test + public void testInvalidOrMissingHostNameFailsBeforeAnyCommand() { + FakeReconciler fake = new FakeReconciler(); + UserVm invalidName = vm("worker; reboot"); + UserVm noName = vm(null); + UserVm blankName = vm(" "); + + assertPhaseFailure("NODE_NAME", () -> fake.readNode(invalidName, access)); + assertPhaseFailure("NODE_NAME", () -> fake.readNode(noName, access)); + assertPhaseFailure("NODE_NAME", () -> fake.startResize(cluster, blankName, access, "uid-1")); + + Assert.assertTrue(fake.controlCommands.isEmpty()); + } + + @Test + public void testReadGuestParsesCpusMemTotalAndOnlineMemoryBlocks() { + FakeReconciler fake = new FakeReconciler(); + fake.guestOutput = "4\n4026532\n8000000\n32\n"; + + GuestResources guest = fake.readGuest(vm, access); + + Assert.assertEquals(4L, guest.getOnlineCpus()); + Assert.assertEquals(4026532L * 1024L, guest.getMemoryBytes()); + Assert.assertEquals(128L * MIB, guest.getMemoryBlockBytes()); + Assert.assertEquals(4L * GIB, guest.getOnlineMemoryBytes()); + Assert.assertEquals(List.of(KubernetesClusterNodeCapacityReconciler.GUEST_RESOURCES_COMMAND), fake.nodeCommands); + } + + @Test + public void testMalformedGuestOutputFails() { + FakeReconciler fake = new FakeReconciler(); + for (String output : List.of("4\n4194304", "0 4194304 8000000 32", "four 4194304 8000000 32", "4 4194304 8000000 32 1", + "4 4194304 0 32", "4 4194304 0x8000000 32", "4 4194304 8000000 0")) { + fake.guestOutput = output; + assertPhaseFailure("READ_GUEST", () -> fake.readGuest(vm, access)); + } + } + + @Test + public void testStartCordonsSchedulableNodeAndRecordsOwnershipInOnePatch() { + FakeReconciler fake = new FakeReconciler(); + + NodeCapacitySnapshot started = fake.startResize(cluster, vm, access, "uid-1"); + + Assert.assertEquals(List.of("sudo /opt/bin/kubectl patch node worker-1 --type merge --request-timeout=20s -p '{\"metadata\":" + + "{\"resourceVersion\":\"7\",\"annotations\":{\"" + PENDING + "\":\"cluster-uuid\",\"" + CORDON + "\":\"cluster-uuid\"}}," + + "\"spec\":{\"unschedulable\":true}}'"), fake.patches); + Assert.assertTrue(started.isUnschedulable()); + Assert.assertTrue(started.isPendingFor(CLUSTER_UUID)); + Assert.assertTrue(started.isCordonOwnedBy(CLUSTER_UUID)); + } + + @Test + public void testStartKeepsOperatorCordonAndAddsOnlyThePendingMarker() { + FakeReconciler fake = new FakeReconciler(); + fake.node.unschedulable = true; + + NodeCapacitySnapshot started = fake.startResize(cluster, vm, access, "uid-1"); + + Assert.assertEquals(1, fake.patches.size()); + Assert.assertFalse(fake.patches.get(0).contains("spec")); + Assert.assertFalse(fake.patches.get(0).contains(CORDON + "\"")); + Assert.assertTrue(started.isUnschedulable()); + Assert.assertTrue(started.isPendingFor(CLUSTER_UUID)); + Assert.assertFalse(started.isCordonOwnedBy(CLUSTER_UUID)); + } + + @Test + public void testStartDoesNothingWhenThePendingWorkIsAlreadyMarked() { + FakeReconciler fake = new FakeReconciler(); + fake.node.unschedulable = true; + fake.node.annotations.put(PENDING, CLUSTER_UUID); + fake.node.annotations.put(CORDON, CLUSTER_UUID); + + NodeCapacitySnapshot started = fake.startResize(cluster, vm, access, "uid-1"); + + Assert.assertTrue(fake.patches.isEmpty()); + Assert.assertTrue(started.isCordonOwnedBy(CLUSTER_UUID)); + } + + @Test + public void testStartNeverChangesMarkersOfAnotherCluster() { + FakeReconciler fake = new FakeReconciler(); + fake.node.annotations.put(PENDING, "another-cluster"); + + assertPhaseFailure("START", () -> fake.startResize(cluster, vm, access, "uid-1")); + + Assert.assertTrue(fake.patches.isEmpty()); + } + + @Test + public void testRejectedPatchFailsAfterBoundedAttempts() { + FakeReconciler fake = new FakeReconciler(); + fake.failingPatches = 10; + + assertPhaseFailure("START", () -> fake.startResize(cluster, vm, access, "uid-1")); + + Assert.assertEquals(KubernetesClusterNodeCapacityReconciler.MAX_NODE_UPDATE_ATTEMPTS, fake.patches.size()); + Assert.assertFalse(fake.node.annotations.containsKey(PENDING)); + Assert.assertFalse(fake.node.unschedulable); + } + + @Test + public void testLostPatchReplyIsSettledByReadingTheNode() { + FakeReconciler fake = new FakeReconciler(); + fake.loseNextReply = true; + + NodeCapacitySnapshot started = fake.startResize(cluster, vm, access, "uid-1"); + + Assert.assertEquals(1, fake.patches.size()); + Assert.assertTrue(started.isPendingFor(CLUSTER_UUID)); + Assert.assertTrue(started.isUnschedulable()); + } + + @Test + public void testConflictIsRetriedWithTheNewResourceVersion() { + FakeReconciler fake = new FakeReconciler(); + fake.node.annotations.put(PENDING, CLUSTER_UUID); + fake.node.annotations.put(CORDON, CLUSTER_UUID); + fake.node.unschedulable = true; + // kubelet writes the node status right after its restart + fake.beforeNextPatch = () -> fake.node.resourceVersion++; + + NodeCapacitySnapshot finished = fake.finishResize(cluster, vm, access, "uid-1"); + + Assert.assertEquals(2, fake.patches.size()); + Assert.assertTrue(fake.patches.get(0).contains("\"resourceVersion\":\"7\"")); + Assert.assertTrue(fake.patches.get(1).contains("\"resourceVersion\":\"8\"")); + Assert.assertFalse(finished.isUnschedulable()); + Assert.assertFalse(finished.isPendingFor(CLUSTER_UUID)); + } + + @Test + public void testReplacedNodeStopsTheUpdate() { + FakeReconciler fake = new FakeReconciler(); + + assertPhaseFailure("FINISH", () -> fake.finishResize(cluster, vm, access, "uid-0")); + + Assert.assertTrue(fake.patches.isEmpty()); + } + + @Test + public void testFinishRemovesMarkersAndUncordonsOnlyACksCordon() { + FakeReconciler fake = new FakeReconciler(); + fake.node.unschedulable = true; + fake.node.annotations.put(PENDING, CLUSTER_UUID); + fake.node.annotations.put(CORDON, CLUSTER_UUID); + fake.node.annotations.put("other", "kept"); + + NodeCapacitySnapshot finished = fake.finishResize(cluster, vm, access, "uid-1"); + + Assert.assertEquals(1, fake.patches.size()); + Assert.assertTrue(fake.patches.get(0).contains("\"spec\":{\"unschedulable\":false}")); + Assert.assertFalse(finished.isUnschedulable()); + Assert.assertEquals(Map.of("other", "kept"), fake.node.annotations); + } + + @Test + public void testFinishKeepsAnOperatorCordon() { + FakeReconciler fake = new FakeReconciler(); + fake.node.unschedulable = true; + fake.node.annotations.put(PENDING, CLUSTER_UUID); + + NodeCapacitySnapshot finished = fake.finishResize(cluster, vm, access, "uid-1"); + + Assert.assertFalse(fake.patches.get(0).contains("spec")); + Assert.assertTrue(finished.isUnschedulable()); + Assert.assertFalse(finished.isPendingFor(CLUSTER_UUID)); + } + + @Test + public void testFinishDoesNothingWithoutMarkersOfThisCluster() { + FakeReconciler fake = new FakeReconciler(); + fake.node.annotations.put(PENDING, "another-cluster"); + + fake.finishResize(cluster, vm, access, "uid-1"); + + Assert.assertTrue(fake.patches.isEmpty()); + Assert.assertEquals("another-cluster", fake.node.annotations.get(PENDING)); + } + + @Test + public void testReleaseCordonKeepsThePendingMarker() { + FakeReconciler fake = new FakeReconciler(); + fake.node.unschedulable = true; + fake.node.annotations.put(PENDING, CLUSTER_UUID); + fake.node.annotations.put(CORDON, CLUSTER_UUID); + + NodeCapacitySnapshot released = fake.releaseCordon(cluster, vm, access, "uid-1"); + + Assert.assertFalse(released.isUnschedulable()); + Assert.assertFalse(released.isCordonOwnedBy(CLUSTER_UUID)); + Assert.assertTrue(released.isPendingFor(CLUSTER_UUID)); + } + + @Test + public void testOwnershipMarkerOnASchedulableNodeIsNotACksCordon() { + FakeNode node = new FakeNode(); + node.annotations.put(CORDON, CLUSTER_UUID); + + NodeCapacitySnapshot snapshot = KubernetesClusterNodeCapacityReconciler.parseNode(node.toJson()); + + Assert.assertTrue(snapshot.hasOwnershipMarker(CLUSTER_UUID)); + Assert.assertFalse(snapshot.isCordonOwnedBy(CLUSTER_UUID)); + } + + @Test + public void testFinishRemovesALeftoverOwnershipMarkerWithoutTouchingScheduling() { + FakeReconciler fake = new FakeReconciler(); + fake.node.annotations.put(PENDING, CLUSTER_UUID); + fake.node.annotations.put(CORDON, CLUSTER_UUID); + + NodeCapacitySnapshot finished = fake.finishResize(cluster, vm, access, "uid-1"); + + Assert.assertFalse(fake.patches.get(0).contains("spec")); + Assert.assertFalse(finished.hasOwnershipMarker(CLUSTER_UUID)); + Assert.assertFalse(finished.isUnschedulable()); + } + + @Test + public void testReleaseRemovesALeftoverOwnershipMarkerWithoutTouchingScheduling() { + FakeReconciler fake = new FakeReconciler(); + fake.node.annotations.put(PENDING, CLUSTER_UUID); + fake.node.annotations.put(CORDON, CLUSTER_UUID); + + NodeCapacitySnapshot released = fake.releaseCordon(cluster, vm, access, "uid-1"); + + Assert.assertFalse(fake.patches.get(0).contains("spec")); + Assert.assertFalse(released.hasOwnershipMarker(CLUSTER_UUID)); + Assert.assertTrue(released.isPendingFor(CLUSTER_UUID)); + } + + @Test + public void testAlreadyResizedNeedsReadyNodeMatchingGuestAndAllocatableWithinCapacity() { + GuestResources guest = guest(4, 4L * 1024L * 1024L, 32); + + Assert.assertTrue(reconciler.isKubernetesNodeAlreadyResized(snapshot(true, 4000, 4 * GIB, 3900, 3 * GIB), guest)); + // reservations may use up all of the capacity + Assert.assertTrue(reconciler.isKubernetesNodeAlreadyResized(snapshot(true, 4000, 4 * GIB, 0, 0), guest)); + Assert.assertFalse(reconciler.isKubernetesNodeAlreadyResized(snapshot(false, 4000, 4 * GIB, 3900, 3 * GIB), guest)); + Assert.assertFalse(reconciler.isKubernetesNodeAlreadyResized(snapshot(true, 2000, 4 * GIB, 1900, 3 * GIB), guest)); + Assert.assertFalse(reconciler.isKubernetesNodeAlreadyResized(snapshot(true, 4000, 2 * GIB, 3900, GIB), guest)); + Assert.assertFalse(reconciler.isKubernetesNodeAlreadyResized(snapshot(true, 4000, 4 * GIB, 4100, 3 * GIB), guest)); + Assert.assertFalse(reconciler.isKubernetesNodeAlreadyResized(snapshot(true, 4000, 4 * GIB, 3900, 5 * GIB), guest)); + } + + @Test + public void testGuestMemoryIsCountedInOnlineMemoryBlocks() { + ServiceOffering target = offering(4, 4608); + + Assert.assertTrue(KubernetesClusterNodeCapacityReconciler.guestMatchesTarget(guest(4, 4568788L, 36), target)); + // 512 MiB of the hot-added memory is not online yet + Assert.assertFalse(KubernetesClusterNodeCapacityReconciler.guestMatchesTarget(guest(4, 4044500L, 32), target)); + Assert.assertFalse(KubernetesClusterNodeCapacityReconciler.guestMatchesTarget(guest(2, 4568788L, 36), target)); + Assert.assertFalse(KubernetesClusterNodeCapacityReconciler.guestMatchesTarget(guest(4, 4699860L, 37), target)); + Assert.assertTrue(KubernetesClusterNodeCapacityReconciler.isGuestAboveTarget(guest(4, 4699860L, 37), target)); + Assert.assertTrue(KubernetesClusterNodeCapacityReconciler.isGuestAboveTarget(guest(6, 4568788L, 36), target)); + Assert.assertFalse(KubernetesClusterNodeCapacityReconciler.isGuestAboveTarget(guest(4, 4568788L, 36), target)); + } + + @Test + public void testGuestMemoryCheckDoesNotDependOnTheMemoryTheKernelKeeps() { + // An 8 GiB VM reports about 230 MiB less MemTotal than its RAM; its memory blocks still add up to 8 GiB. + Assert.assertTrue(KubernetesClusterNodeCapacityReconciler.guestMatchesTarget(guest(4, 8152860L, 64), offering(4, 8192))); + // RAM that is not a whole number of blocks ends in a partly filled block, counted as a whole one. + Assert.assertTrue(KubernetesClusterNodeCapacityReconciler.guestMatchesTarget(guest(1, 950000L, 8), offering(1, 1000))); + Assert.assertFalse(KubernetesClusterNodeCapacityReconciler.isGuestAboveTarget(guest(1, 950000L, 8), offering(1, 1000))); + } + + @Test + public void testGuestVerificationWaitsForHotpluggedResources() { + FakeReconciler fake = new FakeReconciler(); + fake.guestOutputs.add("2\n2029380\n8000000\n16\n"); + fake.guestOutput = "4\n4026532\n8000000\n32\n"; + ServiceOffering target = offering(4, 4096); + + GuestResources guest = fake.waitForGuestResources(vm, target, access, 60000L); + + Assert.assertEquals(4L, guest.getOnlineCpus()); + Assert.assertEquals(List.of(KubernetesClusterNodeCapacityReconciler.POLL_INTERVAL_MS), fake.sleeps); + } + + @Test + public void testGuestAboveTheTargetFailsAtOnce() { + FakeReconciler fake = new FakeReconciler(); + fake.guestOutput = "4\n8152860\n8000000\n64\n"; + ServiceOffering target = offering(4, 4096); + + assertPhaseFailure("GUEST_VERIFY", () -> fake.waitForGuestResources(vm, target, access, 60000L)); + + Assert.assertTrue(fake.sleeps.isEmpty()); + } + + @Test + public void testGuestVerificationStopsAtTheDeadline() { + FakeReconciler fake = new FakeReconciler(); + ServiceOffering target = offering(4, 4096); + + assertPhaseFailure("GUEST_VERIFY", () -> fake.waitForGuestResources(vm, target, access, 12000L)); + + Assert.assertEquals(List.of(5000L, 5000L, 2000L), fake.sleeps); + Assert.assertEquals(4, fake.nodeCommands.size()); + } + + @Test + public void testExpiredDeadlineAllowsOneObservationWithoutSleeping() { + FakeReconciler fake = new FakeReconciler(); + fake.now = 1000L; + GuestResources guest = guest(4, 4194304L, 32); + + assertPhaseFailure("CAPACITY_VERIFY", () -> fake.waitForKubernetesResources(vm, guest, access, 999L)); + + Assert.assertEquals(1, fake.controlCommands.size()); + Assert.assertTrue(fake.sleeps.isEmpty()); + } + + @Test + public void testWaitForKubernetesResourcesReturnsTheUpdatedNode() { + FakeReconciler fake = new FakeReconciler(); + fake.node.capacityCpu = "4"; + fake.node.capacityMemory = "4Gi"; + fake.node.allocatableCpu = "3900m"; + + NodeCapacitySnapshot node = fake.waitForKubernetesResources(vm, guest(4, 4194304L, 32), access, 60000L); + + Assert.assertEquals(4000L, node.getCapacity().getCpuMillis()); + Assert.assertTrue(fake.sleeps.isEmpty()); + } + + @Test + public void testRestartKubeletWaitsUntilActive() { + FakeReconciler fake = new FakeReconciler(); + fake.kubeletStates.add("activating"); + + fake.restartKubelet(vm, access, 60000L); + + Assert.assertEquals(List.of(KubernetesClusterNodeCapacityReconciler.RESTART_KUBELET_COMMAND, + KubernetesClusterNodeCapacityReconciler.KUBELET_STATE_COMMAND, + KubernetesClusterNodeCapacityReconciler.KUBELET_STATE_COMMAND), fake.nodeCommands); + Assert.assertEquals(1, fake.sleeps.size()); + } + + @Test + public void testKubeletIsNotRestartedAfterTheDeadline() { + FakeReconciler fake = new FakeReconciler(); + fake.now = 1000L; + + assertPhaseFailure("KUBELET_RESTART", () -> fake.restartKubelet(vm, access, 1000L)); + + Assert.assertTrue(fake.nodeCommands.isEmpty()); + } + + @Test + public void testInactiveKubeletIsReportedWithoutFailingTheCommand() { + FakeReconciler fake = new FakeReconciler(); + fake.kubeletStates.add("inactive"); + + Assert.assertFalse(fake.isKubeletActive(access)); + Assert.assertTrue(fake.isKubeletActive(access)); + Assert.assertTrue(KubernetesClusterNodeCapacityReconciler.KUBELET_STATE_COMMAND.endsWith("|| true")); + Assert.assertFalse(KubernetesClusterNodeCapacityReconciler.KUBELET_STATE_COMMAND.contains("sudo")); + } + + @Test + public void testOnlineGuestResourcesActivatesHotpluggedCpuAndMemory() { + FakeReconciler fake = new FakeReconciler(); + + fake.onlineGuestResources(access); + + Assert.assertEquals(List.of(KubernetesClusterNodeCapacityReconciler.ONLINE_HOTPLUGGED_RESOURCES), fake.nodeCommands); + } + + @Test + public void testNodeAccessKeepsControlAndNodeEndpointsApart() { + File key = new File("test-only-key"); + NodeAccess endpoints = new NodeAccess("control", 2222, "worker", 2223, "cloud", key); + + Assert.assertEquals("control", endpoints.getControlAddress()); + Assert.assertEquals(2222, endpoints.getControlPort()); + Assert.assertEquals("worker", endpoints.getNodeAddress()); + Assert.assertEquals(2223, endpoints.getNodePort()); + Assert.assertEquals("cloud", endpoints.getUser()); + Assert.assertSame(key, endpoints.getSshKeyFile()); + } + + @Test + public void testSshSuccessReturnsOutput() { + try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { + ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) + .thenReturn(new Pair<>(true, "output")); + Assert.assertEquals("output", reconciler.executeNode(access, "command")); + } + } + + @Test + public void testFailedSshCommandReportsItsOutput() { + try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { + ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) + .thenReturn(new Pair<>(false, "remote error")); + NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, + () -> reconciler.executeNode(access, "command")); + Assert.assertTrue(error.getMessage().contains("remote error")); + } + } + + @Test + public void testSshExceptionIsTranslatedAtTheTransportBoundary() { + Exception failure = new Exception("SSH authentication failed"); + try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { + ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) + .thenThrow(failure); + NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, + () -> reconciler.executeNode(access, "command")); + Assert.assertSame(failure, error.getCause()); + } + } + + @Test + public void testSshInterruptionKeepsTheInterruptFlag() { + InterruptedException failure = new InterruptedException("cancelled"); + try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { + ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) + .thenThrow(failure); + NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, + () -> reconciler.executeNode(access, "command")); + Assert.assertSame(failure, error.getCause()); + Assert.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + @Test + public void testSshProgrammingErrorsAreNotTranslated() { + NullPointerException failure = new NullPointerException("programming error"); + try (MockedStatic ssh = Mockito.mockStatic(SshHelper.class)) { + ssh.when(() -> SshHelper.sshExecute("node", 22, "root", null, null, "command", 10000, 10000, 30000)) + .thenThrow(failure); + Assert.assertSame(failure, Assert.assertThrows(NullPointerException.class, + () -> reconciler.executeNode(access, "command"))); + } + } + + @Test + public void testInterruptedPollingKeepsTheInterruptFlag() { + KubernetesClusterNodeCapacityReconciler inactiveKubelet = new KubernetesClusterNodeCapacityReconciler() { + @Override + protected String executeNode(NodeAccess nodeAccess, String command) { + return "inactive"; + } + }; + long deadline = System.currentTimeMillis() + 10000L; + Thread.currentThread().interrupt(); + try { + NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, + () -> inactiveKubelet.restartKubelet(vm, access, deadline)); + Assert.assertTrue(error.getCause() instanceof InterruptedException); + Assert.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + private void assertPhaseFailure(String phase, ThrowingRunnable action) { + NodeCapacityReconciliationException error = Assert.assertThrows(NodeCapacityReconciliationException.class, action); + Assert.assertTrue(error.getMessage(), error.getMessage().contains(phase + " failed for VM vm-uuid")); + } + + private static NodeCapacitySnapshot snapshot(boolean ready, long capacityCpuMillis, long capacityMemory, + long allocatableCpuMillis, long allocatableMemory) { + return new NodeCapacitySnapshot("uid-1", + "7", + false, + new HashMap<>(), + ready, + new NodeResources(capacityCpuMillis, capacityMemory), + new NodeResources(allocatableCpuMillis, allocatableMemory)); + } + + private static GuestResources guest(long cpus, long memTotalKiB, long onlineMemoryBlocks) { + return new GuestResources(cpus, memTotalKiB, onlineMemoryBlocks * 128L * MIB, 128L * MIB); + } + + private static ServiceOffering offering(int cpu, int memory) { + ServiceOffering offering = Mockito.mock(ServiceOffering.class); + Mockito.when(offering.getCpu()).thenReturn(cpu); + Mockito.when(offering.getRamSize()).thenReturn(memory); + return offering; + } + + private static KubernetesCluster cluster() { + KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); + Mockito.when(cluster.getUuid()).thenReturn(CLUSTER_UUID); + return cluster; + } + + private static UserVm vm(String hostname) { + UserVm vm = Mockito.mock(UserVm.class); + Mockito.when(vm.getHostName()).thenReturn(hostname); + Mockito.when(vm.getUuid()).thenReturn("vm-uuid"); + return vm; + } + + /** A Node as kubectl returns it, changed by the merge patches the reconciler sends. */ + private static class FakeNode { + private final Map annotations = new LinkedHashMap<>(); + private final String uid = "uid-1"; + private int resourceVersion = 7; + private boolean unschedulable; + private String capacityCpu = "2"; + private String capacityMemory = "2Gi"; + private String allocatableCpu = "1900m"; + private String allocatableMemory = "1900Mi"; + + String toJson() { + JsonObject metadata = new JsonObject(); + metadata.addProperty("name", "worker-1"); + metadata.addProperty("uid", uid); + metadata.addProperty("resourceVersion", String.valueOf(resourceVersion)); + JsonObject annotationsJson = new JsonObject(); + annotations.forEach(annotationsJson::addProperty); + metadata.add("annotations", annotationsJson); + JsonObject spec = new JsonObject(); + if (unschedulable) { + spec.addProperty("unschedulable", true); + } + JsonObject capacity = new JsonObject(); + capacity.addProperty("cpu", capacityCpu); + capacity.addProperty("memory", capacityMemory); + JsonObject allocatable = new JsonObject(); + allocatable.addProperty("cpu", allocatableCpu); + allocatable.addProperty("memory", allocatableMemory); + JsonObject readyCondition = new JsonObject(); + readyCondition.addProperty("type", "Ready"); + readyCondition.addProperty("status", "True"); + JsonArray conditions = new JsonArray(); + conditions.add(readyCondition); + JsonObject status = new JsonObject(); + status.add("capacity", capacity); + status.add("allocatable", allocatable); + status.add("conditions", conditions); + JsonObject node = new JsonObject(); + node.add("metadata", metadata); + node.add("spec", spec); + node.add("status", status); + return node.toString(); + } + } + + private static class FakeReconciler extends KubernetesClusterNodeCapacityReconciler { + private final FakeNode node = new FakeNode(); + private final List controlCommands = new ArrayList<>(); + private final List nodeCommands = new ArrayList<>(); + private final List patches = new ArrayList<>(); + private final List sleeps = new ArrayList<>(); + private final Deque guestOutputs = new ArrayDeque<>(); + private final Deque kubeletStates = new ArrayDeque<>(); + private String guestOutput = "2\n2029380\n8000000\n16\n"; + private String nodeOutput; + private int failingPatches; + private boolean loseNextReply; + private Runnable beforeNextPatch; + private long now; + + @Override + protected String executeControl(NodeAccess nodeAccess, String command) { + controlCommands.add(command); + if (command.contains(" patch node ")) { + return patch(command); + } + return nodeOutput != null ? nodeOutput : node.toJson(); + } + + @Override + protected String executeNode(NodeAccess nodeAccess, String command) { + nodeCommands.add(command); + if (KubernetesClusterNodeCapacityReconciler.GUEST_RESOURCES_COMMAND.equals(command)) { + return guestOutputs.isEmpty() ? guestOutput : guestOutputs.poll(); + } + if (KubernetesClusterNodeCapacityReconciler.KUBELET_STATE_COMMAND.equals(command)) { + return kubeletStates.isEmpty() ? "active\n" : kubeletStates.poll(); + } + return ""; + } + + @Override + protected long currentTimeMillis() { + return now; + } + + @Override + protected void sleep(long millis) { + sleeps.add(millis); + now += millis; + } + + private String patch(String command) { + patches.add(command); + if (beforeNextPatch != null) { + Runnable change = beforeNextPatch; + beforeNextPatch = null; + change.run(); + } + if (failingPatches > 0) { + failingPatches--; + throw new NodeCapacityReconciliationException("Error from server (Forbidden)"); + } + JsonObject patch = JsonParser.parseString(command.substring(command.indexOf("-p '") + 4, command.length() - 1)).getAsJsonObject(); + JsonObject metadata = patch.getAsJsonObject("metadata"); + if (!String.valueOf(node.resourceVersion).equals(metadata.get("resourceVersion").getAsString())) { + throw new NodeCapacityReconciliationException("Error from server (Conflict): the object has been modified"); + } + for (Map.Entry annotation : metadata.getAsJsonObject("annotations").entrySet()) { + if (annotation.getValue().isJsonNull()) { + node.annotations.remove(annotation.getKey()); + } else { + node.annotations.put(annotation.getKey(), annotation.getValue().getAsString()); + } + } + if (patch.has("spec")) { + node.unschedulable = patch.getAsJsonObject("spec").get("unschedulable").getAsBoolean(); + } + node.resourceVersion++; + if (loseNextReply) { + loseNextReply = false; + throw new NodeCapacityReconciliationException("Unable to run command: connection reset"); + } + return "node/worker-1 patched"; + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java index 329f9b0e42ad..e0fc2b8d5b84 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java @@ -16,10 +16,12 @@ // under the License. package com.cloud.kubernetes.cluster.utils; +import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) @@ -52,4 +54,20 @@ public void testClusterNodeVersionMatches() { executeThrowAndTestVersionMatch(); } + + @Test + public void testKubernetesNodeNameIsTheLowercaseHostName() { + UserVm vm = Mockito.mock(UserVm.class); + Mockito.when(vm.getHostName()).thenReturn("Worker-1"); + Assert.assertEquals("worker-1", KubernetesClusterUtil.getKubernetesNodeName(vm)); + Mockito.when(vm.getHostName()).thenReturn(null); + Assert.assertNull(KubernetesClusterUtil.getKubernetesNodeName(vm)); + } + + @Test + public void testNodePatchCommandAppliesOneMergePatch() { + Assert.assertEquals("sudo /opt/bin/kubectl patch node worker-1 --type merge --request-timeout=20s -p '{\"spec\":{\"unschedulable\":true}}'", + KubernetesClusterUtil.getKubernetesNodePatchCommand("worker-1", "{\"spec\":{\"unschedulable\":true}}")); + Assert.assertThrows(IllegalArgumentException.class, () -> KubernetesClusterUtil.getKubernetesNodePatchCommand("worker-1", "{\"a\":\"'\"}")); + } }