diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 0711444c061b..9e5e16f5fb51 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1324,6 +1324,7 @@ public class ApiConstants { public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; public static final String SUPPORTS_HA = "supportsha"; public static final String SUPPORTS_AUTOSCALING = "supportsautoscaling"; + public static final String SUPPORTS_ETCD = "supportsetcd"; public static final String AUTOSCALING_ENABLED = "autoscalingenabled"; public static final String MIN_SIZE = "minsize"; public static final String MAX_SIZE = "maxsize"; diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/CheckFileInIsoAnswer.java b/core/src/main/java/org/apache/cloudstack/storage/command/CheckFileInIsoAnswer.java new file mode 100644 index 000000000000..c0fe46bfb8e7 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/CheckFileInIsoAnswer.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.command; + +import com.cloud.agent.api.Answer; + +public class CheckFileInIsoAnswer extends Answer { + + private boolean filePresent; + + public CheckFileInIsoAnswer() { + super(); + } + + /** + * The ISO was read and the file was or was not found in it. + */ + public CheckFileInIsoAnswer(final CheckFileInIsoCommand cmd, final boolean filePresent) { + super(cmd, true, null); + this.filePresent = filePresent; + } + + /** + * The ISO could not be read, so nothing is known about the file. + */ + public CheckFileInIsoAnswer(final CheckFileInIsoCommand cmd, final String errorDetails) { + super(cmd, false, errorDetails); + } + + public boolean isFilePresent() { + return filePresent; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/CheckFileInIsoCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/CheckFileInIsoCommand.java new file mode 100644 index 000000000000..a747b9eec1d1 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/CheckFileInIsoCommand.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.command; + +import com.cloud.agent.api.to.DataStoreTO; + +/** + * Asks the secondary storage VM whether an ISO on the given image store contains a file. + */ +public class CheckFileInIsoCommand extends StorageSubSystemCommand { + + private DataStoreTO store; + private String isoPath; + private String filePath; + + public CheckFileInIsoCommand() { + super(); + } + + public CheckFileInIsoCommand(final DataStoreTO store, final String isoPath, final String filePath) { + super(); + this.store = store; + this.isoPath = isoPath; + this.filePath = filePath; + } + + public DataStoreTO getStore() { + return store; + } + + /** + * @return path of the ISO, relative to the root of the image store + */ + public String getIsoPath() { + return isoPath; + } + + /** + * @return path of the file to look for, relative to the root of the ISO + */ + public String getFilePath() { + return filePath; + } + + @Override + public boolean executeInSequence() { + return false; + } + + @Override + public void setExecuteInSequence(boolean inSeq) {} +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index eb4ebb78ee48..ab1b942899ce 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -180,6 +180,7 @@ import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; import com.cloud.kubernetes.version.KubernetesVersionManagerImpl; +import com.cloud.kubernetes.version.KubernetesVersionService; import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; import com.cloud.network.IpAddress; import com.cloud.network.Network; @@ -326,6 +327,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject public KubernetesSupportedVersionDao kubernetesSupportedVersionDao; @Inject + protected KubernetesVersionService kubernetesVersionService; + @Inject protected SSHKeyPairDao sshKeyPairDao; @Inject protected DataCenterDao dataCenterDao; @@ -1094,6 +1097,29 @@ public boolean isCommandSupported(KubernetesCluster cluster, String cmdName) { } } + /** + * Dedicated etcd nodes install etcd from the Kubernetes ISO, and not every ISO has it. Fail the request when + * the ISO is known not to have it. When the ISO could not be checked the request is let through, and a failed + * etcd setup is then caught while the cluster starts. + */ + protected void validateEtcdNodesSupportedByKubernetesVersion(final long etcdNodes, final KubernetesSupportedVersion kubernetesVersion, final long zoneId) { + if (etcdNodes <= 0) { + return; + } + Boolean etcdIncluded = kubernetesVersionService.isEtcdIncludedInIso(kubernetesVersion, zoneId); + if (etcdIncluded == null) { + logger.warn("Unable to tell whether the ISO of Kubernetes version {} includes the etcd binaries, allowing the request for {} etcd node(s)", + kubernetesVersion, etcdNodes); + return; + } + if (!etcdIncluded) { + throw new InvalidParameterValueException(String.format("Kubernetes version %s (ID: %s) cannot be used for a cluster with dedicated etcd nodes " + + "as its ISO does not include the etcd binaries (%s). Use a Kubernetes version whose ISO was built with etcd, " + + "or create the cluster without etcd nodes", kubernetesVersion.getName(), kubernetesVersion.getUuid(), + KubernetesVersionService.ETCD_ARCHIVE_PATH_IN_ISO)); + } + } + private void validateManagedKubernetesClusterCreateParameters(final CreateKubernetesClusterCmd cmd) throws CloudRuntimeException { ApiServiceConfiguration.validateEndpointUrl(); final String name = cmd.getName(); @@ -1168,6 +1194,8 @@ private void validateManagedKubernetesClusterCreateParameters(final CreateKubern throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state for datacenter ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); } + validateEtcdNodesSupportedByKubernetesVersion(cmd.getEtcdNodes(), clusterKubernetesVersion, zone.getId()); + validateServiceOfferingsForNodeTypes(serviceOfferingNodeTypeMap, defaultServiceOfferingId, cmd.getEtcdNodes(), clusterKubernetesVersion); validateSshKeyPairForKubernetesCreateParameters(sshKeyPair, owner); @@ -3089,7 +3117,8 @@ public ConfigKey[] getConfigKeys() { KubernetesControlNodeInstallReattempts, KubernetesWorkerNodeInstallAttemptWait, KubernetesWorkerNodeInstallReattempts, - KubernetesEtcdNodeStartPort + KubernetesEtcdNodeStartPort, + KubernetesEtcdNodeSetupTimeout }; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 1d19127dc54a..2e54b87c03a6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -143,6 +143,13 @@ public interface KubernetesClusterService extends PluggableService, Configurable "Start port for Port forwarding rules for etcd nodes", true, KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesEtcdNodeSetupTimeout = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.etcd.node.setup.timeout", + "600", + "Timeout interval (in seconds) in which etcd should be running on each dedicated etcd node of a Kubernetes cluster that is starting. " + + "The cluster start fails when it is not, instead of waiting for the cluster start timeout", + true, + KubernetesServiceEnabled.key()); KubernetesCluster findById(final Long id); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index a15086c60d6c..58b90a9f2af5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -694,6 +694,39 @@ protected void setupKubernetesEtcdNetworkRules(List etcdVms, Network net } } + /** + * The control nodes cannot bring up the API server without etcd, so a cluster whose etcd nodes failed to set up + * would only fail when the cluster start timeout is over. Check the etcd nodes first and fail with the reason. + */ + protected void verifyEtcdRunningOnEtcdNodes(final List etcdVms, final List clusterVMs, final Network network, final long startTimeoutTime) { + if (CollectionUtils.isEmpty(etcdVms)) { + return; + } + final long timeoutTime = Math.min(startTimeoutTime, + System.currentTimeMillis() + KubernetesClusterService.KubernetesEtcdNodeSetupTimeout.value() * 1000); + final boolean directAccess = manager.isDirectAccess(network); + final int etcdNodeStartPort = KubernetesClusterService.KubernetesEtcdNodeStartPort.value(); + for (int i = 0; i < etcdVms.size(); i++) { + final UserVm etcdVm = etcdVms.get(i); + // Same route as the port forwarding rules of setupKubernetesEtcdNetworkRules + final String ipAddress = directAccess ? etcdVm.getPrivateIpAddress() : publicIpAddress; + final int port = directAccess ? DEFAULT_SSH_PORT : etcdNodeStartPort + i; + final String failure = waitForEtcdOnEtcdNode(etcdVm, ipAddress, port, timeoutTime); + if (failure != null) { + List vmsWithIso = new ArrayList<>(etcdVms); + vmsWithIso.addAll(clusterVMs); + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s as etcd is not running on etcd node %s: %s", + kubernetesCluster.getName(), etcdVm.getDisplayName(), failure), kubernetesCluster, vmsWithIso, KubernetesCluster.Event.CreateFailed, null); + } + } + logger.info("etcd is running on all {} etcd node(s) of Kubernetes cluster: {}", etcdVms.size(), kubernetesCluster); + } + + protected String waitForEtcdOnEtcdNode(final UserVm etcdVm, final String ipAddress, final int port, final long timeoutTime) { + return KubernetesClusterUtil.waitForEtcdOnEtcdNode(kubernetesCluster, etcdVm.getDisplayName(), ipAddress, port, + CLUSTER_NODE_VM_USER, sshKeyFile, timeoutTime, 10000); + } + private void startKubernetesClusterVMs(Long domainId, Long accountId) { List clusterVms = getKubernetesClusterVMs(); for (final UserVm vm : clusterVms) { @@ -860,6 +893,7 @@ public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId, Lon } attachIsoKubernetesVMs(etcdVms); attachIsoKubernetesVMs(clusterVMs); + verifyEtcdRunningOnEtcdNodes(etcdVms, clusterVMs, network, startTimeoutTime); if (!KubernetesClusterUtil.isKubernetesClusterControlVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), startTimeoutTime)) { String msg = String.format("Failed to setup Kubernetes cluster : %s is not in usable state as the system is unable to access control node VMs of the cluster", kubernetesCluster.getName()); if (kubernetesCluster.getControlNodeCount() > 1 && manager.isDirectAccess(network)) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 023fc9ec7fad..2668d37a8471 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -269,6 +269,59 @@ public static boolean isKubernetesClusterServerRunning(final KubernetesCluster k return k8sApiServerSetup; } + // Written by the setup script of an etcd node (conf/etcd-node.yml) with the reason when the setup fails + public static final String ETCD_NODE_SETUP_FAILED_FILE = "/home/cloud/etcd-setup-failed"; + protected static final String ETCD_NODE_ACTIVE = "ACTIVE"; + protected static final String ETCD_NODE_FAILED = "FAILED:"; + protected static final String ETCD_NODE_STATUS_COMMAND = String.format( + "if [ -f %1$s ]; then echo \"%2$s $(cat %1$s)\"; elif sudo systemctl is-active --quiet etcd; then echo %3$s; else echo WAITING; fi", + ETCD_NODE_SETUP_FAILED_FILE, ETCD_NODE_FAILED, ETCD_NODE_ACTIVE); + + protected static Pair getEtcdNodeStatus(final String ipAddress, final int port, final String user, final File sshKeyFile) throws Exception { + return SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, ETCD_NODE_STATUS_COMMAND, 10000, 10000, 20000); + } + + /** + * Waits for etcd to run on a dedicated etcd node. etcd has to be seen running on two checks in a row, so that + * an etcd which keeps crashing and restarting is not taken as running. + * + * @return null once etcd is running, otherwise the reason why the node is not usable + */ + public static String waitForEtcdOnEtcdNode(final KubernetesCluster kubernetesCluster, final String nodeName, final String ipAddress, + final int port, final String user, final File sshKeyFile, + final long timeoutTime, final long waitDuration) { + String lastStatus = "the node was not reachable over SSH"; + boolean activeOnLastCheck = false; + while (System.currentTimeMillis() < timeoutTime) { + boolean active = false; + try { + Pair result = getEtcdNodeStatus(ipAddress, port, user, sshKeyFile); + String output = result.second() == null ? "" : result.second().trim(); + if (Boolean.TRUE.equals(result.first()) && output.startsWith(ETCD_NODE_FAILED)) { + return output.substring(ETCD_NODE_FAILED.length()).trim(); + } + active = Boolean.TRUE.equals(result.first()) && output.startsWith(ETCD_NODE_ACTIVE); + lastStatus = active ? "etcd did not stay running" : "the etcd service was not running"; + } catch (Exception e) { + lastStatus = "the node was not reachable over SSH"; + LOGGER.debug("Unable to check etcd on node {} of Kubernetes cluster {}: {}", nodeName, kubernetesCluster, e.getMessage()); + } + if (active && activeOnLastCheck) { + return null; + } + activeOnLastCheck = active; + LOGGER.info("Waiting for etcd to run on node {} of Kubernetes cluster {}", nodeName, kubernetesCluster); + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ie) { + LOGGER.warn("Interrupted while waiting for etcd on node {} of Kubernetes cluster {}", nodeName, kubernetesCluster, ie); + Thread.currentThread().interrupt(); + break; + } + } + return String.format("etcd did not start in time, on the last check %s", lastStatus); + } + public static boolean isKubernetesClusterControlVmRunning(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final long timeoutTime) { boolean controlVmRunning = false; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index aef020335f2d..545a7f2a1663 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -19,9 +19,17 @@ import java.net.MalformedURLException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.inject.Inject; +import javax.naming.ConfigurationException; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -37,11 +45,21 @@ import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.command.CheckFileInIsoAnswer; +import org.apache.cloudstack.storage.command.CheckFileInIsoCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import com.cloud.agent.api.Answer; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.cpu.CPU; @@ -55,8 +73,11 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.VMTemplateDetailVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.template.TemplateApiService; import com.cloud.template.VirtualMachineTemplate; @@ -65,7 +86,9 @@ import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -91,6 +114,21 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne private ImageStoreDao imageStoreDao; @Inject private TemplateApiService templateService; + @Inject + private VMTemplateDetailsDao templateDetailsDao; + @Inject + private TemplateDataStoreDao templateDataStoreDao; + @Inject + private DataStoreManager dataStoreManager; + @Inject + private EndPointSelector endPointSelector; + + protected static final int ETCD_IN_ISO_CHECK_INTERVAL_SECONDS = 300; + protected static final int MAX_FAILED_BACKGROUND_ETCD_IN_ISO_CHECKS = 3; + + private ScheduledExecutorService etcdInIsoChecker; + // ISO ID -> number of background checks that could not be run for it + private final Map failedEtcdInIsoChecks = new ConcurrentHashMap<>(); public static final String MINIMUN_AUTOSCALER_SUPPORTED_VERSION = "1.15.0"; @@ -131,6 +169,7 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo response.setSupportsHA(compareSemanticVersions(kubernetesSupportedVersion.getSemanticVersion(), KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT)>=0); response.setSupportsAutoscaling(versionSupportsAutoscaling(kubernetesSupportedVersion)); + response.setSupportsEtcd(getStoredEtcdIncludedInIso(kubernetesSupportedVersion.getIsoId())); updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, response, isRootAdmin); response.setCreated(kubernetesSupportedVersion.getCreated()); return response; @@ -539,6 +578,148 @@ public KubernetesSupportedVersionResponse updateKubernetesSupportedVersion(final return createKubernetesSupportedVersionResponse(version, true); } + /** + * @return what an earlier check found for the ISO, or null if the ISO has not been checked yet + */ + protected Boolean getStoredEtcdIncludedInIso(long isoId) { + VMTemplateDetailVO detail = templateDetailsDao.findDetail(isoId, ETCD_INCLUDED_ISO_DETAIL); + if (detail == null || StringUtils.isBlank(detail.getValue())) { + return null; + } + return Boolean.parseBoolean(detail.getValue()); + } + + /** + * Asks the secondary storage VM whether the ISO has the etcd binaries and stores the answer as a detail of + * the ISO, so the ISO is read only once. + * + * @param zoneId the zone whose copy of the ISO is read, or null to read any copy that is ready + * @return null if the check could not be run, in which case nothing is stored + */ + protected Boolean checkAndStoreEtcdIncludedInIso(long isoId, Long zoneId) { + VMTemplateVO iso = templateDao.findById(isoId); + if (iso == null || iso.isDirectDownload()) { + // A direct download ISO is not kept on an image store, so there is nothing the secondary storage VM can read + return null; + } + TemplateDataStoreVO isoOnStore = zoneId == null ? templateDataStoreDao.findReadyByTemplate(isoId, DataStoreRole.Image) : + templateDataStoreDao.findByTemplateZoneReady(isoId, zoneId); + if (isoOnStore == null || StringUtils.isBlank(isoOnStore.getInstallPath())) { + logger.debug("Kubernetes ISO {} is not ready on an image store, unable to check it for the etcd binaries", iso); + return null; + } + DataStore store = dataStoreManager.getDataStore(isoOnStore.getDataStoreId(), DataStoreRole.Image); + EndPoint endPoint = store == null ? null : endPointSelector.select(store); + if (endPoint == null) { + logger.debug("No secondary storage VM available to check Kubernetes ISO {} for the etcd binaries", iso); + return null; + } + CheckFileInIsoCommand cmd = new CheckFileInIsoCommand(store.getTO(), isoOnStore.getInstallPath(), ETCD_ARCHIVE_PATH_IN_ISO); + Answer answer = endPoint.sendMessage(cmd); + if (!(answer instanceof CheckFileInIsoAnswer) || !answer.getResult()) { + logger.warn("Unable to check Kubernetes ISO {} for the etcd binaries: {}", iso, answer == null ? "no answer" : answer.getDetails()); + return null; + } + boolean etcdIncluded = ((CheckFileInIsoAnswer)answer).isFilePresent(); + templateDetailsDao.addDetail(isoId, ETCD_INCLUDED_ISO_DETAIL, String.valueOf(etcdIncluded), true); + logger.info("Kubernetes ISO {} {} the etcd binaries needed for dedicated etcd nodes", iso, etcdIncluded ? "includes" : "does not include"); + return etcdIncluded; + } + + @Override + public Boolean isEtcdIncludedInIso(KubernetesSupportedVersion kubernetesSupportedVersion, long zoneId) { + long isoId = kubernetesSupportedVersion.getIsoId(); + Boolean etcdIncluded = getStoredEtcdIncludedInIso(isoId); + if (etcdIncluded != null) { + return etcdIncluded; + } + try { + return checkAndStoreEtcdIncludedInIso(isoId, zoneId); + } catch (CloudRuntimeException e) { + logger.warn("Unable to check the ISO of Kubernetes version {} for the etcd binaries", kubernetesSupportedVersion, e); + return null; + } + } + + /** + * Checks the Kubernetes ISOs that are ready but have not been checked for the etcd binaries yet. This covers + * both newly registered ISOs and the ones registered before this check existed. + */ + protected void checkEtcdIncludedInKubernetesIsos() { + Set isoIds = new HashSet<>(); + for (KubernetesSupportedVersionVO version : kubernetesSupportedVersionDao.listAll()) { + long isoId = version.getIsoId(); + if (!isoIds.add(isoId)) { + continue; + } + if (getStoredEtcdIncludedInIso(isoId) != null) { + failedEtcdInIsoChecks.remove(isoId); + continue; + } + if (failedEtcdInIsoChecks.getOrDefault(isoId, 0) >= MAX_FAILED_BACKGROUND_ETCD_IN_ISO_CHECKS) { + // Stop trying in the background, the ISO is still checked when a cluster with etcd nodes is requested + continue; + } + VMTemplateVO iso = templateDao.findById(isoId); + if (iso == null || iso.isDirectDownload() || templateDataStoreDao.findReadyByTemplate(isoId, DataStoreRole.Image) == null) { + continue; + } + try { + if (checkAndStoreEtcdIncludedInIso(isoId, null) == null) { + failedEtcdInIsoChecks.merge(isoId, 1, Integer::sum); + } + } catch (CloudRuntimeException e) { + logger.warn("Unable to check the ISO of Kubernetes version {} for the etcd binaries", version, e); + failedEtcdInIsoChecks.merge(isoId, 1, Integer::sum); + } + } + failedEtcdInIsoChecks.keySet().retainAll(isoIds); + } + + public class EtcdInKubernetesIsoChecker extends ManagedContextRunnable { + @Override + protected void runInContext() { + GlobalLock lock = GlobalLock.getInternLock("KubernetesVersion.EtcdInIsoCheck.Lock"); + try { + if (lock.lock(3)) { + try { + checkEtcdIncludedInKubernetesIsos(); + } catch (Exception e) { + logger.warn("Failed to check the Kubernetes ISOs for the etcd binaries", e); + } finally { + lock.unlock(); + } + } + } finally { + lock.releaseRef(); + } + } + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + etcdInIsoChecker = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-ISO-Etcd-Checker")); + return true; + } + + @Override + public boolean start() { + if (KubernetesClusterService.KubernetesServiceEnabled.value()) { + etcdInIsoChecker.scheduleWithFixedDelay(new EtcdInKubernetesIsoChecker(), ETCD_IN_ISO_CHECK_INTERVAL_SECONDS, + ETCD_IN_ISO_CHECK_INTERVAL_SECONDS, TimeUnit.SECONDS); + } + return true; + } + + @Override + public boolean stop() { + if (etcdInIsoChecker != null) { + etcdInIsoChecker.shutdownNow(); + } + return true; + } + @Override public List> getCommands() { List> cmdList = new ArrayList>(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java index cbe33e94d1ef..6bcac43c8267 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java @@ -31,6 +31,18 @@ public interface KubernetesVersionService extends PluggableService { static final String MIN_KUBERNETES_VERSION = "1.11.0"; + // Detail of a Kubernetes ISO which records whether the ISO has the etcd binaries needed for dedicated etcd nodes + static final String ETCD_INCLUDED_ISO_DETAIL = "cks.etcd.included"; + static final String ETCD_ARCHIVE_PATH_IN_ISO = "etcd/etcd-linux-amd64.tar.gz"; + + /** + * Tells whether the ISO of a Kubernetes version has the etcd binaries needed for dedicated etcd nodes. The ISO + * is checked on the first call and the result is stored with the ISO. + * + * @return null if it is not known because the ISO could not be checked + */ + Boolean isEtcdIncludedInIso(KubernetesSupportedVersion kubernetesSupportedVersion, long zoneId); + ListResponse listKubernetesSupportedVersions(ListKubernetesSupportedVersionsCmd cmd); KubernetesSupportedVersionResponse addKubernetesSupportedVersion(AddKubernetesSupportedVersionCmd cmd) throws CloudRuntimeException; GetUploadParamsResponse registerKubernetesSupportedVersionForPostUpload(GetUploadParamsForKubernetesSupportedVersionCmd cmd); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java index f6e1ee859449..8c22231ba4a3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -78,6 +78,10 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { @Param(description = "Whether Kubernetes supported version supports Autoscaling") private Boolean supportsAutoscaling; + @SerializedName(ApiConstants.SUPPORTS_ETCD) + @Param(description = "Whether the ISO of the Kubernetes supported version has the etcd binaries needed for dedicated etcd nodes. Not returned if the ISO has not been checked yet", since = "24.0.0") + private Boolean supportsEtcd; + @SerializedName(ApiConstants.STATE) @Param(description = "The enabled or disabled state of the Kubernetes supported version") private String state; @@ -218,6 +222,14 @@ public void setSupportsAutoscaling(Boolean supportsAutoscaling) { this.supportsAutoscaling = supportsAutoscaling; } + public Boolean getSupportsEtcd() { + return supportsEtcd; + } + + public void setSupportsEtcd(Boolean supportsEtcd) { + this.supportsEtcd = supportsEtcd; + } + public void setCreated(Date created) { this.created = created; } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml index d455441279dd..4d4cfb6d00c0 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml @@ -36,6 +36,16 @@ write_files: exit 0 fi + # The management server reads this file to report why the etcd setup failed + SETUP_FAILED_FILE=/home/cloud/etcd-setup-failed + rm -f "${SETUP_FAILED_FILE}" + fail_setup() { + echo "Error: $1" >&2 + echo "$1" > "${SETUP_FAILED_FILE}" + chmod 644 "${SETUP_FAILED_FILE}" + exit 1 + } + ISO_MOUNT_DIR=/mnt/etcddisk BINARIES_DIR=${ISO_MOUNT_DIR}/ ATTEMPT_ONLINE_INSTALL=false @@ -97,10 +107,18 @@ write_files: if [ -d "$BINARIES_DIR" ]; then ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" + ETCD_ARCHIVE=${BINARIES_DIR}/etcd/etcd-linux-amd64.tar.gz + if [ ! -f "${ETCD_ARCHIVE}" ]; then + fail_setup "The Kubernetes ISO does not include the etcd binaries (etcd/etcd-linux-amd64.tar.gz), dedicated etcd nodes need an ISO built with etcd" + fi mkdir -p /opt/bin/ - tar -zxf ${BINARIES_DIR}/etcd/etcd-linux-amd64.tar.gz -C /opt/bin/ + if ! tar -zxf "${ETCD_ARCHIVE}" -C /opt/bin/; then + fail_setup "Unable to extract the etcd binaries from etcd/etcd-linux-amd64.tar.gz of the Kubernetes ISO" + fi mv /opt/bin/etcd*/etcd* /opt/bin/ sudo rm -rf /opt/bin/etcd-* + else + fail_setup "The Kubernetes ISO with the etcd binaries did not become available on the etcd node" fi - path: /etc/systemd/system/etcd.service @@ -129,6 +147,4 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh - - /opt/bin/setup-etcd-node - - systemctl daemon-reload - - systemctl enable --now etcd + - [ sh, -c, "/opt/bin/setup-etcd-node && systemctl daemon-reload && systemctl enable --now etcd" ] diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index 1fab5420c3c3..40555acfe2af 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -30,6 +30,7 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesVersionService; import com.cloud.network.Network; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.rules.FirewallRule; @@ -85,6 +86,8 @@ @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterManagerImplTest { + private static final long ZONE_ID = 1L; + @Mock FirewallRulesDao firewallRulesDao; @@ -118,6 +121,9 @@ public class KubernetesClusterManagerImplTest { @Mock private HostDao hostDao; + @Mock + private KubernetesVersionService kubernetesVersionService; + @Spy @InjectMocks KubernetesClusterManagerImpl kubernetesClusterManager; @@ -914,4 +920,39 @@ public void testValidateNodeAffinityGroupsAffinityNoExistingWorkers() { Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()); } + @Test + public void testValidateEtcdNodesSupportedByKubernetesVersionNoEtcdNodes() { + KubernetesSupportedVersion version = Mockito.mock(KubernetesSupportedVersion.class); + kubernetesClusterManager.validateEtcdNodesSupportedByKubernetesVersion(0, version, ZONE_ID); + Mockito.verifyNoInteractions(kubernetesVersionService); + } + + @Test + public void testValidateEtcdNodesSupportedByKubernetesVersionEtcdIncluded() { + KubernetesSupportedVersion version = Mockito.mock(KubernetesSupportedVersion.class); + Mockito.when(kubernetesVersionService.isEtcdIncludedInIso(version, ZONE_ID)).thenReturn(true); + kubernetesClusterManager.validateEtcdNodesSupportedByKubernetesVersion(3, version, ZONE_ID); + } + + @Test + public void testValidateEtcdNodesSupportedByKubernetesVersionNotKnown() { + KubernetesSupportedVersion version = Mockito.mock(KubernetesSupportedVersion.class); + Mockito.when(kubernetesVersionService.isEtcdIncludedInIso(version, ZONE_ID)).thenReturn(null); + kubernetesClusterManager.validateEtcdNodesSupportedByKubernetesVersion(1, version, ZONE_ID); + } + + @Test + public void testValidateEtcdNodesSupportedByKubernetesVersionEtcdNotIncluded() { + KubernetesSupportedVersion version = Mockito.mock(KubernetesSupportedVersion.class); + Mockito.when(version.getName()).thenReturn("v1.28.4"); + Mockito.when(kubernetesVersionService.isEtcdIncludedInIso(version, ZONE_ID)).thenReturn(false); + try { + kubernetesClusterManager.validateEtcdNodesSupportedByKubernetesVersion(1, version, ZONE_ID); + Assert.fail("A cluster with etcd nodes should be rejected when the ISO has no etcd"); + } catch (InvalidParameterValueException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("v1.28.4")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("does not include the etcd binaries")); + } + } + } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorkerTest.java new file mode 100644 index 000000000000..193876a7e92f --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorkerTest.java @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.network.Network; +import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterStartWorkerTest { + + private static final String PUBLIC_IP = "203.0.113.10"; + + @Mock + private KubernetesCluster kubernetesCluster; + @Mock + private KubernetesClusterManagerImpl clusterManager; + @Mock + private Network network; + + private KubernetesClusterStartWorker worker; + private int etcdNodeStartPort; + + @Before + public void setUp() { + worker = Mockito.spy(new KubernetesClusterStartWorker(kubernetesCluster, clusterManager)); + worker.publicIpAddress = PUBLIC_IP; + etcdNodeStartPort = KubernetesClusterService.KubernetesEtcdNodeStartPort.value(); + } + + private long startTimeoutTime() { + return System.currentTimeMillis() + 3600 * 1000L; + } + + private List mockEtcdVms(int count) { + List vms = new ArrayList<>(); + for (int i = 0; i < count; i++) { + vms.add(Mockito.mock(UserVm.class)); + } + return vms; + } + + @Test + public void testVerifyEtcdRunningOnEtcdNodesWithoutEtcdNodes() { + worker.verifyEtcdRunningOnEtcdNodes(new ArrayList<>(), new ArrayList<>(), network, startTimeoutTime()); + + Mockito.verify(worker, Mockito.never()).waitForEtcdOnEtcdNode(Mockito.any(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong()); + } + + @Test + public void testVerifyEtcdRunningOnEtcdNodesUsesPortForwardingRules() { + List etcdVms = mockEtcdVms(3); + Mockito.when(clusterManager.isDirectAccess(network)).thenReturn(false); + Mockito.doReturn(null).when(worker).waitForEtcdOnEtcdNode(Mockito.any(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong()); + + worker.verifyEtcdRunningOnEtcdNodes(etcdVms, new ArrayList<>(), network, startTimeoutTime()); + + for (int i = 0; i < etcdVms.size(); i++) { + Mockito.verify(worker).waitForEtcdOnEtcdNode(Mockito.eq(etcdVms.get(i)), Mockito.eq(PUBLIC_IP), Mockito.eq(etcdNodeStartPort + i), Mockito.anyLong()); + } + } + + @Test + public void testVerifyEtcdRunningOnEtcdNodesOnDirectAccessNetwork() { + List etcdVms = mockEtcdVms(1); + Mockito.when(etcdVms.get(0).getPrivateIpAddress()).thenReturn("10.1.1.20"); + Mockito.when(clusterManager.isDirectAccess(network)).thenReturn(true); + Mockito.doReturn(null).when(worker).waitForEtcdOnEtcdNode(Mockito.any(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong()); + + worker.verifyEtcdRunningOnEtcdNodes(etcdVms, new ArrayList<>(), network, startTimeoutTime()); + + Mockito.verify(worker).waitForEtcdOnEtcdNode(Mockito.eq(etcdVms.get(0)), Mockito.eq("10.1.1.20"), + Mockito.eq(KubernetesClusterActionWorker.DEFAULT_SSH_PORT), Mockito.anyLong()); + } + + @Test + public void testVerifyEtcdRunningOnEtcdNodesWaitIsBoundedBySetting() { + List etcdVms = mockEtcdVms(1); + Mockito.doReturn(null).when(worker).waitForEtcdOnEtcdNode(Mockito.any(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong()); + long before = System.currentTimeMillis(); + + worker.verifyEtcdRunningOnEtcdNodes(etcdVms, new ArrayList<>(), network, startTimeoutTime()); + + ArgumentCaptor timeoutTime = ArgumentCaptor.forClass(Long.class); + Mockito.verify(worker).waitForEtcdOnEtcdNode(Mockito.any(), Mockito.any(), Mockito.anyInt(), timeoutTime.capture()); + long setupTimeoutMs = KubernetesClusterService.KubernetesEtcdNodeSetupTimeout.value() * 1000; + Assert.assertTrue(timeoutTime.getValue() <= System.currentTimeMillis() + setupTimeoutMs); + Assert.assertTrue(timeoutTime.getValue() >= before + setupTimeoutMs); + } + + @Test + public void testVerifyEtcdRunningOnEtcdNodesWaitNeverExceedsClusterStartTimeout() { + List etcdVms = mockEtcdVms(1); + Mockito.doReturn(null).when(worker).waitForEtcdOnEtcdNode(Mockito.any(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong()); + long startTimeoutTime = System.currentTimeMillis() + 5000; + + worker.verifyEtcdRunningOnEtcdNodes(etcdVms, new ArrayList<>(), network, startTimeoutTime); + + Mockito.verify(worker).waitForEtcdOnEtcdNode(Mockito.any(), Mockito.any(), Mockito.anyInt(), Mockito.eq(startTimeoutTime)); + } + + @Test + public void testVerifyEtcdRunningOnEtcdNodesFailsClusterWithReason() { + List etcdVms = mockEtcdVms(2); + Mockito.when(etcdVms.get(0).getDisplayName()).thenReturn("cluster-etcd-1"); + List clusterVMs = mockEtcdVms(1); + Mockito.when(kubernetesCluster.getName()).thenReturn("cluster"); + Mockito.when(kubernetesCluster.getId()).thenReturn(7L); + String reason = "The Kubernetes ISO does not include the etcd binaries"; + Mockito.doReturn(reason).when(worker).waitForEtcdOnEtcdNode(Mockito.any(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong()); + Mockito.doReturn(true).when(worker).stateTransitTo(7L, KubernetesCluster.Event.CreateFailed); + Mockito.doNothing().when(worker).detachIsoKubernetesVMs(Mockito.anyList()); + + try { + worker.verifyEtcdRunningOnEtcdNodes(etcdVms, clusterVMs, network, startTimeoutTime()); + Assert.fail("The cluster start should fail when etcd is not running on an etcd node"); + } catch (CloudRuntimeException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("cluster-etcd-1")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains(reason)); + } + + // Fails on the first etcd node without waiting for the others, and the ISO is detached from every VM + Mockito.verify(worker, Mockito.times(1)).waitForEtcdOnEtcdNode(Mockito.any(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong()); + Mockito.verify(worker).stateTransitTo(7L, KubernetesCluster.Event.CreateFailed); + ArgumentCaptor> detached = ArgumentCaptor.forClass(List.class); + Mockito.verify(worker).detachIsoKubernetesVMs(detached.capture()); + Assert.assertEquals(3, detached.getValue().size()); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java index 329f9b0e42ad..4aa85625e3c6 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java @@ -16,10 +16,14 @@ // under the License. package com.cloud.kubernetes.cluster.utils; +import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.utils.Pair; +import com.cloud.utils.ssh.SshHelper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) @@ -52,4 +56,68 @@ public void testClusterNodeVersionMatches() { executeThrowAndTestVersionMatch(); } + + private String waitForEtcd(MockedStatic sshHelper, long timeoutMs) { + KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); + return KubernetesClusterUtil.waitForEtcdOnEtcdNode(cluster, "etcd-1", "10.1.1.10", 50000, "cloud", null, + System.currentTimeMillis() + timeoutMs, 1); + } + + private void mockEtcdNodeStatus(MockedStatic sshHelper, Pair first, Pair... next) { + sshHelper.when(() -> SshHelper.sshExecute(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.any(), Mockito.any(), + Mockito.eq(KubernetesClusterUtil.ETCD_NODE_STATUS_COMMAND), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt())) + .thenReturn(first, next); + } + + @Test + @SuppressWarnings("unchecked") + public void testWaitForEtcdOnEtcdNodeRunning() { + try (MockedStatic sshHelper = Mockito.mockStatic(SshHelper.class)) { + mockEtcdNodeStatus(sshHelper, new Pair<>(true, "WAITING\n"), new Pair<>(true, "ACTIVE\n"), new Pair<>(true, "ACTIVE\n")); + Assert.assertNull(waitForEtcd(sshHelper, 10000)); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testWaitForEtcdOnEtcdNodeSetupFailed() { + try (MockedStatic sshHelper = Mockito.mockStatic(SshHelper.class)) { + String reason = "The Kubernetes ISO does not include the etcd binaries"; + mockEtcdNodeStatus(sshHelper, new Pair<>(true, "WAITING\n"), new Pair<>(true, "FAILED: " + reason + "\n")); + Assert.assertEquals(reason, waitForEtcd(sshHelper, 10000)); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testWaitForEtcdOnEtcdNodeNeverStarts() { + try (MockedStatic sshHelper = Mockito.mockStatic(SshHelper.class)) { + mockEtcdNodeStatus(sshHelper, new Pair<>(true, "WAITING\n")); + String failure = waitForEtcd(sshHelper, 100); + Assert.assertNotNull(failure); + Assert.assertTrue(failure, failure.contains("the etcd service was not running")); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testWaitForEtcdOnEtcdNodeKeepsRestarting() { + try (MockedStatic sshHelper = Mockito.mockStatic(SshHelper.class)) { + // Running on one check, down on the next: never running twice in a row + mockEtcdNodeStatus(sshHelper, new Pair<>(true, "ACTIVE\n"), new Pair<>(true, "WAITING\n"), new Pair<>(true, "ACTIVE\n"), + new Pair<>(true, "WAITING\n")); + Assert.assertNotNull(waitForEtcd(sshHelper, 100)); + } + } + + @Test + public void testWaitForEtcdOnEtcdNodeNotReachable() { + try (MockedStatic sshHelper = Mockito.mockStatic(SshHelper.class)) { + sshHelper.when(() -> SshHelper.sshExecute(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.any(), Mockito.any(), + Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt())).thenThrow(new java.io.IOException("Connection refused")); + String failure = waitForEtcd(sshHelper, 100); + Assert.assertNotNull(failure); + Assert.assertTrue(failure, failure.contains("not reachable over SSH")); + } + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java index 35f8e66e0452..62e62f495a7d 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java @@ -22,9 +22,17 @@ import java.util.UUID; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.storage.command.CheckFileInIsoAnswer; +import org.apache.cloudstack.storage.command.CheckFileInIsoCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -34,12 +42,19 @@ import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +import com.cloud.agent.api.Answer; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.cpu.CPU; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.VMTemplateDetailVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateDetailsDao; @RunWith(MockitoJUnitRunner.class) public class KubernetesVersionManagerImplTest { @@ -53,6 +68,24 @@ public class KubernetesVersionManagerImplTest { @Mock DataCenterDao dataCenterDao; + @Mock + KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + + @Mock + VMTemplateDao templateDao; + + @Mock + VMTemplateDetailsDao templateDetailsDao; + + @Mock + TemplateDataStoreDao templateDataStoreDao; + + @Mock + DataStoreManager dataStoreManager; + + @Mock + EndPointSelector endPointSelector; + @InjectMocks KubernetesVersionManagerImpl kubernetesVersionManager = new KubernetesVersionManagerImpl(); @@ -144,4 +177,214 @@ public void testValidateImageStoreForAllZonesWithSomeMissingStorage() { ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", (Long) null, false); } + + private static final long ISO_ID = 200L; + private static final long ZONE_ID = 1L; + + private KubernetesSupportedVersionVO mockVersionWithIso() { + KubernetesSupportedVersionVO version = Mockito.mock(KubernetesSupportedVersionVO.class); + Mockito.when(version.getIsoId()).thenReturn(ISO_ID); + return version; + } + + private void mockStoredEtcdDetail(String value) { + VMTemplateDetailVO detail = Mockito.mock(VMTemplateDetailVO.class); + Mockito.when(detail.getValue()).thenReturn(value); + Mockito.when(templateDetailsDao.findDetail(ISO_ID, KubernetesVersionService.ETCD_INCLUDED_ISO_DETAIL)).thenReturn(detail); + } + + private EndPoint mockIsoReadyOnImageStore() { + return mockIsoReadyOnImageStore(true); + } + + private EndPoint mockIsoReadyOnImageStore(boolean inZone) { + VMTemplateVO iso = Mockito.mock(VMTemplateVO.class); + Mockito.when(iso.isDirectDownload()).thenReturn(false); + Mockito.when(templateDao.findById(ISO_ID)).thenReturn(iso); + TemplateDataStoreVO isoOnStore = Mockito.mock(TemplateDataStoreVO.class); + Mockito.when(isoOnStore.getInstallPath()).thenReturn("template/tmpl/1/200/200-1-uuid.iso"); + Mockito.when(isoOnStore.getDataStoreId()).thenReturn(5L); + if (inZone) { + Mockito.when(templateDataStoreDao.findByTemplateZoneReady(ISO_ID, ZONE_ID)).thenReturn(isoOnStore); + } else { + Mockito.when(templateDataStoreDao.findReadyByTemplate(ISO_ID, DataStoreRole.Image)).thenReturn(isoOnStore); + } + DataStore store = Mockito.mock(DataStore.class); + Mockito.when(dataStoreManager.getDataStore(5L, DataStoreRole.Image)).thenReturn(store); + EndPoint endPoint = Mockito.mock(EndPoint.class); + Mockito.when(endPointSelector.select(store)).thenReturn(endPoint); + return endPoint; + } + + private void verifyNothingStored() { + Mockito.verify(templateDetailsDao, Mockito.never()).addDetail(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean()); + } + + @Test + public void testIsEtcdIncludedInIsoUsesStoredResult() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + mockStoredEtcdDetail("true"); + Assert.assertEquals(Boolean.TRUE, kubernetesVersionManager.isEtcdIncludedInIso(version, ZONE_ID)); + mockStoredEtcdDetail("false"); + Assert.assertEquals(Boolean.FALSE, kubernetesVersionManager.isEtcdIncludedInIso(version, ZONE_ID)); + Mockito.verifyNoInteractions(endPointSelector); + verifyNothingStored(); + } + + @Test + public void testIsEtcdIncludedInIsoChecksAndStoresWhenEtcdPresent() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + EndPoint endPoint = mockIsoReadyOnImageStore(); + Mockito.when(endPoint.sendMessage(Mockito.any(CheckFileInIsoCommand.class))).thenAnswer(invocation -> { + CheckFileInIsoCommand cmd = invocation.getArgument(0); + Assert.assertEquals("template/tmpl/1/200/200-1-uuid.iso", cmd.getIsoPath()); + Assert.assertEquals(KubernetesVersionService.ETCD_ARCHIVE_PATH_IN_ISO, cmd.getFilePath()); + return new CheckFileInIsoAnswer(cmd, true); + }); + + Assert.assertEquals(Boolean.TRUE, kubernetesVersionManager.isEtcdIncludedInIso(version, ZONE_ID)); + + Mockito.verify(templateDetailsDao).addDetail(ISO_ID, KubernetesVersionService.ETCD_INCLUDED_ISO_DETAIL, "true", true); + } + + @Test + public void testIsEtcdIncludedInIsoChecksAndStoresWhenEtcdMissing() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + EndPoint endPoint = mockIsoReadyOnImageStore(); + Mockito.when(endPoint.sendMessage(Mockito.any(CheckFileInIsoCommand.class))) + .thenAnswer(invocation -> new CheckFileInIsoAnswer(invocation.getArgument(0), false)); + + Assert.assertEquals(Boolean.FALSE, kubernetesVersionManager.isEtcdIncludedInIso(version, ZONE_ID)); + + Mockito.verify(templateDetailsDao).addDetail(ISO_ID, KubernetesVersionService.ETCD_INCLUDED_ISO_DETAIL, "false", true); + } + + @Test + public void testIsEtcdIncludedInIsoUnknownWhenIsoCouldNotBeRead() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + EndPoint endPoint = mockIsoReadyOnImageStore(); + Mockito.when(endPoint.sendMessage(Mockito.any(CheckFileInIsoCommand.class))) + .thenAnswer(invocation -> new CheckFileInIsoAnswer((CheckFileInIsoCommand)invocation.getArgument(0), "isoinfo: command not found")); + + Assert.assertNull(kubernetesVersionManager.isEtcdIncludedInIso(version, ZONE_ID)); + + verifyNothingStored(); + } + + @Test + public void testIsEtcdIncludedInIsoUnknownWhenAgentDoesNotKnowTheCommand() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + EndPoint endPoint = mockIsoReadyOnImageStore(); + Mockito.when(endPoint.sendMessage(Mockito.any(CheckFileInIsoCommand.class))) + .thenAnswer(invocation -> Answer.createUnsupportedCommandAnswer(invocation.getArgument(0))); + + Assert.assertNull(kubernetesVersionManager.isEtcdIncludedInIso(version, ZONE_ID)); + + verifyNothingStored(); + } + + @Test + public void testIsEtcdIncludedInIsoUnknownWhenNoSecondaryStorageVm() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + mockIsoReadyOnImageStore(); + Mockito.when(endPointSelector.select(Mockito.any(DataStore.class))).thenReturn(null); + + Assert.assertNull(kubernetesVersionManager.isEtcdIncludedInIso(version, ZONE_ID)); + + verifyNothingStored(); + } + + @Test + public void testIsEtcdIncludedInIsoUnknownForDirectDownloadIso() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + VMTemplateVO iso = Mockito.mock(VMTemplateVO.class); + Mockito.when(iso.isDirectDownload()).thenReturn(true); + Mockito.when(templateDao.findById(ISO_ID)).thenReturn(iso); + + Assert.assertNull(kubernetesVersionManager.isEtcdIncludedInIso(version, ZONE_ID)); + + Mockito.verifyNoInteractions(endPointSelector); + verifyNothingStored(); + } + + @Test + public void testIsEtcdIncludedInIsoReadsOnlyTheCopyInTheZone() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + VMTemplateVO iso = Mockito.mock(VMTemplateVO.class); + Mockito.when(templateDao.findById(ISO_ID)).thenReturn(iso); + Mockito.when(templateDataStoreDao.findByTemplateZoneReady(ISO_ID, ZONE_ID)).thenReturn(null); + + Assert.assertNull(kubernetesVersionManager.isEtcdIncludedInIso(version, ZONE_ID)); + + // A copy of the ISO in another zone is not used + Mockito.verify(templateDataStoreDao, Mockito.never()).findReadyByTemplate(Mockito.anyLong(), Mockito.any()); + Mockito.verifyNoInteractions(endPointSelector); + verifyNothingStored(); + } + + @Test + public void testBackgroundCheckSkipsCheckedIsos() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + Mockito.when(kubernetesSupportedVersionDao.listAll()).thenReturn(List.of(version)); + mockStoredEtcdDetail("false"); + + kubernetesVersionManager.checkEtcdIncludedInKubernetesIsos(); + + Mockito.verifyNoInteractions(endPointSelector); + verifyNothingStored(); + } + + @Test + public void testBackgroundCheckStoresResultForUncheckedIso() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + // Two versions sharing one ISO are checked once + Mockito.when(kubernetesSupportedVersionDao.listAll()).thenReturn(List.of(version, version)); + EndPoint endPoint = mockIsoReadyOnImageStore(false); + Mockito.when(endPoint.sendMessage(Mockito.any(CheckFileInIsoCommand.class))) + .thenAnswer(invocation -> new CheckFileInIsoAnswer(invocation.getArgument(0), true)); + + kubernetesVersionManager.checkEtcdIncludedInKubernetesIsos(); + + Mockito.verify(endPoint, Mockito.times(1)).sendMessage(Mockito.any(CheckFileInIsoCommand.class)); + Mockito.verify(templateDetailsDao).addDetail(ISO_ID, KubernetesVersionService.ETCD_INCLUDED_ISO_DETAIL, "true", true); + } + + @Test + public void testBackgroundCheckGivesUpAfterRepeatedFailures() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + Mockito.when(kubernetesSupportedVersionDao.listAll()).thenReturn(List.of(version)); + EndPoint endPoint = mockIsoReadyOnImageStore(false); + Mockito.when(endPoint.sendMessage(Mockito.any(CheckFileInIsoCommand.class))) + .thenAnswer(invocation -> new CheckFileInIsoAnswer((CheckFileInIsoCommand)invocation.getArgument(0), "isoinfo: command not found")); + + for (int i = 0; i < KubernetesVersionManagerImpl.MAX_FAILED_BACKGROUND_ETCD_IN_ISO_CHECKS + 2; i++) { + kubernetesVersionManager.checkEtcdIncludedInKubernetesIsos(); + } + + Mockito.verify(endPoint, Mockito.times(KubernetesVersionManagerImpl.MAX_FAILED_BACKGROUND_ETCD_IN_ISO_CHECKS)) + .sendMessage(Mockito.any(CheckFileInIsoCommand.class)); + verifyNothingStored(); + } + + @Test + public void testBackgroundCheckWaitsForIsoToBeReady() { + KubernetesSupportedVersionVO version = mockVersionWithIso(); + Mockito.when(kubernetesSupportedVersionDao.listAll()).thenReturn(List.of(version)); + VMTemplateVO iso = Mockito.mock(VMTemplateVO.class); + Mockito.when(templateDao.findById(ISO_ID)).thenReturn(iso); + Mockito.when(templateDataStoreDao.findReadyByTemplate(ISO_ID, DataStoreRole.Image)).thenReturn(null); + + // An ISO that is still downloading is not counted as a failed check, however long it takes + for (int i = 0; i < KubernetesVersionManagerImpl.MAX_FAILED_BACKGROUND_ETCD_IN_ISO_CHECKS + 2; i++) { + kubernetesVersionManager.checkEtcdIncludedInKubernetesIsos(); + } + Mockito.verifyNoInteractions(endPointSelector); + + EndPoint endPoint = mockIsoReadyOnImageStore(false); + Mockito.when(endPoint.sendMessage(Mockito.any(CheckFileInIsoCommand.class))) + .thenAnswer(invocation -> new CheckFileInIsoAnswer(invocation.getArgument(0), true)); + kubernetesVersionManager.checkEtcdIncludedInKubernetesIsos(); + + Mockito.verify(templateDetailsDao).addDetail(ISO_ID, KubernetesVersionService.ETCD_INCLUDED_ISO_DETAIL, "true", true); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java index 7ba35169b9ef..0bb831f7f7d7 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java @@ -67,8 +67,10 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.storage.VMTemplateDetailVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.template.TemplateApiService; import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.Pair; @@ -104,6 +106,8 @@ public class KubernetesVersionServiceTest { @Mock private TemplateApiService templateService; @Mock + private VMTemplateDetailsDao templateDetailsDao; + @Mock private Account accountMock; AutoCloseable closeable; @@ -176,6 +180,49 @@ public void listKubernetesSupportedVersionsTest() { } } + private Boolean listSupportsEtcd(String storedDetailValue) { + final long isoId = 200L; + CallContext callContextMock = Mockito.mock(CallContext.class); + try (MockedStatic callContextMockedStatic = Mockito.mockStatic(CallContext.class)) { + callContextMockedStatic.when(CallContext::current).thenReturn(callContextMock); + when(callContextMock.getCallingAccount()).thenReturn(accountMock); + ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); + KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); + when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + when(versionVO.getIsoId()).thenReturn(isoId); + List versionVOs = List.of(versionVO); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); + when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class))) + .thenReturn(new Pair<>(versionVOs, versionVOs.size())); + VMTemplateDetailVO detail = null; + if (storedDetailValue != null) { + detail = Mockito.mock(VMTemplateDetailVO.class); + when(detail.getValue()).thenReturn(storedDetailValue); + } + when(templateDetailsDao.findDetail(isoId, KubernetesVersionService.ETCD_INCLUDED_ISO_DETAIL)).thenReturn(detail); + + ListResponse response = kubernetesVersionService.listKubernetesSupportedVersions(cmd); + + Mockito.verify(templateDetailsDao).findDetail(isoId, KubernetesVersionService.ETCD_INCLUDED_ISO_DETAIL); + return response.getResponses().get(0).getSupportsEtcd(); + } + } + + @Test + public void listKubernetesSupportedVersionsSupportsEtcdNotCheckedYet() { + Assert.assertNull(listSupportsEtcd(null)); + } + + @Test + public void listKubernetesSupportedVersionsSupportsEtcdTrue() { + Assert.assertEquals(Boolean.TRUE, listSupportsEtcd("true")); + } + + @Test + public void listKubernetesSupportedVersionsSupportsEtcdFalse() { + Assert.assertEquals(Boolean.FALSE, listSupportsEtcd("false")); + } + @Test public void listKubernetesSupportedVersionsTestWhenAdmin() { CallContext callContextMock = Mockito.mock(CallContext.class); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 4345ff24e3c5..cea39fd81837 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -58,6 +58,8 @@ import org.apache.cloudstack.framework.security.keystore.KeystoreManager; import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.command.BackupDeleteAnswer; +import org.apache.cloudstack.storage.command.CheckFileInIsoAnswer; +import org.apache.cloudstack.storage.command.CheckFileInIsoCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DeleteCommand; @@ -342,11 +344,72 @@ public Answer executeRequest(Command cmd) { return execute((ListDataStoreObjectsCommand)cmd); } else if (cmd instanceof QuerySnapshotZoneCopyCommand) { return execute((QuerySnapshotZoneCopyCommand)cmd); + } else if (cmd instanceof CheckFileInIsoCommand) { + return execute((CheckFileInIsoCommand)cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } } + protected Answer execute(CheckFileInIsoCommand cmd) { + DataStoreTO store = cmd.getStore(); + if (!(store instanceof NfsTO)) { + return new CheckFileInIsoAnswer(cmd, "Checking the content of an ISO is supported only on NFS image stores"); + } + if (org.apache.commons.lang3.StringUtils.isAnyBlank(cmd.getIsoPath(), cmd.getFilePath())) { + return new CheckFileInIsoAnswer(cmd, "ISO path and file path are required to check the content of an ISO"); + } + String rootDir = getRootDir(((NfsTO)store).getUrl(), _nfsVersion); + File iso = new File(rootDir, cmd.getIsoPath()); + if (!iso.isFile()) { + return new CheckFileInIsoAnswer(cmd, String.format("ISO %s not found on the image store", cmd.getIsoPath())); + } + String listing; + try { + listing = listFilesInIso(iso); + } catch (CloudRuntimeException e) { + logger.warn("Unable to list the files in ISO {}: {}", iso.getAbsolutePath(), e.getMessage()); + return new CheckFileInIsoAnswer(cmd, e.getMessage()); + } + boolean filePresent = isFileInIsoListing(listing, cmd.getFilePath()); + logger.debug("File {} is {} in ISO {}", cmd.getFilePath(), filePresent ? "present" : "not present", iso.getAbsolutePath()); + return new CheckFileInIsoAnswer(cmd, filePresent); + } + + /** + * Lists all the files in an ISO, one full path per line, without mounting it. + */ + protected String listFilesInIso(File iso) { + Script command = new Script("isoinfo", _timeout, logger); + command.add("-R"); + command.add("-f"); + command.add("-i", iso.getAbsolutePath()); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = command.execute(parser); + if (result != null) { + throw new CloudRuntimeException(String.format("Unable to list the files in ISO %s: %s", iso.getName(), result)); + } + return parser.getLines(); + } + + /** + * Looks for a file in the output of {@link #listFilesInIso(File)}. The comparison ignores the case and the + * ";1" version suffix, which are what an ISO without Rock Ridge names would give. + */ + protected static boolean isFileInIsoListing(String listing, String filePath) { + if (org.apache.commons.lang3.StringUtils.isAnyBlank(listing, filePath)) { + return false; + } + String wanted = org.apache.commons.lang3.StringUtils.strip(filePath.trim(), "/"); + for (String line : listing.split("\\R")) { + String entry = org.apache.commons.lang3.StringUtils.strip(line.trim().replaceFirst(";\\d+$", ""), "/"); + if (entry.equalsIgnoreCase(wanted)) { + return true; + } + } + return false; + } + private Answer execute(ListDataStoreObjectsCommand cmd) { return listFilesAtPath(getRootDir(cmd.getStore().getUrl(), _nfsVersion), cmd.getPath(), cmd.getStartIndex(), cmd.getPageSize()); } diff --git a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java index 4ed6001d3b7e..01ba5d5af1fc 100644 --- a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java @@ -39,6 +39,8 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.to.NfsTO; import org.apache.cloudstack.storage.command.BackupDeleteAnswer; +import org.apache.cloudstack.storage.command.CheckFileInIsoAnswer; +import org.apache.cloudstack.storage.command.CheckFileInIsoCommand; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; @@ -59,6 +61,7 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.EncryptionUtil; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @RunWith(MockitoJUnitRunner.class) @@ -407,4 +410,93 @@ public void deleteScreenshotTestPathStartsWithSeparator() { verify(resource).deleteLocalFile("/root/" + "path/to/file"); } + + private static final String ETCD_ARCHIVE = "etcd/etcd-linux-amd64.tar.gz"; + + @Test + public void testIsFileInIsoListingFindsFile() { + String listing = "/docker\n/docker/etcd:3.5.21-0.tar\n/etcd\n/etcd/etcd-linux-amd64.tar.gz\n/k8s/kubeadm\n"; + assertTrue(NfsSecondaryStorageResource.isFileInIsoListing(listing, ETCD_ARCHIVE)); + assertTrue(NfsSecondaryStorageResource.isFileInIsoListing(listing, "/" + ETCD_ARCHIVE)); + } + + @Test + public void testIsFileInIsoListingFileMissing() { + // What an ISO built without ETCD_VERSION has: only the etcd image that kubeadm runs on control nodes + String listing = "/docker\n/docker/etcd:3.5.21-0.tar\n/k8s/kubeadm\n"; + assertFalse(NfsSecondaryStorageResource.isFileInIsoListing(listing, ETCD_ARCHIVE)); + assertFalse(NfsSecondaryStorageResource.isFileInIsoListing("", ETCD_ARCHIVE)); + assertFalse(NfsSecondaryStorageResource.isFileInIsoListing(null, ETCD_ARCHIVE)); + } + + @Test + public void testIsFileInIsoListingWithoutRockRidgeNames() { + String listing = "/ETCD\r\n/ETCD/ETCD-LINUX-AMD64.TAR.GZ;1\r\n"; + assertTrue(NfsSecondaryStorageResource.isFileInIsoListing(listing, ETCD_ARCHIVE)); + } + + private CheckFileInIsoCommand prepareCheckFileInIsoCommand(Path root) throws Exception { + Path isoDir = Files.createDirectories(root.resolve("template/tmpl/1/200")); + Files.createFile(isoDir.resolve("200-1-uuid.iso")); + NfsTO store = Mockito.mock(NfsTO.class); + Mockito.when(store.getUrl()).thenReturn("nfs://host/export"); + doReturn(root.toString()).when(resource).getRootDir(any(), any()); + return new CheckFileInIsoCommand(store, "template/tmpl/1/200/200-1-uuid.iso", ETCD_ARCHIVE); + } + + @Test + public void testExecuteCheckFileInIsoCommandFilePresent() throws Exception { + Path root = Files.createTempDirectory("checkiso"); + CheckFileInIsoCommand cmd = prepareCheckFileInIsoCommand(root); + doReturn("/etcd\n/etcd/etcd-linux-amd64.tar.gz\n").when(resource).listFilesInIso(any()); + + Answer answer = resource.execute(cmd); + + assertTrue(answer.getResult()); + assertTrue(((CheckFileInIsoAnswer)answer).isFilePresent()); + } + + @Test + public void testExecuteCheckFileInIsoCommandFileNotPresent() throws Exception { + Path root = Files.createTempDirectory("checkiso"); + CheckFileInIsoCommand cmd = prepareCheckFileInIsoCommand(root); + doReturn("/k8s\n/k8s/kubeadm\n").when(resource).listFilesInIso(any()); + + Answer answer = resource.execute(cmd); + + assertTrue(answer.getResult()); + assertFalse(((CheckFileInIsoAnswer)answer).isFilePresent()); + } + + @Test + public void testExecuteCheckFileInIsoCommandIsoCannotBeListed() throws Exception { + Path root = Files.createTempDirectory("checkiso"); + CheckFileInIsoCommand cmd = prepareCheckFileInIsoCommand(root); + doThrow(new CloudRuntimeException("isoinfo: command not found")).when(resource).listFilesInIso(any()); + + Answer answer = resource.execute(cmd); + + assertFalse(answer.getResult()); + assertFalse(((CheckFileInIsoAnswer)answer).isFilePresent()); + } + + @Test + public void testExecuteCheckFileInIsoCommandIsoMissing() throws Exception { + Path root = Files.createTempDirectory("checkiso"); + NfsTO store = Mockito.mock(NfsTO.class); + Mockito.when(store.getUrl()).thenReturn("nfs://host/export"); + doReturn(root.toString()).when(resource).getRootDir(any(), any()); + + Answer answer = resource.execute(new CheckFileInIsoCommand(store, "template/tmpl/1/200/missing.iso", ETCD_ARCHIVE)); + + assertFalse(answer.getResult()); + verify(resource, times(0)).listFilesInIso(any()); + } + + @Test + public void testExecuteCheckFileInIsoCommandNonNfsStore() { + Answer answer = resource.execute(new CheckFileInIsoCommand(Mockito.mock(DataStoreTO.class), "x.iso", ETCD_ARCHIVE)); + + assertFalse(answer.getResult()); + } } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 1c193e96d490..c9f3124f5d85 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -3945,6 +3945,7 @@ "message.ip.v6.prefix.delete": "IPv6 prefix deleted", "message.iso.arch": "Please select an ISO architecture", "message.iso.desc": "Disc image containing data or bootable media for OS.", +"message.kubernetes.version.etcd.not.included": "The ISO of the selected Kubernetes version does not include etcd, so a Cluster with dedicated etcd nodes cannot be created with it. Select another version or leave the etcd nodes empty.", "message.kubernetes.cluster.add.nodes": "Please confirm that you want to add the following nodes to the cluster", "message.kubernetes.cluster.delete": "Please confirm that you want to destroy the Cluster.", "message.kubeconfig.cluster.not.available": "Kubernetes Cluster kubeconfig not available currently.", diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue index 618e46cb72d9..b820634f5a15 100644 --- a/ui/src/views/compute/CreateKubernetesCluster.vue +++ b/ui/src/views/compute/CreateKubernetesCluster.vue @@ -297,6 +297,12 @@ +