From e67aa865f6e8af386d434b1a7b7ea5e99ae3751b Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Mon, 4 May 2026 15:42:13 +0200 Subject: [PATCH 01/14] linstor: support shared (thick LVM) storage pools Resources in a LINSTOR shared storage pool live on one shared LUN and are active on at most one node; there is no DRBD replication or dual-primary. Handle them in the KVM storage adaptor and MS driver: - resolve the actual resource-definition name (template resources can be shared under a different name) before connect/copy operations - on live migration create and activate the resource on the destination node instead of setting DRBD allow-two-primaries - on disconnect delete the local resource only if another active resource exists (during live migration two resources are active at the same time) - getDevicePath: only accept resources that expose a device path, inactive shared resources have none - createResourceBase returns the actually used resource name --- .../kvm/storage/LinstorStorageAdaptor.java | 133 +++++++++++++----- .../LinstorPrimaryDataStoreDriverImpl.java | 13 +- .../storage/datastore/util/LinstorUtil.java | 12 +- 3 files changed, 114 insertions(+), 44 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index c0d61e64e399..ac66ac5f212a 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -61,6 +61,7 @@ import com.linbit.linstor.api.model.ProviderKind; import com.linbit.linstor.api.model.Resource; import com.linbit.linstor.api.model.ResourceConnectionModify; +import com.linbit.linstor.api.model.ResourceCreate; import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionCreate; import com.linbit.linstor.api.model.ResourceDefinitionModify; @@ -468,21 +469,22 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, Qemu String foundRscName = resourceDefinition != null ? resourceDefinition.getName() : rscName; + makeResourceAvailable(api, foundRscName, false); + // query linstor for the device path List resources = api.viewResources( - Collections.emptyList(), + Collections.singletonList(localNodeName), Collections.singletonList(foundRscName), Collections.emptyList(), null, null, null); - makeResourceAvailable(api, foundRscName, false); - if (!resources.isEmpty() && !resources.get(0).getVolumes().isEmpty()) { final String devPath = LinstorUtil.getDevicePathFromResource(resources.get(0)); logger.info("Linstor: Created drbd device: " + devPath); - final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath, name, pool); + final String diskName = foundRscName.substring(LinstorUtil.RSC_PREFIX.length()); + final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath, diskName, pool); kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW); long allocatedKib = resources.get(0).getVolumes().get(0).getAllocatedSizeKib() != null ? resources.get(0).getVolumes().get(0).getAllocatedSizeKib() : 0; @@ -550,6 +552,37 @@ private void allow2PrimariesIfInUse(DevelopersApi api, String rscName) throws Ap } } + private boolean usesDrbd(DevelopersApi api, String rscName) throws ApiException { + List rscs = api.viewResources( + Collections.emptyList(), + Collections.singletonList(rscName), + null, + null, + null, + null); + + for (ResourceWithVolumes rsc : rscs) { + if (rsc.getLayerObject().getDrbd() != null) { + return true; + } + } + return false; + } + + + private String findCorrectTemplateName(DevelopersApi api, String rscName, KVMStoragePool pool ) { + String templateName = rscName; + LinstorStoragePool lpool = (LinstorStoragePool) pool; + try { + ResourceDefinition resourceDefinition = LinstorUtil.findResourceDefinition( + api, rscName, lpool.getResourceGroup()); + templateName = resourceDefinition != null ? resourceDefinition.getName() : rscName; + } catch (ApiException e) { + logger.error("Error finding resource definition for {}", rscName); + } + return templateName; + } + @Override public boolean connectPhysicalDisk( String volumePath, KVMStoragePool pool, Map details, boolean isVMMigration) @@ -562,20 +595,36 @@ public boolean connectPhysicalDisk( final DevelopersApi api = getLinstorAPI(pool); String rscName; + boolean usesDrbd; try { - rscName = getLinstorRscName(volumePath); - - ResourceMakeAvailable rma = new ResourceMakeAvailable(); - ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma); - checkLinstorAnswersThrow(answers); + rscName = findCorrectTemplateName(api, getLinstorRscName(volumePath), pool); + usesDrbd = usesDrbd(api, rscName); + + if (isVMMigration && !usesDrbd) { + // create the resource on the new node, noop if already exists + ResourceCreate rc = new ResourceCreate(); + Resource rsc = new Resource(); + rsc.setNodeName(localNodeName); + rc.setResource(rsc); + ApiCallRcList answers = api.resourceCreateOnNode(rscName, localNodeName, rc); + checkLinstorAnswersThrow(answers); + // activate resource if there already was an inactive resource here + answers = api.activateRsc(rscName, localNodeName); + checkLinstorAnswersThrow(answers); + } else { + // DRBD resources just work with a diskless and don't have the activate/deactivate concept + ResourceMakeAvailable rma = new ResourceMakeAvailable(); + ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma); + checkLinstorAnswersThrow(answers); + } waitForLocalDevice(api, rscName); } catch (ApiException apiEx) { logger.error(apiEx); throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); } - if (isVMMigration) { + if (isVMMigration && usesDrbd) { try { allow2PrimariesIfInUse(api, rscName); } catch (ApiException apiEx) { @@ -622,13 +671,26 @@ private void removeTwoPrimariesProps(DevelopersApi api, String inUseNode, String removeTwoPrimariesRcProps(api, rscName, inUseNode, deleteProps); } + private void deleteUnusedSharedResources(DevelopersApi api, Resource nodeRsc) { + try { + List rscs = api.resourceList(nodeRsc.getName(), null, null); + if (rscs != null && rscs.stream() + .filter(rsc -> rsc.getFlags() == null || !rsc.getFlags().contains(ApiConsts.FLAG_RSC_INACTIVE)) + .count() > 1) { + api.resourceDelete(nodeRsc.getName(), localNodeName, true); + } + } catch (ApiException apiEx) { + logger.error(apiEx.getBestMessage()); + } + } + private boolean tryDisconnectLinstor(String volumePath, KVMStoragePool pool) { if (volumePath == null) { return false; } - logger.debug("Linstor: Using storage pool: " + pool.getUuid()); + logger.debug("Linstor: Using storage pool: {}", pool.getUuid()); final DevelopersApi api = getLinstorAPI(pool); Optional optRsc; @@ -652,33 +714,40 @@ private boolean tryDisconnectLinstor(String volumePath, KVMStoragePool pool) if (optRsc.isPresent()) { Resource rsc = optRsc.get(); - try { - String inUseNode = LinstorUtil.isResourceInUse(api, rsc.getName()); - if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) { - removeTwoPrimariesProps(api, inUseNode, rsc.getName()); + + // if it is a non DRBD resource(shared) we need to check if we are the last active one + // on live migrate, we can have 2 active resources at the same time + if (rsc.getLayerObject().getDrbd() == null) { + deleteUnusedSharedResources(api, rsc); + } else { + try { + String inUseNode = LinstorUtil.isResourceInUse(api, rsc.getName()); + if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) { + removeTwoPrimariesProps(api, inUseNode, rsc.getName()); + } + } catch (ApiException apiEx) { + logger.error(apiEx.getBestMessage()); + // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal } - } catch (ApiException apiEx) { - logger.error(apiEx.getBestMessage()); - // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal - } - try { - // if diskless resource remove it, in the worst case it will be transformed to a tiebreaker - if (rsc.getFlags() != null && - rsc.getFlags().contains(ApiConsts.FLAG_DRBD_DISKLESS) && - !rsc.getFlags().contains(ApiConsts.FLAG_TIE_BREAKER)) { - ApiCallRcList delAnswers = api.resourceDelete(rsc.getName(), localNodeName, true); - logLinstorAnswers(delAnswers); + try { + // if diskless resource remove it, in the worst case it will be transformed to a tiebreaker + if (rsc.getFlags() != null && + rsc.getFlags().contains(ApiConsts.FLAG_DRBD_DISKLESS) && + !rsc.getFlags().contains(ApiConsts.FLAG_TIE_BREAKER)) { + ApiCallRcList delAnswers = api.resourceDelete(rsc.getName(), localNodeName, true); + logLinstorAnswers(delAnswers); + } + } catch (ApiException apiEx) { + logger.error(apiEx.getBestMessage()); + // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal } - } catch (ApiException apiEx) { - logger.error(apiEx.getBestMessage()); - // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal } return true; } - logger.warn("Linstor: Couldn't find resource for this path: " + volumePath); + logger.warn("Linstor: Couldn't find resource for this path: {}", volumePath); return false; } @@ -927,14 +996,14 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt name, QemuImg.PhysicalDiskFormat.RAW, provisioningType, disk.getVirtualSize(), null); final DevelopersApi api = getLinstorAPI(destPools); - setRscDfnAuxProperties(api, disk, destPools, name); + setRscDfnAuxProperties(api, disk, destPools, dstDisk.getName()); logger.debug("Linstor.copyPhysicalDisk: dstPath: {}", dstDisk.getPath()); final QemuImgFile destFile = new QemuImgFile(dstDisk.getPath()); destFile.setFormat(dstDisk.getFormat()); destFile.setSize(disk.getVirtualSize()); - boolean zeroedDevice = LinstorUtil.resourceSupportZeroBlocks(destPools, getLinstorRscName(name)); + boolean zeroedDevice = LinstorUtil.resourceSupportZeroBlocks(destPools, getLinstorRscName(dstDisk.getName())); try { final QemuImg qemu = new QemuImg(timeout, zeroedDevice, true); qemu.convert(srcFile, destFile); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 02a7530c0595..64e7149654f5 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -973,8 +973,7 @@ private Answer copyTemplate(DataObject srcData, DataObject dstData) { TemplateInfo tInfo = (TemplateInfo) dstData; final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId()); final DevelopersApi api = getLinstorAPI(pool); - final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid(); - boolean newCreated = LinstorUtil.createResourceBase( + Pair crResult = LinstorUtil.createResourceBase( LinstorUtil.RSC_PREFIX + dstData.getUuid(), tInfo.getSize(), tInfo.getName(), @@ -988,7 +987,9 @@ private Answer copyTemplate(DataObject srcData, DataObject dstData) { false); Answer answer; - if (newCreated) { + boolean newRscCreated = crResult.second(); + if (newRscCreated) { + String newRscName = crResult.first(); int nMaxExecutionMinutes = NumbersUtil.parseInt( _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); CopyCommand cmd = new CopyCommand( @@ -998,16 +999,16 @@ private Answer copyTemplate(DataObject srcData, DataObject dstData) { VirtualMachineManager.ExecuteInSequence.value()); try { - Optional optEP = getLinstorEP(api, pool, rscName); + Optional optEP = getLinstorEP(api, pool, newRscName); if (optEP.isPresent()) { answer = optEP.get().sendMessage(cmd); } else { - deleteResourceDefinition(pool, rscName); + deleteResourceDefinition(pool, newRscName); throw new CloudRuntimeException("Unable to get matching Linstor endpoint."); } } catch (ApiException exc) { logger.error("copy template failed: ", exc); - deleteResourceDefinition(pool, rscName); + deleteResourceDefinition(pool, newRscName); throw new CloudRuntimeException(exc.getBestMessage()); } } else { diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index bcbd1537d09d..97a44dfb5496 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -415,12 +415,12 @@ public static String getDevicePath(DevelopersApi api, String rscName) throws Api null, null); for (ResourceWithVolumes rsc : resources) { - if (!rsc.getVolumes().isEmpty()) { + if (!rsc.getVolumes().isEmpty() && rsc.getVolumes().get(0).getDevicePath() != null) { return LinstorUtil.getDevicePathFromResource(rsc); } } - final String errMsg = "viewResources didn't return resources or volumes for " + rscName; + final String errMsg = "viewResources didn't return resources or volumes for " + rscName + " with a device path"; LOGGER.error(errMsg); throw new CloudRuntimeException("Linstor: " + errMsg); } @@ -858,7 +858,7 @@ private static void spawnResource( * @param isTemplate indicates if the resource is a template * @return true if a new resource was created, false if it already existed or was reused. */ - public static boolean createResourceBase( + public static Pair createResourceBase( String rscName, long sizeInBytes, String volName, String vmName, @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, String rscGrp, long poolId, boolean isTemplate, boolean exactSize) @@ -874,14 +874,14 @@ public static boolean createResourceBase( existingRDs.stream().anyMatch(p -> p.first().getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrp))); if (!alreadyCreated) { boolean createNewRsc = !foundShareableTemplate(api, rscName, rscGrp, existingRDs); + String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; if (createNewRsc) { - String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; spawnResource(api, newRscName, sizeInBytes, isTemplate, rscGrp, volName, vmName, passPhraseId, passPhrase, exactSize); } - return createNewRsc; + return new Pair<>(newRscName, createNewRsc); } - return false; + return new Pair<>(rscName, false); } catch (ApiException apiEx) { LOGGER.error("Linstor: ApiEx - {}", apiEx.getMessage()); From d22f0208c3b8921f67c474d8ea6f1cbfebe0b13b Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Fri, 31 Jul 2026 12:42:12 +0200 Subject: [PATCH 02/14] linstor: update java-linstor to 0.8.1 Client for LINSTOR REST API 1.29.1 (shipped with linstor-server 1.35.0): make-available gained the auto_manage_dual_primary option, a new unmake-available endpoint was added, resourceSnapshotDelete grew a delete_empty_resource_definition parameter and the resource-definition clone request accepts volume_sizes. Adapt the snapshot delete call sites to the new signature (passing null keeps the previous behavior). --- .../datastore/driver/LinstorPrimaryDataStoreDriverImpl.java | 2 +- .../cloudstack/storage/snapshot/LinstorVMSnapshotStrategy.java | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 64e7149654f5..0f7d8378f77c 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -277,7 +277,7 @@ private void deleteSnapshot(@Nonnull DataStore dataStore, @Nonnull String rscDef try { - ApiCallRcList answers = linstorApi.resourceSnapshotDelete(rscDefName, snapshotName, Collections.emptyList()); + ApiCallRcList answers = linstorApi.resourceSnapshotDelete(rscDefName, snapshotName, Collections.emptyList(), null); if (answers.hasError()) { for (ApiCallRc answer : answers) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/snapshot/LinstorVMSnapshotStrategy.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/snapshot/LinstorVMSnapshotStrategy.java index d0f82484694b..ee3e17b64090 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/snapshot/LinstorVMSnapshotStrategy.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/snapshot/LinstorVMSnapshotStrategy.java @@ -214,7 +214,7 @@ private StrategyPriority allVolumesOnLinstor(Long vmId) { private String linstorDeleteSnapshot(final DevelopersApi api, final String rscName, final String snapshotName) { String resultMsg = null; try { - ApiCallRcList answers = api.resourceSnapshotDelete(rscName, snapshotName, Collections.emptyList()); + ApiCallRcList answers = api.resourceSnapshotDelete(rscName, snapshotName, Collections.emptyList(), null); if (answers.hasError()) { resultMsg = LinstorUtil.getBestErrorMessage(answers); } diff --git a/pom.xml b/pom.xml index ccecbbf99f99..3bede6344078 100644 --- a/pom.xml +++ b/pom.xml @@ -179,7 +179,7 @@ 10.1 2.6.6 0.6.0 - 0.7.0 + 0.8.1 0.10.2 3.4.4_1 4.0.1 From 7f53032824b8a7eca3112545ec15d8cc2eff7281 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Fri, 31 Jul 2026 12:42:52 +0200 Subject: [PATCH 03/14] linstor: use make-available/unmake-available for live migration LINSTOR REST API 1.29.0 moves the live-migration preparation into the controller: make-available with auto_manage_dual_primary sets DRBD allow-two-primaries (and protocol C) between the migration source and destination, or activates shared storage pool resources on both nodes; unmake-available reverts this on the source node and deletes diskless or redundant shared copies while keeping tiebreaker and diskful resources. connectPhysicalDisk now issues a single make-available with auto_manage_dual_primary set for migrations, and tryDisconnectLinstor a single unmake-available. The controller REST API version is queried per operation (one GET against the controller); against controllers older than 1.29 the previous manual dual-primary, activate and delete handling is used unchanged. --- .../kvm/storage/LinstorStorageAdaptor.java | 71 ++++++++++++++----- .../storage/datastore/util/LinstorUtil.java | 43 +++++++++++ .../storage/LinstorStorageAdaptorTest.java | 9 +++ .../datastore/util/LinstorUtilTest.java | 39 ++++++++++ 4 files changed, 144 insertions(+), 18 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index ac66ac5f212a..4fd32a62a24b 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -594,29 +594,40 @@ public boolean connectPhysicalDisk( } final DevelopersApi api = getLinstorAPI(pool); + final boolean liveMigrateApi = LinstorUtil.supportsLiveMigrateApi(api); String rscName; - boolean usesDrbd; + boolean usesDrbd = false; try { rscName = findCorrectTemplateName(api, getLinstorRscName(volumePath), pool); - usesDrbd = usesDrbd(api, rscName); - - if (isVMMigration && !usesDrbd) { - // create the resource on the new node, noop if already exists - ResourceCreate rc = new ResourceCreate(); - Resource rsc = new Resource(); - rsc.setNodeName(localNodeName); - rc.setResource(rsc); - ApiCallRcList answers = api.resourceCreateOnNode(rscName, localNodeName, rc); - checkLinstorAnswersThrow(answers); - // activate resource if there already was an inactive resource here - answers = api.activateRsc(rscName, localNodeName); - checkLinstorAnswersThrow(answers); - } else { - // DRBD resources just work with a diskless and don't have the activate/deactivate concept + + if (liveMigrateApi) { + // controller >= 1.29 prepares the live migration itself: + // dual-primary for DRBD, activation on both nodes for shared storage pools ResourceMakeAvailable rma = new ResourceMakeAvailable(); + rma.autoManageDualPrimary(isVMMigration); ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma); checkLinstorAnswersThrow(answers); + } else { + usesDrbd = usesDrbd(api, rscName); + + if (isVMMigration && !usesDrbd) { + // create the resource on the new node, noop if already exists + ResourceCreate rc = new ResourceCreate(); + Resource rsc = new Resource(); + rsc.setNodeName(localNodeName); + rc.setResource(rsc); + ApiCallRcList answers = api.resourceCreateOnNode(rscName, localNodeName, rc); + checkLinstorAnswersThrow(answers); + // activate resource if there already was an inactive resource here + answers = api.activateRsc(rscName, localNodeName); + checkLinstorAnswersThrow(answers); + } else { + // DRBD resources just work with a diskless and don't have the activate/deactivate concept + ResourceMakeAvailable rma = new ResourceMakeAvailable(); + ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma); + checkLinstorAnswersThrow(answers); + } } waitForLocalDevice(api, rscName); } catch (ApiException apiEx) { @@ -624,7 +635,7 @@ public boolean connectPhysicalDisk( throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); } - if (isVMMigration && usesDrbd) { + if (!liveMigrateApi && isVMMigration && usesDrbd) { try { allow2PrimariesIfInUse(api, rscName); } catch (ApiException apiEx) { @@ -671,6 +682,28 @@ private void removeTwoPrimariesProps(DevelopersApi api, String inUseNode, String removeTwoPrimariesRcProps(api, rscName, inUseNode, deleteProps); } + /** + * Revert a make-available after e.g. a live migration: removes dual-primary settings and + * deletes the resource on this node if that is possible without losing data + * (diskless or a redundant shared storage pool copy; tiebreaker and diskful resources are kept). + * No-op if the resource isn't deployed here, so it is safe to call on every disconnect. + */ + private void unmakeAvailable(DevelopersApi api, String rscName) { + try { + ApiCallRcList answers = api.resourceUnmakeAvailableOnNode(rscName, localNodeName); + if (answers.hasError()) { + // e.g. FAIL_IN_USE: the resource is still in use on this node and is kept + logger.warn("Linstor: unmake-available {} on {}: {}", + rscName, localNodeName, LinstorUtil.getBestErrorMessage(answers)); + } else { + logLinstorAnswers(answers); + } + } catch (ApiException apiEx) { + logger.error(apiEx.getBestMessage()); + // do not fail here, cleaning up after a disconnect isn't fatal + } + } + private void deleteUnusedSharedResources(DevelopersApi api, Resource nodeRsc) { try { List rscs = api.resourceList(nodeRsc.getName(), null, null); @@ -715,9 +748,11 @@ private boolean tryDisconnectLinstor(String volumePath, KVMStoragePool pool) if (optRsc.isPresent()) { Resource rsc = optRsc.get(); + if (LinstorUtil.supportsLiveMigrateApi(api)) { + unmakeAvailable(api, rsc.getName()); // if it is a non DRBD resource(shared) we need to check if we are the last active one // on live migrate, we can have 2 active resources at the same time - if (rsc.getLayerObject().getDrbd() == null) { + } else if (rsc.getLayerObject().getDrbd() == null) { deleteUnusedSharedResources(api, rsc); } else { try { diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 97a44dfb5496..8c42902daf36 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -103,6 +103,49 @@ public static DevelopersApi getLinstorAPI(String linstorUrl, String apiToken, bo return new DevelopersApi(client); } + /** + * The REST API version of the connected controller, e.g. "1.29.1", or null if it could not be queried. + */ + @Nullable + public static String getRestApiVersion(DevelopersApi api) { + try { + return api.controllerVersion().getRestApiVersion(); + } catch (ApiException apiExc) { + LOGGER.warn("Unable to query controller API version: {}", apiExc.getBestMessage()); + return null; + } + } + + /** + * Check if the connected controller supports the live-migration attach/detach API: + * make-available with auto_manage_dual_primary and unmake-available, added with REST API 1.29.0. + */ + public static boolean supportsLiveMigrateApi(DevelopersApi api) { + return isVersionAtLeast(getRestApiVersion(api), 1, 29, 0); + } + + static boolean isVersionAtLeast(String version, int major, int minor, int patch) { + if (version == null || version.isEmpty()) { + return false; + } + String[] parts = version.split("\\."); + try { + int maj = Integer.parseInt(parts[0]); + int min = parts.length > 1 ? Integer.parseInt(parts[1]) : 0; + int pat = parts.length > 2 ? Integer.parseInt(parts[2]) : 0; + if (maj != major) { + return maj > major; + } + if (min != minor) { + return min > minor; + } + return pat >= patch; + } catch (NumberFormatException nfExc) { + LOGGER.warn("Unable to parse controller API version '{}'", version); + return false; + } + } + public static String getBestErrorMessage(ApiCallRcList answers) { return answers != null && !answers.isEmpty() ? answers.stream() diff --git a/plugins/storage/volume/linstor/src/test/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptorTest.java b/plugins/storage/volume/linstor/src/test/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptorTest.java index 054d6b8eac16..c29d8bcf67e4 100644 --- a/plugins/storage/volume/linstor/src/test/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptorTest.java +++ b/plugins/storage/volume/linstor/src/test/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptorTest.java @@ -51,6 +51,7 @@ import com.linbit.linstor.api.model.ApiCallRc; import com.linbit.linstor.api.model.ApiCallRcList; import com.linbit.linstor.api.model.AutoPlaceRequest; +import com.linbit.linstor.api.model.ControllerVersion; import com.linbit.linstor.api.model.DrbdResource; import com.linbit.linstor.api.model.LayerType; import com.linbit.linstor.api.model.Properties; @@ -106,6 +107,12 @@ private void mockDefinitionsCreated() throws ApiException { when(api.volumeDefinitionCreate(anyString(), any())).thenReturn(success()); } + private void mockControllerVersion(String restApiVersion) throws ApiException { + ControllerVersion version = new ControllerVersion(); + version.setRestApiVersion(restApiVersion); + when(api.controllerVersion()).thenReturn(version); + } + private static ResourceWithVolumes drbdResource() { ResourceLayer layer = new ResourceLayer(); layer.setDrbd(new DrbdResource()); @@ -239,6 +246,7 @@ public void deleteBackupScratchFailsWhenStillInUse() throws ApiException { @Test public void connectPhysicalDiskWaitsForTheLocalDevice() throws ApiException { + mockControllerVersion("1.29.0"); when(api.resourceMakeAvailableOnNode(eq("cs-vol-1"), eq(localNode), any())).thenReturn(success()); when(api.viewResources(Collections.singletonList(localNode), Collections.singletonList("cs-vol-1"), Collections.emptyList(), null, null, null)).thenReturn(Collections.singletonList(drbdResource())); @@ -269,6 +277,7 @@ public void createPhysicalDiskWaitsForTheLocalDevice() throws ApiException { @Test(expected = CloudRuntimeException.class) public void connectPhysicalDiskFailsWithoutLocalVolume() throws ApiException { + mockControllerVersion("1.29.0"); when(api.resourceMakeAvailableOnNode(eq("cs-vol-1"), eq(localNode), any())).thenReturn(success()); when(api.viewResources(Collections.singletonList(localNode), Collections.singletonList("cs-vol-1"), Collections.emptyList(), null, null, null)).thenReturn(Collections.emptyList()); diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java index f1786d98ff06..d35c039f8467 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -20,6 +20,7 @@ import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.ControllerVersion; import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ProviderKind; @@ -189,6 +190,44 @@ public void testDiskfulStoragePoolsIgnoresDiskless() throws ApiException { Assert.assertNull(LinstorUtil.getDiskfulStoragePool(api, "cs-test")); } + @Test + public void testIsVersionAtLeast() { + Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.29.1", 1, 29, 1)); + Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.29.2", 1, 29, 1)); + Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.30.0", 1, 29, 1)); + Assert.assertTrue(LinstorUtil.isVersionAtLeast("2.0.0", 1, 29, 1)); + Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.30", 1, 29, 1)); + + Assert.assertFalse(LinstorUtil.isVersionAtLeast("1.29.0", 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast("1.29", 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast("1.28.5", 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast("0.99.9", 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast(null, 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast("", 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast("garbage", 1, 29, 1)); + } + + private ControllerVersion controllerVersion(String restApiVersion) { + ControllerVersion version = new ControllerVersion(); + version.setRestApiVersion(restApiVersion); + return version; + } + + @Test + public void testSupportsLiveMigrateApi() throws ApiException { + DevelopersApi newCtrl = mock(DevelopersApi.class); + when(newCtrl.controllerVersion()).thenReturn(controllerVersion("1.29.0")); + Assert.assertTrue(LinstorUtil.supportsLiveMigrateApi(newCtrl)); + + DevelopersApi oldCtrl = mock(DevelopersApi.class); + when(oldCtrl.controllerVersion()).thenReturn(controllerVersion("1.28.3")); + Assert.assertFalse(LinstorUtil.supportsLiveMigrateApi(oldCtrl)); + + DevelopersApi unreachable = mock(DevelopersApi.class); + when(unreachable.controllerVersion()).thenThrow(new ApiException(503, "unavailable")); + Assert.assertFalse(LinstorUtil.supportsLiveMigrateApi(unreachable)); + } + @Test public void testGetRscGroupStoragePools() throws ApiException { List storagePools = LinstorUtil.getRscGroupStoragePools(api, "cloudstack"); From f9f4c02030e7dd66a9d36986db1ea1adcf505983 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 21 Oct 2026 12:55:38 +0200 Subject: [PATCH 04/14] linstor: support snapshots on thick LVM (shared) storage pools - getSnapshotPath: thick LVM snapshot LVs use the same naming scheme as thin ones - backup: thick LVM snapshot LVs of inactive origins are inactive and carry the skip-activation flag; activate them (lvchange -ay -K) for the copy to secondary storage and deactivate them again afterwards --- .../LinstorBackupSnapshotCommandWrapper.java | 23 +++++++++++++++++++ .../storage/datastore/util/LinstorUtil.java | 1 + .../datastore/util/LinstorUtilTest.java | 11 +++++++++ 3 files changed, 35 insertions(+) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java index 1c48eb2b22f4..9b6772814a76 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java @@ -66,6 +66,19 @@ private String zfsSnapdev(boolean hide, String zfsUrl) { return script.execute(); } + private String lvmSetActive(boolean activate, String devMapperPath) { + // lvm resolves /dev/mapper/vg-lv names textually, works also for inactive LVs + Script script = new Script("lvchange", Duration.millis(5000)); + if (activate) { + script.add("-ay"); + script.add("-K"); // snapshot LVs carry the skip-activation flag + } else { + script.add("-an"); + } + script.add(devMapperPath); + return script.execute(); + } + private String qemuShrink(String path, long sizeByte, long timeout) { Script qemuImg = new Script("qemu-img", Duration.millis(timeout)); qemuImg.add("resize"); @@ -176,6 +189,7 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); KVMStoragePool linstorPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.Linstor, src.getDataStore().getUuid()); boolean zfsHidden = false; + String lvmSnapDev = null; // vg/lv we activated for the copy, deactivated on cleanup String srcPath = src.getPath(); if (linstorPool == null) { @@ -199,6 +213,12 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR return new CopyCmdAnswer("Unable to unhide zfs snapshot device."); } srcPath = "/dev/zvol/" + srcPath.substring(6); + } else if (srcPath.startsWith("/dev/mapper/") && !new File(srcPath).exists()) { + // thick LVM snapshot LVs are created inactive with the skip-activation flag + if (lvmSetActive(true, srcPath) != null) { + return new CopyCmdAnswer("Unable to activate LVM snapshot device " + srcPath); + } + lvmSnapDev = srcPath; } secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl()); @@ -262,6 +282,9 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR if (zfsHidden) { zfsSnapdev(true, src.getPath()); } + if (lvmSnapDev != null) { + lvmSetActive(false, lvmSnapDev); + } } } } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 8c42902daf36..db7e39f0a1a4 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -273,6 +273,7 @@ public static String getSnapshotPath(com.linbit.linstor.api.model.StoragePool sp final String backingPool = sp.getProps().get("StorDriver/StorPoolName"); final String path; switch (sp.getProviderKind()) { + case LVM: // thick LVM snapshot LVs use the same naming scheme as thin case LVM_THIN: // device-mapper doubles every dash in the VG and LV name, so the VG part needs escaping too final String vgName = backingPool.split("/")[0]; diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java index d35c039f8467..3fe3b470c729 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -121,6 +121,17 @@ public void testGetSnapshotPath() { snapPath); } + { + StoragePool spLVM = new StoragePool(); + Properties lvmProps = new Properties(); + lvmProps.put("StorDriver/StorPoolName", "shared"); + spLVM.setProps(lvmProps); + spLVM.setProviderKind(ProviderKind.LVM); + String snapPath = LinstorUtil.getSnapshotPath(spLVM, "cs-cb32532a-dd8f-47e0-a81c-8a75573d3545", "cs-6c6b4e95"); + Assert.assertEquals( + "/dev/mapper/shared-cs--cb32532a--dd8f--47e0--a81c--8a75573d3545_00000_cs--6c6b4e95", snapPath); + } + { StoragePool spZFS = new StoragePool(); Properties zfsProps = new Properties(); From 3bc806cb8a0cd44f6bffe2dc35e754fd0893103c Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Fri, 31 Jul 2026 12:43:49 +0200 Subject: [PATCH 05/14] linstor: activate inactive resources for snapshot backup and revert Shared storage pool resources of stopped VMs are INACTIVE and expose no device on any node. Activate them through the Linstor controller (which takes the shared-space activation lock) before sending the backup or revert copy command to the host and deactivate them again afterwards. The agent itself must only ever activate the snapshot LV, never the resource, to not bypass the shared-space locking. --- .../LinstorPrimaryDataStoreDriverImpl.java | 69 ++++++++++++++----- .../storage/datastore/util/LinstorUtil.java | 14 ++++ 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 0f7d8378f77c..66312f3e4d13 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -687,18 +687,19 @@ private String revertSnapshotFromImageStore( VirtualMachineManager.ExecuteInSequence.value()); final StoragePool pool = (StoragePool) volumeInfo.getDataStore(); - Optional optEP = getDiskfullEP(linstorApi, pool, rscName); - if (optEP.isEmpty()) { - optEP = getLinstorEP(linstorApi, pool, rscName); - } - - if (optEP.isPresent()) { - Answer answer = optEP.get().sendMessage(cmd); - if (!answer.getResult()) { - resultMsg = answer.getDetails(); - } + Host diskfulHost = getDiskfullHost(linstorApi, pool, rscName); + Answer answer; + if (diskfulHost != null) { + answer = sendWithRscActivated(linstorApi, rscName, diskfulHost, cmd); } else { - resultMsg = "Unable to get matching Linstor endpoint."; + Optional optEP = getLinstorEP(linstorApi, pool, rscName); + if (optEP.isEmpty()) { + return "Unable to get matching Linstor endpoint."; + } + answer = optEP.get().sendMessage(cmd); + } + if (!answer.getResult()) { + resultMsg = answer.getDetails(); } return resultMsg; } @@ -905,7 +906,7 @@ private Optional getLinstorEP(DevelopersApi api, StoragePool return Optional.empty(); } - private Optional getDiskfullEP(DevelopersApi api, StoragePool storagePool, String rscName) + private Host getDiskfullHost(DevelopersApi api, StoragePool storagePool, String rscName) throws ApiException { // Take the first diskful copy whose host can be used: the copies are ordered by how // suited they are (in use, then active, then inactive), and for shared storage pools @@ -914,13 +915,46 @@ private Optional getDiskfullEP(DevelopersApi api, StoragePoo : LinstorUtil.getDiskfulStoragePoolsByPreference(api, rscName)) { Host host = getEnabledClusterHost(storagePool, Collections.singletonList(linSP.getNodeName())); if (host != null) { - return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host)); + return host; } logger.debug("Linstor: no usable cloudstack host for diskful node {}, trying next copy", linSP.getNodeName()); } logger.error("Linstor: No diskfull host found."); - return Optional.empty(); + return null; + } + + /** + * Send the given command to the host, temporarily activating the resource there if it is + * inactive (e.g. a shared storage pool resource of a stopped VM). Activation goes through + * the Linstor controller, so the shared-space activation lock is honored; the agent itself + * must only activate/deactivate the snapshot LV, never the resource. + */ + private Answer sendWithRscActivated(DevelopersApi api, String rscName, Host host, CopyCommand cmd) + throws ApiException { + boolean activated = false; + if (LinstorUtil.isResourceInactiveOnNode(api, rscName, host.getName())) { + ApiCallRcList answers = api.activateRsc(rscName, host.getName()); + if (answers.hasError()) { + throw new CloudRuntimeException(String.format( + "Linstor: Unable to activate resource %s on node %s: %s", + rscName, host.getName(), LinstorUtil.getBestErrorMessage(answers))); + } + activated = true; + } + try { + return RemoteHostEndPoint.getHypervisorHostEndPoint(host).sendMessage(cmd); + } finally { + if (activated) { + ApiCallRcList answers = api.deactivateRsc(rscName, host.getName()); + if (answers.hasError()) { + // don't fail the copy over cleanup, but a stray active resource on a shared + // storage pool must be visible in the logs + logger.error("Linstor: Unable to deactivate resource {} on node {}: {}", + rscName, host.getName(), LinstorUtil.getBestErrorMessage(answers)); + } + } + } } private String restoreResourceFromSnapshot( @@ -1163,11 +1197,10 @@ protected Answer copySnapshot(DataObject srcData, DataObject destData) { VirtualMachineManager.ExecuteInSequence.value()); cmd.setOptions(options); - Optional optEP = encrypted ? - Optional.empty() : getDiskfullEP(api, pool, rscName); + Host diskfulHost = encrypted ? null : getDiskfullHost(api, pool, rscName); Answer answer; - if (optEP.isPresent()) { - answer = optEP.get().sendMessage(cmd); + if (diskfulHost != null) { + answer = sendWithRscActivated(api, rscName, diskfulHost, cmd); } else { logger.debug("No diskfull endpoint used to copy image (encrypted={}), using temporary resource", encrypted); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index db7e39f0a1a4..de0098c17431 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -409,6 +409,20 @@ public static String isResourceInUse(KVMStoragePool pool, String resName) { } } + /** + * Check if the resource is deployed but inactive on the given node (e.g. a shared + * storage pool resource of a stopped VM). + */ + public static boolean isResourceInactiveOnNode(DevelopersApi api, String rscName, String nodeName) + throws ApiException { + List rscs = api.resourceList(rscName, null, null); + if (rscs == null) { + return false; + } + return rscs.stream().anyMatch(rsc -> nodeName.equalsIgnoreCase(rsc.getNodeName()) && + rsc.getFlags() != null && rsc.getFlags().contains(ApiConsts.FLAG_RSC_INACTIVE)); + } + /** * Check if the given resources are diskless. * From cf676cb46912c803b4763c3027e99b36696a4348 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Fri, 31 Jul 2026 12:44:36 +0200 Subject: [PATCH 06/14] linstor: return expected device path for inactive shared resources The KVM start flow builds the domain XML (createVbd) before connectPhysicalDisk runs make-available, so getDevicePath must not fail for resources that are inactive on every node - the start would spin on the missing device path and never reach the activating connect. Return the deterministic storage layer path instead; make-available activates the resource before the domain is started. Also add the plugin CHANGELOG entries for shared storage pool support. --- plugins/storage/volume/linstor/CHANGELOG.md | 14 ++++++++++++++ .../storage/datastore/util/LinstorUtil.java | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index b4d0aedda141..42f3f5439125 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -35,6 +35,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 "shares '' must be in range [1, 10000]" for VMs with more than 10000 cpus * MHz, and smaller VMs ended up with an unscaled CPU weight; the CPU shares calculated by the target host are now used. +## [2026-07-31] + +### Added + +- Support for shared (thick LVM) storage pools: resources on a shared LUN are + activated on at most one node; live migration uses the LINSTOR 1.29 + make-available/unmake-available API (with fallback to the previous handling + for older controllers), snapshots work on thick LVM pools, and resources of + stopped VMs are activated on demand for start, snapshot backup and revert + +### Changed + +- java-linstor client updated to 0.8.1 (LINSTOR REST API 1.29.1) + ## [2026-07-30] ### Added diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index de0098c17431..677bebfd70cf 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -478,6 +478,17 @@ public static String getDevicePath(DevelopersApi api, String rscName) throws Api } } + // Shared storage pool resources of stopped VMs are INACTIVE and report no device path. + // Callers like createVbd build the domain XML before connectPhysicalDisk activates the + // resource, so return the deterministic storage layer path here. + com.linbit.linstor.api.model.StoragePool sp = getDiskfulStoragePool(api, rscName); + if (sp != null && (sp.getProviderKind() == ProviderKind.LVM || sp.getProviderKind() == ProviderKind.LVM_THIN)) { + final String vg = sp.getProps().get("StorDriver/StorPoolName").split("/")[0]; + final String path = String.format("/dev/%s/%s_00000", vg, rscName); + LOGGER.info("Linstor: no active device path for {}, using expected LVM path {}", rscName, path); + return path; + } + final String errMsg = "viewResources didn't return resources or volumes for " + rscName + " with a device path"; LOGGER.error(errMsg); throw new CloudRuntimeException("Linstor: " + errMsg); From 909b9671e2f1714cd9acfb6b18fa5d2dbb1b3146 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Mon, 3 Aug 2026 20:47:31 +0200 Subject: [PATCH 07/14] linstor: don't register backup snapshot LVs with dmeventd Activating a thick LVM snapshot LV for the backup copy registers it with dmeventd by default; that registration can block lvchange indefinitely (observed stuck in pipe_read waiting for a dmeventd reply), and a blocked lvm command holds the node-wide LVM lock. Pass --monitor n on the temporary activation and deactivation: monitoring only exists to auto-extend snapshots, and Linstor creates thick snapshot LVs at full origin size, so they can never overflow. Also raise the command timeout. --- .../wrapper/LinstorBackupSnapshotCommandWrapper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java index 9b6772814a76..62f4d3da1b6d 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java @@ -68,13 +68,18 @@ private String zfsSnapdev(boolean hide, String zfsUrl) { private String lvmSetActive(boolean activate, String devMapperPath) { // lvm resolves /dev/mapper/vg-lv names textually, works also for inactive LVs - Script script = new Script("lvchange", Duration.millis(5000)); + Script script = new Script("lvchange", Duration.millis(30000)); if (activate) { script.add("-ay"); script.add("-K"); // snapshot LVs carry the skip-activation flag } else { script.add("-an"); } + // never talk to dmeventd: registering the snapshot for monitoring can block lvchange + // indefinitely (and with it the whole LVM lock on the node); a full-size thick COW + // snapshot cannot overflow, so monitoring is not needed for the backup window + script.add("--monitor"); + script.add("n"); script.add(devMapperPath); return script.execute(); } From 7d1df06f1d1c605b0114463a0e0fad0a51773a4e Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Mon, 3 Aug 2026 21:04:49 +0200 Subject: [PATCH 08/14] linstor: activate inactive template resources for cloning Shared storage pool templates are INACTIVE while no clone is running and Linstor's clone source selection skips inactive resources, failing with 'No suitable storage pools found for cloning'. Activate a diskful template resource for the duration of the clone and deactivate it again afterwards. --- .../LinstorPrimaryDataStoreDriverImpl.java | 10 +++++++ .../storage/datastore/util/LinstorUtil.java | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 66312f3e4d13..e42985fc5b0f 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -462,6 +462,16 @@ private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolV cloneRequest.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); } } + // Shared storage pool templates are INACTIVE while unused (e.g. after a node + // reboot) and the clone source selection skips inactive resources. Activate one + // diskful resource and leave it active: concurrent clones share the source, so + // deactivating after a clone would race with other clones still using it. + final String activateNode = LinstorUtil.getDiskfulNodeToActivate(linstorApi, cloneRes); + if (activateNode != null) { + logger.info("Linstor: activating template resource {} on {} for cloning", + cloneRes, activateNode); + LinstorUtil.checkLinstorAnswersThrow(linstorApi.activateRsc(cloneRes, activateNode)); + } ResourceDefinitionCloneStarted cloneStarted = linstorApi.resourceDefinitionClone( cloneRes, cloneRequest); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 677bebfd70cf..e39106730bdd 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -409,6 +409,32 @@ public static String isResourceInUse(KVMStoragePool pool, String resName) { } } + /** + * If the resource has no active diskful deployment, return a node with an inactive + * diskful resource that could be activated (e.g. an unused template on a shared + * storage pool). Returns null if an active diskful resource exists or none is deployed. + */ + public static String getDiskfulNodeToActivate(DevelopersApi api, String rscName) throws ApiException { + List rscs = api.resourceList(rscName, null, null); + if (rscs == null) { + return null; + } + String inactiveDiskfulNode = null; + for (Resource rsc : rscs) { + List flags = rsc.getFlags(); + boolean diskless = flags != null && + (flags.contains(ApiConsts.FLAG_DISKLESS) || flags.contains(ApiConsts.FLAG_DRBD_DISKLESS)); + if (diskless) { + continue; + } + if (flags == null || !flags.contains(ApiConsts.FLAG_RSC_INACTIVE)) { + return null; // active diskful resource exists + } + inactiveDiskfulNode = rsc.getNodeName(); + } + return inactiveDiskfulNode; + } + /** * Check if the resource is deployed but inactive on the given node (e.g. a shared * storage pool resource of a stopped VM). From 855dac05bc917b7104672ca7f808ea123845ca30 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Tue, 4 Aug 2026 13:45:09 +0200 Subject: [PATCH 09/14] kvm: connect source volume for copy to secondary storage copyVolumeFromPrimaryToSecondary read the source device without connecting it first. Detached volumes being migrated between pools are not connected anywhere, and storage drivers that expose devices on demand (e.g. Linstor shared storage pools, where resources of stopped VMs are inactive without a device path) fail the copy. Connect the source volume for the duration of the copy, mirroring what VM start and attach flows do. --- .../cloud/hypervisor/kvm/storage/KVMStorageProcessor.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 235949502c71..c50048f380b1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -714,10 +714,14 @@ public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) { final String secondaryStorageUrl = nfsStore.getUrl(); KVMStoragePool secondaryStoragePool = null; + boolean srcConnected = false; try { final String volumeName = UUID.randomUUID().toString(); final String destVolumeName = volumeName + "." + ImageFormat.QCOW2.getFileExtension(); + // connect: a detached source volume may have no device yet on shared pools + srcConnected = storagePoolMgr.connectPhysicalDisk( + primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath, null); final KVMPhysicalDisk volume = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath); volume.setFormat(PhysicalDiskFormat.valueOf(srcFormat.toString())); @@ -736,6 +740,9 @@ public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) { } finally { srcVol.clearPassphrase(); destVol.clearPassphrase(); + if (srcConnected) { + storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath); + } if (secondaryStoragePool != null) { storagePoolMgr.deleteStoragePool(secondaryStoragePool.getType(), secondaryStoragePool.getUuid()); } From 4782dd9d6f191df29e653d1ef4847b9eb81b63a7 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Tue, 4 Aug 2026 14:09:54 +0200 Subject: [PATCH 10/14] linstor: pin clone target layer stack to the resource group Clone requests without an explicit layer list can produce a target with a default DRBD layer stack while the source on a shared storage pool is STORAGE only, failing with 'no common clone strategy found'. Set the resource group's configured layer stack on the clone request for unencrypted volumes, mirroring what is already done for encrypted ones. --- .../LinstorPrimaryDataStoreDriverImpl.java | 9 +++++++ .../storage/datastore/util/LinstorUtil.java | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index e42985fc5b0f..71fede8703e1 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -461,6 +461,15 @@ private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolV String utf8Passphrase = new String(volumeInfo.getPassphrase(), StandardCharsets.UTF_8); cloneRequest.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); } + } else { + // pin the clone target to the resource group's layer stack (e.g. STORAGE on + // shared pools); otherwise the target can end up with a default DRBD stack + // that has no common clone strategy with the source + List rgLayers = LinstorUtil.getRscGrpLayerList( + linstorApi, LinstorUtil.getRscGrp(storagePoolVO)); + if (!rgLayers.isEmpty()) { + cloneRequest.setLayerList(rgLayers); + } } // Shared storage pool templates are INACTIVE while unused (e.g. after a node // reboot) and the clone source selection skips inactive resources. Activate one diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index e39106730bdd..54bbc8bf431b 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -867,6 +867,30 @@ private static boolean foundShareableTemplate( * @param resourceGroup Resource group to get the encryption layer list * @return layer list with LUKS added */ + /** + * The layer stack configured on the resource group, empty if none is set. + */ + public static List getRscGrpLayerList(DevelopersApi api, String resourceGroup) { + try { + List rscGrps = api.resourceGroupList( + Collections.singletonList(resourceGroup), Collections.emptyList(), null, null); + + if (CollectionUtils.isEmpty(rscGrps)) { + throw new CloudRuntimeException( + String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); + } + + List layerStack = rscGrps.get(0).getSelectFilter() != null ? + rscGrps.get(0).getSelectFilter().getLayerStack() : Collections.emptyList(); + return CollectionUtils.isNotEmpty(layerStack) ? + layerStack.stream().map(LayerType::valueOf).collect(Collectors.toList()) : + Collections.emptyList(); + } catch (ApiException apiEx) { + LOGGER.error(apiEx.getBestMessage()); + throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); + } + } + public static List getEncryptedLayerList(DevelopersApi api, String resourceGroup) { try { List rscGrps = api.resourceGroupList( From ed946f2355f73d99f12acdb5e38fab27a2c812d8 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 5 Aug 2026 07:57:35 +0200 Subject: [PATCH 11/14] linstor: spawn resources with the resource group's layer stack Rely on the explicit layer stack instead of the resource group default when spawning unencrypted resources. A template spawned with a DRBD stack cannot be used as a clone source for STORAGE or LUKS,STORAGE resources of a shared storage pool later on (the clone either fails with 'no common clone strategy found' or hangs in state CLONING). --- .../cloudstack/storage/datastore/util/LinstorUtil.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 54bbc8bf431b..b285090b5662 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -946,6 +946,16 @@ private static void spawnResource( String utf8Passphrase = new String(passPhrase, StandardCharsets.UTF_8); rscGrpSpawn.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); } + } else { + // spell out the resource group's layer stack instead of relying on the implicit + // default: a template spawned with a DRBD stack cannot be cloned into a + // STORAGE/LUKS resource of a shared storage pool later on + List rgLayers = getRscGrpLayerList(api, rscGrpName); + if (!rgLayers.isEmpty()) { + AutoSelectFilter asf = new AutoSelectFilter(); + asf.setLayerStack(rgLayers.stream().map(LayerType::toString).collect(Collectors.toList())); + rscGrpSpawn.setSelectFilter(asf); + } } Properties props = new Properties(); From ebbc9d74a13bafc3e641b0faca2d67dedb717c0c Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 5 Aug 2026 10:18:03 +0200 Subject: [PATCH 12/14] linstor: count shared storage only once for capacity Every node accessing a shared space (e.g. a LUN with a shared LVM volume group) reports the full capacity of that space, so summing all storage pools of the resource group multiplied the reported capacity by the number of nodes: a 100 GiB shared LUN on three nodes was reported as 300 GiB (and 600 GiB after the over-provisioning factor). Deduplicate storage pools by their shared space when calculating total, used and free capacity. --- .../kvm/storage/LinstorStorageAdaptor.java | 17 +--- .../storage/datastore/util/LinstorUtil.java | 78 ++++++++++++++----- .../datastore/util/LinstorUtilTest.java | 39 ++++++++++ 3 files changed, 101 insertions(+), 33 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 4fd32a62a24b..8ea5e8674466 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -58,7 +58,6 @@ import com.linbit.linstor.api.model.LayerType; import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.Properties; -import com.linbit.linstor.api.model.ProviderKind; import com.linbit.linstor.api.model.Resource; import com.linbit.linstor.api.model.ResourceConnectionModify; import com.linbit.linstor.api.model.ResourceCreate; @@ -68,7 +67,6 @@ import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceMakeAvailable; import com.linbit.linstor.api.model.ResourceWithVolumes; -import com.linbit.linstor.api.model.StoragePool; import com.linbit.linstor.api.model.Volume; import com.linbit.linstor.api.model.VolumeDefinition; import com.linbit.linstor.api.model.VolumeDefinitionCreate; @@ -1145,12 +1143,7 @@ public long getAvailable(LinstorStoragePool pool) { DevelopersApi linstorApi = getLinstorAPI(pool); final String rscGroupName = pool.getResourceGroup(); try { - List storagePools = LinstorUtil.getRscGroupStoragePools(linstorApi, rscGroupName); - - final long free = storagePools.stream() - .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS) - .mapToLong(sp -> sp.getFreeCapacity() != null ? sp.getFreeCapacity() : 0L).sum() * 1024; // linstor uses KiB - + final long free = LinstorUtil.getFreeCapacityBytes(linstorApi, rscGroupName); logger.debug("Linstor: getAvailable() -> " + free); return free; } catch (ApiException apiEx) { @@ -1163,13 +1156,7 @@ public long getUsed(LinstorStoragePool pool) { DevelopersApi linstorApi = getLinstorAPI(pool); final String rscGroupName = pool.getResourceGroup(); try { - List storagePools = LinstorUtil.getRscGroupStoragePools(linstorApi, rscGroupName); - - final long used = storagePools.stream() - .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS) - .mapToLong(sp -> sp.getTotalCapacity() != null && sp.getFreeCapacity() != null ? - sp.getTotalCapacity() - sp.getFreeCapacity() : 0L) - .sum() * 1024; // linstor uses Kib + final long used = LinstorUtil.getUsedCapacityBytes(linstorApi, rscGroupName); logger.debug("Linstor: getUsed() -> " + used); return used; } catch (ApiException apiEx) { diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index b285090b5662..a18985daa974 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -46,10 +46,12 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; @@ -60,6 +62,7 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -315,15 +318,62 @@ public static List getRscGroupStoragePools(DevelopersApi api, Strin ); } + /** + * Diskful storage pools that contribute capacity, counting storage shared by multiple + * nodes only once. Nodes accessing the same shared space (e.g. a LUN with a shared LVM + * volume group) all report the full capacity of that space, so summing them as-is + * multiplies the real capacity by the number of nodes. + * + * Deduplication uses the free space manager name: node local storage pools get a + * per-node name ("node;pool"), pools sharing a space all report the shared space name. + */ + public static List getCapacityStoragePools( + @Nonnull DevelopersApi api, String rscGroupName) throws ApiException { + List storagePools = getRscGroupStoragePools(api, rscGroupName); + List distinct = new ArrayList<>(); + Set seenSpaces = new HashSet<>(); + for (StoragePool sp : storagePools) { + if (sp.getProviderKind() == ProviderKind.DISKLESS) { + continue; + } + if (StringUtils.isEmpty(sp.getFreeSpaceMgrName()) || seenSpaces.add(sp.getFreeSpaceMgrName())) { + distinct.add(sp); + } + } + return distinct; + } + + private static long sumTotalCapacity(List storagePools) { + return storagePools.stream() + .mapToLong(sp -> sp.getTotalCapacity() != null ? sp.getTotalCapacity() : 0L) + .sum() * 1024; // linstor uses KiB + } + + private static long sumUsedCapacity(List storagePools) { + return storagePools.stream() + .mapToLong(sp -> sp.getTotalCapacity() != null && sp.getFreeCapacity() != null ? + sp.getTotalCapacity() - sp.getFreeCapacity() : 0L) + .sum() * 1024; // linstor uses KiB + } + + private static long sumFreeCapacity(List storagePools) { + return storagePools.stream() + .mapToLong(sp -> sp.getFreeCapacity() != null ? sp.getFreeCapacity() : 0L) + .sum() * 1024; // linstor uses KiB + } + + public static long getFreeCapacityBytes(@Nonnull DevelopersApi api, String rscGroupName) throws ApiException { + return sumFreeCapacity(getCapacityStoragePools(api, rscGroupName)); + } + + public static long getUsedCapacityBytes(@Nonnull DevelopersApi api, String rscGroupName) throws ApiException { + return sumUsedCapacity(getCapacityStoragePools(api, rscGroupName)); + } + public static long getCapacityBytes(String linstorUrl, String rscGroupName, String apiToken, boolean insecureSsl) { DevelopersApi linstorApi = getLinstorAPI(linstorUrl, apiToken, insecureSsl); try { - List storagePools = getRscGroupStoragePools(linstorApi, rscGroupName); - - return storagePools.stream() - .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS) - .mapToLong(sp -> sp.getTotalCapacity() != null ? sp.getTotalCapacity() : 0L) - .sum() * 1024; // linstor uses kiB + return sumTotalCapacity(getCapacityStoragePools(linstorApi, rscGroupName)); } catch (ApiException apiEx) { LOGGER.error(apiEx.getMessage()); throw new CloudRuntimeException(apiEx); @@ -333,18 +383,10 @@ public static long getCapacityBytes(String linstorUrl, String rscGroupName, Stri public static Pair getStorageStats(String linstorUrl, String rscGroupName, String apiToken, boolean insecureSsl) { DevelopersApi linstorApi = getLinstorAPI(linstorUrl, apiToken, insecureSsl); try { - List storagePools = LinstorUtil.getRscGroupStoragePools(linstorApi, rscGroupName); - - long capacity = storagePools.stream() - .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS) - .mapToLong(sp -> sp.getTotalCapacity() != null ? sp.getTotalCapacity() : 0L) - .sum() * 1024; // linstor uses kiB - - long used = storagePools.stream() - .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS) - .mapToLong(sp -> sp.getTotalCapacity() != null && sp.getFreeCapacity() != null ? - sp.getTotalCapacity() - sp.getFreeCapacity() : 0L) - .sum() * 1024; // linstor uses Kib + List storagePools = getCapacityStoragePools(linstorApi, rscGroupName); + + long capacity = sumTotalCapacity(storagePools); + long used = sumUsedCapacity(storagePools); LOGGER.debug( String.format("Linstor(%s;%s): storageStats -> %d/%d", linstorUrl, rscGroupName, capacity, used)); return new Pair<>(capacity, used); diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java index 3fe3b470c729..fcfd4810f79d 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -65,6 +65,15 @@ private StoragePool mockStoragePool(String name, String node, ProviderKind kind) return sp; } + private StoragePool mockStoragePool( + String name, String node, ProviderKind kind, String freeSpaceMgrName, long totalKib, long freeKib) { + StoragePool sp = mockStoragePool(name, node, kind); + sp.setFreeSpaceMgrName(freeSpaceMgrName); + sp.setTotalCapacity(totalKib); + sp.setFreeCapacity(freeKib); + return sp; + } + @Before public void setUp() throws ApiException { api = mock(DevelopersApi.class); @@ -201,6 +210,36 @@ public void testDiskfulStoragePoolsIgnoresDiskless() throws ApiException { Assert.assertNull(LinstorUtil.getDiskfulStoragePool(api, "cs-test")); } + @Test + public void testGetCapacityStoragePoolsCountsSharedSpaceOnce() throws ApiException { + ResourceGroup sharedGroup = new ResourceGroup(); + sharedGroup.setName("shared"); + AutoSelectFilter asf = new AutoSelectFilter(); + asf.setPlaceCount(1); + sharedGroup.setSelectFilter(asf); + when(api.resourceGroupList(Collections.singletonList("shared"), null, null, null)) + .thenReturn(Collections.singletonList(sharedGroup)); + + // 3 nodes accessing the same shared space, each reporting the full 100 GiB, plus one + // node-local pool and a diskless pool which must be ignored + when(api.viewStoragePools(Collections.emptyList(), null, null, null, null, true)) + .thenReturn(Arrays.asList( + mockStoragePool("sharedpool", "nodeA", ProviderKind.LVM, "cs-shared", 104857600L, 52428800L), + mockStoragePool("sharedpool", "nodeB", ProviderKind.LVM, "cs-shared", 104857600L, 52428800L), + mockStoragePool("sharedpool", "nodeC", ProviderKind.LVM, "cs-shared", 104857600L, 52428800L), + mockStoragePool("local", "nodeA", ProviderKind.LVM_THIN, "nodeA;local", 10485760L, 10485760L), + mockStoragePool("diskless", "nodeB", ProviderKind.DISKLESS, "nodeB;diskless", 0L, 0L) + )); + + List pools = LinstorUtil.getCapacityStoragePools(api, "shared"); + Assert.assertEquals(2, pools.size()); + // 100 GiB shared (once) + 10 GiB local + Assert.assertEquals((104857600L + 10485760L) * 1024, LinstorUtil.getFreeCapacityBytes(api, "shared") + + LinstorUtil.getUsedCapacityBytes(api, "shared")); + // half of the shared space is used, the local pool is empty + Assert.assertEquals(52428800L * 1024, LinstorUtil.getUsedCapacityBytes(api, "shared")); + } + @Test public void testIsVersionAtLeast() { Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.29.1", 1, 29, 1)); From 8fc4688511c8d764010200537c5016c08e6aa0a7 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 5 Aug 2026 10:43:52 +0200 Subject: [PATCH 13/14] linstor: no over-provisioning for thick storage pools Thickly provisioned backends (thick LVM, ZFS with reservation, file) allocate the full volume size, so the pool capacity is a hard limit and the storage.overprovisioning.factor default of 2 makes CloudStack allocate twice the space that exists. The Linstor storage pool type has to allow over-provisioning for the thin backends, so set the factor to 1.0 per pool when all storage pools of the resource group are thick. Mixed or unknown setups keep the default. --- .../LinstorPrimaryDataStoreLifeCycleImpl.java | 10 +++++++ .../storage/datastore/util/LinstorUtil.java | 27 +++++++++++++++++++ .../datastore/util/LinstorUtilTest.java | 14 ++++++++++ 3 files changed, 51 insertions(+) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java index 7c1330b9088e..4883de102159 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java @@ -207,6 +207,16 @@ public DataStore initialize(Map dsInfos) { LinstorConfigurationManager.ApiToken.key(), DBEncryptionUtil.encrypt(apiToken), false); } + // Thickly provisioned storage (e.g. thick LVM, also used for shared storage pools) allocates + // the full volume size, so the pool capacity is a hard limit. The Linstor storage pool type + // allows over-provisioning for thin backends, so disable it per pool here instead. + if (dataStore != null && LinstorUtil.isThicklyProvisioned(url, resourceGroup, apiToken, insecureSsl)) { + logger.info("Linstor: {} is thickly provisioned, setting {} to 1.0", + storagePoolName, CapacityManager.StorageOverprovisioningFactor.key()); + storagePoolDetailsDao.addDetail(dataStore.getId(), + CapacityManager.StorageOverprovisioningFactor.key(), "1.0", true); + } + return dataStore; } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index a18985daa974..3abfaf9de3c5 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -343,6 +343,33 @@ public static List getCapacityStoragePools( return distinct; } + /** + * Storage provider kinds that allocate the full volume size up front. Such pools cannot be + * over-provisioned: their capacity is the real limit. + */ + private static final Collection THICK_PROVIDER_KINDS = Arrays.asList( + ProviderKind.LVM, ProviderKind.ZFS, ProviderKind.FILE); + + /** + * Check if all storage pools backing the resource group are thickly provisioned. + * Unknown/mixed setups are reported as thin, keeping the over-provisioning default. + */ + public static boolean isThicklyProvisioned(String linstorUrl, String rscGroupName, String apiToken, + boolean insecureSsl) { + DevelopersApi api = getLinstorAPI(linstorUrl, apiToken, insecureSsl); + try { + return isThicklyProvisionedPools(getCapacityStoragePools(api, rscGroupName)); + } catch (ApiException apiEx) { + LOGGER.warn("Unable to query storage pool provider kinds: {}", apiEx.getBestMessage()); + return false; + } + } + + static boolean isThicklyProvisionedPools(List storagePools) { + return !storagePools.isEmpty() && storagePools.stream() + .allMatch(sp -> THICK_PROVIDER_KINDS.contains(sp.getProviderKind())); + } + private static long sumTotalCapacity(List storagePools) { return storagePools.stream() .mapToLong(sp -> sp.getTotalCapacity() != null ? sp.getTotalCapacity() : 0L) diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java index fcfd4810f79d..1d5c795a225c 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -240,6 +240,20 @@ public void testGetCapacityStoragePoolsCountsSharedSpaceOnce() throws ApiExcepti Assert.assertEquals(52428800L * 1024, LinstorUtil.getUsedCapacityBytes(api, "shared")); } + @Test + public void testThickProviderKindDetection() { + Assert.assertTrue(LinstorUtil.isThicklyProvisionedPools(Arrays.asList( + mockStoragePool("sharedpool", "nodeA", ProviderKind.LVM, "cs-shared", 1L, 1L), + mockStoragePool("sharedpool", "nodeB", ProviderKind.LVM, "cs-shared", 1L, 1L)))); + Assert.assertFalse(LinstorUtil.isThicklyProvisionedPools(Arrays.asList( + mockStoragePool("thinpool", "nodeA", ProviderKind.LVM_THIN, "nodeA;thinpool", 1L, 1L)))); + // mixed setups keep the over-provisioning default + Assert.assertFalse(LinstorUtil.isThicklyProvisionedPools(Arrays.asList( + mockStoragePool("sharedpool", "nodeA", ProviderKind.LVM, "cs-shared", 1L, 1L), + mockStoragePool("thinpool", "nodeB", ProviderKind.LVM_THIN, "nodeB;thinpool", 1L, 1L)))); + Assert.assertFalse(LinstorUtil.isThicklyProvisionedPools(Collections.emptyList())); + } + @Test public void testIsVersionAtLeast() { Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.29.1", 1, 29, 1)); From b5305a0db8a21f33e83facf8b7ee1df785183e1c Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 5 Aug 2026 14:15:45 +0200 Subject: [PATCH 14/14] linstor: only send auto_manage_dual_primary when migrating Linstor controllers older than 1.29 reject requests with unknown properties, so a make-available carrying auto_manage_dual_primary fails against them - which broke every make-available call (and with it template copies and VM starts) even though the live-migrate API is version gated: the generated client initializes fields with their schema default (false here) and serialized everything non-null, so the property was always sent. --- .../cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 8ea5e8674466..fa112a23a572 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -603,7 +603,10 @@ public boolean connectPhysicalDisk( // controller >= 1.29 prepares the live migration itself: // dual-primary for DRBD, activation on both nodes for shared storage pools ResourceMakeAvailable rma = new ResourceMakeAvailable(); - rma.autoManageDualPrimary(isVMMigration); + if (isVMMigration) { + // only set when needed: older controllers reject the unknown property + rma.autoManageDualPrimary(true); + } ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma); checkLinstorAnswersThrow(answers); } else {