diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index 32e756607e55..06906f3b8595 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -48,6 +48,7 @@ import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.hypervisor.kvm.storage.LibvirtStorageAdaptor; import com.cloud.hypervisor.kvm.storage.MultipathNVMeOFPool; import com.cloud.hypervisor.kvm.storage.MultipathSCSIPool; import com.cloud.resource.CommandWrapper; @@ -187,7 +188,9 @@ notify the guest in one step (no passphrase needed, qemu already has the secret /* fetch new size as seen from libvirt, don't want to assume anything */ pool = storagePoolMgr.getStoragePool(spool.getType(), spool.getUuid()); pool.refresh(); - final long finalSize = pool.getPhysicalDisk(volumeId).getVirtualSize(); + // an encrypted RBD image keeps its LUKS2 header in front of the data, which is not part of the size of the volume + final long finalSize = rbdEncrypted ? LibvirtStorageAdaptor.getEncryptedRbdVolumeSize(pool, volumeId) + : pool.getPhysicalDisk(volumeId).getVirtualSize(); logger.debug("after resize, size reports as: " + toHumanReadableSize(finalSize) + ", requested: " + toHumanReadableSize(newSize)); return new ResizeVolumeAnswer(command, true, "success", finalSize); } catch (final CloudRuntimeException e) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 2c642efb9cfb..c0b8510dcf96 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -97,8 +97,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private static final int RBD_FEATURE_DEEP_FLATTEN = 32; public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN; private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */ - /* Space reserved at the front of an encrypted RBD image for the LUKS2 header/keyslots so the - usable (decrypted) size still matches the requested volume size. */ + /* Size of the LUKS2 header (and keyslots) librbd keeps at the front of an encrypted RBD image with objects of + up to 16 MiB, see getRbdLuks2HeaderSize(long). The image is made larger than the volume by the size of its + header, so that the Instance sees the size of the volume, as it does on every other pool type. */ private static final long LUKS2_HEADER_RESERVE_BYTES = 16L << 20; // 16 MiB /* libvirt's VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS, not exposed as a constant by libvirt-java */ private static final int VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS = 2; @@ -756,6 +757,88 @@ private KVMPhysicalDisk getRbdPhysicalDisk(String volumeUuid, LibvirtStoragePool } } + /** + * Returns the size of the LUKS2 header librbd keeps at the front of an encrypted RBD image with objects of the given + * size: 16 MiB, or one object if the objects are larger, as librbd starts the data of the image on an object boundary. + */ + public static long getRbdLuks2HeaderSize(long objectSize) { + return Math.max(LUKS2_HEADER_RESERVE_BYTES, objectSize); + } + + /** + * Returns the size of an encrypted RBD volume as the Instance sees it: the size of its image without the LUKS2 header + * librbd keeps at the front of the image. + */ + public static long getEncryptedRbdVolumeSize(KVMStoragePool pool, String image) { + try { + Rados r = connectToCeph(pool); + try { + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + try { + Rbd rbd = new Rbd(io); + // The image is only stat'ed, so it is opened read only: a running Instance holds its exclusive lock. + RbdImage rbdImage = rbd.openReadOnly(image); + try { + RbdImageInfo info = rbdImage.stat(); + return info.size - getRbdLuks2HeaderSize(info.obj_size); + } finally { + try { + rbd.close(rbdImage); + } catch (RbdException ignored) { + // the size was read already, a failing close must not hide it + } + } + } finally { + r.ioCtxDestroy(io); + } + } finally { + r.shutDown(); + } + } catch (RadosException | RbdException e) { + throw new CloudRuntimeException(String.format("Failed to get the size of the encrypted RBD volume %s/%s: %s", + pool.getSourceDir(), image, e.getMessage()), e); + } + } + + /** + * Makes room for the LUKS2 header in front of the given size in a new RBD image, which was created with room for a + * header of {@link #LUKS2_HEADER_RESERVE_BYTES}: the image is grown if its objects are larger, as the header then + * takes one object (see {@link #getRbdLuks2HeaderSize(long)}). + */ + private void reserveRbdLuks2Header(KVMStoragePool pool, String image, long size) { + try { + Rados r = connectToCeph(pool); + try { + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + try { + reserveRbdLuks2Header(new Rbd(io), image, size); + } finally { + r.ioCtxDestroy(io); + } + } finally { + r.shutDown(); + } + } catch (RadosException | RbdException e) { + throw new CloudRuntimeException(String.format("Failed to make room for the encryption header in RBD image %s/%s: %s", + pool.getSourceDir(), image, e.getMessage()), e); + } + } + + private void reserveRbdLuks2Header(Rbd rbd, String image, long size) throws RbdException { + RbdImage rbdImage = rbd.open(image); + try { + RbdImageInfo info = rbdImage.stat(); + long imageSize = size + getRbdLuks2HeaderSize(info.obj_size); + if (info.size < imageSize) { + logger.debug("Growing RBD image {} to {}, as the encryption header of an image with objects of {} takes one object", + image, toHumanReadableSize(imageSize), toHumanReadableSize(info.obj_size)); + rbdImage.resize(imageSize); + } + } finally { + closeRbdImage(rbd, rbdImage, image); + } + } + /** * Closes an RBD image without throwing, so that a failure to close cannot discard a * successful result or hide the exception that is already on its way out. @@ -1079,11 +1162,16 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, // Create the raw RBD image first. For encrypted volumes we apply a native librbd LUKS header // afterwards via `rbd encryption format` (engine='librbd'). We deliberately do NOT hand the // passphrase to qemu-img, which would instead produce a qemu-native LUKS container. + // librbd keeps the header in the image, in front of the data, so the image of an encrypted volume + // is larger than the volume by the size of the header: the Instance then sees the size of the volume. + boolean encrypted = passphrase != null && passphrase.length > 0; + long imageSize = encrypted ? size + LUKS2_HEADER_RESERVE_BYTES : size; KVMPhysicalDisk disk = (dataPool == null) ? - createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) : - createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, null); + createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, imageSize) : + createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, imageSize, null); - if (passphrase != null && passphrase.length > 0) { + if (encrypted) { + reserveRbdLuks2Header(pool, name, size); formatRbdImageEncryption(pool, name, passphrase); disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); } @@ -1268,6 +1356,64 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag } } + static String maskRbdKey(String message) { + return message == null ? null : message.replaceAll("key=[^:\\s]*", "key=******"); + } + + private static Rados connectToCeph(KVMStoragePool pool) throws RadosException { + Rados r = new Rados(pool.getAuthUserName()); + r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); + // librados aborts the process rather than returning an error if it is handed a null key, which a pool without + // a cephx user has + if (pool.getAuthUserName() != null) { + r.confSet("key", pool.getAuthSecret()); + } else { + r.confSet("auth_client_required", "none"); + } + r.confSet("client_mount_timeout", "30"); + r.connect(); + return r; + } + + private boolean rbdImageExists(KVMStoragePool pool, String imageName) throws RadosException, RbdException { + Rados r = connectToCeph(pool); + try { + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + try { + return Arrays.asList(new Rbd(io).list()).contains(imageName); + } finally { + r.ioCtxDestroy(io); + } + } finally { + r.shutDown(); + } + } + + /** + * Removes an RBD image left behind by a failed copy, so that it does not make every later copy fail with "File exists" + */ + private void removeRbdImageQuietly(KVMStoragePool pool, String imageName) { + try { + Rados r = connectToCeph(pool); + try { + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + try { + Rbd rbd = new Rbd(io); + if (Arrays.asList(rbd.list()).contains(imageName)) { + rbd.remove(imageName); + logger.info("Removed the incomplete RBD image {}/{} left by the failed copy", pool.getSourceDir(), imageName); + } + } finally { + r.ioCtxDestroy(io); + } + } finally { + r.shutDown(); + } + } catch (RadosException | RbdException e) { + logger.warn("Failed to remove the incomplete RBD image {}/{} left by the failed copy, remove it manually: {}", pool.getSourceDir(), imageName, e.getMessage()); + } + } + /** * This function copies a physical disk from Secondary Storage to Primary Storage * or from Primary to Primary Storage @@ -1565,9 +1711,10 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, /** * Option A (thin CoW encrypted root), used when the template already lives on the same RBD cluster * as the destination pool. Per the Ceph "Image Encryption" clone recipe: grow the template base to - * reserve LUKS2-header space, snapshot+protect that grown state, clone from it, apply a LUKS2 header, - * then resize the clone to the requested size. The inherited (plaintext) template data stays readable - * through the clone's encryption, and the clone is a thin CoW image (only the header is written). + * reserve LUKS2-header space, snapshot+protect that grown state (the template itself is shrunk back), + * clone from it, apply a LUKS2 header, then resize the clone to the requested size. The inherited + * (plaintext) template data stays readable through the clone's encryption, and the clone is a thin CoW + * image (only the header is written). * * @return the encrypted CoW clone, or {@code null} if the Ceph operations failed */ @@ -1578,6 +1725,7 @@ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KV IoCTX ioContext = null; Rbd rbdClient = null; RbdImage templateImage = null; + long cloneSize; try { radosConnection = new Rados(destPool.getAuthUserName()); radosConnection.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); @@ -1587,20 +1735,35 @@ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KV ioContext = radosConnection.ioCtxCreate(destPool.getSourceDir()); rbdClient = new Rbd(ioContext); templateImage = rbdClient.open(template.getName()); - boolean luksSnapshotExists = false; + Long luksSnapshotSize = null; for (RbdSnapInfo snapshotInfo : templateImage.snapList()) { if (luksReservedSnapshotName.equals(snapshotInfo.name)) { - luksSnapshotExists = true; + luksSnapshotSize = snapshotInfo.size; break; } } - if (!luksSnapshotExists) { - templateImage.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES); - templateImage.snapCreate(luksReservedSnapshotName); - templateImage.snapProtect(luksReservedSnapshotName); + if (luksSnapshotSize == null) { + // The template is grown by the size of the header only for the snapshot the encrypted clones are made of, + // and shrunk back right after: its unencrypted clones are made of its base snapshot, which may not exist + // yet, and must keep the size of the template. + RbdImageInfo templateInfo = templateImage.stat(); + luksSnapshotSize = templateInfo.size + getRbdLuks2HeaderSize(templateInfo.obj_size); + templateImage.resize(luksSnapshotSize); + try { + templateImage.snapCreate(luksReservedSnapshotName); + templateImage.snapProtect(luksReservedSnapshotName); + } finally { + templateImage.resize(templateInfo.size); + } logger.debug("Prepared LUKS-reserved template snapshot {}@{}", template.getName(), luksReservedSnapshotName); } rbdClient.clone(template.getName(), luksReservedSnapshotName, ioContext, newUuid, RBD_FEATURES, rbdOrder); + RbdImage cloneImage = rbdClient.open(newUuid); + try { + cloneSize = luksSnapshotSize - getRbdLuks2HeaderSize(cloneImage.stat().obj_size); + } finally { + closeRbdImage(rbdClient, cloneImage, newUuid); + } } catch (RadosException | RbdException e) { logger.error("Failed to create encrypted CoW clone {}: {}", newUuid, e.getMessage()); return null; @@ -1617,7 +1780,7 @@ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KV } } formatRbdImageEncryption(destPool, newUuid, passphrase); - if (disk.getVirtualSize() > template.getVirtualSize()) { + if (disk.getVirtualSize() > cloneSize) { // grow the clone to the requested root size (encryption-aware) new RbdEncryption().resize(destPool.getSourceHost(), destPool.getSourcePort(), destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(), @@ -1648,6 +1811,7 @@ private KVMPhysicalDisk createEncryptedRootFullCopy(KVMStoragePool srcPool, KVMP ioContext = radosConnection.ioCtxCreate(destPool.getSourceDir()); Rbd rbdClient = new Rbd(ioContext); rbdClient.create(newUuid, imageSizeWithLuksHeader, RBD_FEATURES, rbdOrder); + reserveRbdLuks2Header(rbdClient, newUuid, disk.getVirtualSize()); } catch (RadosException | RbdException e) { logger.error("Failed to create encrypted RBD image {}: {}", newUuid, e.getMessage()); return null; @@ -1808,12 +1972,21 @@ to support snapshots(backuped) as qcow2 files. */ * To do so it's mandatory that librbd on the system is at least 0.67.7 (Ceph Dumpling) */ logger.debug("The source image is not RBD, but the destination is. We will convert into RBD format 2"); + String rbdDestPath = destPool.getSourceDir() + "/" + name; + // Only an image created by this copy may be removed again when the copy fails + boolean destCreatedByCopy = false; try { srcFile = new QemuImgFile(sourcePath, sourceFormat); - String rbdDestPath = destPool.getSourceDir() + "/" + name; String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool, rbdDestPath); destFile = new QemuImgFile(rbdDestFile, destFormat); + if (rbdImageExists(destPool, name)) { + // qemu-img can't convert into an existing image, and it can't be told if it is complete, so don't touch it + logger.error("RBD image {} already exists, it will not be overwritten by the copy of {}", rbdDestPath, srcFile.getFileName()); + return null; + } + destCreatedByCopy = true; + logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); qemu.convert(srcFile, destFile); logger.debug("Successfully converted source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); @@ -1839,15 +2012,23 @@ to support snapshots(backuped) as qcow2 files. */ r.ioCtxDestroy(io); } catch (QemuImgException | LibvirtException e) { String srcFilename = srcFile != null ? srcFile.getFileName() : null; - String destFilename = destFile != null ? destFile.getFileName() : null; - logger.error(String.format("Failed to convert from %s to %s the error was: %s", srcFilename, destFilename, e.getMessage())); + logger.error("Failed to convert from {} to RBD image {} the error was: {}", srcFilename, rbdDestPath, maskRbdKey(e.getMessage())); newDisk = null; + if (destCreatedByCopy) { + removeRbdImageQuietly(destPool, name); + } } catch (RadosException e) { logger.error("A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; + if (destCreatedByCopy) { + removeRbdImageQuietly(destPool, name); + } } catch (RbdException e) { logger.error("A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; + if (destCreatedByCopy) { + removeRbdImageQuietly(destPool, name); + } } } else { /** diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java index 9b8e1fd2d748..761f52a51e19 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java @@ -27,6 +27,7 @@ import java.util.UUID; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -347,4 +348,32 @@ public void testCLVMPoolDetailsSecureZeroFillDefault() { assert !secureZeroEnabled; } + + @Test + public void maskRbdKeyHidesTheCephKey() { + String message = "qemu-img: rbd:pool/image:mon_host=10.0.0.1\\:6789:auth_client_required=cephx:id=pool:key=AQCsecret==:rbd_default_format=2: error while converting raw: error rbd create: File exists"; + String masked = LibvirtStorageAdaptor.maskRbdKey(message); + Assert.assertFalse(masked.contains("AQCsecret")); + Assert.assertTrue(masked.contains("key=******:rbd_default_format=2")); + Assert.assertTrue(masked.endsWith("File exists")); + } + + @Test + public void maskRbdKeyKeepsMessagesWithoutKey() { + Assert.assertNull(LibvirtStorageAdaptor.maskRbdKey(null)); + Assert.assertEquals("timeout", LibvirtStorageAdaptor.maskRbdKey("timeout")); + } + + /** + * Measured with `rbd encryption format` (Ceph 20.2): the data of a LUKS2 encrypted image starts at 16 MiB with + * objects of 1, 4 or 8 MiB, and at 32 MiB with objects of 32 MiB. + */ + @Test + public void testGetRbdLuks2HeaderSize() { + Assert.assertEquals(16L << 20, LibvirtStorageAdaptor.getRbdLuks2HeaderSize(1L << 20)); + Assert.assertEquals(16L << 20, LibvirtStorageAdaptor.getRbdLuks2HeaderSize(4L << 20)); + Assert.assertEquals(16L << 20, LibvirtStorageAdaptor.getRbdLuks2HeaderSize(8L << 20)); + Assert.assertEquals(16L << 20, LibvirtStorageAdaptor.getRbdLuks2HeaderSize(16L << 20)); + Assert.assertEquals(32L << 20, LibvirtStorageAdaptor.getRbdLuks2HeaderSize(32L << 20)); + } }