Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
import com.cloud.hypervisor.kvm.storage.LibvirtStorageAdaptor;
import com.cloud.hypervisor.kvm.storage.MultipathNVMeOFPool;
import com.cloud.hypervisor.kvm.storage.MultipathSCSIPool;
import com.cloud.resource.CommandWrapper;
Expand Down Expand Up @@ -187,7 +188,9 @@ notify the guest in one step (no passphrase needed, qemu already has the secret
/* fetch new size as seen from libvirt, don't want to assume anything */
pool = storagePoolMgr.getStoragePool(spool.getType(), spool.getUuid());
pool.refresh();
final long finalSize = pool.getPhysicalDisk(volumeId).getVirtualSize();
// an encrypted RBD image keeps its LUKS2 header in front of the data, which is not part of the size of the volume
final long finalSize = rbdEncrypted ? LibvirtStorageAdaptor.getEncryptedRbdVolumeSize(pool, volumeId)
: pool.getPhysicalDisk(volumeId).getVirtualSize();
logger.debug("after resize, size reports as: " + toHumanReadableSize(finalSize) + ", requested: " + toHumanReadableSize(newSize));
return new ResizeVolumeAnswer(command, true, "success", finalSize);
} catch (final CloudRuntimeException e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
private static final int RBD_FEATURE_DEEP_FLATTEN = 32;
public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN;
private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */
/* Space reserved at the front of an encrypted RBD image for the LUKS2 header/keyslots so the
usable (decrypted) size still matches the requested volume size. */
/* Size of the LUKS2 header (and keyslots) librbd keeps at the front of an encrypted RBD image with objects of
up to 16 MiB, see getRbdLuks2HeaderSize(long). The image is made larger than the volume by the size of its
header, so that the Instance sees the size of the volume, as it does on every other pool type. */
private static final long LUKS2_HEADER_RESERVE_BYTES = 16L << 20; // 16 MiB
/* libvirt's VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS, not exposed as a constant by libvirt-java */
private static final int VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS = 2;
Expand Down Expand Up @@ -756,6 +757,88 @@ private KVMPhysicalDisk getRbdPhysicalDisk(String volumeUuid, LibvirtStoragePool
}
}

/**
* Returns the size of the LUKS2 header librbd keeps at the front of an encrypted RBD image with objects of the given
* size: 16 MiB, or one object if the objects are larger, as librbd starts the data of the image on an object boundary.
*/
public static long getRbdLuks2HeaderSize(long objectSize) {
return Math.max(LUKS2_HEADER_RESERVE_BYTES, objectSize);
}

/**
* Returns the size of an encrypted RBD volume as the Instance sees it: the size of its image without the LUKS2 header
* librbd keeps at the front of the image.
*/
public static long getEncryptedRbdVolumeSize(KVMStoragePool pool, String image) {
try {
Rados r = connectToCeph(pool);
try {
IoCTX io = r.ioCtxCreate(pool.getSourceDir());
try {
Rbd rbd = new Rbd(io);
// The image is only stat'ed, so it is opened read only: a running Instance holds its exclusive lock.
RbdImage rbdImage = rbd.openReadOnly(image);
try {
RbdImageInfo info = rbdImage.stat();
return info.size - getRbdLuks2HeaderSize(info.obj_size);
} finally {
try {
rbd.close(rbdImage);
} catch (RbdException ignored) {
// the size was read already, a failing close must not hide it
}
}
} finally {
r.ioCtxDestroy(io);
}
} finally {
r.shutDown();
}
} catch (RadosException | RbdException e) {
throw new CloudRuntimeException(String.format("Failed to get the size of the encrypted RBD volume %s/%s: %s",
pool.getSourceDir(), image, e.getMessage()), e);
}
}

/**
* Makes room for the LUKS2 header in front of the given size in a new RBD image, which was created with room for a
* header of {@link #LUKS2_HEADER_RESERVE_BYTES}: the image is grown if its objects are larger, as the header then
* takes one object (see {@link #getRbdLuks2HeaderSize(long)}).
*/
private void reserveRbdLuks2Header(KVMStoragePool pool, String image, long size) {
try {
Rados r = connectToCeph(pool);
try {
IoCTX io = r.ioCtxCreate(pool.getSourceDir());
try {
reserveRbdLuks2Header(new Rbd(io), image, size);
} finally {
r.ioCtxDestroy(io);
}
} finally {
r.shutDown();
}
} catch (RadosException | RbdException e) {
throw new CloudRuntimeException(String.format("Failed to make room for the encryption header in RBD image %s/%s: %s",
pool.getSourceDir(), image, e.getMessage()), e);
}
}

private void reserveRbdLuks2Header(Rbd rbd, String image, long size) throws RbdException {
RbdImage rbdImage = rbd.open(image);
try {
RbdImageInfo info = rbdImage.stat();
long imageSize = size + getRbdLuks2HeaderSize(info.obj_size);
if (info.size < imageSize) {
logger.debug("Growing RBD image {} to {}, as the encryption header of an image with objects of {} takes one object",
image, toHumanReadableSize(imageSize), toHumanReadableSize(info.obj_size));
rbdImage.resize(imageSize);
}
} finally {
closeRbdImage(rbd, rbdImage, image);
}
}

/**
* Closes an RBD image without throwing, so that a failure to close cannot discard a
* successful result or hide the exception that is already on its way out.
Expand Down Expand Up @@ -1079,11 +1162,16 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool,
// Create the raw RBD image first. For encrypted volumes we apply a native librbd LUKS header
// afterwards via `rbd encryption format` (engine='librbd'). We deliberately do NOT hand the
// passphrase to qemu-img, which would instead produce a qemu-native LUKS container.
// librbd keeps the header in the image, in front of the data, so the image of an encrypted volume
// is larger than the volume by the size of the header: the Instance then sees the size of the volume.
boolean encrypted = passphrase != null && passphrase.length > 0;
long imageSize = encrypted ? size + LUKS2_HEADER_RESERVE_BYTES : size;
KVMPhysicalDisk disk = (dataPool == null) ?
createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) :
createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, null);
createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, imageSize) :
createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, imageSize, null);

if (passphrase != null && passphrase.length > 0) {
if (encrypted) {
reserveRbdLuks2Header(pool, name, size);
formatRbdImageEncryption(pool, name, passphrase);
disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2);
}
Expand Down Expand Up @@ -1268,6 +1356,64 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag
}
}

static String maskRbdKey(String message) {
return message == null ? null : message.replaceAll("key=[^:\\s]*", "key=******");
}

private static Rados connectToCeph(KVMStoragePool pool) throws RadosException {
Rados r = new Rados(pool.getAuthUserName());
r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort());
// librados aborts the process rather than returning an error if it is handed a null key, which a pool without
// a cephx user has
if (pool.getAuthUserName() != null) {
r.confSet("key", pool.getAuthSecret());
} else {
r.confSet("auth_client_required", "none");
}
r.confSet("client_mount_timeout", "30");
r.connect();
return r;
}

private boolean rbdImageExists(KVMStoragePool pool, String imageName) throws RadosException, RbdException {
Rados r = connectToCeph(pool);
try {
IoCTX io = r.ioCtxCreate(pool.getSourceDir());
try {
return Arrays.asList(new Rbd(io).list()).contains(imageName);
} finally {
r.ioCtxDestroy(io);
}
} finally {
r.shutDown();
}
}

/**
* Removes an RBD image left behind by a failed copy, so that it does not make every later copy fail with "File exists"
*/
private void removeRbdImageQuietly(KVMStoragePool pool, String imageName) {
try {
Rados r = connectToCeph(pool);
try {
IoCTX io = r.ioCtxCreate(pool.getSourceDir());
try {
Rbd rbd = new Rbd(io);
if (Arrays.asList(rbd.list()).contains(imageName)) {
rbd.remove(imageName);
logger.info("Removed the incomplete RBD image {}/{} left by the failed copy", pool.getSourceDir(), imageName);
}
} finally {
r.ioCtxDestroy(io);
}
} finally {
r.shutDown();
}
} catch (RadosException | RbdException e) {
logger.warn("Failed to remove the incomplete RBD image {}/{} left by the failed copy, remove it manually: {}", pool.getSourceDir(), imageName, e.getMessage());
}
}

/**
* This function copies a physical disk from Secondary Storage to Primary Storage
* or from Primary to Primary Storage
Expand Down Expand Up @@ -1565,9 +1711,10 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template,
/**
* Option A (thin CoW encrypted root), used when the template already lives on the same RBD cluster
* as the destination pool. Per the Ceph "Image Encryption" clone recipe: grow the template base to
* reserve LUKS2-header space, snapshot+protect that grown state, clone from it, apply a LUKS2 header,
* then resize the clone to the requested size. The inherited (plaintext) template data stays readable
* through the clone's encryption, and the clone is a thin CoW image (only the header is written).
* reserve LUKS2-header space, snapshot+protect that grown state (the template itself is shrunk back),
* clone from it, apply a LUKS2 header, then resize the clone to the requested size. The inherited
* (plaintext) template data stays readable through the clone's encryption, and the clone is a thin CoW
* image (only the header is written).
*
* @return the encrypted CoW clone, or {@code null} if the Ceph operations failed
*/
Expand All @@ -1578,6 +1725,7 @@ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KV
IoCTX ioContext = null;
Rbd rbdClient = null;
RbdImage templateImage = null;
long cloneSize;
try {
radosConnection = new Rados(destPool.getAuthUserName());
radosConnection.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort());
Expand All @@ -1587,20 +1735,35 @@ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KV
ioContext = radosConnection.ioCtxCreate(destPool.getSourceDir());
rbdClient = new Rbd(ioContext);
templateImage = rbdClient.open(template.getName());
boolean luksSnapshotExists = false;
Long luksSnapshotSize = null;
for (RbdSnapInfo snapshotInfo : templateImage.snapList()) {
if (luksReservedSnapshotName.equals(snapshotInfo.name)) {
luksSnapshotExists = true;
luksSnapshotSize = snapshotInfo.size;
break;
}
}
if (!luksSnapshotExists) {
templateImage.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES);
templateImage.snapCreate(luksReservedSnapshotName);
templateImage.snapProtect(luksReservedSnapshotName);
if (luksSnapshotSize == null) {
// The template is grown by the size of the header only for the snapshot the encrypted clones are made of,
// and shrunk back right after: its unencrypted clones are made of its base snapshot, which may not exist
// yet, and must keep the size of the template.
RbdImageInfo templateInfo = templateImage.stat();
luksSnapshotSize = templateInfo.size + getRbdLuks2HeaderSize(templateInfo.obj_size);
templateImage.resize(luksSnapshotSize);
try {
templateImage.snapCreate(luksReservedSnapshotName);
templateImage.snapProtect(luksReservedSnapshotName);
} finally {
templateImage.resize(templateInfo.size);
}
logger.debug("Prepared LUKS-reserved template snapshot {}@{}", template.getName(), luksReservedSnapshotName);
}
rbdClient.clone(template.getName(), luksReservedSnapshotName, ioContext, newUuid, RBD_FEATURES, rbdOrder);
RbdImage cloneImage = rbdClient.open(newUuid);
try {
cloneSize = luksSnapshotSize - getRbdLuks2HeaderSize(cloneImage.stat().obj_size);
} finally {
closeRbdImage(rbdClient, cloneImage, newUuid);
}
} catch (RadosException | RbdException e) {
logger.error("Failed to create encrypted CoW clone {}: {}", newUuid, e.getMessage());
return null;
Expand All @@ -1617,7 +1780,7 @@ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KV
}
}
formatRbdImageEncryption(destPool, newUuid, passphrase);
if (disk.getVirtualSize() > template.getVirtualSize()) {
if (disk.getVirtualSize() > cloneSize) {
// grow the clone to the requested root size (encryption-aware)
new RbdEncryption().resize(destPool.getSourceHost(), destPool.getSourcePort(),
destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(),
Expand Down Expand Up @@ -1648,6 +1811,7 @@ private KVMPhysicalDisk createEncryptedRootFullCopy(KVMStoragePool srcPool, KVMP
ioContext = radosConnection.ioCtxCreate(destPool.getSourceDir());
Rbd rbdClient = new Rbd(ioContext);
rbdClient.create(newUuid, imageSizeWithLuksHeader, RBD_FEATURES, rbdOrder);
reserveRbdLuks2Header(rbdClient, newUuid, disk.getVirtualSize());
} catch (RadosException | RbdException e) {
logger.error("Failed to create encrypted RBD image {}: {}", newUuid, e.getMessage());
return null;
Expand Down Expand Up @@ -1808,12 +1972,21 @@ to support snapshots(backuped) as qcow2 files. */
* To do so it's mandatory that librbd on the system is at least 0.67.7 (Ceph Dumpling)
*/
logger.debug("The source image is not RBD, but the destination is. We will convert into RBD format 2");
String rbdDestPath = destPool.getSourceDir() + "/" + name;
// Only an image created by this copy may be removed again when the copy fails
boolean destCreatedByCopy = false;
try {
srcFile = new QemuImgFile(sourcePath, sourceFormat);
String rbdDestPath = destPool.getSourceDir() + "/" + name;
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool, rbdDestPath);
destFile = new QemuImgFile(rbdDestFile, destFormat);

if (rbdImageExists(destPool, name)) {
// qemu-img can't convert into an existing image, and it can't be told if it is complete, so don't touch it
logger.error("RBD image {} already exists, it will not be overwritten by the copy of {}", rbdDestPath, srcFile.getFileName());
return null;
}
destCreatedByCopy = true;

logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath);
qemu.convert(srcFile, destFile);
logger.debug("Successfully converted source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath);
Expand All @@ -1839,15 +2012,23 @@ to support snapshots(backuped) as qcow2 files. */
r.ioCtxDestroy(io);
} catch (QemuImgException | LibvirtException e) {
String srcFilename = srcFile != null ? srcFile.getFileName() : null;
String destFilename = destFile != null ? destFile.getFileName() : null;
logger.error(String.format("Failed to convert from %s to %s the error was: %s", srcFilename, destFilename, e.getMessage()));
logger.error("Failed to convert from {} to RBD image {} the error was: {}", srcFilename, rbdDestPath, maskRbdKey(e.getMessage()));
newDisk = null;
if (destCreatedByCopy) {
removeRbdImageQuietly(destPool, name);
}
} catch (RadosException e) {
logger.error("A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage());
newDisk = null;
if (destCreatedByCopy) {
removeRbdImageQuietly(destPool, name);
}
} catch (RbdException e) {
logger.error("A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage());
newDisk = null;
if (destCreatedByCopy) {
removeRbdImageQuietly(destPool, name);
}
}
} else {
/**
Expand Down
Loading
Loading