#263: add --no-sparse-detection option

Author: abbbi

.github/workflows/ci-nosparsedetect.yml

@@ -0,0 +1,49 @@
+name: nosparsedetect CI on ubuntu-22.04
+
+on: [push, pull_request]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-22.04
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: abbbi/github-actions-tune@v1
+    - name: Set up libvirt
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          apparmor-profiles \
+          bridge-utils \
+          dnsmasq-base \
+          ebtables \
+          libarchive-tools \
+          libguestfs-tools \
+          libvirt-clients \
+          libvirt-daemon \
+          libvirt-daemon-system \
+          qemu-kvm \
+          qemu-utils \
+          python3-libnbd \
+          python3-tqdm \
+          python3-lz4 \
+          python3-libvirt \
+          python3-lxml \
+          python3-paramiko\
+          python3-scp \
+          python3-colorlog \
+          nbdkit \
+          nbdkit-plugin-python \
+          unzip \
+          libnbd-bin \
+        ;
+        # start daemon
+        echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf
+        sudo aa-teardown
+        sudo rm -f /etc/apparmor.d/libvirt/libvirt*
+        sudo systemctl start libvirtd
+        sudo systemctl restart libvirtd
+        sudo modprobe nbd max_partitions=10
+    - name: Execute tests (nosparsedetect)
+      run: cd t && sudo -E make nosparsedetect.tests

Changelog

@@ -1,3 +1,18 @@
+Version 2.24
+---------
+ * virnbdbackup:
+  - Add option --no-sparse-detection (#263): The feature implemented as
+  with version 2.21 requires to query the complete base:allocation
+  bitmap during incremental backup.
+
+  Depending on the Hypervisor hardware and the virtual machine disk size,
+  querying the base:allocation bitmap may take longer than to simply treat
+  fstrimmed blocks the same way as changed blocks and include them in the
+  backup.
+
+  The option introduced can be used to disable the new behavior and can
+  enhance backup times for specific users.
+
 Version 2.23
 ---------
  * virtnbdbackup:

README.md

@@ -73,6 +73,7 @@ of your `kvm/qemu` virtual machines.
   - [Backup fails with "Timed out during operation: cannot acquire state change lock"](#backup-fails-with-timed-out-during-operation-cannot-acquire-state-change-lock)
   - [Backup fails with "Failed to bind socket to /var/tmp/virtnbdbackup.XX: Permission denied"](#backup-fails-with-failed-to-bind-socket-to-vartmpvirtnbdbackupxx-permission-denied)
   - [High memory usage during backup](#high-memory-usage-during-backup)
+  - [fstrim and (incremental) backup sizes](#fstrim-and-incremental-backup-sizes)
   - [Test your backups!](#test-your-backups)
   - [Links](#links)
 
@@ -1220,6 +1221,25 @@ version if using `virtnbdbackup` on any other distribution.
 
 See also: https://github.com/abbbi/virtnbdbackup/issues/8
 
+## fstrim and (incremental) backup sizes
+
+If virtual machines have configured disks with discard option and fstrim is
+running frequently, trimmed blocks are detected during backup operation by
+default.
+
+If, for example, the fstrim operation frees 30 GiB of disk space, and the
+virtual machine has only 5 GiB of changed data blocks, 30 GiB of data
+will be skipped during incremental backup.
+
+This works by comparing the complete allocation bitmap of the virtual machine
+disk images during incremental backup.
+
+Depending on your hardware and the size of the virtual disks, the operation to
+query the complete allocation bitmap during incremental backup, may take longer
+than to backup the complete changeset. Therefore, you can disable the detection
+of sparse/fstrimmed blocks using the `--no-sparse-detection` option.
+
+
 ## Test your backups!
 
 The utility is provided "as is", i take no responsibility or warranty if you

libvirtnbdbackup/__init__.py

@@ -15,4 +15,4 @@
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
 
-__version__ = "2.23"
+__version__ = "2.24"

libvirtnbdbackup/backup/disk.py

@@ -54,10 +54,14 @@ def _getExtentHandler(args: Namespace, nbdClient):
     if args.qemu:
         logging.info("Using qemu tools to query extents")
         extentHandler = extenthandler.ExtentHandler(
-            qemu.util(nbdClient.cType.exportName), nbdClient.cType
+            qemu.util(nbdClient.cType.exportName),
+            nbdClient.cType,
+            args.no_sparse_detection,
         )
     else:
-        extentHandler = extenthandler.ExtentHandler(nbdClient, nbdClient.cType)
+        extentHandler = extenthandler.ExtentHandler(
+            nbdClient, nbdClient.cType, args.no_sparse_detection
+        )
 
     return extentHandler
 

libvirtnbdbackup/backup/server.py

@@ -75,7 +75,7 @@ def connect(  # pylint: disable=too-many-arguments
     else:
         cType = nbdcli.Unix(disk.target, metaContext, socket)
 
-    nbdClient = nbdcli.client(cType)
+    nbdClient = nbdcli.client(cType, args.no_sparse_detection)
 
     try:
         return nbdClient.connect()

libvirtnbdbackup/extenthandler/extenthandler.py

@@ -32,7 +32,7 @@ class ExtentHandler:
     extent information as nbdinfo or qemu-img map
     """
 
-    def __init__(self, nbdFh, cType) -> None:
+    def __init__(self, nbdFh, cType, no_sparse_detection: bool) -> None:
         self.useQemu = False
         self._maxRequestBlock: int = 4294967295
         self._align: int = 512
@@ -44,6 +44,7 @@ def __init__(self, nbdFh, cType) -> None:
         self._nbdFh = nbdFh
         self._cType = cType
         self._extentEntries: Dict = {}
+        self.no_sparse_detection = no_sparse_detection
 
         if cType.metaContext == "":
             self._metaContext = CONTEXT_BASE_ALLOCATION
@@ -55,9 +56,12 @@ def __init__(self, nbdFh, cType) -> None:
             log.debug("NBD server exports [%d] metacontexts:", contexts)
             for i in range(0, contexts):
                 ctx = self._nbdFh.nbd.get_meta_context(i)
+                if self.no_sparse_detection is True and ctx == CONTEXT_BASE_ALLOCATION:
+                    continue
                 self._extentEntries[ctx] = []
         else:
-            self._extentEntries[CONTEXT_BASE_ALLOCATION] = []
+            if self.no_sparse_detection is False:
+                self._extentEntries[CONTEXT_BASE_ALLOCATION] = []
             self._extentEntries[self._metaContext] = []
 
         log.debug("Primary meta context for backup: %s", self._metaContext)
@@ -264,6 +268,10 @@ def queryBlockStatus(self) -> List[Extent]:
                 extObj.offset,
                 extObj.offset + extObj.length,
             )
+        if self.no_sparse_detection is True:
+            log.info("Skipping detection of sparse/fstrimmed blocks.")
+            return extentList
+
         if self._metaContext != CONTEXT_BASE_ALLOCATION:
             log.debug("Detected [%d] bytes of changed data regions.", totalLength)
             extentList = self.overlap(extentList)

libvirtnbdbackup/nbdcli/client.py

@@ -24,10 +24,11 @@
 log = logging.getLogger("nbd")
 
 
+# pylint: disable=too-many-instance-attributes
 class client:
     """Helper functions for NBD"""
 
-    def __init__(self, cType):
+    def __init__(self, cType, no_sparse_detection: bool):
         """
         Connect NBD backend
         """
@@ -40,6 +41,7 @@ def __init__(self, cType):
             self._metaContext = nbd.CONTEXT_BASE_ALLOCATION
         self.maxRequestSize = 33554432
         self.minRequestSize = 65536
+        self.no_sparse_detection = no_sparse_detection
         self.nbd = nbd.NBD()
 
         def debug(func, args):
@@ -72,7 +74,8 @@ def _connect(self) -> nbd.NBD:
         try:
             if self.cType.tls:
                 self.nbd.set_tls(nbd.TLS_REQUIRE)
-            self.nbd.add_meta_context(nbd.CONTEXT_BASE_ALLOCATION)
+            if self.no_sparse_detection is False:
+                self.nbd.add_meta_context(nbd.CONTEXT_BASE_ALLOCATION)
             if self._metaContext != "":
                 log.debug(
                     "Adding meta context to NBD connection: [%s]", self._metaContext

libvirtnbdbackup/restore/server.py

@@ -48,7 +48,7 @@ def setup(args: Namespace, exportName: str, targetFile: str, virtClient: virt.cl
         proc = qFh.startRemoteRestoreNbdServer(args, targetFile)
         cType = nbdcli.TCP(exportName, "", remoteIP, args.tls, args.nbd_port)
 
-    nbdClient = nbdcli.client(cType)
+    nbdClient = nbdcli.client(cType, False)
     logging.info("Started NBD server, PID: [%s]", proc.pid)
     return nbdClient.connect()
 

man/virtnbdbackup.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.3.
-.TH VIRTNBDBACKUP "1" "March 2025" "virtnbdbackup 2.23" "User Commands"
+.TH VIRTNBDBACKUP "1" "March 2025" "virtnbdbackup 2.24" "User Commands"
 .SH NAME
 virtnbdbackup \- backup utility for libvirt
 .SH DESCRIPTION
@@ -8,8 +8,9 @@ usage: virtnbdbackup [\-h] \fB\-d\fR DOMAIN [\-l {copy,full,inc,diff,auto}]
 [\-t {stream,raw}] [\-r] \fB\-o\fR OUTPUT [\-C CHECKPOINTDIR]
 [\-\-scratchdir SCRATCHDIR] [\-S] [\-i INCLUDE] [\-x EXCLUDE]
 [\-f SOCKETFILE] [\-n] [\-z [COMPRESS]] [\-w WORKER]
-[\-F FREEZE_MOUNTPOINT] [\-e] [\-T THRESHOLD] [\-U URI]
-[\-\-user USER] [\-\-ssh\-user SSH_USER] [\-\-ssh\-port SSH_PORT]
+[\-F FREEZE_MOUNTPOINT] [\-e] [\-\-no\-sparse\-detection]
+[\-T THRESHOLD] [\-U URI] [\-\-user USER]
+[\-\-ssh\-user SSH_USER] [\-\-ssh\-port SSH_PORT]
 [\-\-password PASSWORD] [\-P NBD_PORT] [\-I NBD_IP] [\-\-tls]
 [\-\-tls\-cert TLS_CERT] [\-L] [\-\-quiet] [\-\-nocolor] [\-q]
 [\-s] [\-k] [\-p] [\-v] [\-V]
@@ -52,7 +53,7 @@ Backup only disk with target dev name (\fB\-i\fR vda)
 Exclude disk(s) with target dev name (\fB\-x\fR vda,vdb)
 .TP
 \fB\-f\fR SOCKETFILE, \fB\-\-socketfile\fR SOCKETFILE
-Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.117051\/\fP)
+Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.535505\/\fP)
 .TP
 \fB\-n\fR, \fB\-\-noprogress\fR
 Disable progress bar
@@ -69,6 +70,9 @@ If qemu agent available, freeze only filesystems on specified mountpoints within
 \fB\-e\fR, \fB\-\-strict\fR
 Change exit code if warnings occur during backup operation. (default: False)
 .TP
+\fB\-\-no\-sparse\-detection\fR
+Skip detection of sparse ranges during incremental or differential backup. (default: False)
+.TP
 \fB\-T\fR THRESHOLD, \fB\-\-threshold\fR THRESHOLD
 Execute backup only if threshold is reached.
 .SS "Remote Backup options:"

man/virtnbdmap.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.3.
-.TH VIRTNBDMAP "1" "March 2025" "virtnbdmap 2.23" "User Commands"
+.TH VIRTNBDMAP "1" "March 2025" "virtnbdmap 2.24" "User Commands"
 .SH NAME
 virtnbdmap \- map virtnbdbackup image files to nbd devices
 .SH DESCRIPTION

man/virtnbdrestore.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.3.
-.TH VIRTNBDRESTORE "1" "March 2025" "virtnbdrestore 2.23" "User Commands"
+.TH VIRTNBDRESTORE "1" "March 2025" "virtnbdrestore 2.24" "User Commands"
 .SH NAME
 virtnbdrestore \- restore utility for libvirt
 .SH DESCRIPTION
@@ -41,7 +41,7 @@ Process only disk matching target dev name. (default: None)
 Disable progress bar
 .TP
 \fB\-f\fR SOCKETFILE, \fB\-\-socketfile\fR SOCKETFILE
-Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.117056\/\fP)
+Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.535510\/\fP)
 .TP
 \fB\-r\fR, \fB\-\-raw\fR
 Copy raw images as is during restore. (default: False)

t/Makefile

@@ -15,13 +15,15 @@ vm4.tests: $(bats)
 vm5.tests: $(bats)
 	$(call clean)
 	export TEST=vm5 ; ./bats-core/bin/bats tests.bats
-
 fstrim.tests: $(bats)
 	$(call clean)
 	export TEST=fstrim ; ./bats-core/bin/bats fstrim.bats
+nosparsedetect.tests: $(bats)
+	$(call clean)
+	export TEST=nosparsedetect ; ./bats-core/bin/bats nosparsedetect.bats
 
 
-all: | $(bats) vm1.tests vm2.tests vm3.tests vm4.tests fstrim.tests
+all: | $(bats) vm1.tests vm2.tests vm3.tests vm4.tests fstrim.tests nosparsedetect.tests
 
 clean:
 	$(call clean)
@@ -35,7 +37,7 @@ define clean
 	@rm -rf checkpoints
 	@rm -f vmconfig*.xml
 	@umount -lf /empty || true
-	for vm in vm1 restore_vm1 vm2 vm3 vm4 vm5 fstrim; do \
+	for vm in vm1 restore_vm1 vm2 vm3 vm4 vm5 fstrim nosparsedetect restored; do \
 		if virsh list --all | grep $$vm; then \
 			virsh destroy $$vm; \
 			virsh undefine $$vm --checkpoints-metadata --nvram --managed-save; \