fixup! [PAL,LibOS,common] Add file recovery support for encrypted files

!TODO: use below commit msg

[LibOS,common] Add file recovery support for encrypted files

Previously, a fatal error during writes to encrypted files could cause
file corruption due to incorrect GMACs and/or encryption keys.

To address this, we introduce a file recovery mechanism using a "shadow"
recovery file that stores data about to change and a `has_pending_write`
flag in the metadata node indicating the start of a write transaction.
During file flush, all cached blocks that are about to change are saved
to the recovery file in the format of physical node numbers (offsets)
plus encrypted block data. Before saving the main file contents, the
`has_pending_write` flag is set in the file's metadata node and cleared
only when the transaction is complete. If an encrypted file is opened
and the `has_pending_write` flag is set, a recovery process starts to
revert partial changes using the recovery file, returning to the last
known good state. The "shadow" recovery file is cleaned up on file
close.

This commit adds a new mount parameter `enable_recovery = [true|false]`
for encrypted files mounts to optionally enable this feature. We extend
the file flush logic of protected files (pf) to include the recovery
file dump and the setting/unsetting of the `has_pending_write` flag.
We also extend `pf_open()` to make the pf aware of the underlying
recovery file managed by LibOS, and to include an optional recovery
check and initiate recovery if needed.

Signed-off-by: Kailun Qin <[email protected]>

Documentation/pal/host-abi.rst

@@ -372,6 +372,3 @@ random bits, to obtain an attestation report and quote, etc.
 
 .. doxygenfunction:: PalFreeThenLazyReallocCommittedPages
    :project: pal
-
-.. doxygenfunction:: PalRecoverEncryptedFile
-   :project: pal

common/src/protected_files/protected_files.c

@@ -56,7 +56,7 @@ static const char* g_pf_error_list[] = {
     [-PF_STATUS_NOT_IMPLEMENTED] = "Functionality not implemented",
     [-PF_STATUS_CALLBACK_FAILED] = "Callback failed",
     [-PF_STATUS_PATH_TOO_LONG] = "Path is too long",
-    [-PF_STATUS_RECOVERY_NEEDED] = "File recovery needed",
+    [-PF_STATUS_RECOVERY_NEEDED] = "File recovery needed but failed",
     [-PF_STATUS_FLUSH_ERROR] = "Flush error",
     [-PF_STATUS_CRYPTO_ERROR] = "Crypto error",
     [-PF_STATUS_CORRUPTED] = "File is corrupted",
@@ -848,11 +848,11 @@ static bool ipf_init_fields(pf_context_t* pf) {
 
     ipf_init_root_mht(&pf->root_mht_node);
 
-    pf->host_file_handle = NULL;
+    pf->host_file_handle          = NULL;
     pf->host_recovery_file_handle = NULL;
-    pf->need_writing     = false;
-    pf->file_status      = PF_STATUS_UNINITIALIZED;
-    pf->last_error       = PF_STATUS_SUCCESS;
+    pf->need_writing              = false;
+    pf->file_status               = PF_STATUS_UNINITIALIZED;
+    pf->last_error                = PF_STATUS_SUCCESS;
 
     pf->cache = lruc_create();
     return true;
@@ -950,9 +950,67 @@ static void ipf_try_clear_error(pf_context_t* pf) {
     }
 }
 
+static bool ipf_check_recovery_needed(pf_context_t* pf) {
+    // read metadata node
+    if (!ipf_read_node(pf, /*physical_node_number=*/0, (uint8_t*)&pf->metadata_node))
+        return pf->last_error;
+
+    return pf->metadata_node.plaintext_part.has_pending_write == 1;
+}
+
+/* Reads each recovery node from the recovery file and apply the embedded pf node
+ * (recovery_node.bytes) to the corresponding offset (recovery_node.physical_node_number) in the
+ * main file. */
+static bool ipf_recover(pf_context_t* pf, uint64_t recovery_file_size) {
+    pf_status_t status;
+
+    if (!pf->host_recovery_file_handle) {
+        DEBUG_PF("file recovery needed but recovery file handle not set; please consider setting "
+                 "'enable_recovery = true' for the mount");
+        pf->last_error = PF_STATUS_RECOVERY_NEEDED;
+        return false;
+    }
+
+    if (recovery_file_size == 0 || recovery_file_size % sizeof(recovery_node_t) != 0) {
+        DEBUG_PF("recovery file size is not right [%lu]", recovery_file_size);
+        pf->last_error = PF_STATUS_RECOVERY_NEEDED;
+        return false;
+    }
+
+    size_t recovery_nodes_count = recovery_file_size / sizeof(recovery_node_t);
+
+    for (size_t i = 0; i < recovery_nodes_count; i++) {
+        recovery_node_t recovery_node;
+
+        status = g_cb_read(pf->host_recovery_file_handle, &recovery_node,
+                           i * sizeof(recovery_node_t), sizeof(recovery_node_t));
+        if (PF_FAILURE(status)) {
+            pf->last_error = status;
+            return false;
+        }
+
+        size_t offset = recovery_node.physical_node_number;
+        status = g_cb_write(pf->host_file_handle, recovery_node.bytes,
+                            offset * sizeof(recovery_node.bytes), sizeof(recovery_node.bytes));
+        if (PF_FAILURE(status)) {
+            pf->last_error = status;
+            return false;
+        }
+    }
+
+    status = g_cb_fsync(pf->host_file_handle);
+    if (PF_FAILURE(status)) {
+        pf->last_error = status;
+        return false;
+    }
+
+    return true;
+}
+
 static pf_context_t* ipf_open(const char* path, pf_file_mode_t mode, bool create, pf_handle_t file,
                               uint64_t real_size, const pf_key_t* kdk_key,
-                              pf_handle_t recovery_file_handle, pf_status_t* status) {
+                              pf_handle_t recovery_file_handle, uint64_t recovery_file_size,
+                              bool try_recover, pf_status_t* status) {
     *status = PF_STATUS_NO_MEMORY;
     pf_context_t* pf = calloc(1, sizeof(*pf));
 
@@ -998,6 +1056,20 @@ static pf_context_t* ipf_open(const char* path, pf_file_mode_t mode, bool create
         if (!ipf_init_existing_file(pf, path))
             goto out;
 
+        if (try_recover && ipf_check_recovery_needed(pf)) {
+            DEBUG_PF("%s: starting file recovery", path);
+
+            if (!ipf_recover(pf, recovery_file_size))
+                goto out;
+
+            if (ipf_check_recovery_needed(pf)) {
+                DEBUG_PF("%s: attempted but failed", path);
+                pf->last_error = PF_STATUS_RECOVERY_NEEDED;
+                goto out;
+            }
+
+            DEBUG_PF("%s: file recovery completed", path);
+        }
     } else {
         if (!ipf_init_new_file(pf, path))
             goto out;
@@ -1229,13 +1301,14 @@ void pf_set_callbacks(pf_read_f read_f, pf_write_f write_f, pf_fsync_f fsync_f,
 
 pf_status_t pf_open(pf_handle_t handle, const char* path, uint64_t underlying_size,
                     pf_file_mode_t mode, bool create, const pf_key_t* key,
-                    pf_handle_t recovery_file_handle, pf_context_t** context) {
+                    pf_handle_t recovery_file_handle, uint64_t recovery_file_size,
+                    bool try_recover, pf_context_t** context) {
     if (!g_initialized)
         return PF_STATUS_UNINITIALIZED;
 
     pf_status_t status;
     *context = ipf_open(path, mode, create, handle, underlying_size, key, recovery_file_handle,
-                        &status);
+                        recovery_file_size, try_recover, &status);
     return status;
 }
 
@@ -1401,19 +1474,3 @@ pf_status_t pf_flush(pf_context_t* pf) {
 
     return PF_STATUS_SUCCESS;
 }
-
-pf_status_t pf_get_recovery_info(pf_context_t* pf, bool* out_recovery_needed,
-                                 size_t* out_node_size) {
-    if (out_recovery_needed) {
-        // read metadata node
-        if (!ipf_read_node(pf, /*physical_node_number=*/0, (uint8_t*)&pf->metadata_node))
-            return pf->last_error;
-
-        *out_recovery_needed = (pf->metadata_node.plaintext_part.has_pending_write == 1);
-    }
-
-    if (out_node_size)
-        *out_node_size = sizeof(((recovery_node_t*)0)->bytes);
-
-    return PF_STATUS_SUCCESS;
-}

common/src/protected_files/protected_files.h

@@ -210,7 +210,7 @@ void pf_set_callbacks(pf_read_f read_f, pf_write_f write_f, pf_fsync_f fsync_f,
 const char* pf_strerror(int err);
 
 /*!
- * \brief Open a protected file.
+ * \brief Open a protected file, with optional recovery check and process.
  *
  * \param      handle                Open underlying file handle.
  * \param      path                  Path to the file. If NULL and \p create is false, don't check path
@@ -220,13 +220,16 @@ const char* pf_strerror(int err);
  * \param      create                Overwrite file contents if true.
  * \param      key                   Wrap key.
  * \param      recovery_file_handle  (optional) Underlying recovery file handle.
+ * \param      recovery_file_size    Recovery file size.
+ * \param      try_recover           Whether to check for and perform file recovery if needed.
  * \param[out] context               PF context for later calls.
  *
  * \returns PF status.
  */
 pf_status_t pf_open(pf_handle_t handle, const char* path, uint64_t underlying_size,
                     pf_file_mode_t mode, bool create, const pf_key_t* key,
-                    pf_handle_t recovery_file_handle, pf_context_t** context);
+                    pf_handle_t recovery_file_handle, uint64_t recovery_file_size,
+                    bool try_cover, pf_context_t** context);
 
 /*!
  * \brief Close a protected file and commit all changes to disk.
@@ -304,15 +307,3 @@ pf_status_t pf_rename(pf_context_t* pf, const char* new_path);
  * \returns PF status.
  */
 pf_status_t pf_flush(pf_context_t* pf);
-
-/*!
- * \brief Get the recovery info of a PF.
- *
- * \param      pf                   PF context.
- * \param[out] out_recovery_needed  (optional) Whether recovery is needed for \p pf.
- * \param[out] out_node_size        (optional) Size of the \p pf node.
- *
- * \returns PF status.
- */
-pf_status_t pf_get_recovery_info(pf_context_t* pf, bool* out_recovery_needed,
-                                 size_t* out_node_size);

libos/src/fs/libos_fs_encrypted.c

@@ -166,7 +166,8 @@ static int encrypted_file_internal_open(struct libos_encrypted_file* enc, PAL_HA
     int ret;
     char* normpath = NULL;
     PAL_HANDLE recovery_file_pal_handle = NULL;
-    bool maybe_recovery_needed = !create && !pal_handle;
+    size_t recovery_file_size = 0;
+    bool try_recover = !create && !pal_handle;
 
     if (!pal_handle) {
         enum pal_create_mode create_mode = create ? PAL_CREATE_ALWAYS : PAL_CREATE_NEVER;
@@ -199,6 +200,15 @@ static int encrypted_file_internal_open(struct libos_encrypted_file* enc, PAL_HA
                 ret = pal_to_unix_errno(ret);
                 goto out;
             }
+
+            PAL_STREAM_ATTR pal_attr;
+            ret = PalStreamAttributesQueryByHandle(recovery_file_pal_handle, &pal_attr);
+            if (ret < 0) {
+                log_warning("PalStreamAttributesQueryByHandle failed: %s", pal_strerror(ret));
+                ret = pal_to_unix_errno(ret);
+                goto out;
+            }
+            recovery_file_size = pal_attr.pending_size;
         }
     }
     assert(enc->enable_recovery == (recovery_file_pal_handle != NULL));
@@ -236,58 +246,15 @@ static int encrypted_file_internal_open(struct libos_encrypted_file* enc, PAL_HA
         goto out;
     }
     pf_status_t pfs = pf_open(pal_handle, normpath, size, PF_FILE_MODE_READ | PF_FILE_MODE_WRITE,
-                              create, &enc->key->pf_key, recovery_file_pal_handle, &pf);
+                              create, &enc->key->pf_key, recovery_file_pal_handle,
+                              recovery_file_size, try_recover, &pf);
     unlock(&g_keys_lock);
     if (PF_FAILURE(pfs)) {
         log_warning("pf_open failed: %s", pf_strerror(pfs));
         ret = -EACCES;
         goto out;
     }
 
-    if (maybe_recovery_needed) {
-        bool recovery_needed;
-        size_t node_size;
-        pfs = pf_get_recovery_info(pf, &recovery_needed, &node_size);
-        if (PF_FAILURE(pfs)) {
-            log_warning("get file recovery info failed: %s", pf_strerror(pfs));
-            ret = -EACCES;
-            goto out;
-        }
-
-        if (recovery_needed) {
-            if (!enc->enable_recovery) {
-                log_warning("%s: file recovery needed but feature disabled; please consider "
-                            "setting 'enable_recovery = true' for the mount", normpath);
-                ret = -EACCES;
-                goto out;
-            }
-
-            log_debug("%s: starting file recovery", normpath);
-
-            ret = PalRecoverEncryptedFile(pal_handle, recovery_file_pal_handle, node_size);
-            if (ret < 0) {
-                log_warning("PalRecoverEncryptedFile failed: %s", pal_strerror(ret));
-                ret = pal_to_unix_errno(ret);
-                goto out;
-            }
-
-            pfs = pf_get_recovery_info(pf, &recovery_needed, /*out_node_size=*/NULL);
-            if (PF_FAILURE(pfs)) {
-                log_warning("get file recovery info: %s", pf_strerror(pfs));
-                ret = -EACCES;
-                goto out;
-            }
-
-            if (recovery_needed) {
-                log_warning("%s: file recovery attempted but failed", normpath);
-                ret = -EACCES;
-                goto out;
-            }
-
-            log_debug("%s: file recovery completed", normpath);
-        }
-    }
-
     enc->pf = pf;
     enc->pal_handle = pal_handle;
     enc->recovery_file_pal_handle = recovery_file_pal_handle;

pal/include/host/linux-common/linux_utils.h

@@ -55,5 +55,3 @@ void file_attrcopy(PAL_STREAM_ATTR* attr, struct stat* stat);
 int create_reserved_mem_ranges_fd(void* reserved_mem_ranges, size_t reserved_mem_ranges_size);
 
 void probe_stack(size_t pages_count);
-
-int recover_encrypted_file(int file_fd, int recovery_file_fd, size_t node_size);

pal/include/pal/pal.h

@@ -1043,16 +1043,4 @@ void PalGetLazyCommitPages(uintptr_t addr, size_t size, uint8_t* bitvector);
  */
 int PalFreeThenLazyReallocCommittedPages(void* addr, size_t size);
 
-/*!
- * \brief Recover an encrypted file.
- *
- * \param handle                Handle to the file.
- * \param recovery_file_handle  Handle to the recovery file.
- * \param node_size             Size of the pf node.
- *
- * \returns 0 on success, negative error code on failure.
- */
-int PalRecoverEncryptedFile(PAL_HANDLE file_handle, PAL_HANDLE recovery_file_handle,
-                            size_t node_size);
-
 #undef INSIDE_PAL_H

pal/include/pal_internal.h

@@ -318,6 +318,3 @@ extern int (*g_mem_bkeep_get_vma_info_upcall)(uintptr_t addr, pal_prot_flags_t*
 
 void _PalGetLazyCommitPages(uintptr_t addr, size_t size, uint8_t* bitvector);
 int _PalFreeThenLazyReallocCommittedPages(void* addr, uint64_t size);
-
-int _PalRecoverEncryptedFile(PAL_HANDLE file_handle, PAL_HANDLE recovery_file_handle,
-                             size_t node_size);

pal/src/host/linux-common/file_utils.c

@@ -168,71 +168,3 @@ out:;
 
     return ret;
 }
-
-/* The recovery file is in the format of pairs of 8-byte offsets (physical node numbers) and
- * encrypted file nodes. This function reads each pair from the recovery file and applies the
- * encrypted file nodes to the corresponding offsets in the main file. */
-int recover_encrypted_file(int file_fd, int recovery_file_fd, size_t node_size) {
-    long ret;
-    size_t recovery_node_size = sizeof(uint64_t) + node_size;
-    char* recovery_node = NULL;
-
-    ret = DO_SYSCALL(lseek, recovery_file_fd, 0, SEEK_END);
-    if (ret < 0) {
-        log_error("lseek failed: %s", unix_strerror(ret));
-        goto out;
-    }
-
-    if (ret == 0 || ret % recovery_node_size != 0) {
-        log_error("recovery file size is not right [%lu]", ret);
-        ret = -EINVAL;
-        goto out;
-    }
-
-    size_t nodes_count = ret / recovery_node_size;
-
-    ret = DO_SYSCALL(lseek, recovery_file_fd, 0, SEEK_SET);
-    if (ret < 0) {
-        log_error("lseek failed: %s", unix_strerror(ret));
-        goto out;
-    }
-
-    recovery_node = malloc(recovery_node_size);
-    if (!recovery_node) {
-        ret = -ENOMEM;
-        goto out;
-    }
-
-    for (size_t i = 0; i < nodes_count; i++) {
-        ret = read_all(recovery_file_fd, recovery_node, recovery_node_size);
-        if (ret < 0) {
-            log_error("read_all failed: %s", unix_strerror(ret));
-            goto out;
-        }
-
-        size_t offset = 0;
-        memcpy(&offset, recovery_node, sizeof(uint64_t));
-        ret = DO_SYSCALL(lseek, file_fd, offset * node_size, SEEK_SET);
-        if (ret < 0) {
-            log_error("lseek failed: %s", unix_strerror(ret));
-            goto out;
-        }
-
-        ret = write_all(file_fd, recovery_node + sizeof(uint64_t), node_size);
-        if (ret < 0) {
-            log_error("write_all failed: %s", unix_strerror(ret));
-            goto out;
-        }
-    }
-
-    ret = DO_SYSCALL(fsync, file_fd);
-    if (ret < 0) {
-        log_error("fsync failed: %s", unix_strerror(ret));
-        goto out;
-    }
-
-    ret = 0;
-out:
-    free(recovery_node);
-    return (int)ret;
-}