Merge pull request #316 from hsiangkao/turbooci_erofs

EROFS TurboOCI enhancement

pkg/snapshot/overlay.go

@@ -21,8 +21,10 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 	"unsafe"
@@ -119,8 +121,8 @@ func DefaultBootConfig() *BootConfig {
 		RootfsQuota:       "",
 		Tenant:            -1,
 		TurboFsType: []string{
-			"ext4",
 			"erofs",
+			"ext4",
 		},
 	}
 }
@@ -606,9 +608,9 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind,
 				if isTurboOCI, _, _ := o.checkTurboOCI(obdInfo.Labels); isTurboOCI {
 					_, fsType = o.turboOCIFsMeta(obdID)
 				} else {
+					log.G(ctx).Warnf("cannot get fs type from label, %v, using %s", obdInfo.Labels, fsType)
 					fsType = o.defaultFsType
 				}
-				log.G(ctx).Warnf("cannot get fs type from label, %v, using %s", obdInfo.Labels, fsType)
 			}
 			log.G(ctx).Debugf("attachAndMountBlockDevice (obdID: %s, writeType: %s, fsType %s, targetPath: %s)",
 				obdID, writeType, fsType, o.overlaybdTargetPath(obdID))
@@ -1471,16 +1473,31 @@ func (o *snapshotter) blockPath(id string) string {
 	return filepath.Join(o.root, "snapshots", id, "block")
 }
 
+var erofsSupported = false
+var erofsSupportedOnce sync.Once
+
+// If EROFS fsmeta exists and is prioritized, check and modprobe erofs
 func IsErofsSupported() bool {
-	fs, err := os.ReadFile("/proc/filesystems")
-	if err != nil || !bytes.Contains(fs, []byte("\terofs\n")) {
-		return false
-	}
-	return true
+	erofsSupportedOnce.Do(func() {
+		fs, err := os.ReadFile("/proc/filesystems")
+		if err != nil || !bytes.Contains(fs, []byte("\terofs\n")) {
+			// Try to `modprobe erofs` again
+			cmd := exec.Command("modprobe", "erofs")
+			_, err = cmd.CombinedOutput()
+			if err != nil {
+				return
+			}
+			fs, err = os.ReadFile("/proc/filesystems")
+			if err != nil || !bytes.Contains(fs, []byte("\terofs\n")) {
+				return
+			}
+		}
+		erofsSupported = true
+	})
+	return erofsSupported
 }
 
 func (o *snapshotter) turboOCIFsMeta(id string) (string, string) {
-	// TODO: make the priority order (multi-meta exists) configurable later if needed
 	for _, fsType := range o.turboFsType {
 		fsmeta := filepath.Join(o.root, "snapshots", id, "fs", fsType+".fs.meta")
 		if _, err := os.Stat(fsmeta); err == nil {

pkg/snapshot/storage.go

@@ -66,6 +66,9 @@ const (
 	// param used to restrict tcmu data area size
 	// it is worked by setting max_data_area_mb for devices in configfs.
 	obdMaxDataAreaMB = 4
+
+	// Just in case someone really needs to force to ext4
+	ext4FSFallbackFile = ".TurboOCI_ext4"
 )
 
 type mountMatcherFunc func(fields []string, separatorIndex int) bool
@@ -542,7 +545,15 @@ func (o *snapshotter) constructOverlayBDSpec(ctx context.Context, key string, wr
 
 		configJSON.RepoBlobURL = blobPrefixURL
 		if isTurboOCI, dataDgst, compType := o.checkTurboOCI(info.Labels); isTurboOCI {
-			fsmeta, _ := o.turboOCIFsMeta(id)
+			var fsmeta string
+
+			// If parent layers exist, follow the meta choice from the bottom layer
+			if info.Parent != "" {
+				_, fsmeta = filepath.Split(configJSON.Lowers[0].File)
+				fsmeta = filepath.Join(o.root, "snapshots", id, "fs", fsmeta)
+			} else {
+				fsmeta, _ = o.turboOCIFsMeta(id)
+			}
 			lower := sn.OverlayBDBSConfigLower{
 				Dir: o.upperPath(id),
 				// keep this to support ondemand turboOCI loading.
@@ -651,6 +662,22 @@ func (o *snapshotter) constructOverlayBDSpec(ctx context.Context, key string, wr
 			Data:  o.overlaybdWritableDataPath(id),
 		}
 	}
+
+	if isTurboOCI, _, _ := o.checkTurboOCI(info.Labels); isTurboOCI {
+		// If the fallback file exists, enforce TurboOCI fstype to EXT4
+		ext4FSFallbackPath := filepath.Join(o.root, ext4FSFallbackFile)
+		_, err = os.Stat(ext4FSFallbackPath)
+		if err == nil && configJSON.Lowers[0].File != "" {
+			var newLowers []sn.OverlayBDBSConfigLower
+			log.G(ctx).Infof("fallback to EXT4 since %s exists", ext4FSFallbackPath)
+			for _, l := range configJSON.Lowers {
+				s, _ := filepath.Split(l.File)
+				l.File = filepath.Join(s, "ext4.fs.meta")
+				newLowers = append(newLowers, l)
+			}
+			configJSON.Lowers = newLowers
+		}
+	}
 	configBuffer, _ := json.MarshalIndent(configJSON, "", "  ")
 	log.G(ctx).Infoln(string(configBuffer))
 	return o.atomicWriteOverlaybdTargetConfig(id, &configJSON)