Parallelize library discovery phase in compile

Author: cmaglie

internal/arduino/builder/internal/detector/cache.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
 	"github.com/arduino/go-paths-helper"
 )
 
@@ -28,17 +29,18 @@ type detectorCache struct {
 }
 
 type detectorCacheEntry struct {
-	AddedIncludePath *paths.Path `json:"added_include_path,omitempty"`
-	Compile          *sourceFile `json:"compile,omitempty"`
-	MissingIncludeH  *string     `json:"missing_include_h,omitempty"`
+	AddedIncludePath *paths.Path  `json:"added_include_path,omitempty"`
+	Compile          *sourceFile  `json:"compile,omitempty"`
+	CompileTask      *runner.Task `json:"compile_task,omitempty"`
+	MissingIncludeH  *string      `json:"missing_include_h,omitempty"`
 }
 
 func (e *detectorCacheEntry) String() string {
 	if e.AddedIncludePath != nil {
 		return "Added include path: " + e.AddedIncludePath.String()
 	}
-	if e.Compile != nil {
-		return "Compiling: " + e.Compile.String()
+	if e.Compile != nil && e.CompileTask != nil {
+		return "Compiling: " + e.Compile.String() + " / " + e.CompileTask.String()
 	}
 	if e.MissingIncludeH != nil {
 		if *e.MissingIncludeH == "" {
@@ -109,6 +111,14 @@ func (c *detectorCache) Peek() *detectorCacheEntry {
 	return nil
 }
 
+// EntriesAhead returns the entries that are ahead of the current cache position.
+func (c *detectorCache) EntriesAhead() []*detectorCacheEntry {
+	if c.curr < len(c.entries) {
+		return c.entries[c.curr:]
+	}
+	return nil
+}
+
 // Save writes the current cache to the given file.
 func (c *detectorCache) Save(cacheFile *paths.Path) error {
 	// Cut off the cache if it is not fully consumed

internal/arduino/builder/internal/detector/detector.go

@@ -58,6 +58,7 @@ type SketchLibrariesDetector struct {
 	includeFolders                paths.PathList
 	logger                        *logger.BuilderLogger
 	diagnosticStore               *diagnostics.Store
+	preRunner                     *runner.Runner
 }
 
 // NewSketchLibrariesDetector todo
@@ -236,6 +237,18 @@ func (l *SketchLibrariesDetector) findIncludes(
 		l.logger.Warn(i18n.Tr("Failed to load library discovery cache: %[1]s", err))
 	}
 
+	// Pre-run cache entries
+	l.preRunner = runner.New(ctx)
+	for _, entry := range l.cache.EntriesAhead() {
+		if entry.Compile != nil && entry.CompileTask != nil {
+			upToDate, _ := entry.Compile.ObjFileIsUpToDate()
+			if !upToDate {
+				l.preRunner.Enqueue(entry.CompileTask)
+			}
+		}
+	}
+	defer l.preRunner.Cancel()
+
 	l.addIncludeFolder(buildCorePath)
 	if buildVariantPath != nil {
 		l.addIncludeFolder(buildVariantPath)
@@ -263,6 +276,15 @@ func (l *SketchLibrariesDetector) findIncludes(
 				cachePath.Remove()
 				return err
 			}
+
+			// Create a new pre-runner if the previous one was cancelled
+			if l.preRunner == nil {
+				l.preRunner = runner.New(ctx)
+				// Push in the remainder of the queue
+				for _, sourceFile := range *sourceFileQueue {
+					l.preRunner.Enqueue(l.gccPreprocessTask(sourceFile, buildProperties))
+				}
+			}
 		}
 
 		// Finalize the cache
@@ -326,9 +348,9 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 
 	first := true
 	for {
-		l.cache.Expect(&detectorCacheEntry{Compile: sourceFile})
-
 		preprocTask := l.gccPreprocessTask(sourceFile, buildProperties)
+		l.cache.Expect(&detectorCacheEntry{Compile: sourceFile, CompileTask: preprocTask})
+
 		var preprocErr error
 		var preprocResult *runner.Result
 
@@ -340,8 +362,27 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 			}
 			first = false
 		} else {
-			preprocResult = preprocTask.Run(ctx)
-			preprocErr = preprocResult.Error
+			if l.preRunner != nil {
+				if r := l.preRunner.Results(preprocTask); r != nil {
+					preprocResult = r
+					preprocErr = preprocResult.Error
+				}
+			}
+			if preprocResult == nil {
+				// The pre-runner missed this task, maybe the cache is outdated
+				// or maybe the source code changed.
+
+				// Stop the pre-runner
+				if l.preRunner != nil {
+					preRunner := l.preRunner
+					l.preRunner = nil
+					go preRunner.Cancel()
+				}
+
+				// Run the actual preprocessor
+				preprocResult = preprocTask.Run(ctx)
+				preprocErr = preprocResult.Error
+			}
 			if l.logger.Verbose() {
 				l.logger.WriteStdout(preprocResult.Stdout)
 			}