disable self-contained and file protocol templates as default (projec…

…tdiscovery#5825) * disable self-contained and file protocol templates as default * make excluding default * add config funcs * fix wrn display * fix integration tests * enable self-contained templates when code templates are enabled --------- Co-authored-by: sandeep <[email protected]>
alban-stourbe-wmx · Nov 19, 2024 · 63687c2 · 63687c2
1 parent abfd432
commit 63687c2
Show file tree

Hide file tree

Showing 15 changed files with 62 additions and 9 deletions.
diff --git a/cmd/integration-test/code.go b/cmd/integration-test/code.go
@@ -99,7 +99,7 @@ type codePreCondition struct{}
 
 // Execute executes a test case and returns an error if occurred
 func (h *codePreCondition) Execute(filePath string) error {
-	results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code")
+	results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code", "-esc")
 	if err != nil {
 		return err
 	}

diff --git a/cmd/integration-test/file.go b/cmd/integration-test/file.go
@@ -15,7 +15,7 @@ type fileWithOrMatcher struct{}
 
 // Execute executes a test case and returns an error if occurred
 func (h *fileWithOrMatcher) Execute(filePath string) error {
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug, "-file")
 	if err != nil {
 		return err
 	}
@@ -27,7 +27,7 @@ type fileWithAndMatcher struct{}
 
 // Execute executes a test case and returns an error if occurred
 func (h *fileWithAndMatcher) Execute(filePath string) error {
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug, "-file")
 	if err != nil {
 		return err
 	}
@@ -39,7 +39,7 @@ type fileWithExtractor struct{}
 
 // Execute executes a test case and returns an error if occurred
 func (h *fileWithExtractor) Execute(filePath string) error {
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug, "-file")
 	if err != nil {
 		return err
 	}

diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go
@@ -952,7 +952,7 @@ func (h *httpRequestSelfContained) Execute(filePath string) error {
 	}()
 	defer server.Close()
 
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-esc")
 	if err != nil {
 		return err
 	}
@@ -988,7 +988,7 @@ func (h *httpRequestSelfContainedWithParams) Execute(filePath string) error {
 	}()
 	defer server.Close()
 
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-esc")
 	if err != nil {
 		return err
 	}
@@ -1031,7 +1031,7 @@ func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error {
 	}
 	defer FileLoc.Close()
 
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-V", "test="+FileLoc.Name())
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-V", "test="+FileLoc.Name(), "-esc")
 	if err != nil {
 		return err
 	}

diff --git a/cmd/integration-test/network.go b/cmd/integration-test/network.go
@@ -119,7 +119,7 @@ func (h *networkRequestSelContained) Execute(filePath string) error {
 		_, _ = conn.Write([]byte("Authentication successful"))
 	})
 	defer ts.Close()
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-esc")
 	if err != nil {
 		return err
 	}

diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
@@ -263,6 +263,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&options.SignTemplates, "sign", false, "signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable"),
 		flagSet.BoolVar(&options.EnableCodeTemplates, "code", false, "enable loading code protocol-based templates"),
 		flagSet.BoolVarP(&options.DisableUnsignedTemplates, "disable-unsigned-templates", "dut", false, "disable running unsigned templates or templates with mismatched signature"),
+		flagSet.BoolVarP(&options.EnableSelfContainedTemplates, "enable-self-contained", "esc", false, "enable loading self-contained templates"),
+		flagSet.BoolVar(&options.EnableFileTemplates, "file", false, "enable loading file templates"),
 	)
 
 	flagSet.CreateGroup("filters", "Filtering",
@@ -492,6 +494,11 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 		options.DAST = true
 	}
 
+	// All cloud-based templates depend on both code and self-contained templates.
+	if options.EnableCodeTemplates {
+		options.EnableSelfContainedTemplates = true
+	}
+
 	// api key hierarchy: cli flag > env var > .pdcp/credential file
 	if pdcpauth == "true" {
 		runner.AuthWithPDCP()

diff --git a/cmd/tmc/main.go b/cmd/tmc/main.go
@@ -69,6 +69,7 @@ func init() {
 	// need to set headless to true for headless templates
 	defaultOpts.Headless = true
 	defaultOpts.EnableCodeTemplates = true
+	defaultOpts.EnableSelfContainedTemplates = true
 	if err := protocolstate.Init(defaultOpts); err != nil {
 		gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
 	}

diff --git a/internal/runner/runner.go b/internal/runner/runner.go
@@ -730,6 +730,8 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 		stats.ForceDisplayWarning(templates.ExcludedCodeTmplStats)
 		stats.ForceDisplayWarning(templates.ExludedDastTmplStats)
 		stats.ForceDisplayWarning(templates.TemplatesExcludedStats)
+		stats.ForceDisplayWarning(templates.ExcludedFileStats)
+		stats.ForceDisplayWarning(templates.ExcludedSelfContainedStats)
 	}
 
 	if tmplCount == 0 && workflowCount == 0 {

diff --git a/lib/config.go b/lib/config.go
@@ -380,6 +380,23 @@ func WithSandboxOptions(allowLocalFileAccess bool, restrictLocalNetworkAccess bo
 func EnableCodeTemplates() NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
 		e.opts.EnableCodeTemplates = true
+		e.opts.EnableSelfContainedTemplates = true
+		return nil
+	}
+}
+
+// EnableSelfContainedTemplates allows loading/executing self-contained templates
+func EnableSelfContainedTemplates() NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.opts.EnableSelfContainedTemplates = true
+		return nil
+	}
+}
+
+// EnableFileTemplates allows loading/executing file protocol templates
+func EnableFileTemplates() NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.opts.EnableFileTemplates = true
 		return nil
 	}
 }

diff --git a/lib/sdk_private.go b/lib/sdk_private.go
@@ -3,11 +3,12 @@ package nuclei
 import (
 	"context"
 	"fmt"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/input"
+
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"

diff --git a/lib/tests/sdk_test.go b/lib/tests/sdk_test.go
@@ -144,6 +144,7 @@ func TestWithVarsNuclei(t *testing.T) {
 		}()
 		ne, err := nuclei.NewNucleiEngineCtx(
 			context.TODO(),
+			nuclei.EnableSelfContainedTemplates(),
 			nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{Templates: []string{"http/token-spray/api-1forge.yaml"}}),
 			nuclei.WithVars([]string{"token=foobar"}),
 			nuclei.WithVerbosity(nuclei.VerbosityOptions{Debug: true}),

diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go
@@ -489,6 +489,17 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
 						stats.Increment(templates.SkippedUnsignedStats)
 						return
 					}
+
+					if parsed.SelfContained && !store.config.ExecutorOptions.Options.EnableSelfContainedTemplates {
+						stats.Increment(templates.ExcludedSelfContainedStats)
+						return
+					}
+
+					if parsed.HasFileProtocol() && !store.config.ExecutorOptions.Options.EnableFileTemplates {
+						stats.Increment(templates.ExcludedFileStats)
+						return
+					}
+
 					// if template has request signature like aws then only signed and verified templates are allowed
 					if parsed.UsesRequestSignature() && !parsed.Verified {
 						stats.Increment(templates.SkippedRequestSignatureStats)

diff --git a/pkg/templates/parser_stats.go b/pkg/templates/parser_stats.go
@@ -10,5 +10,7 @@ const (
 	ExcludedCodeTmplStats        = "code-flag-missing-warnings"
 	ExludedDastTmplStats         = "fuzz-flag-missing-warnings"
 	SkippedUnsignedStats         = "skipped-unsigned-stats" // tracks loading of unsigned templates
+	ExcludedSelfContainedStats   = "excluded-self-contained-stats"
+	ExcludedFileStats            = "excluded-file-stats"
 	SkippedRequestSignatureStats = "skipped-request-signature-stats"
 )
diff --git a/pkg/templates/stats.go b/pkg/templates/stats.go
@@ -9,6 +9,8 @@ func init() {
 	stats.NewEntry(SkippedCodeTmplTamperedStats, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)")
 	stats.NewEntry(ExcludedHeadlessTmplStats, "Excluded %d headless template[s] (disabled as default), use -headless option to run headless templates.")
 	stats.NewEntry(ExcludedCodeTmplStats, "Excluded %d code template[s] (disabled as default), use -code option to run code templates.")
+	stats.NewEntry(ExcludedSelfContainedStats, "Excluded %d self-contained template[s] (disabled as default), use -esc option to run self-contained templates.")
+	stats.NewEntry(ExcludedFileStats, "Excluded %d file template[s] (disabled as default), use -file option to run file templates.")
 	stats.NewEntry(TemplatesExcludedStats, "Excluded %d template[s] with known weak matchers / tags excluded from default run using .nuclei-ignore")
 	stats.NewEntry(ExludedDastTmplStats, "Excluded %d dast template[s] (disabled as default), use -dast option to run dast templates.")
 	stats.NewEntry(SkippedUnsignedStats, "Skipping %d unsigned template[s]")

diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go
@@ -555,3 +555,8 @@ func (template *Template) UnmarshalJSON(data []byte) error {
 	}
 	return nil
 }
+
+// HasFileProtocol returns true if the template has a file protocol section
+func (template *Template) HasFileProtocol() bool {
+	return len(template.RequestsFile) > 0
+}
diff --git a/pkg/types/types.go b/pkg/types/types.go
@@ -383,6 +383,10 @@ type Options struct {
 	EnableCodeTemplates bool
 	// DisableUnsignedTemplates disables processing of unsigned templates
 	DisableUnsignedTemplates bool
+	// EnableSelfContainedTemplates disables processing of self-contained templates
+	EnableSelfContainedTemplates bool
+	// EnableFileTemplates enables file templates
+	EnableFileTemplates bool
 	// Disables cloud upload
 	EnableCloudUpload bool
 	// ScanID is the scan ID to use for cloud upload