Detect potential leaks in dart pub publish. (#3049)

Detect potential leaks in `dart pub publish`.

Inspired by the paper [How Bad Can It Git?][1] the
`LeakDetectionValidator` uses regular expressions and entropy thresholds
to scan files for possible secrets about to be published. If files about
to be published matches any of the patterns for known secret keys an
error is thrown and the publishing event is aborted.

To ignore a _false-positive_ a `false_secrets` field is introduced in
`pubspec.yaml`. This field is a list of git-ignore style patterns for
files to ignore when scanning for potential leaks. This allows package
authors to explicitly ignore files known to contain _false positives_.

This commit includes API keys and secrets from various systems, these
have all been revoked immediately following creation and included for
testing and entropy threshold computation.

[1]: https://www.ndss-symposium.org/wp-content/uploads/2019/02/ndss2019_04B-3_Meli_paper.pdf

Author: jonasfj

.gitallowed

@@ -0,0 +1,7 @@
+# Allow file for git-secrets, see:
+# https://github.com/awslabs/git-secrets#ignoring-false-positives
+
+# Ignore secrets from leak detection, these secrets have all been revoked and
+# are only used for testing the detection logic.
+lib/src/validator/leak_detection.dart
+test/validator/leak_detection_test.dart

lib/src/ignore.dart

@@ -150,16 +150,18 @@ class Ignore {
         path.endsWith('/') ? path.substring(0, path.length - 1) : path;
     return listFiles(
       beneath: pathWithoutSlash,
-      includeDirs: true,
+      includeDirs: true, // because we are listing below pathWithoutSlash
       listDir: (dir) {
         // List the next part of path:
         if (dir == pathWithoutSlash) return [];
         final startOfNext = dir.isEmpty ? 0 : dir.length + 1;
         final nextSlash = path.indexOf('/', startOfNext);
         return [path.substring(startOfNext, nextSlash)];
       },
-      ignoreForDir: (dir) => dir == '' ? this : null,
+      ignoreForDir: (dir) => dir == '.' || dir.isEmpty ? this : null,
       isDir: (candidate) =>
+          candidate == '.' ||
+          candidate.isEmpty ||
           path.length > candidate.length && path[candidate.length] == '/',
     ).isEmpty;
   }

lib/src/pubspec.dart

@@ -388,6 +388,43 @@ class Pubspec {
   bool _parsedPublishTo = false;
   String _publishTo;
 
+  /// The list of patterns covering _false-positive secrets_ in the package.
+  ///
+  /// This is a list of git-ignore style patterns for files that should be
+  /// ignored when trying to detect possible leaks of secrets during
+  /// package publication.
+  List<String> get falseSecrets {
+    if (_falseSecrets == null) {
+      final falseSecrets = <String>[];
+
+      // Throws a [PubspecException]
+      void _falseSecretsError(SourceSpan span) => _error(
+            '"false_secrets" field must be a list of git-ignore style patterns',
+            span,
+          );
+
+      final falseSecretsNode = fields.nodes['false_secrets'];
+      if (falseSecretsNode != null) {
+        if (falseSecretsNode is YamlList) {
+          for (final node in falseSecretsNode.nodes) {
+            final value = node.value;
+            if (value is! String) {
+              _falseSecretsError(node.span);
+            }
+            falseSecrets.add(value);
+          }
+        } else {
+          _falseSecretsError(falseSecretsNode.span);
+        }
+      }
+
+      _falseSecrets = List.unmodifiable(falseSecrets);
+    }
+    return _falseSecrets;
+  }
+
+  List<String> _falseSecrets;
+
   /// The executables that should be placed on the user's PATH when this
   /// package is globally activated.
   ///
@@ -581,6 +618,7 @@ class Pubspec {
     _collectError(() => publishTo);
     _collectError(() => features);
     _collectError(() => executables);
+    _collectError(() => falseSecrets);
     _collectError(_ensureEnvironment);
     return errors;
   }

lib/src/validator.dart

@@ -23,6 +23,7 @@ import 'validator/flutter_constraint.dart';
 import 'validator/flutter_plugin_format.dart';
 import 'validator/gitignore.dart';
 import 'validator/language_version.dart';
+import 'validator/leak_detection.dart';
 import 'validator/license.dart';
 import 'validator/name.dart';
 import 'validator/null_safety_mixed_mode.dart';
@@ -145,7 +146,8 @@ abstract class Validator {
       LanguageVersionValidator(entrypoint),
       RelativeVersionNumberingValidator(entrypoint, serverUrl),
       NullSafetyMixedModeValidator(entrypoint),
-      PubspecTypoValidator(entrypoint)
+      PubspecTypoValidator(entrypoint),
+      LeakDetectionValidator(entrypoint),
     ];
     if (packageSize != null) {
       validators.add(SizeValidator(entrypoint, packageSize));