fix: hard-coded denylist fixed to return 550 error to reduce flooding…

…, added parseLog hook to model to speed up querying for users (more real-time)
forwardemail · Feb 27, 2025 · 515ee9b · 515ee9b
1 parent 5ba8958
commit 515ee9b
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 9 deletions.
diff --git a/app/models/logs.js b/app/models/logs.js
@@ -52,6 +52,7 @@ const isEmail = require('#helpers/is-email');
 const logger = require('#helpers/logger');
 const parseAddresses = require('#helpers/parse-addresses');
 const parseRootDomain = require('#helpers/parse-root-domain');
+const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address');
 
 // headers that we store values for
 const KEYWORD_HEADERS = new Set([
@@ -666,6 +667,23 @@ function getQueryHash(log) {
     //
     // if (log?.message) $and.push({ $text: { $search: log.message } });
 
+    //
+    // NOTE: we need to use envelope to make it unique across domains
+    //       (otherwise two domains won't see individual logs for denylisting)
+    //
+    if (
+      log?.meta?.session?.envelope?.rcptTo &&
+      Array.isArray(log.meta.session.envelope.rcptTo) &&
+      log.meta.session.envelope.rcptTo.length > 0
+    ) {
+      for (const rcptTo of log.meta.session.envelope.rcptTo) {
+        if (isEmail(rcptTo?.address))
+          set.add(
+            parseRootDomain(parseHostFromDomainOrAddress(rcptTo.address))
+          );
+      }
+    }
+
     let hasErrorWithUniqueMessage = false;
     for (const errorName of [
       'BSONObjectTooLarge',
@@ -925,7 +943,7 @@ Logs.pre('save', async function (next) {
   // if the log was newly created then we don't want to parse the DNS yet
   // (it runs in background job via bree, otherwise DNS requests would flood API)
   //
-  if (this.isNew) return next();
+  // if (this.isNew) return next();
 
   try {
     await parseLog(this);

diff --git a/helpers/is-denylisted.js b/helpers/is-denylisted.js
@@ -16,14 +16,14 @@ const isAllowlisted = require('#helpers/is-allowlisted');
 const parseRootDomain = require('#helpers/parse-root-domain');
 const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address');
 
-function createDenylistError(val) {
+function createDenylistError(val, code = 421) {
   let str = 'value';
   if (isEmail(val)) str = 'address';
   else if (isIP(val)) str = 'IP';
   else if (isFQDN(val)) str = 'domain';
   return new DenylistError(
     `The ${str} ${val} is denylisted by ${config.urls.web} ; To request removal, you must visit ${config.urls.web}/denylist?q=${val} ;`,
-    421,
+    code,
     val
   );
 }
@@ -37,23 +37,23 @@ async function isDenylisted(value, client, resolver) {
 
   for (const v of value) {
     // if the value was in hard-coded denylist then exit early
-    if (config.denylist.has(v)) throw createDenylistError(v);
+    if (config.denylist.has(v)) throw createDenylistError(v, 550);
 
     // if it was an email address then check domain and root domain (if differs) against hard-coded denylist
     if (isEmail(v)) {
       const domain = parseHostFromDomainOrAddress(v);
-      if (config.denylist.has(domain)) throw createDenylistError(domain);
+      if (config.denylist.has(domain)) throw createDenylistError(domain, 550);
       const root = parseRootDomain(domain);
       if (domain !== root && config.denylist.has(root))
-        throw createDenylistError(root);
+        throw createDenylistError(root, 550);
     }
 
     // if it was a domain name then check root domain against hard-coded denylist if differs
     if (isFQDN(v)) {
-      if (config.denylist.has(v)) throw createDenylistError(v);
+      if (config.denylist.has(v)) throw createDenylistError(v, 550);
       const root = parseRootDomain(v);
       if (v !== root && config.denylist.has(root))
-        throw createDenylistError(root);
+        throw createDenylistError(root, 550);
       //
       // TODO: we need to ensure we're not adding this in `jobs/update-umbrella.js`
       //
@@ -79,7 +79,7 @@ async function isDenylisted(value, client, resolver) {
       );
 
       if (root !== v) {
-        if (config.denylist.has(root)) throw createDenylistError(root);
+        if (config.denylist.has(root)) throw createDenylistError(root, 550);
 
         const isRootDomainAllowlisted = client
           ? // eslint-disable-next-line no-await-in-loop

diff --git a/helpers/logger.js b/helpers/logger.js
@@ -244,6 +244,8 @@ async function hook(err, message, meta) {
           // unique hash (already exists)
           if (err.code === 11000 || err.message === 'Hash is not unique.')
             return;
+          // duplicate denylist log
+          if (err.is_denylist_without_domains) return;
           //
           // NOTE: this allows us to log mongodb timeout issues (e.g. due to slow queries)
           //