Add filesystem cache

.mocharc.json

@@ -0,0 +1,3 @@
+{
+  "ignore": "test/setup-cache.js"
+}

lib/caching-queued-fetch.js

@@ -1,5 +1,4 @@
-const {END_EVENT, ITEM_EVENT, default:RequestQueue} = require('limited-request-queue');
-
+const fsCacheFetch = require('fetch-filecache-for-crawling');
 
 let ghToken = "";
 module.exports.DEFAULT_INTERVAL = 50;
@@ -13,7 +12,7 @@ class Queue {
   maxRetry = 2;
   cache = {};
   originQueue = {};
-  async request(url, options, {verbose}, attempts = 0) {
+  async request(url, options, {verbose, fsCachePath}, attempts = 0) {
     if (!this.cache[url]) {
       if (attempts > this.maxRetry) {
         const error = `HTTP error ${this.cache[url].status} ${this.cache[url].statusText} while fetching ${url} (tried ${attempts} times)`;
@@ -25,7 +24,12 @@ class Queue {
       if (verbose) {
         console.log("fetching " + url);
       }
-      const r = await fetch(url, options);
+      const _f = fsCachePath ? fsCacheFetch : fetch;
+      if (fsCachePath) {
+        options.logToConsole = verbose;
+        options.cacheFolder = fsCachePath;
+      }
+      const r = await _f(url, options);
       this.cache[url] = { status: r.status, statusText: r.statusText, headers: r.headers, body: await r.text()};
     }
     if (this.cache[url].status >= 400) {
@@ -44,6 +48,9 @@ class Queue {
         );
       }
       if (retryAfter) {
+        if (verbose) {
+          console.log(`Asked to retry fetching ${url} in ${retryAfter}s`);
+        }
         delete this.cache[url];
         await wait(retryAfter);
         await this.request(url, options, { verbose }, attempts++);
@@ -70,6 +77,10 @@ class Queue {
         })
         .then(async (ret) => {
           res(ret);
+          // no need to wait if we're not hitting the network
+          if (ret.headers?.get("cache-status")?.match(/fetch-filecache-for-crawling; hit/)) {
+            return;
+          }
           return wait(queueOptions.interval);
         }).catch(rej);
     });

monitor.js

@@ -18,7 +18,7 @@ const authedFetch = (url) => {
     // Roughly matching github API rate limit of 5000 requests per hour
     interval = 750;
   }
-  return queuedFetch(url, { headers }, { interval, verbose: true });
+  return queuedFetch(url, { headers }, { interval, verbose: true, fsCachePath: ".cache" });
 };