Fix Contributors List (ionic-team#952)

* very WIP

* grabbing only the data we need

* completing the contributor processing

* ignore future changes to commits data

* commenting out unecessary verbose task

* removing unecssary data

* adding the ignore back in

Author: perrygovier

.gitignore

@@ -8,5 +8,6 @@ src/l10n/pages/**/*.json
 src/components/menu/data/
 src/components/page/data/
 src/components/search/data/
+scripts/data/github-commits.json
 .idea
 /.env

package-lock.json

@@ -28,6 +28,7 @@
     "express": "^4.16.4",
     "helmet": "^3.16.0",
     "ionicons": "^4.6.1",
+    "simple-git": "^1.126.0",
     "throng": "^4.0.0"
   },
   "devDependencies": {

package.json

@@ -5,6 +5,11 @@ import buildData from '../build-data';
 
 const tasks = new Listr({ collapse: false });
 
+tasks.add({
+  title: 'Data',
+  task: () => buildData
+});
+
 tasks.add({
   title: 'Pages',
   task: () => buildPages
@@ -15,11 +20,6 @@ tasks.add({
   task: () => buildMenus
 });
 
-tasks.add({
-  title: 'Data',
-  task: () => buildData
-});
-
 tasks.run().catch(err => {
   console.error(err);
   process.exit(1);

scripts/build-all/index.ts

@@ -0,0 +1,56 @@
+import dotenv from 'dotenv';
+import { resolve } from 'path';
+import fetch from 'node-fetch';
+import url from 'url';
+import { outputJson } from 'fs-extra';
+
+dotenv.config();
+
+const OUTPUT_PATH = resolve(
+  __dirname,
+  '../data/github-commits.json'
+);
+
+export default {
+  title: 'Getting a list of past commits',
+  task: async (_, task) => {
+    const History = await getAllGHCommits(task);
+    if (Object.keys(History).length > 0) {
+      outputJson(OUTPUT_PATH, History, { spaces: 2 });
+    }
+  }
+};
+
+async function getAllGHCommits(task, page = 1) {
+  try {
+    task.output = `Getting commits: ${page * 100 - 99} - ${page * 100}`;
+    const request = await fetch(url.format({
+      protocol: 'https',
+      hostname: 'api.github.com',
+      pathname: 'repos/ionic-team/ionic-docs/commits',
+      query: {
+        access_token: process.env.GITHUB_TOKEN,
+        per_page: 100,
+        page
+      }
+    }));
+
+    let commits = await request.json().then(list => list.reduce((obj, commit) => {
+      obj[commit.sha] = {
+        id: commit.author.login,
+        avatar: commit.author.avatar_url,
+        time: commit.commit.committer.date
+      };
+      return obj;
+    }, {}));
+
+    // recursively get more commits if there are more to get, limit 2k
+    if (Object.keys(commits).length === 100 && page < 20) {
+      commits = { ...commits, ...await getAllGHCommits(task, page + 1) };
+    }
+
+    return commits;
+  } catch {
+    return {};
+  }
+}

scripts/build-data/file-contributors.ts

@@ -2,12 +2,16 @@ import Listr from 'listr';
 import buildApiReference from './api-reference';
 import buildReleaseNotes from './release-notes';
 import buildSearchIndex from './search-index';
+import buildContributors from './file-contributors';
 
 const tasks = new Listr([
   buildApiReference,
+  buildContributors,
   buildReleaseNotes,
-  buildSearchIndex
-]);
+  buildSearchIndex,
+],
+  // { renderer: 'verbose' }
+);
 
 export default tasks;
 

scripts/build-data/index.ts

@@ -9,7 +9,9 @@ import CLI from './page-types/cli';
 import Native from './page-types/native';
 import { convertHtmlToHypertextData } from './html-to-hypertext-data';
 
-const tasks = new Listr();
+const tasks = new Listr(
+  // { renderer: 'verbose' }
+);
 tasks.add(Static);
 tasks.add(API);
 tasks.add(CLI);
@@ -126,7 +128,9 @@ export function updatePageHtmlToHypertext(page: Page) {
 }
 
 function writePage(page: Page): Promise<any> {
-  if (listrStatus) listrStatus.output = `Writing Page: ${page.path}`;
+  if (listrStatus && listrStatus._task && listrStatus._task.output !== 'Writing Pages') {
+    listrStatus.output = 'Writing Pages';
+  }
   return fs.outputJson(toFilePath(page.path), page, {
     spaces: 2
   });

scripts/build-pages/index.ts

@@ -11,6 +11,11 @@ import glob from 'fast-glob';
 import fetch from 'node-fetch';
 import frontMatter from 'front-matter';
 import markdownRenderer from '../markdown-renderer';
+import simplegit from 'simple-git/promise';
+
+// ingored by git
+// generated in build-data/file-contrbutors.ts by build-data npm task
+import * as GITHUB_COMMITS from '../../data/github-commits.json';
 
 export default {
   title: 'Build static pages',
@@ -54,24 +59,10 @@ const readMarkdown = (path: string): Promise<string> =>
   });
 
 const getGitHubData = async (filePath: string) => {
-  const [, path] = /^.+\/(src\/(l10n\/)?pages\/.+\.md)$/.exec(filePath);
-  const since = new Date('2019-01-23').toISOString();
+  const [, path] = /^.+\/(src\/pages\/.+\.md)$/.exec(filePath);
 
   try {
-    const request = await fetch(url.format({
-      protocol: 'https',
-      hostname: 'api.github.com',
-      pathname: 'repos/ionic-team/ionic-docs/commits',
-      query: {
-        access_token: process.env.GITHUB_TOKEN,
-        since,
-        path
-      }
-    }));
-
-    const commits = await request.json();
-    const contributors = Array.from(new Set(commits.map(commit => commit.author.login)));
-    const lastUpdated = commits.length ? commits[0].commit.author.date : since;
+    const { contributors, lastUpdated } = await getFileContributors(filePath);
     return {
       path,
       contributors,
@@ -81,7 +72,19 @@ const getGitHubData = async (filePath: string) => {
     return {
       path,
       contributors: [],
-      lastUpdated: since
+      lastUpdated: new Date('2019-01-23').toISOString()
     };
   }
 };
+
+async function getFileContributors(filename) {
+  return simplegit().log({ file: filename }).then(status => ({
+      contributors: Array.from(new Set(status.all.map(commit =>
+        // only add the user ID if we can find it based on the commit hash
+        GITHUB_COMMITS[commit.hash] ? GITHUB_COMMITS[commit.hash].id : null
+      // filter out null users
+      ).filter(user => !!user))),
+      lastUpdated: status.latest.date
+    })
+  );
+}