diff --git a/client-admin/src/components/conversation-admin/comment-moderation/comment.js b/client-admin/src/components/conversation-admin/comment-moderation/comment.js index af51b377d..ec9a5e478 100644 --- a/client-admin/src/components/conversation-admin/comment-moderation/comment.js +++ b/client-admin/src/components/conversation-admin/comment-moderation/comment.js @@ -27,6 +27,7 @@ class Comment extends React.Component { return ( + {this.props.comment.active ? null : 'Comment flagged as toxic by Jigsaw Perspective API. Comment not shown to participants. Accept to override.'} {this.props.comment.txt} * We use Amazon Web Services Simple Email Service for sending account confirmation and notification emails to users. We also use AWS Simple Storage Service for. Information about how AWS handles: * We use Google Translate for automatic machine translation of comments, and Google Analytics for site usage statistics. Information about how Google uses this data when you use our Site can be found at: +* We use Jigsaw Perspective API for comment moderation. This service analyzes the content of comments to detect potentially toxic or inappropriate language. Information about how Perspective API handles data can be found at: * We pass all Site web requests through a Cloudflare caching proxy. No personal information is cached at this level. Information about how Cloudflare uses your data can be found at: ## Public Areas and Syndicated Services diff --git a/client-admin/src/content/tos.md b/client-admin/src/content/tos.md index cab7a6ef0..f4546a0b0 100644 --- a/client-admin/src/content/tos.md +++ b/client-admin/src/content/tos.md @@ -52,6 +52,7 @@ These Terms of Use are a legally binding contract between you and The Computatio 5. **User Content Disclaimer.**   We are under no obligation to edit or control User Content that you or other users post or publish, and will not be in any way responsible or liable for User Content. The Computational Democracy Project may, however, at any time and without prior notice, screen, remove, edit, or block any User Content that in our sole judgment violates these Terms or is otherwise objectionable. + We may use automated systems, including but not limited to the Jigsaw Perspective API, to moderate User Content for potentially toxic or inappropriate language. You understand that when using the Service you will be exposed to User Content from a variety of sources and acknowledge that User Content may be inaccurate, offensive, indecent or objectionable. You agree to waive, and hereby do waive, any legal or equitable rights or remedies you have or may have against The Computational Democracy Project with respect to User Content. We expressly disclaim any and all liability in connection with User Content. diff --git a/client-report/src/components/app.js b/client-report/src/components/app.js index 2617aa73c..44f2ca227 100644 --- a/client-report/src/components/app.js +++ b/client-report/src/components/app.js @@ -16,7 +16,7 @@ import Footer from "./framework/Footer"; import Overview from "./overview"; import MajorityStrict from "./lists/majorityStrict"; import Uncertainty from "./lists/uncertainty"; -import AllCommentsModeratedIn from "./lists/allCommentsModeratedIn" +import AllCommentsModeratedIn from "./lists/allCommentsModeratedIn"; import ParticipantGroups from "./lists/participantGroups"; // import CommentsGraph from "./commentsGraph/commentsGraph"; import ParticipantsGraph from "./participantsGraph/participantsGraph"; @@ -24,14 +24,13 @@ import ParticipantsGraph from "./participantsGraph/participantsGraph"; import Beeswarm from "./beeswarm/beeswarm"; import Controls from "./controls/controls"; -import net from "../util/net" +import net from "../util/net"; -import $ from 'jquery'; +import $ from "jquery"; var pathname = window.location.pathname; // "/report/2arcefpshi" var report_id = pathname.split("/")[2]; - function assertExists(obj, key) { if (typeof obj[key] === "undefined") { console.error("assertExists failed. Missing: ", key); @@ -39,7 +38,6 @@ function assertExists(obj, key) { } class App extends React.Component { - constructor(props) { super(props); this.state = { @@ -52,7 +50,7 @@ class App extends React.Component { colorBlindMode: false, dimensions: { width: window.innerWidth, - height: window.innerHeight + height: window.innerHeight, }, shouldPoll: false, voteColors: { @@ -64,15 +62,17 @@ class App extends React.Component { } getMath(conversation_id) { - return net.polisGet("/api/v3/math/pca2", { - lastVoteTimestamp: 0, - conversation_id: conversation_id, - }).then((data) => { - if (!data) { - return {}; - } - return data; - }); + return net + .polisGet("/api/v3/math/pca2", { + lastVoteTimestamp: 0, + conversation_id: conversation_id, + }) + .then((data) => { + if (!data) { + return {}; + } + return data; + }); } getComments(conversation_id, isStrictMod) { @@ -98,14 +98,16 @@ class App extends React.Component { }); } getReport(report_id) { - return net.polisGet("/api/v3/reports", { - report_id: report_id, - }).then((reports) => { - if (reports.length) { - return reports[0]; - } - return null; - }); + return net + .polisGet("/api/v3/reports", { + report_id: report_id, + }) + .then((reports) => { + if (reports.length) { + return reports[0]; + } + return null; + }); } getGroupDemographics(conversation_id) { return net.polisGet("/api/v3/group_demographics", { @@ -128,30 +130,40 @@ class App extends React.Component { }); return new Promise((resolve, reject) => { - attemptResponse.then((response) => { - if (response.status && response.status === "pending") { - this.corMatRetries = _.isNumber(this.corMatRetries) ? this.corMatRetries + 1 : 1; - setTimeout(() => { - this.getCorrelationMatrix(math_tick).then(resolve, reject); - }, this.corMatRetries < 10 ? 200 : 3000); // try to get a quick response, but don't keep polling at that rate for more than 10 seconds. - } else if (globals.enableMatrix && response && response.status === "polis_report_needs_comment_selection") { - this.setState({ - errorText: "Select some comments", - }); - reject("Currently, No comments are selected for display in the matrix."); - } else { - resolve(response); + attemptResponse.then( + (response) => { + if (response.status && response.status === "pending") { + this.corMatRetries = _.isNumber(this.corMatRetries) ? this.corMatRetries + 1 : 1; + setTimeout( + () => { + this.getCorrelationMatrix(math_tick).then(resolve, reject); + }, + this.corMatRetries < 10 ? 200 : 3000 + ); // try to get a quick response, but don't keep polling at that rate for more than 10 seconds. + } else if ( + globals.enableMatrix && + response && + response.status === "polis_report_needs_comment_selection" + ) { + this.setState({ + errorText: "Select some comments", + }); + reject("Currently, No comments are selected for display in the matrix."); + } else { + resolve(response); + } + }, + (err) => { + reject(err); } - }, (err) => { - reject(err); - }); + ); }); } getData() { const reportPromise = this.getReport(report_id); // debug initial report data fetch - reportPromise.then((report) => console.log("report received:", report)) + reportPromise.then((report) => console.log("report received:", report)); const mathPromise = reportPromise.then((report) => { return this.getMath(report.conversation_id); }); @@ -164,15 +176,17 @@ class App extends React.Component { return this.getGroupDemographics(report.conversation_id); }); //const conversationStatsPromise = reportPromise.then((report) => { - //return this.getConversationStats(report.conversation_id) + //return this.getConversationStats(report.conversation_id) //}); const participantsOfInterestPromise = reportPromise.then((report) => { return this.getParticipantsOfInterest(report.conversation_id); }); - const matrixPromise = globals.enableMatrix ? mathPromise.then((math) => { - const math_tick = math.math_tick; - return this.getCorrelationMatrix(math_tick); - }) : Promise.resolve(); + const matrixPromise = globals.enableMatrix + ? mathPromise.then((math) => { + const math_tick = math.math_tick; + return this.getCorrelationMatrix(math_tick); + }) + : Promise.resolve(); const conversationPromise = reportPromise.then((report) => { return this.getConversation(report.conversation_id); }); @@ -186,191 +200,197 @@ class App extends React.Component { matrixPromise, conversationPromise, //conversationStatsPromise, - ]).then((a) => { - let [ - report, - mathResult, - comments, - groupDemographics, - participants, - correlationHClust, - conversation, - //conversationstats, - ] = a; - - assertExists(mathResult, "base-clusters"); - assertExists(mathResult, "consensus"); - assertExists(mathResult, "group-aware-consensus"); - assertExists(mathResult, "group-clusters"); - assertExists(mathResult, "group-votes"); - assertExists(mathResult, "n-cmts"); - assertExists(mathResult, "repness"); - assertExists(mathResult, "pca"); - assertExists(mathResult, "tids"); - assertExists(mathResult, "user-vote-counts"); - assertExists(mathResult, "votes-base"); - assertExists(mathResult.pca, "center"); - assertExists(mathResult.pca, "comment-extremity"); - assertExists(mathResult.pca, "comment-projection"); - assertExists(mathResult.pca, "comps"); - - let indexToTid = mathResult.tids; - - // # ptpts that voted - var ptptCountTotal = conversation.participant_count; - - - // # ptpts that voted enough to be included in math - var ptptCount = 0; - _.each(mathResult["group-votes"], (val/*, key*/) => { - ptptCount += val["n-members"]; - }); + ]) + .then((a) => { + let [ + report, + mathResult, + comments, + groupDemographics, + participants, + correlationHClust, + conversation, + //conversationstats, + ] = a; + + assertExists(mathResult, "base-clusters"); + assertExists(mathResult, "consensus"); + assertExists(mathResult, "group-aware-consensus"); + assertExists(mathResult, "group-clusters"); + assertExists(mathResult, "group-votes"); + assertExists(mathResult, "n-cmts"); + assertExists(mathResult, "repness"); + assertExists(mathResult, "pca"); + assertExists(mathResult, "tids"); + assertExists(mathResult, "user-vote-counts"); + assertExists(mathResult, "votes-base"); + assertExists(mathResult.pca, "center"); + assertExists(mathResult.pca, "comment-extremity"); + assertExists(mathResult.pca, "comment-projection"); + assertExists(mathResult.pca, "comps"); + + let indexToTid = mathResult.tids; + + // # ptpts that voted + var ptptCountTotal = conversation.participant_count; + + // # ptpts that voted enough to be included in math + var ptptCount = 0; + _.each(mathResult["group-votes"], (val /*, key*/) => { + ptptCount += val["n-members"]; + }); - var badTids = {}; - var filteredTids = {}; - var filteredProbabilities = {}; - - // prep Correlation matrix. - if (globals.enableMatrix) { - var probabilities = correlationHClust.matrix; - var tids = correlationHClust.comments; - for (let row = 0; row < probabilities.length; row++) { - if (probabilities[row][0] === "NaN") { - let tid = correlationHClust.comments[row]; - badTids[tid] = true; - // console.log("bad", tid); + var badTids = {}; + var filteredTids = {}; + var filteredProbabilities = {}; + + // prep Correlation matrix. + if (globals.enableMatrix) { + var probabilities = correlationHClust.matrix; + var tids = correlationHClust.comments; + for (let row = 0; row < probabilities.length; row++) { + if (probabilities[row][0] === "NaN") { + let tid = correlationHClust.comments[row]; + badTids[tid] = true; + // console.log("bad", tid); + } } - } - filteredProbabilities = probabilities.map((row) => { - return row.filter((cell, colNum) => { - let colTid = correlationHClust.comments[colNum]; - return badTids[colTid] !== true; + filteredProbabilities = probabilities + .map((row) => { + return row.filter((cell, colNum) => { + let colTid = correlationHClust.comments[colNum]; + return badTids[colTid] !== true; + }); + }) + .filter((row, rowNum) => { + let rowTid = correlationHClust.comments[rowNum]; + return badTids[rowTid] !== true; + }); + filteredTids = tids.filter((tid /*, index*/) => { + return badTids[tid] !== true; }); - }).filter((row, rowNum) => { - let rowTid = correlationHClust.comments[rowNum]; - return badTids[rowTid] !== true; - }); - filteredTids = tids.filter((tid/*, index*/) => { - return badTids[tid] !== true; - }); - } + } - var maxTid = -1; - for (let i = 0; i < comments.length; i++) { - if (comments[i].tid > maxTid) { - maxTid = comments[i].tid; + var maxTid = -1; + for (let i = 0; i < comments.length; i++) { + if (comments[i].tid > maxTid) { + maxTid = comments[i].tid; + } } - } - var tidWidth = ("" + maxTid).length + var tidWidth = ("" + maxTid).length; - function pad(n, width, z) { - z = z || '0'; - n = n + ''; - return n.length >= width ? n : new Array(width - n.length + 1).join(z) + n; - } - function formatTid(tid) { - // let padded = "" + tid; - // return '#' + pad(""+tid, tidWidth); - return pad(""+tid, tidWidth); - } + function pad(n, width, z) { + z = z || "0"; + n = n + ""; + return n.length >= width ? n : new Array(width - n.length + 1).join(z) + n; + } + function formatTid(tid) { + // let padded = "" + tid; + // return '#' + pad(""+tid, tidWidth); + return pad("" + tid, tidWidth); + } - let repfulAgreeTidsByGroup = {}; - let repfulDisageeTidsByGroup = {}; - if (mathResult.repness) { - _.each(mathResult.repness, (entries, gid) => { - entries.forEach((entry) => { - if (entry['repful-for'] === 'agree') { - repfulAgreeTidsByGroup[gid] = repfulAgreeTidsByGroup[gid] || []; - repfulAgreeTidsByGroup[gid].push(entry.tid); - } else if (entry['repful-for'] === 'disagree') { - repfulDisageeTidsByGroup[gid] = repfulDisageeTidsByGroup[gid] || []; - repfulDisageeTidsByGroup[gid].push(entry.tid); - } + let repfulAgreeTidsByGroup = {}; + let repfulDisageeTidsByGroup = {}; + if (mathResult.repness) { + _.each(mathResult.repness, (entries, gid) => { + entries.forEach((entry) => { + if (entry["repful-for"] === "agree") { + repfulAgreeTidsByGroup[gid] = repfulAgreeTidsByGroup[gid] || []; + repfulAgreeTidsByGroup[gid].push(entry.tid); + } else if (entry["repful-for"] === "disagree") { + repfulDisageeTidsByGroup[gid] = repfulDisageeTidsByGroup[gid] || []; + repfulDisageeTidsByGroup[gid].push(entry.tid); + } + }); }); - }); - } - - // ====== REMEMBER: gid's start at zero, (0, 1, 2) but we show them as group 1, 2, 3 in participation view ====== - let groupNames = {}; - for (let i = 0; i <= 9; i++) { - let label = report["label_group_" + i]; - if (label) { - groupNames[i] = label; } - } - - let uncertainty = []; - // let maxCount = _.reduce(comments, (memo, c) => { return Math.max(c.count, memo);}, 1); - comments.map((c) => { - var unc = c.pass_count / c.count - if (unc > .3) { - c.unc = unc; - uncertainty.push(c); + // ====== REMEMBER: gid's start at zero, (0, 1, 2) but we show them as group 1, 2, 3 in participation view ====== + let groupNames = {}; + for (let i = 0; i <= 9; i++) { + let label = report["label_group_" + i]; + if (label) { + groupNames[i] = label; + } } - }); - uncertainty.sort((a, b) => { - return (b.unc*b.unc*b.pass_count - a.unc*a.unc*a.pass_count); - }); - uncertainty = uncertainty.slice(0, 5); - - let extremity = {}; - _.each(mathResult.pca["comment-extremity"], function(e, index) { - extremity[indexToTid[index]] = e; - }); - var uniqueCommenters = {}; - var voteTotals = DataUtils.getVoteTotals(mathResult); - comments = comments.map((c) => { - c["group-aware-consensus"] = mathResult["group-aware-consensus"][c.tid]; - uniqueCommenters[c.pid] = 1; - c = Object.assign(c, voteTotals[c.tid]); - return c; - }); - var numUniqueCommenters = _.keys(uniqueCommenters).length; - var totalVotes = _.reduce(_.values(mathResult["user-vote-counts"]), function(memo, num) { - return memo + num; - }, 0); - const computedStats = { - votesPerVoterAvg: totalVotes / ptptCountTotal, - commentsPerCommenterAvg: comments.length / numUniqueCommenters, - }; + let uncertainty = []; + // let maxCount = _.reduce(comments, (memo, c) => { return Math.max(c.count, memo);}, 1); + comments.map((c) => { + var unc = c.pass_count / c.count; + if (unc > 0.3) { + c.unc = unc; + uncertainty.push(c); + } + }); + uncertainty.sort((a, b) => { + return b.unc * b.unc * b.pass_count - a.unc * a.unc * a.pass_count; + }); + uncertainty = uncertainty.slice(0, 5); - this.setState({ - loading: false, - math: mathResult, - consensus: mathResult.consensus, - extremity: extremity, - uncertainty: uncertainty.map((c) => {return c.tid;}), - comments: comments, - demographics: groupDemographics, - participants: participants, - conversation: conversation, - ptptCount: ptptCount, - ptptCountTotal: ptptCountTotal, - filteredCorrelationMatrix: filteredProbabilities, - filteredCorrelationTids: filteredTids, - badTids: badTids, - groupNames: groupNames, - repfulAgreeTidsByGroup: repfulAgreeTidsByGroup, - repfulDisageeTidsByGroup: repfulDisageeTidsByGroup, - formatTid: formatTid, - report: report, - //conversationStats: conversationstats, - computedStats: computedStats, - nothingToShow: !comments.length || !groupDemographics.length - }); + let extremity = {}; + _.each(mathResult.pca["comment-extremity"], function (e, index) { + extremity[indexToTid[index]] = e; + }); - }).catch((err) => { - this.setState({ - error: true, - errorText: String(err), + var uniqueCommenters = {}; + var voteTotals = DataUtils.getVoteTotals(mathResult); + comments = comments.map((c) => { + c["group-aware-consensus"] = mathResult["group-aware-consensus"][c.tid]; + uniqueCommenters[c.pid] = 1; + c = Object.assign(c, voteTotals[c.tid]); + return c; + }); + var numUniqueCommenters = _.keys(uniqueCommenters).length; + var totalVotes = _.reduce( + _.values(mathResult["user-vote-counts"]), + function (memo, num) { + return memo + num; + }, + 0 + ); + const computedStats = { + votesPerVoterAvg: totalVotes / ptptCountTotal, + commentsPerCommenterAvg: comments.length / numUniqueCommenters, + }; + + this.setState({ + loading: false, + math: mathResult, + consensus: mathResult.consensus, + extremity: extremity, + uncertainty: uncertainty.map((c) => { + return c.tid; + }), + comments: comments, + demographics: groupDemographics, + participants: participants, + conversation: conversation, + ptptCount: ptptCount, + ptptCountTotal: ptptCountTotal, + filteredCorrelationMatrix: filteredProbabilities, + filteredCorrelationTids: filteredTids, + badTids: badTids, + groupNames: groupNames, + repfulAgreeTidsByGroup: repfulAgreeTidsByGroup, + repfulDisageeTidsByGroup: repfulDisageeTidsByGroup, + formatTid: formatTid, + report: report, + //conversationStats: conversationstats, + computedStats: computedStats, + nothingToShow: !comments.length || !groupDemographics.length, + }); + }) + .catch((err) => { + this.setState({ + error: true, + errorText: String(err), + }); }); - }); } - UNSAFE_componentWillMount() { this.getData(); @@ -380,14 +400,17 @@ class App extends React.Component { } }, 20 * 1000); - window.addEventListener("resize", _.throttle(() => { - this.setState({ - dimensions: { - width: window.innerWidth, - height: window.innerHeight - } - }) - }, 500)); + window.addEventListener( + "resize", + _.throttle(() => { + this.setState({ + dimensions: { + width: window.innerWidth, + height: window.innerHeight, + }, + }); + }, 500) + ); } onAutoRefreshEnabled() { @@ -402,7 +425,7 @@ class App extends React.Component { }); } - handleColorblindModeClick () { + handleColorblindModeClick() { var colorBlind = !this.state.colorBlindMode; if (colorBlind) { this.setState({ @@ -423,36 +446,45 @@ class App extends React.Component { render() { if (this.state.error) { - return (
-
Error Loading
-
{this.state.errorText}
-
); + return ( +
+
Error Loading
+
{this.state.errorText}
+
+ ); } if (this.state.nothingToShow) { - return (
-
Nothing to show yet
-
); + return ( +
+
Nothing to show yet
+
+ ); } if (this.state.loading) { - return (
-
Loading ...
-
); + return ( +
+
Loading ...
+
+ ); } - console.log('top level app state and props', this.state, this.props) + console.log("top level app state and props", this.state, this.props); return ( -
- -
+ +
+ }} + > + voteColors={this.state.voteColors} + /> {/* This may eventually need to go back in below */} {/* stats={this.state.conversationStats} */} @@ -464,7 +496,8 @@ class App extends React.Component { ptptCountTotal={this.state.ptptCountTotal} demographics={this.state.demographics} conversation={this.state.conversation} - voteColors={this.state.voteColors}/> + voteColors={this.state.voteColors} + /> + voteColors={this.state.voteColors} + /> {/*

Consensus

Inclusive Majority

*/} + + voteColors={this.state.voteColors} + /> + voteColors={this.state.voteColors} + /> + voteColors={this.state.voteColors} + /> {/* {false ? + voteColors={this.state.voteColors} + /> {/* */} -
+ voteColors={this.state.voteColors} + /> +
); diff --git a/client-report/src/components/globals.js b/client-report/src/components/globals.js index d9cb88100..c5effedcf 100644 --- a/client-report/src/components/globals.js +++ b/client-report/src/components/globals.js @@ -21,17 +21,16 @@ export const brandColors = { export const allCommentsSortDefault = "tid"; - -const fontSizes = { +export const fontSizes = { largest: 36, large: 24, medium: 18, -} +}; const fontWeights = { boldest: 700, - normal: 400 -} + normal: 400, +}; export const primaryHeading = { fontSize: fontSizes.largest, @@ -40,49 +39,46 @@ export const primaryHeading = { export const secondaryHeading = { fontSize: fontSizes.large, - fontWeight: fontWeights.normal -} + fontWeight: fontWeights.normal, +}; export const groupHeader = { fontSize: fontSizes.largest, fontWeight: fontWeights.boldest, -} +}; export const overviewNumber = { fontSize: fontSizes.largest, fontWeight: fontWeights.normal, marginBottom: 0, -} +}; export const overviewLabel = { fontSize: fontSizes.medium, fontWeight: fontWeights.normal, marginTop: 0, - maxWidth: 190 -} + maxWidth: 190, +}; export const tidGrey = "rgb(200,200,200)"; export const paragraph = { width: paragraphWidth, fontFamily: serif, lineHeight: paragraphLineHeight, -} - +}; // Duplicated in: // polisClientParticipation/vis2/components/globals.js // polisReport/src/components/globals.js -export const groupLabels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] -export const groupSymbols = ["○", "◆", "+", "-", "◬", "▮", ] - +export const groupLabels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"]; +export const groupSymbols = ["○", "◆", "+", "-", "◬", "▮"]; export const enableMatrix = false; // export const maxCommentExtremityToShow = 2; /* naive & may not be solid. should be dynamically generated from extremity array probably to pick top 20 or something */ -export const maxCommentExtremityToShow = .6; /* naive & may not be solid. should be dynamically generated from extremity array probably to pick top 20 or something */ +export const maxCommentExtremityToShow = 0.6; /* naive & may not be solid. should be dynamically generated from extremity array probably to pick top 20 or something */ export const labelPadding = 40; export const shouldColorizeTidsByRepfulness = true; - // ====== REMEMBER: gid's start at zero, (0, 1, 2) but we show them as group 1, 2, 3 ======= export const groupColor = (gid) => { diff --git a/client-report/src/components/overview.js b/client-report/src/components/overview.js index 160fd9ebb..470448864 100644 --- a/client-report/src/components/overview.js +++ b/client-report/src/components/overview.js @@ -8,26 +8,39 @@ const computeVoteTotal = (users) => { let voteTotal = 0; _.each(users, (count) => { - voteTotal += count + voteTotal += count; }); return voteTotal; -} +}; // const computeUniqueCommenters = (comments) => { // } -const Number = ({number, label}) => ( -
-

- {number.toLocaleString()} -

-

- {label} -

+const Number = ({ number, label }) => ( +
+

{number.toLocaleString()}

+

{label}

-) +); + +const pathname = window.location.pathname; // "/report/2arcefpshi" +const report_id = pathname.split("/")[2]; + +const getCurrentTimestamp = () => { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, "0"); + const day = String(now.getDate()).padStart(2, "0"); + const hours = String(now.getHours()).padStart(2, "0"); + const minutes = String(now.getMinutes()).padStart(2, "0"); + return `${year}-${month}-${day}-${hours}${minutes}`; +}; + +const getDownloadFilename = (file, conversation) => { + return `${getCurrentTimestamp()}-${conversation.conversation_id}-${file}.csv`; +}; const Overview = ({ conversation, @@ -40,40 +53,144 @@ const Overview = ({ computedStats, }) => { return ( -
-

Overview

-

- Pol.is is a real-time survey system that helps identify the different ways a large group of people think about a divisive or complicated topic. Here’s a basic breakdown of some terms you’ll need to know in order to understand this report. -

-

- Participants: These are the people who participated in the conversation by voting and writing statements. Based on how they voted, each participant is sorted into an opinion group. -

-

- Statements: Participants may submit statements for other participants to vote on. Statements are assigned a number in the order they’re submitted. -

-

- Opinion groups: Groups are made of participants who voted similarly to each other, and differently from the other groups. -

- -

- {conversation && conversation.ownername ? "This pol.is conversation was run by "+conversation.ownername+". " : null} - {conversation && conversation.topic ? "The topic was '"+conversation.topic+"'. " : null} -

-
+
+
+

Overview

+

+ Pol.is is a real-time survey system that helps identify the different ways a large group + of people think about a divisive or complicated topic. Here's a basic breakdown of some + terms you'll need to know in order to understand this report. +

+

+ Participants: These are the people who participated in the conversation + by voting and writing statements. Based on how they voted, each participant is sorted into + an opinion group. +

+

+ Statements: Participants may submit statements for other participants to + vote on. Statements are assigned a number in the order they're submitted. +

+

+ Opinion groups: Groups are made of participants who voted similarly to + each other, and differently from the other groups. +

+ +

+ {conversation && conversation.ownername + ? "This pol.is conversation was run by " + conversation.ownername + ". " + : null} + {conversation && conversation.topic + ? "The topic was '" + conversation.topic + "'. " + : null} +

+
+ +
- + {/* Leaving this out for now until we get smarter conversationStats */} {/* */} - - - + +
+
+

+ Raw Data Export (Anonymous) +

+

+ {`The following data exports are anonymized. Participants are identifed by an integer representing the order in which they first voted. For a full description of files and columns, please see: `} + https://compdemocracy.org/export/ +

+

+ {`--------Summary: `} + + {getDownloadFilename("summary", conversation)} + +

+

+ {`-------Comments: `} + + {getDownloadFilename("comments", conversation)} + + {` (may take up to several minutes)`} +

+

+ {`--Votes history: `} + + {getDownloadFilename("votes", conversation)} + + {` (as event log)`} +

+
+

+ Public API endpoints (read only, Jupyter notebook friendly) +

+

+ {`$ curl http://${window.location.hostname}/api/v3/reportExport/${report_id}/summary.csv`} +

+

+ {`$ curl http://${window.location.hostname}/api/v3/reportExport/${report_id}/comments.csv`} +

+

+ {`$ curl http://${window.location.hostname}/api/v3/reportExport/${report_id}/votes.csv`} +

+
+ {window.location.hostname === "pol.is" || + (window.location.hostname === "localhost" && ( +
+

+ Attribution of Polis Data +

+ +

+ All Polis data is licensed under a Creative Commons Attribution 4.0 International + license: https://creativecommons.org/licenses/by/4.0/ +

+

+ --------------- BEGIN STATEMENT --------------- +

+

{`Data was gathered using the Polis software (see: compdemocracy.org/polis and github.com/compdemocracy/polis) and is sub-licensed + under CC BY 4.0 with Attribution to The Computational Democracy Project. The data and more + information about how the data was collected can be found at the following link: ${window.location.href}`}

+

+ --------------- END STATEMENT--------------- +

+

+ For further information on best practices for Attribution of CC 4.0 licensed content + Please see: + https://wiki.creativecommons.org/wiki/Best_practices_for_attribution#Title.2C_Author.2C_Source.2C_License +

+
+ ))} +
); }; @@ -82,9 +199,8 @@ export default Overview; //

{conversation && conversation.participant_count ? "A total of "+ptptCount+" people participated. " : null}

- // It was presented {conversation ? conversation.medium : "loading"} to an audience of {conversation ? conversation.audiences : "loading"}. // The conversation was run for {conversation ? conversation.duration : "loading"}. - // {demographics ? demographics.foo : "loading"} were women +// {demographics ? demographics.foo : "loading"} were women - // {conversation && conversation.description ? "The specific question was '"+conversation.description+"'. ": null} +// {conversation && conversation.description ? "The specific question was '"+conversation.description+"'. ": null} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d9bb4a030..2c2d290a2 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -32,10 +32,10 @@ services: environment: CHOKIDAR_USEPOLLING: "true" - postgres: - restart: no - ports: - - "${POSTGRES_PORT:-5432}:5432" + # postgres: + # restart: no + # ports: + # - "${POSTGRES_PORT:-5432}:5432" file-server: build: diff --git a/docker-compose.yml b/docker-compose.yml index 438245179..4d6751128 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,20 +34,20 @@ services: labels: polis_tag: ${TAG:-dev} depends_on: - - "postgres" + # - "postgres" - "file-server" networks: - "polis-net" # Scale the server container to a given number of instances. scale: 1 volumes: - # Persist logs to a volume, so they can be accessed after the container is stopped. + # Persist logs to a volume, so they can be accessed after the container is stopped. - server-logs:/app/logs math: image: docker.io/compdem/polis-math:${TAG:-dev} - depends_on: - - "postgres" + # depends_on: + # - "postgres" build: context: ./math labels: @@ -60,23 +60,23 @@ services: networks: - "polis-net" - postgres: - image: docker.io/compdem/polis-postgres:${TAG:-dev} - restart: always - build: - context: ./server - dockerfile: Dockerfile-db - labels: - polis_tag: ${TAG:-dev} - environment: - - POSTGRES_DB=${POSTGRES_DB:?error} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?error} - - POSTGRES_USER=${POSTGRES_USER:?error} - networks: - - "polis-net" - volumes: - - "backups:/backups" - - "postgres:/var/lib/postgresql/data" + # postgres: + # image: docker.io/compdem/polis-postgres:${TAG:-dev} + # restart: always + # build: + # context: ./server + # dockerfile: Dockerfile-db + # labels: + # polis_tag: ${TAG:-dev} + # environment: + # - POSTGRES_DB=${POSTGRES_DB:?error} + # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?error} + # - POSTGRES_USER=${POSTGRES_USER:?error} + # networks: + # - "polis-net" + # volumes: + # - "backups:/backups" + # - "postgres:/var/lib/postgresql/data" nginx-proxy: image: docker.io/compdem/polis-nginx-proxy:${TAG:-dev} @@ -120,9 +120,9 @@ volumes: backups: labels: polis_tag: ${TAG:-dev} - postgres: - labels: - polis_tag: ${TAG:-dev} + # postgres: + # labels: + # polis_tag: ${TAG:-dev} server-logs: labels: polis_tag: ${TAG:-dev} diff --git a/example.env b/example.env index 6b1cbf80b..009be62b4 100644 --- a/example.env +++ b/example.env @@ -117,7 +117,7 @@ AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= # This value is written by the server app if SHOULD_USE_TRANSLATION_API is true. GOOGLE_APPLICATION_CREDENTIALS= - +GOOGLE_JIGSAW_PERSPECTIVE_API_KEY= ###### DEPRECATED ###### # (Deprecated) Used internally by Node.Crypto to encrypt/decrypt IP addresses. diff --git a/server/app.ts b/server/app.ts index 765f31407..52bd8c3b8 100644 --- a/server/app.ts +++ b/server/app.ts @@ -84,6 +84,7 @@ helpersInitialized.then( handle_GET_math_correlationMatrix, handle_GET_dataExport, handle_GET_dataExport_results, + handle_GET_reportExport, handle_GET_domainWhitelist, handle_GET_dummyButton, handle_GET_einvites, @@ -300,6 +301,19 @@ helpersInitialized.then( handle_GET_dataExport ); + app.get( + "/api/v3/reportExport/:report_id/:report_type", + moveToBody, + need( + "report_id", + getReportIdFetchRid, + assignToPCustom("rid") + ), + need("report_id", getStringLimitLength(1, 1000), assignToP), + need("report_type", getStringLimitLength(1, 1000), assignToP), + handle_GET_reportExport + ); + app.get( "/api/v3/dataExport/results", moveToBody, diff --git a/server/package-lock.json b/server/package-lock.json index 1438d8797..92428cdfb 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -21,6 +21,7 @@ "dotenv": "^16.0.3", "express": "~3.21.2", "fb": "~1.0.2", + "googleapis": "^142.0.0", "html-entities": "^2.4.0", "http-proxy": "~1.18.1", "lru-cache": "~3.0.0", @@ -2490,6 +2491,15 @@ "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz", "integrity": "sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==" }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -2668,13 +2678,19 @@ "integrity": "sha512-k9VSlRfRi5JYyQWMylSOgjld96ta1qaQUIvmn+na0BzViclH04PBumewv4z5aeXNkn6Z/gAN5FtPeBLvV20F9w==" }, "node_modules/call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dev": true, + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "license": "MIT", "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -3311,6 +3327,23 @@ "node": ">=0.10.0" } }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/degenerator": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-1.0.4.tgz", @@ -3559,6 +3592,27 @@ "node": ">= 0.6" } }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", @@ -4420,10 +4474,79 @@ "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==" }, "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/gaxios/node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/gaxios/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gaxios/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } }, "node_modules/gcp-metadata": { "version": "0.3.1", @@ -4456,14 +4579,19 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", - "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", - "dev": true, + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "license": "MIT", "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -4627,6 +4755,204 @@ "gp12-pem": "bin/gp12-pem" } }, + "node_modules/googleapis": { + "version": "142.0.0", + "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-142.0.0.tgz", + "integrity": "sha512-LsU1ynez4/KNPwnFMSDI93pBEsETNdQPCrT3kz2qgiNg5H2pW4dKW+1VmENMkZ4u9lMxA89nnXD3nqWBJ0rruQ==", + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^9.0.0", + "googleapis-common": "^7.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/googleapis-common": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-7.2.0.tgz", + "integrity": "sha512-/fhDZEJZvOV3X5jmD+fKxMqma5q2Q9nZNSF3kn1F18tpxmA86BcTxAGBQdM0N89Z3bEaIs+HVznSmFJEAmMTjA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "gaxios": "^6.0.3", + "google-auth-library": "^9.7.0", + "qs": "^6.7.0", + "url-template": "^2.0.8", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/googleapis-common/node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/googleapis-common/node_modules/google-auth-library": { + "version": "9.13.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.13.0.tgz", + "integrity": "sha512-p9Y03Uzp/Igcs36zAaB0XTSwZ8Y0/tpYiz5KIde5By+H9DCVUSYtDWZu6aFXsWTqENMb8BD/pDT3hR8NVrPkfA==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/googleapis-common/node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "license": "MIT", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/googleapis-common/node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/googleapis-common/node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/googleapis-common/node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/googleapis-common/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/googleapis/node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/googleapis/node_modules/google-auth-library": { + "version": "9.13.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.13.0.tgz", + "integrity": "sha512-p9Y03Uzp/Igcs36zAaB0XTSwZ8Y0/tpYiz5KIde5By+H9DCVUSYtDWZu6aFXsWTqENMb8BD/pDT3hR8NVrPkfA==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/googleapis/node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "license": "MIT", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/googleapis/node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/googleapis/node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -4692,11 +5018,35 @@ "node": ">=4" } }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -4704,6 +5054,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/hexoid": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", @@ -5890,6 +6252,15 @@ "node": ">=4" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", @@ -6447,6 +6818,26 @@ "node": ">= 0.4.0" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-forge": { "version": "0.7.6", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz", @@ -6585,10 +6976,13 @@ } }, "node_modules/object-inspect": { - "version": "1.12.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", - "dev": true, + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", + "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -7970,6 +8364,23 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz", "integrity": "sha512-pVEuxHdSGrt8QmQ3LOZXLhSA6MP/iPqKzZeO6Squ7PNGkA/9MBsSfV0/L+bIxkoDmjF4tZcLpcVq/fkqoHvuKg==" }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/setprototypeof": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", @@ -7997,14 +8408,18 @@ } }, "node_modules/side-channel": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", - "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "dev": true, + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "license": "MIT", "dependencies": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -8700,6 +9115,12 @@ "node": ">=0.8" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/triple-beam": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", @@ -9012,6 +9433,12 @@ "querystring": "0.2.0" } }, + "node_modules/url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==", + "license": "BSD" + }, "node_modules/url/node_modules/punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -9115,6 +9542,22 @@ "makeerror": "1.0.12" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -11275,6 +11718,11 @@ "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz", "integrity": "sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==" }, + "bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==" + }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -11420,13 +11868,15 @@ "integrity": "sha512-k9VSlRfRi5JYyQWMylSOgjld96ta1qaQUIvmn+na0BzViclH04PBumewv4z5aeXNkn6Z/gAN5FtPeBLvV20F9w==" }, "call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dev": true, + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", "requires": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" } }, "callsites": { @@ -11927,6 +12377,16 @@ "integrity": "sha512-z2wJZXrmeHdvYJp/Ux55wIjqo81G5Bp4c+oELTW+7ar6SogWHajt5a9gO3s3IDaGSAXjDk0vlQKN3rms8ab3og==", "dev": true }, + "define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "requires": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + } + }, "degenerator": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-1.0.4.tgz", @@ -12139,6 +12599,19 @@ } } }, + "es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "requires": { + "get-intrinsic": "^1.2.4" + } + }, + "es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" + }, "es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", @@ -12795,10 +13268,50 @@ } }, "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" + }, + "gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "requires": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "dependencies": { + "agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "requires": { + "debug": "^4.3.4" + } + }, + "https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "requires": { + "agent-base": "^7.0.2", + "debug": "4" + } + }, + "is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" + }, + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + } + } }, "gcp-metadata": { "version": "0.3.1", @@ -12822,14 +13335,15 @@ "dev": true }, "get-intrinsic": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", - "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", - "dev": true, + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", "requires": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" } }, "get-package-type": { @@ -12958,6 +13472,153 @@ "node-forge": "^0.7.1" } }, + "googleapis": { + "version": "142.0.0", + "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-142.0.0.tgz", + "integrity": "sha512-LsU1ynez4/KNPwnFMSDI93pBEsETNdQPCrT3kz2qgiNg5H2pW4dKW+1VmENMkZ4u9lMxA89nnXD3nqWBJ0rruQ==", + "requires": { + "google-auth-library": "^9.0.0", + "googleapis-common": "^7.0.0" + }, + "dependencies": { + "gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "requires": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + } + }, + "google-auth-library": { + "version": "9.13.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.13.0.tgz", + "integrity": "sha512-p9Y03Uzp/Igcs36zAaB0XTSwZ8Y0/tpYiz5KIde5By+H9DCVUSYtDWZu6aFXsWTqENMb8BD/pDT3hR8NVrPkfA==", + "requires": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + } + }, + "gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "requires": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + } + }, + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + } + } + }, + "googleapis-common": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-7.2.0.tgz", + "integrity": "sha512-/fhDZEJZvOV3X5jmD+fKxMqma5q2Q9nZNSF3kn1F18tpxmA86BcTxAGBQdM0N89Z3bEaIs+HVznSmFJEAmMTjA==", + "requires": { + "extend": "^3.0.2", + "gaxios": "^6.0.3", + "google-auth-library": "^9.7.0", + "qs": "^6.7.0", + "url-template": "^2.0.8", + "uuid": "^9.0.0" + }, + "dependencies": { + "gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "requires": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + } + }, + "google-auth-library": { + "version": "9.13.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.13.0.tgz", + "integrity": "sha512-p9Y03Uzp/Igcs36zAaB0XTSwZ8Y0/tpYiz5KIde5By+H9DCVUSYtDWZu6aFXsWTqENMb8BD/pDT3hR8NVrPkfA==", + "requires": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + } + }, + "gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "requires": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + } + }, + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "requires": { + "side-channel": "^1.0.6" + } + }, + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + } + } + }, + "gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "requires": { + "get-intrinsic": "^1.1.3" + } + }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -13010,11 +13671,31 @@ "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", "dev": true }, + "has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "requires": { + "es-define-property": "^1.0.0" + } + }, + "has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==" + }, "has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "dev": true + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "requires": { + "function-bind": "^1.1.2" + } }, "hexoid": { "version": "1.0.0", @@ -13927,6 +14608,14 @@ "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", "dev": true }, + "json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "requires": { + "bignumber.js": "^9.0.0" + } + }, "json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", @@ -14390,6 +15079,14 @@ "resolved": "https://registry.npmjs.org/netmask/-/netmask-1.0.6.tgz", "integrity": "sha512-3DWDqAtIiPSkBXZyYEjwebfK56nrlQfRGt642fu8RPaL+ePu750+HCMHxjJCG3iEHq/0aeMvX6KIzlv7nuhfrA==" }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, "node-forge": { "version": "0.7.6", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz", @@ -14496,10 +15193,9 @@ "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" }, "object-inspect": { - "version": "1.12.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", - "dev": true + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", + "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==" }, "on-finished": { "version": "2.3.0", @@ -15560,6 +16256,19 @@ } } }, + "set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "requires": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + } + }, "setprototypeof": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", @@ -15581,14 +16290,14 @@ "dev": true }, "side-channel": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", - "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "dev": true, + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", "requires": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" } }, "signal-exit": { @@ -16139,6 +16848,11 @@ "punycode": "^2.1.1" } }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "triple-beam": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", @@ -16348,6 +17062,11 @@ } } }, + "url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==" + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -16431,6 +17150,20 @@ "makeerror": "1.0.12" } }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/server/package.json b/server/package.json index 7f8b4e879..c24349856 100644 --- a/server/package.json +++ b/server/package.json @@ -37,6 +37,7 @@ "dotenv": "^16.0.3", "express": "~3.21.2", "fb": "~1.0.2", + "googleapis": "^142.0.0", "html-entities": "^2.4.0", "http-proxy": "~1.18.1", "lru-cache": "~3.0.0", diff --git a/server/src/config.ts b/server/src/config.ts index ad74b5f3a..c33fc8cd0 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -3,13 +3,18 @@ import isTrue from "boolean"; const devHostname = process.env.API_DEV_HOSTNAME || "localhost:5000"; const devMode = isTrue(process.env.DEV_MODE) as boolean; -const domainOverride = process.env.DOMAIN_OVERRIDE || null as string | null; +const domainOverride = process.env.DOMAIN_OVERRIDE || (null as string | null); const prodHostname = process.env.API_PROD_HOSTNAME || "pol.is"; -const serverPort = parseInt(process.env.API_SERVER_PORT || process.env.PORT || "5000", 10) as number; -const shouldUseTranslationAPI = isTrue(process.env.SHOULD_USE_TRANSLATION_API) as boolean; - -import('source-map-support').then((sourceMapSupport) => { - sourceMapSupport.install(); +const serverPort = parseInt( + process.env.API_SERVER_PORT || process.env.PORT || "5000", + 10 +) as number; +const shouldUseTranslationAPI = isTrue( + process.env.SHOULD_USE_TRANSLATION_API +) as boolean; + +import("source-map-support").then((sourceMapSupport) => { + sourceMapSupport.install(); }); export default { @@ -59,16 +64,22 @@ export default { adminEmailDataExport: process.env.ADMIN_EMAIL_DATA_EXPORT as string, adminEmailDataExportTest: process.env.ADMIN_EMAIL_DATA_EXPORT_TEST as string, adminEmailEmailTest: process.env.ADMIN_EMAIL_EMAIL_TEST as string, - adminEmails: process.env.ADMIN_EMAILS || '[]' as string, - adminUIDs: process.env.ADMIN_UIDS || '[]' as string, - akismetAntispamApiKey: process.env.AKISMET_ANTISPAM_API_KEY || null as string | null, + adminEmails: process.env.ADMIN_EMAILS || ("[]" as string), + adminUIDs: process.env.ADMIN_UIDS || ("[]" as string), + akismetAntispamApiKey: + process.env.AKISMET_ANTISPAM_API_KEY || (null as string | null), + googleJigsawPerspectiveApiKey: + process.env.GOOGLE_JIGSAW_PERSPECTIVE_API_KEY || (null as string | null), awsRegion: process.env.AWS_REGION as string, - backfillCommentLangDetection: isTrue(process.env.BACKFILL_COMMENT_LANG_DETECTION) as boolean, + backfillCommentLangDetection: isTrue( + process.env.BACKFILL_COMMENT_LANG_DETECTION + ) as boolean, cacheMathResults: isTrueOrBlank(process.env.CACHE_MATH_RESULTS) as boolean, databaseURL: process.env.DATABASE_URL as string, - emailTransportTypes: process.env.EMAIL_TRANSPORT_TYPES || null as string | null, + emailTransportTypes: + process.env.EMAIL_TRANSPORT_TYPES || (null as string | null), encryptionPassword: process.env.ENCRYPTION_PASSWORD_00001 as string, - fbAppId: process.env.FB_APP_ID || null as string | null, + fbAppId: process.env.FB_APP_ID || (null as string | null), logLevel: process.env.SERVER_LOG_LEVEL as string, logToFile: isTrue(process.env.SERVER_LOG_TO_FILE) as boolean, mailgunApiKey: process.env.MAILGUN_API_KEY || (null as string | null), @@ -78,14 +89,29 @@ export default { maxmindUserID: process.env.MAXMIND_USER_ID as string, nodeEnv: process.env.NODE_ENV as string, polisFromAddress: process.env.POLIS_FROM_ADDRESS as string, - readOnlyDatabaseURL: process.env.READ_ONLY_DATABASE_URL || process.env.DATABASE_URL as string, - runPeriodicExportTests: isTrue(process.env.RUN_PERIODIC_EXPORT_TESTS) as boolean, + readOnlyDatabaseURL: + process.env.READ_ONLY_DATABASE_URL || (process.env.DATABASE_URL as string), + runPeriodicExportTests: isTrue( + process.env.RUN_PERIODIC_EXPORT_TESTS + ) as boolean, shouldUseTranslationAPI: setGoogleApplicationCredentials() as boolean, - staticFilesAdminPort: parseInt(process.env.STATIC_FILES_ADMIN_PORT || process.env.STATIC_FILES_PORT || '8080', 10) as number, - staticFilesParticipationPort: parseInt(process.env.STATIC_FILES_PARTICIPATION_PORT || process.env.STATIC_FILES_PORT || '8080', 10) as number, + staticFilesAdminPort: parseInt( + process.env.STATIC_FILES_ADMIN_PORT || + process.env.STATIC_FILES_PORT || + "8080", + 10 + ) as number, + staticFilesParticipationPort: parseInt( + process.env.STATIC_FILES_PARTICIPATION_PORT || + process.env.STATIC_FILES_PORT || + "8080", + 10 + ) as number, staticFilesHost: process.env.STATIC_FILES_HOST as string, - twitterConsumerKey: process.env.TWITTER_CONSUMER_KEY || null as string | null, - twitterConsumerSecret: process.env.TWITTER_CONSUMER_SECRET || null as string | null, + twitterConsumerKey: + process.env.TWITTER_CONSUMER_KEY || (null as string | null), + twitterConsumerSecret: + process.env.TWITTER_CONSUMER_SECRET || (null as string | null), webserverPass: process.env.WEBSERVER_PASS as string, webserverUsername: process.env.WEBSERVER_USERNAME as string, @@ -98,12 +124,12 @@ export default { process.env.DOMAIN_WHITELIST_ITEM_06 || null, process.env.DOMAIN_WHITELIST_ITEM_07 || null, process.env.DOMAIN_WHITELIST_ITEM_08 || null, - ].filter(item => item !== null) as string[], + ].filter((item) => item !== null) as string[], }; // Use this function when a value shuould default to true if not set. function isTrueOrBlank(val: string | boolean | undefined): boolean { - return val === undefined || val === '' || isTrue(val); + return val === undefined || val === "" || isTrue(val); } function setGoogleApplicationCredentials(): boolean { @@ -111,13 +137,17 @@ function setGoogleApplicationCredentials(): boolean { return false; } - const googleCredentialsBase64: string | undefined = process.env.GOOGLE_CREDENTIALS_BASE64; - const googleCredsStringified: string | undefined = process.env.GOOGLE_CREDS_STRINGIFIED; + const googleCredentialsBase64: string | undefined = + process.env.GOOGLE_CREDENTIALS_BASE64; + const googleCredsStringified: string | undefined = + process.env.GOOGLE_CREDS_STRINGIFIED; try { // TODO: Consider deprecating GOOGLE_CREDS_STRINGIFIED in future. if (!googleCredentialsBase64 && !googleCredsStringified) { - throw new Error("Missing Google credentials. Translation API will be disabled."); + throw new Error( + "Missing Google credentials. Translation API will be disabled." + ); } const creds_string = googleCredentialsBase64 diff --git a/server/src/server.ts b/server/src/server.ts index e448a2bac..fe4148d5c 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -5,7 +5,7 @@ import akismetLib from "akismet"; import AWS from "aws-sdk"; import badwords from "badwords/object"; -import Promise from "bluebird"; +import { Promise as BluebirdPromise } from "bluebird"; import http from "http"; import httpProxy from "http-proxy"; // const Promise = require('es6-promise').Promise, @@ -13,6 +13,7 @@ import async from "async"; // npm list types-at-fb // @ts-ignore import FB from "fb"; +import { google } from "googleapis"; import fs from "fs"; import bcrypt from "bcryptjs"; import crypto from "crypto"; @@ -114,7 +115,6 @@ const resolveWith = (x: { body?: { user_id: string } }) => { return Promise.resolve(x); }; - //var SegfaultHandler = require('segfault-handler'); //SegfaultHandler.registerHandler("segfault.log"); @@ -125,22 +125,18 @@ const resolveWith = (x: { body?: { user_id: string } }) => { // }; if (devMode) { - Promise.longStackTraces(); + BluebirdPromise.longStackTraces(); } // Bluebird uncaught error handler. -Promise.onPossiblyUnhandledRejection(function (err: any) { - logger.error('onPossiblyUnhandledRejection', err); +BluebirdPromise.onPossiblyUnhandledRejection(function (err: any) { + logger.error("onPossiblyUnhandledRejection", err); // throw err; // not throwing since we're printing stack traces anyway }); -const adminEmails = Config.adminEmails - ? JSON.parse(Config.adminEmails) - : []; +const adminEmails = Config.adminEmails ? JSON.parse(Config.adminEmails) : []; -const polisDevs = Config.adminUIDs - ? JSON.parse(Config.adminUIDs) - : []; +const polisDevs = Config.adminUIDs ? JSON.parse(Config.adminUIDs) : []; function isPolisDev(uid?: any) { return polisDevs.indexOf(uid) >= 0; @@ -611,7 +607,7 @@ function initializePolisHelpers() { conv?: { zid: any }, tid?: any, voteType?: any, - weight?: number, + weight?: number ) { let zid = conv?.zid; weight = weight || 0; @@ -651,7 +647,7 @@ function initializePolisHelpers() { tid?: any, xid?: any, voteType?: any, - weight?: number, + weight?: number ) { return ( pgQueryP_readOnly("select * from conversations where zid = ($1);", [zid]) @@ -695,25 +691,20 @@ function initializePolisHelpers() { }); } if (conv.use_xid_whitelist) { - return isXidWhitelisted(conv.owner, xid).then((is_whitelisted: boolean) => { - if (is_whitelisted) { - return conv; - } else { - throw 'polis_err_xid_not_whitelisted'; + return isXidWhitelisted(conv.owner, xid).then( + (is_whitelisted: boolean) => { + if (is_whitelisted) { + return conv; + } else { + throw "polis_err_xid_not_whitelisted"; + } } - }); + ); } return conv; }) .then(function (conv: any) { - return doVotesPost( - uid, - pid, - conv, - tid, - voteType, - weight, - ); + return doVotesPost(uid, pid, conv, tid, voteType, weight); }) ); } @@ -776,10 +767,17 @@ function initializePolisHelpers() { } function redirectIfNotHttps( - req: { headers: { [x: string]: string; host: string }; method: string; path: string; url: string }, + req: { + headers: { [x: string]: string; host: string }; + method: string; + path: string; + url: string; + }, res: { end: () => any; - status: (arg0: number) => { + status: ( + arg0: number + ) => { send: (arg0: string) => any; }; writeHead: (arg0: number, arg1: { Location: string }) => void; @@ -787,23 +785,23 @@ function initializePolisHelpers() { next: () => any ) { // Exempt dev mode or healthcheck path from HTTPS check - if (devMode || req.path === '/api/v3/testConnection') { + if (devMode || req.path === "/api/v3/testConnection") { return next(); } // Check if the request is already HTTPS - const isHttps = req.headers['x-forwarded-proto'] === 'https'; + const isHttps = req.headers["x-forwarded-proto"] === "https"; if (!isHttps) { - logger.debug('redirecting to https', { headers: req.headers }); + logger.debug("redirecting to https", { headers: req.headers }); // Only redirect GET requests; otherwise, send a 400 error for non-GET methods - if (req.method === 'GET') { + if (req.method === "GET") { res.writeHead(302, { - Location: `https://${req.headers.host}${req.url}` + Location: `https://${req.headers.host}${req.url}`, }); return res.end(); } else { - res.status(400).send('Please use HTTPS when submitting data.'); + res.status(400).send("Please use HTTPS when submitting data."); } } return next(); @@ -1299,8 +1297,15 @@ function initializePolisHelpers() { } res.status(200).json({}); } + + type PcaCacheItem = { + asPOJO: any; + asJSON: string; + asBufferOfGzippedJson: any; + expiration: number; + } let pcaCacheSize = Config.cacheMathResults ? 300 : 1; - let pcaCache = new LruCache({ + let pcaCache = new LruCache({ max: pcaCacheSize, }); @@ -1562,12 +1567,12 @@ function initializePolisHelpers() { return o; } - function getPca(zid?: any, math_tick?: number) { + function getPca(zid?: any, math_tick?: number) :Promise { let cached = pcaCache.get(zid); // Object is of type 'unknown'.ts(2571) // @ts-ignore if (cached && cached.expiration < Date.now()) { - cached = null; + cached = undefined; } // Object is of type 'unknown'.ts(2571) // @ts-ignore @@ -1579,7 +1584,7 @@ function initializePolisHelpers() { cached_math_tick: cachedPOJO.math_tick, query_math_tick: math_tick, }); - return Promise.resolve(null); + return Promise.resolve(undefined); } else { logger.info("math from cache", { zid, math_tick }); return Promise.resolve(cached); @@ -1616,7 +1621,7 @@ function initializePolisHelpers() { math_env: Config.mathEnv, } ); - return null; + return undefined; } let item = rows[0].data; @@ -1629,7 +1634,7 @@ function initializePolisHelpers() { zid, math_tick, }); - return null; + return undefined; } logger.info("after cache miss, found item, adding to cache", { zid, @@ -1638,18 +1643,11 @@ function initializePolisHelpers() { processMathObject(item); - return updatePcaCache(zid, item).then( - function (o: any) { - return o; - }, - function (err: any) { - return err; - } - ); + return updatePcaCache(zid, item); }); } - function updatePcaCache(zid: any, item: { zid: any }) { + function updatePcaCache(zid: any, item: { zid: any }) :Promise { return new Promise(function ( resolve: (arg0: { asPOJO: any; @@ -1680,7 +1678,7 @@ function initializePolisHelpers() { }); } function redirectIfHasZidButNoConversationId( - req: { body: { zid: any; conversation_id: any }, headers?: any }, + req: { body: { zid: any; conversation_id: any }; headers?: any }, res: { writeHead: (arg0: number, arg1: { Location: string }) => void; end: () => any; @@ -2074,6 +2072,149 @@ function initializePolisHelpers() { // }); // return res.end(); } + + async function handle_GET_reportExport( + req: { + p: { rid: string, report_type: string }, + headers: { host: string, "x-forwarded-proto": string } + }, + res: { send: (data :string) => void, setHeader: (key: string, value: string) => void } + ) { + function formatCSV(colFns :Record string>, rows: object[]) :string { + const fns = Object.values(colFns); + const sep = "\n"; + let csv = Object.keys(colFns).join(",") + sep; + if (rows.length > 0) { + for (const row of rows) { + // we append to a single string here (instead of creating an array of strings and joining + // them) to reduce the amount of garbage created; we may have millions of rows, I wish we + // could stream directly to the response... + for (let ii = 0; ii < fns.length; ii += 1) { + if (ii > 0) csv += ","; + csv += fns[ii](row); + } + csv += sep + } + } + return csv; + } + + async function loadConversationSummary (zid :number) { + const [zinvite, convoRows, commentersRow, pca] = await Promise.all([ + getZinvite(zid), + pgQueryP_readOnly( + `SELECT topic, description FROM conversations WHERE zid = $1`, [zid] + ), + pgQueryP_readOnly( + `SELECT COUNT(DISTINCT pid) FROM comments WHERE zid = $1`, [zid] + ), + getPca(zid) + ]); + if (!zinvite || !convoRows || !commentersRow || !pca) { + throw new Error("polis_error_data_unknown_report"); + } + + const convo = (convoRows as {topic: string, description: string}[])[0]; + const commenters = (commentersRow as { count: number }[])[0].count; + + type PcaData = { + "in-conv": number[], + "user-vote-counts": Record, + "group-clusters": Record, + "n-cmts": number, + } + const data = pca.asPOJO as PcaData + const siteUrl = `${req.headers["x-forwarded-proto"]}://${req.headers.host}` + + const escapeQuotes = (s: string) => s.replace(/"/g, "\"\""); + return [ + [ "topic", `"${escapeQuotes(convo.topic)}"` ], + [ "url", `${siteUrl}/${zinvite}` ], + [ "voters", Object.keys(data["user-vote-counts"]).length ], + [ "voters-in-conv", data["in-conv"].length ], + [ "commenters", commenters ], + [ "comments", data["n-cmts"] ], + [ "groups", Object.keys(data["group-clusters"]).length ], + [ "conversation-description", `"${escapeQuotes(convo.description)}"` ], + ].map(row => row.join(",")); + } + + const loadCommentSummary = (zid: number) => pgQueryP_readOnly( + `SELECT + created, + tid, + pid, + COALESCE((SELECT count(*) FROM votes WHERE votes.tid = comments.tid AND vote = 1), 0) as agrees, + COALESCE((SELECT count(*) FROM votes WHERE votes.tid = comments.tid AND vote = -1), 0) as disagrees, + mod, + txt + FROM comments + WHERE zid = $1`, + [zid]); + + const loadVotes = (zid: number) => pgQueryP_readOnly( + `SELECT created as timestamp, tid, pid, vote FROM votes WHERE zid = $1 order by tid, pid`, + [zid]); + + const formatDatetime = (timestamp: string) => new Date(parseInt(timestamp)).toString(); + + const { rid, report_type } = req.p; + try { + const zid = await getZidForRid(rid); + if (!zid) { + fail(res, 404, "polis_error_data_unknown_report"); + return; + } + + switch (report_type) { + case "summary.csv": + res.setHeader('content-type', 'text/csv'); + res.send((await loadConversationSummary(zid)).join("\n")); + break; + + case "comments.csv": + const rows = await loadCommentSummary(zid) as object[] | undefined; + console.log(rows) + if (rows) { + res.setHeader('content-type', 'text/csv'); + res.send(formatCSV({ + "timestamp": (row) => String(Math.floor(row.created/1000)), + "datetime": (row) => formatDatetime(row.created), + "comment-id": (row) => String(row.tid), + "author-id": (row) => String(row.pid), + agrees: (row) => String(row.agrees), + disagrees: (row) => String(row.disagrees), + moderated: (row) => String(row.mod), + "comment-body": (row) => String(row.txt), + }, rows)); + } else fail(res, 500, "polis_err_data_export"); + break; + + case "votes.csv": + const votes = await loadVotes(zid) as object[] | undefined; + if (votes) { + res.setHeader('content-type', 'text/csv'); + res.send(formatCSV({ + timestamp: (row) => String(Math.floor(row.timestamp/1000)), + datetime: (row) => formatDatetime(row.timestamp), + "comment-id": (row) => String(row.tid), + "voter-id": (row) => String(row.pid), + vote: (row) => String(row.vote), + }, votes)); + } else fail(res, 500, "polis_err_data_export"); + break; + + default: + fail(res, 404, "polis_error_data_unknown_report"); + break; + } + } catch (err) { + const msg = err instanceof Error && err.message && err.message.startsWith("polis_") ? + err.message : "polis_err_data_export"; + fail(res, 500, msg, err); + } + } + function getBidIndexToPidMapping(zid: number, math_tick: number) { math_tick = math_tick || -1; return pgQueryP_readOnly( @@ -2368,7 +2509,8 @@ function initializePolisHelpers() { ); } ); - }); + } + ); } const getServerNameWithProtocol = Config.getServerNameWithProtocol; @@ -2407,7 +2549,12 @@ function initializePolisHelpers() { server, function (err: any) { if (err) { - fail(res, 500, "Error: Couldn't send password reset email.", err); + fail( + res, + 500, + "Error: Couldn't send password reset email.", + err + ); return; } finish(); @@ -2470,7 +2617,7 @@ Feel free to reply to this email if you need help.`; ) { res?.clearCookie?.(cookieName, { path: "/", - domain: cookies.cookieDomain(req) + domain: cookies.cookieDomain(req), }); } @@ -3719,12 +3866,14 @@ ${serverName}/pwreset/${pwresettoken} userInfo.email, "Polis Password Reset", body - ).then(function () { - callback?.(); - }).catch(function (err: any) { - logger.error("polis_err_failed_to_email_password_reset_code", err); - callback?.(err); - }); + ) + .then(function () { + callback?.(); + }) + .catch(function (err: any) { + logger.error("polis_err_failed_to_email_password_reset_code", err); + callback?.(err); + }); } ); } @@ -4303,7 +4452,7 @@ Email verified! You can close this tab or hit the back button. // @ts-ignore pid_to_ptpt[c.pid] = c; }); - return Promise.mapSeries( + return BluebirdPromise.mapSeries( candidates, (item: { zid: any; pid: any }, index: any, length: any) => { return getNumberOfCommentsRemaining(item.zid, item.pid).then( @@ -4403,7 +4552,7 @@ Email verified! You can close this tab or hit the back button. } ); - return Promise.each( + return BluebirdPromise.each( needNotification, ( item: { pid: string | number; remaining: any }, @@ -4741,9 +4890,10 @@ Email verified! You can close this tab or hit the back button. // lti_user_image: any; lti_context_id: any; tool_consumer_instance_guid?: any; afterJoinRedirectUrl: any; }; }' but required in type // '{ cookies: { [x: string]: any; }; }'.ts(2345) // @ts-ignore - addCookies(req, res, token, uid).then(function () { - res.json(response_data); - }) + addCookies(req, res, token, uid) + .then(function () { + res.json(response_data); + }) .catch(function (err: any) { fail(res, 500, "polis_err_adding_cookies", err); }); @@ -5271,7 +5421,11 @@ Email verified! You can close this tab or hit the back button. // Type 'unknown' is not assignable to type 'any[]'.ts(2345) // @ts-ignore ).then(function (rows: string | any[]) { - logger.debug("isParentDomainWhitelisted", { domain, zid, isWithinIframe }); + logger.debug("isParentDomainWhitelisted", { + domain, + zid, + isWithinIframe, + }); if (!rows || !rows.length || !rows[0].domain_whitelist.length) { // there is no whitelist, so any domain is ok. logger.debug("isParentDomainWhitelisted : no whitelist"); @@ -6373,7 +6527,6 @@ Email verified! You can close this tab or hit the back button. const _getCommentsList = Comment._getCommentsList; const getNumberOfCommentsRemaining = Comment.getNumberOfCommentsRemaining; - function handle_GET_participation( req: { p: { zid: any; uid?: any; strict: any } }, res: { @@ -7010,400 +7163,304 @@ Email verified! You can close this tab or hit the back button. }); } - function handle_POST_comments( - req: { - p: { - zid?: any; - xid?: any; - uid?: any; - txt?: any; - pid?: any; - vote?: any; - twitter_tweet_id?: any; - quote_twitter_screen_name?: any; - quote_txt?: any; - quote_src_url?: any; - anon?: any; - is_seed?: any; + const GOOGLE_DISCOVERY_URL = + "https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1"; + + async function analyzeComment(txt: string) { + try { + const client = await google.discoverAPI(GOOGLE_DISCOVERY_URL); + + const analyzeRequest = { + comment: { + text: txt, + }, + requestedAttributes: { + TOXICITY: {}, + }, }; - headers?: Headers; - connection?: { remoteAddress: any; socket: { remoteAddress: any } }; - socket?: { remoteAddress: any }; - }, - res: { json: (arg0: { tid: any; currentPid: any }) => void } - ) { - let zid = req.p.zid; - let xid = req.p.xid; - let uid = req.p.uid; - let txt = req.p.txt; - let pid = req.p.pid; // PID_FLOW may be undefined - let currentPid = pid; - let vote = req.p.vote; - let twitter_tweet_id = req.p.twitter_tweet_id; - let quote_twitter_screen_name = req.p.quote_twitter_screen_name; - let quote_txt = req.p.quote_txt; - let quote_src_url = req.p.quote_src_url; - let anon = req.p.anon; - let is_seed = req.p.is_seed; - let mustBeModerator = !!quote_txt || !!twitter_tweet_id || anon; - - // either include txt, or a tweet id - if ( - (_.isUndefined(txt) || txt === "") && - (_.isUndefined(twitter_tweet_id) || twitter_tweet_id === "") && - (_.isUndefined(quote_txt) || quote_txt === "") - ) { - fail(res, 400, "polis_err_param_missing_txt"); - return; + + const response = await client.comments.analyze({ + key: Config.googleJigsawPerspectiveApiKey, + resource: analyzeRequest, + }); + + return response.data; + } catch (err) { + console.error("Error:", err); } + } + + /* this is a concept and can be generalized to other handlers */ + interface PolisRequestParams { + zid?: string; + xid?: string; + uid?: string; + txt?: string; + pid?: string; + vote?: number; + anon?: boolean; + is_seed?: boolean; + } + + interface PolisRequest extends Request { + p: PolisRequestParams; + } + + async function handle_POST_comments( + req: PolisRequest, + res: Response + ): Promise { + const { zid, xid, uid, txt, pid: initialPid, vote, anon, is_seed } = req.p; + + console.log("============= handle_POST_comments ==========="); + console.log(zid, xid, uid, txt, initialPid, vote, anon, is_seed); + /* + 2024-08-20 15:44:25 ============= handle_POST_comments =========== + 2024-08-20 15:44:25 37436 undefined 186 a lovely comment 3 undefined -1 undefined undefined + */ - if (quote_txt && _.isUndefined(quote_src_url)) { - fail(res, 400, "polis_err_param_missing_quote_src_url"); + let pid = initialPid; + let currentPid = pid; + const mustBeModerator = anon; + + if (!txt || txt === "") { + fail(res, 400, "polis_err_param_missing_txt"); return; } - function doGetPid() { - // PID_FLOW + async function doGetPid(): Promise { if (_.isUndefined(pid)) { - return getPidPromise(req.p.zid, req.p.uid, true).then((pid: number) => { - if (pid === -1) { - // Argument of type '(rows: any[]) => number' is not assignable to parameter of type '(value: unknown) => number | PromiseLike'. - // Types of parameters 'rows' and 'value' are incompatible. - // Type 'unknown' is not assignable to type 'any[]'.ts(2345) - // @ts-ignore - return addParticipant(req.p.zid, req.p.uid).then(function ( - rows: any[] - ) { - let ptpt = rows[0]; - pid = ptpt.pid; - currentPid = pid; - return pid; - }); - } else { - return pid; - } - }); + const newPid = await getPidPromise(zid!, uid!, true); + if (newPid === -1) { + const rows = await addParticipant(zid!, uid!); + const ptpt = rows[0]; + pid = ptpt.pid; + currentPid = pid; + return pid; + } else { + return newPid; + } } - return Promise.resolve(pid); - } - let twitterPrepPromise = Promise.resolve(); - if (twitter_tweet_id) { - twitterPrepPromise = prepForTwitterComment(twitter_tweet_id, zid); - } else if (quote_twitter_screen_name) { - twitterPrepPromise = prepForQuoteWithTwitterUser( - quote_twitter_screen_name, - zid - ); + return Number(pid); } - twitterPrepPromise - .then( - // No overload matches this call. - // Overload 1 of 2, '(onFulfill?: ((value: void) => any) | undefined, onReject?: ((error: any) => any) | undefined): Bluebird', gave the following error. - // Argument of type '(info: { ptpt: any; tweet: any; }) => Bluebird' is not assignable to parameter of type '(value: void) => any'. - // Types of parameters 'info' and 'value' are incompatible. - // Type 'void' is not assignable to type '{ ptpt: any; tweet: any; }'. - // Overload 2 of 2, '(onfulfilled?: ((value: void) => any) | null | undefined, onrejected?: ((reason: any) => Resolvable) | null | undefined): Bluebird', gave the following error. - // Argument of type '(info: { ptpt: any; tweet: any; }) => Bluebird' is not assignable to parameter of type '(value: void) => any'. - // Types of parameters 'info' and 'value' are incompatible. - // Type 'void' is not assignable to type '{ ptpt: any; tweet: any; }'.ts(2769) - // @ts-ignore - function (info: { ptpt: any; tweet: any }) { - let ptpt = info && info.ptpt; - // let twitterUser = info && info.twitterUser; - let tweet = info && info.tweet; - - if (tweet) { - logger.debug("Post comments tweet", { txt, tweetTxt: tweet.txt }); - txt = tweet.text; - } else if (quote_txt) { - logger.debug("Post comments quote_txt", { txt, quote_txt }); - txt = quote_txt; - } else { - logger.debug("Post comments txt", {zid, pid, txt}); - } - - let ip = - req?.headers?.["x-forwarded-for"] || // TODO This header may contain multiple IP addresses. Which should we report? - req?.connection?.remoteAddress || - req?.socket?.remoteAddress || - req?.connection?.socket.remoteAddress; - - let isSpamPromise = isSpam({ - comment_content: txt, - comment_author: uid, - permalink: "https://pol.is/" + zid, - user_ip: ip, - user_agent: req?.headers?.["user-agent"], - referrer: req?.headers?.referer, - }); - isSpamPromise.catch(function (err: any) { - logger.error("isSpam failed", err); - }); - // let isSpamPromise = Promise.resolve(false); - let isModeratorPromise = isModerator(zid, uid); + try { + logger.debug("Post comments txt", { zid, pid, txt }); + + const ip = + req.headers["x-forwarded-for"] || + req.connection?.remoteAddress || + req.socket?.remoteAddress || + req.connection?.socket?.remoteAddress; + + const isSpamPromise = isSpam({ + comment_content: txt, + comment_author: uid!, + permalink: `https://pol.is/${zid}`, + user_ip: ip as string, + user_agent: req.headers["user-agent"], + referrer: req.headers.referer, + }).catch((err) => { + logger.error("isSpam failed", err); + return false; + }); - let conversationInfoPromise = getConversationInfo(zid); + const jigsawModerationPromise = analyzeComment(txt); - // return xidUserPromise.then(function(xidUser) { + const isModeratorPromise = isModerator(zid!, uid!); + const conversationInfoPromise = getConversationInfo(zid!); - let shouldCreateXidRecord = false; + let shouldCreateXidRecord = false; - let pidPromise; - if (ptpt) { - pidPromise = Promise.resolve(ptpt.pid); - } else { - let xidUserPromise = - !_.isUndefined(xid) && !_.isNull(xid) - ? getXidStuff(xid, zid) - : Promise.resolve(); - pidPromise = xidUserPromise.then((xidUser: UserType | "noXidRecord") => { - shouldCreateXidRecord = xidUser === "noXidRecord"; - if (typeof xidUser === 'object') { - uid = xidUser.uid; - pid = xidUser.pid; - return pid; - } else { - return doGetPid().then((pid: any) => { - if (shouldCreateXidRecord) { - // Expected 6 arguments, but got 3.ts(2554) - // conversation.ts(34, 3): An argument for 'x_profile_image_url' was not provided. - // @ts-ignore - return createXidRecordByZid(zid, uid, xid).then(() => { - return pid; - }); - } - return pid; - }); - } - }); + const pidPromise = (async () => { + if (xid) { + const xidUser = await getXidStuff(xid, zid!); + shouldCreateXidRecord = xidUser === "noXidRecord"; + if (typeof xidUser === "object") { + uid = xidUser.uid; + pid = xidUser.pid; + return pid; } + } + const newPid = await doGetPid(); + if (shouldCreateXidRecord) { + await createXidRecordByZid(zid!, uid!, xid!); + } + return newPid; + })(); + + const commentExistsPromise = commentExists(zid!, txt); + + const [ + finalPid, + conv, + is_moderator, + commentExistsAlready, + spammy, + jigsawResponse, + ] = await Promise.all([ + pidPromise, + conversationInfoPromise, + isModeratorPromise, + commentExistsPromise, + isSpamPromise, + jigsawModerationPromise, + ]); - let commentExistsPromise = commentExists(zid, txt); + if (!is_moderator && mustBeModerator) { + fail(res, 403, "polis_err_post_comment_auth"); + return; + } - return Promise.all([ - pidPromise, - conversationInfoPromise, - isModeratorPromise, - commentExistsPromise, - ]).then( - function (results: any[]) { - let pid = results[0]; - let conv = results[1]; - let is_moderator = results[2]; - let commentExists = results[3]; - - if (!is_moderator && mustBeModerator) { - fail(res, 403, "polis_err_post_comment_auth"); - return; - } + if (finalPid < 0) { + fail(res, 500, "polis_err_post_comment_bad_pid"); + return; + } - if (pid < 0) { - // NOTE: this API should not be called in /demo mode - fail(res, 500, "polis_err_post_comment_bad_pid"); - return; - } + if (commentExistsAlready) { + fail(res, 409, "polis_err_post_comment_duplicate"); + return; + } - if (commentExists) { - fail(res, 409, "polis_err_post_comment_duplicate"); - return; - } + if (!conv.is_active) { + fail(res, 403, "polis_err_conversation_is_closed"); + return; + } - if (!conv.is_active) { - fail(res, 403, "polis_err_conversation_is_closed"); - return; - } + const bad = hasBadWords(txt); - if (_.isUndefined(txt)) { - logger.error("polis_err_post_comments_missing_txt"); - throw "polis_err_post_comments_missing_txt"; - } - let bad = hasBadWords(txt); + const velocity = 1; + const jigsawToxicityThreshold = 0.8; + let active = true; + const classifications = []; - return isSpamPromise - .then( - function (spammy: any) { - return spammy; - }, - function (err: any) { - logger.error("spam check failed", err); - return false; // spam check failed, continue assuming "not spammy". - } - ) - .then(function (spammy: any) { - let velocity = 1; - let active = true; - let classifications = []; - if (bad && conv.profanity_filter) { - active = false; - classifications.push("bad"); - logger.info("active=false because (bad && conv.profanity_filter)"); - } - if (spammy && conv.spam_filter) { - active = false; - classifications.push("spammy"); - logger.info("active=false because (spammy && conv.spam_filter)"); - } - if (conv.strict_moderation) { - active = false; - logger.info("active=false because (conv.strict_moderation)"); - } + console.log("JIGSAW RESPONSE", txt); + console.log( + `Jigsaw toxicty Score for comment "${txt}": ${jigsawResponse?.attributeScores?.TOXICITY?.summaryScore?.value}` + ); - let mod = 0; // hasn't yet been moderated. + if ( + conv.profanity_filter && + jigsawResponse?.attributeScores?.TOXICITY?.summaryScore?.value > + jigsawToxicityThreshold + ) { + active = false; + classifications.push("bad"); + logger.info("active=false because (bad && conv.profanity_filter)"); + } + if (spammy && conv.spam_filter) { + active = false; + classifications.push("spammy"); + logger.info("active=false because (spammy && conv.spam_filter)"); + } + if (conv.strict_moderation) { + active = false; + logger.info("active=false because (conv.strict_moderation)"); + } - // moderators' comments are automatically in (when prepopulating). - if (is_moderator && is_seed) { - mod = polisTypes.mod.ok; - active = true; - } - let authorUid = ptpt ? ptpt.uid : uid; - - Promise.all([detectLanguage(txt)]).then((a: any[]) => { - let detections = a[0]; - let detection = Array.isArray(detections) - ? detections[0] - : detections; - let lang = detection.language; - let lang_confidence = detection.confidence; - - return pgQueryP( - "INSERT INTO COMMENTS " + - "(pid, zid, txt, velocity, active, mod, uid, tweet_id, quote_src_url, anon, is_seed, created, tid, lang, lang_confidence) VALUES " + - "($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, default, null, $12, $13) RETURNING *;", - [ - pid, - zid, - txt, - velocity, - active, - mod, - authorUid, - twitter_tweet_id || null, - quote_src_url || null, - anon || false, - is_seed || false, - lang, - lang_confidence, - ] - ).then( - // Argument of type '(docs: any[]) => any' is not assignable to parameter of type '(value: unknown) => any'. - // Types of parameters 'docs' and 'value' are incompatible. - // Type 'unknown' is not assignable to type 'any[]'.ts(2345) - // @ts-ignore - function (docs: any[]) { - let comment = docs && docs[0]; - let tid = comment && comment.tid; - // let createdTime = comment && comment.created; - - if (bad || spammy || conv.strict_moderation) { - getNumberOfCommentsWithModerationStatus( - zid, - polisTypes.mod.unmoderated - ) - .catch(function (err: any) { - logger.error( - "polis_err_getting_modstatus_comment_count", - err - ); - return void 0; - }) - .then(function (n: number) { - if (n === 0) { - return; - } - pgQueryP_readOnly( - "select * from users where site_id = (select site_id from page_ids where zid = ($1)) UNION select * from users where uid = ($2);", - [zid, conv.owner] - ).then(function (users: any) { - let uids = _.pluck(users, "uid"); - // also notify polis team for moderation - uids.forEach(function (uid?: any) { - sendCommentModerationEmail(req, uid, zid, n); - }); - }); - }); - } else { - addNotificationTask(zid); - } + let mod = 0; + if (is_moderator && is_seed) { + mod = polisTypes.mod.ok; + active = true; + } - // It should be safe to delete this. Was added to postpone the no-auto-vote change for old conversations. - if (is_seed && _.isUndefined(vote) && zid <= 17037) { - vote = 0; - } + const [detections] = await Promise.all([detectLanguage(txt)]); + const detection = Array.isArray(detections) ? detections[0] : detections; + const lang = detection.language; + const lang_confidence = detection.confidence; - let createdTime = comment.created; - let votePromise = _.isUndefined(vote) - ? Promise.resolve() - : votesPost(uid, pid, zid, tid, xid, vote, 0); - - return ( - votePromise - // This expression is not callable. - //Each member of the union type '{ (onFulfill?: ((value: void) => Resolvable) | undefined, onReject?: ((error: any) => Resolvable) | undefined): Bluebird; (onfulfilled?: ((value: void) => Resolvable<...>) | ... 1 more ... | undefined, onrejected?: ((reason: any) => Resolvable<...>) | ... 1 more ... | u...' has signatures, but none of those signatures are compatible with each other.ts(2349) - // @ts-ignore - .then( - function (o: { vote: { created: any } }) { - if (o && o.vote && o.vote.created) { - createdTime = o.vote.created; - } - - setTimeout(function () { - updateConversationModifiedTime( - zid, - createdTime - ); - updateLastInteractionTimeForConversation( - zid, - uid - ); - if (!_.isUndefined(vote)) { - updateVoteCount(zid, pid); - } - }, 100); - - res.json({ - tid: tid, - currentPid: currentPid, - }); - }, - function (err: any) { - fail(res, 500, "polis_err_vote_on_create", err); - } - ) - ); - }, - function (err: { code: string | number }) { - if (err.code === "23505" || err.code === 23505) { - // duplicate comment - fail(res, 409, "polis_err_post_comment_duplicate", err); - } else { - fail(res, 500, "polis_err_post_comment", err); - } - } - ); // insert - }); // lang - }); - }, - function (errors: any[]) { - if (errors[0]) { - fail(res, 500, "polis_err_getting_pid", errors[0]); - return; - } - if (errors[1]) { - fail(res, 500, "polis_err_getting_conv_info", errors[1]); - return; - } - } + const insertedComment = await pgQueryP( + `INSERT INTO COMMENTS + (pid, zid, txt, velocity, active, mod, uid, anon, is_seed, created, tid, lang, lang_confidence) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, default, null, $10, $11) + RETURNING *;`, + [ + finalPid, + zid, + txt, + velocity, + active, + mod, + uid, + anon || false, + is_seed || false, + lang, + lang_confidence, + ] + ); + + const comment = insertedComment[0]; + const tid = comment.tid; + + if (bad || spammy || conv.strict_moderation) { + try { + const n = await getNumberOfCommentsWithModerationStatus( + zid!, + polisTypes.mod.unmoderated ); - }, - function (err: any) { - fail(res, 500, "polis_err_fetching_tweet", err); + if (n !== 0) { + const users = await pgQueryP_readOnly( + "SELECT * FROM users WHERE site_id = (SELECT site_id FROM page_ids WHERE zid = $1) UNION SELECT * FROM users WHERE uid = $2;", + [zid, conv.owner] + ); + const uids = users.map((user: { uid: string }) => user.uid); + uids.forEach((uid: string) => + sendCommentModerationEmail(req, uid, zid!, n) + ); + } + } catch (err) { + logger.error("polis_err_getting_modstatus_comment_count", err); } - ) - .catch(function (err: any) { - fail(res, 500, "polis_err_post_comment_misc", err); + } else { + addNotificationTask(zid!); + } + + if (is_seed && _.isUndefined(vote) && Number(zid) <= 17037) { + vote = 0; + } + + let createdTime = comment.created; + + if (!_.isUndefined(vote)) { + try { + const voteResult = await votesPost( + uid!, + finalPid, + zid!, + tid, + xid!, + vote, + 0 + ); + if (voteResult?.vote?.created) { + createdTime = voteResult.vote.created; + } + } catch (err) { + fail(res, 500, "polis_err_vote_on_create", err); + return; + } + } + + setTimeout(() => { + updateConversationModifiedTime(zid!, createdTime); + updateLastInteractionTimeForConversation(zid!, uid!); + if (!_.isUndefined(vote)) { + updateVoteCount(zid!, finalPid); + } + }, 100); + + res.json({ + tid: tid, + currentPid: currentPid, }); - } // end POST /api/v3/comments + } catch (err: any) { + if (err.code === "23505" || err.code === 23505) { + fail(res, 409, "polis_err_post_comment_duplicate", err); + } else { + fail(res, 500, "polis_err_post_comment", err); + } + } + } function handle_GET_votes_me( req: { p: { zid: any; uid?: any; pid: any } }, @@ -7503,8 +7560,11 @@ Email verified! You can close this tab or hit the back button. let comments = results[0]; let math = results[1]; let numberOfCommentsRemainingRows = results[2]; - logger.debug("getNextPrioritizedComment intermediate results:", - {zid, pid, numberOfCommentsRemainingRows}); + logger.debug("getNextPrioritizedComment intermediate results:", { + zid, + pid, + numberOfCommentsRemainingRows, + }); if (!comments || !comments.length) { return null; } else if ( @@ -7959,7 +8019,7 @@ Email verified! You can close this tab or hit the back button. req.p.tid, req.p.xid, req.p.vote, - req.p.weight, + req.p.weight ); }) .then(function (o: { vote: any }) { @@ -7983,7 +8043,11 @@ Email verified! You can close this tab or hit the back button. return getNextComment(zid, pid, [], true, lang); }) .then(function (nextComment: any) { - logger.debug("handle_POST_votes nextComment:", {zid, pid, nextComment}); + logger.debug("handle_POST_votes nextComment:", { + zid, + pid, + nextComment, + }); let result: PidReadyResult = {}; if (nextComment) { result.nextComment = nextComment; @@ -8022,8 +8086,8 @@ Email verified! You can close this tab or hit the back button. fail(res, 403, "polis_err_conversation_is_closed", err); } else if (err === "polis_err_post_votes_social_needed") { fail(res, 403, "polis_err_post_votes_social_needed", err); - } else if (err === 'polis_err_xid_not_whitelisted') { - fail(res, 403, 'polis_err_xid_not_whitelisted', err); + } else if (err === "polis_err_xid_not_whitelisted") { + fail(res, 403, "polis_err_xid_not_whitelisted", err); } else { fail(res, 500, "polis_err_vote", err); } @@ -8460,7 +8524,7 @@ Email verified! You can close this tab or hit the back button. pgQueryP( "update conversations set is_active = false where zid = ($1);", [conv.zid] - ) + ); }) .catch(function (err: any) { fail(res, 500, "polis_err_closing_conversation", err); @@ -10913,17 +10977,18 @@ Thanks for using Polis! // * create a twitter record } - function addParticipant(zid: any, uid?: any) { - return pgQueryP( + const addParticipant = async (zid: string, uid?: string): Promise => { + await pgQueryP( "INSERT INTO participants_extended (zid, uid) VALUES ($1, $2);", [zid, uid] - ).then(() => { - return pgQueryP( - "INSERT INTO participants (pid, zid, uid, created) VALUES (NULL, $1, $2, default) RETURNING *;", - [zid, uid] - ); - }); - } + ); + + return pgQueryP( + "INSERT INTO participants (pid, zid, uid, created) VALUES (NULL, $1, $2, default) RETURNING *;", + [zid, uid] + ); + }; + function getAndInsertTwitterUser(o: any, uid?: any) { return getTwitterUserInfo(o, false).then(function (userString: string) { const u: UserType = JSON.parse(userString)[0]; @@ -12510,7 +12575,6 @@ Thanks for using Polis! }); } - function handle_POST_einvites( req: { p: { email: any } }, res: { @@ -13536,7 +13600,8 @@ Thanks for using Polis! // serve up index.html in response to anything starting with a number let hostname: string = Config.staticFilesHost; - let staticFilesParticipationPort: number = Config.staticFilesParticipationPort; + let staticFilesParticipationPort: number = + Config.staticFilesParticipationPort; let staticFilesAdminPort: number = Config.staticFilesAdminPort; let fetchUnsupportedBrowserPage = makeFileFetcher( hostname, @@ -13611,14 +13676,16 @@ Thanks for using Polis! function ifDefinedFirstElseSecond(first: any, second: boolean) { return _.isUndefined(first) ? second : first; } - let fetch404Page = makeFileFetcher(hostname, staticFilesAdminPort, "/404.html", { - "Content-Type": "text/html", - }); + let fetch404Page = makeFileFetcher( + hostname, + staticFilesAdminPort, + "/404.html", + { + "Content-Type": "text/html", + } + ); - function fetchIndexForConversation( - req: { path: string; }, - res: any - ) { + function fetchIndexForConversation(req: { path: string }, res: any) { logger.debug("fetchIndexForConversation", req.path); let match = req.path.match(/[0-9][0-9A-Za-z]+/); let conversation_id: any; @@ -13654,12 +13721,7 @@ Thanks for using Polis! conversation: x, // Nothing user-specific can go here, since we want to cache these per-conv index files on the CDN. }; - fetchIndex( - req, - res, - preloadData, - staticFilesParticipationPort - ); + fetchIndex(req, res, preloadData, staticFilesParticipationPort); }) .catch(function (err: any) { logger.error("polis_err_fetching_conversation_info", err); @@ -13948,6 +14010,7 @@ Thanks for using Polis! handle_GET_math_correlationMatrix, handle_GET_dataExport, handle_GET_dataExport_results, + handle_GET_reportExport, handle_GET_domainWhitelist, handle_GET_dummyButton, handle_GET_einvites, diff --git a/server/tsconfig.json b/server/tsconfig.json index c38392e11..5e35cc18d 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -4,7 +4,7 @@ /* Basic Options */ // "incremental": true, /* Enable incremental compilation */ - "target": "es5" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, + "target": "es2017" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, // "lib": [], /* Specify library files to be included in the compilation. */ "allowJs": true /* Allow javascript files to be compiled. */,