|
| 1 | +import * as fs from "fs"; |
| 2 | +import * as stream from "stream"; |
| 3 | +import * as http from "http"; |
| 4 | +import * as https from "https"; |
| 5 | +import * as path from "path"; |
| 6 | +import mimeDB from "mime-db"; |
| 7 | +import { chromium } from "playwright-extra"; |
| 8 | +import StealthPlugin from "puppeteer-extra-plugin-stealth"; |
| 9 | + |
| 10 | +chromium.use(StealthPlugin()); |
| 11 | + |
| 12 | +/** |
| 13 | + * @typedef {import('playwright').Browser} Browser |
| 14 | + */ |
| 15 | + |
| 16 | +/** |
| 17 | + * @typedef {{id: number, url: string}|{id: number, doi: string}} ParsedReference |
| 18 | + */ |
| 19 | + |
| 20 | +/** |
| 21 | + * @param browser {Browser} |
| 22 | + * @param url {string} |
| 23 | + * @return {Promise<string>} |
| 24 | + */ |
| 25 | +async function getPageAsMHTML(browser, url) { |
| 26 | + const page = await browser.newPage(); |
| 27 | + try { |
| 28 | + await page.goto(url); |
| 29 | + await page.waitForLoadState('networkidle'); |
| 30 | + // Extra timeout bc some pages fade content in with an animation |
| 31 | + await page.waitForTimeout(1000); |
| 32 | + const session = await page.context().newCDPSession(page); |
| 33 | + const doc = await session.send('Page.captureSnapshot', {format: 'mhtml'}); |
| 34 | + return doc.data; |
| 35 | + } finally { |
| 36 | + await page.close(); |
| 37 | + } |
| 38 | +} |
| 39 | + |
| 40 | +/** |
| 41 | + * @param browser {Browser} |
| 42 | + * @param doi {string} |
| 43 | + * @param filename {string} |
| 44 | + * @return {Promise<string>} |
| 45 | + */ |
| 46 | +async function saveDOIAsPDFFromSciHub(browser, doi, filename) { |
| 47 | + const url = `https://sci-hub.st/${doi}`; |
| 48 | + const page = await browser.newPage(); |
| 49 | + try { |
| 50 | + await page.goto(url); |
| 51 | + await page.waitForLoadState('domcontentloaded'); |
| 52 | + await page.locator('button', { hasText: 'save' }).click(); |
| 53 | + await page.pdf({ path: filename }); |
| 54 | + } catch (e) { |
| 55 | + if (e.name === 'TimeoutError') { |
| 56 | + throw new Error(`TimeoutError while accessing sci-hub - this might be because of a captcha. |
| 57 | + Alternatively, visit https://sci-hub.st/${doi} and download the file manually.`); |
| 58 | + } |
| 59 | + throw e; |
| 60 | + } finally { |
| 61 | + await page.close(); |
| 62 | + } |
| 63 | +} |
| 64 | + |
| 65 | +/** |
| 66 | + * @param browser {Browser} |
| 67 | + * @param url {string} |
| 68 | + * @param filename {string} |
| 69 | + * @return {Promise<void>} |
| 70 | + */ |
| 71 | +async function savePageAsMHTML(browser, url, filename) { |
| 72 | + const mhtml = await getPageAsMHTML(browser, url); |
| 73 | + await fs.promises.writeFile(filename, mhtml, 'utf-8'); |
| 74 | +} |
| 75 | + |
| 76 | +/** |
| 77 | + * @param url {string} |
| 78 | + * @param filename {string} |
| 79 | + * @return {Promise<void>} |
| 80 | + */ |
| 81 | +async function downloadFile(url, filename) { |
| 82 | + const ws = fs.createWriteStream(filename); |
| 83 | + /** @type {import('http')|import('https')} */ |
| 84 | + let httplib = https; |
| 85 | + if (url.startsWith('http:')) { |
| 86 | + httplib = http; |
| 87 | + } |
| 88 | + |
| 89 | + await /** @type {Promise<void>} */(new Promise((resolve, reject) => { |
| 90 | + httplib.get(url, res => { |
| 91 | + stream.pipeline(res, ws, err => { |
| 92 | + if (err) { |
| 93 | + reject(err); |
| 94 | + } else { |
| 95 | + resolve(); |
| 96 | + } |
| 97 | + }); |
| 98 | + }); |
| 99 | + })); |
| 100 | +} |
| 101 | + |
| 102 | +/** |
| 103 | + * @param browser {Browser} |
| 104 | + * @param url {string} |
| 105 | + * @param filename {string} |
| 106 | + * @return {Promise<void>} |
| 107 | + */ |
| 108 | +async function savePage(browser, url, filename) { |
| 109 | + if (filename.endsWith('.mhtml')) { |
| 110 | + await savePageAsMHTML(browser, url, filename); |
| 111 | + } else { |
| 112 | + await downloadFile(url, filename); |
| 113 | + } |
| 114 | +} |
| 115 | + |
| 116 | +/** |
| 117 | + * Make a HEAD request to a URL and return the value of the Content-Type response header. |
| 118 | + * Return null if the request fails or the response has a non-ok status code or no |
| 119 | + * Content-Type header. |
| 120 | + * @param url {string} |
| 121 | + * @return {Promise<string>} |
| 122 | + */ |
| 123 | +async function getMimeTypeForURL(url) { |
| 124 | + const response = await fetch(url, {method: 'HEAD'}); |
| 125 | + if (!response.ok) { |
| 126 | + throw new Error(`HEAD ${url} returned non-ok response status code: ${response.status}`); |
| 127 | + } |
| 128 | + const contentType = response.headers.get('content-type'); |
| 129 | + if (contentType === null) { |
| 130 | + throw new Error(`HEAD ${url} returned response with status code ${response.status} and no Content-Type header`); |
| 131 | + } |
| 132 | + return contentType.split(';')[0]; |
| 133 | +} |
| 134 | + |
| 135 | +/** |
| 136 | + * @param id {number} |
| 137 | + * @param mimeType {string} |
| 138 | + * @param targetDirectory {string} |
| 139 | + * @return {string} |
| 140 | + */ |
| 141 | +function getPageTargetFilename(id, mimeType, targetDirectory) { |
| 142 | + if (mimeType === 'text/html') { |
| 143 | + return path.join(targetDirectory, `${id}.mhtml`); |
| 144 | + } |
| 145 | + const extension = mimeDB[mimeType]?.extensions?.[0]; |
| 146 | + if (extension) { |
| 147 | + return path.join(targetDirectory, `${id}.${extension}`); |
| 148 | + } |
| 149 | + return path.join(targetDirectory, `${id}`); |
| 150 | +} |
| 151 | + |
| 152 | +/** |
| 153 | + * Return true if the given string is valid URL. |
| 154 | + * This is equivalent to URL.canParse (which is only available in node>=20) |
| 155 | + * @param url {string} |
| 156 | + * @return {boolean} |
| 157 | + */ |
| 158 | +function isValidURL(url) { |
| 159 | + try { |
| 160 | + new URL(url); |
| 161 | + return true; |
| 162 | + } catch { |
| 163 | + return false; |
| 164 | + } |
| 165 | +} |
| 166 | + |
| 167 | +/** |
| 168 | + * Find references with https: URLs or DOIs in a text. |
| 169 | + * Yield objects with the reference ID and URL or DOI. |
| 170 | + * @param text {string} |
| 171 | + * @return {Generator<ParsedReference>} |
| 172 | + */ |
| 173 | +function *extractURLsFromText(text) { |
| 174 | + const re = /^\[(?<id>[0-9]+)].*?((?<url>https?:\/\/\S+)|(?<doi>10\.[0-9]+\/\S+))/gm; |
| 175 | + let match = null; |
| 176 | + while ((match = re.exec(text)) !== null) { |
| 177 | + const id = parseInt(match.groups.id); |
| 178 | + if (!isNaN(id)) { |
| 179 | + const { url, doi } = match.groups; |
| 180 | + if (isValidURL(url)) { |
| 181 | + yield {id, url}; |
| 182 | + } else if (doi) { |
| 183 | + yield {id, doi} |
| 184 | + } |
| 185 | + } |
| 186 | + } |
| 187 | +} |
| 188 | + |
| 189 | +/** |
| 190 | + * @param file {string} |
| 191 | + * @return {Promise<boolean>} |
| 192 | + */ |
| 193 | +async function fileExists(file) { |
| 194 | + try { |
| 195 | + const stat = await fs.promises.stat(file); |
| 196 | + if (stat) { |
| 197 | + return stat.isFile(); |
| 198 | + } |
| 199 | + } catch {} |
| 200 | + |
| 201 | + return false; |
| 202 | +} |
| 203 | + |
| 204 | +/** |
| 205 | + * @param reference {ParsedReference} |
| 206 | + * @param targetDirectory {string} |
| 207 | + * @param browser {Browser} |
| 208 | + * @return {Promise<void>} |
| 209 | + */ |
| 210 | +async function saveReference(reference, targetDirectory, browser) { |
| 211 | + const {id, url, doi} = reference; |
| 212 | + if (url) { |
| 213 | + const mimeType = await getMimeTypeForURL(url); |
| 214 | + const targetFile = getPageTargetFilename(id, mimeType, targetDirectory); |
| 215 | + if (await fileExists(targetFile)) { |
| 216 | + console.log(`[${id}]: ${targetFile} exists, skipping`); |
| 217 | + return; |
| 218 | + } |
| 219 | + await savePage(browser, url, targetFile); |
| 220 | + console.log(`[${id}]: ${targetFile} downloaded`); |
| 221 | + } else if (doi) { |
| 222 | + const targetFile = getPageTargetFilename(id, 'application/pdf', targetDirectory); |
| 223 | + await saveDOIAsPDFFromSciHub(browser, doi, targetFile); |
| 224 | + console.log(`[${id}]: ${targetFile} downloaded`); |
| 225 | + } |
| 226 | +} |
| 227 | + |
| 228 | +/** |
| 229 | + * @param input {string} |
| 230 | + * @param targetDirectory {string} |
| 231 | + * @param browser {Browser} |
| 232 | + * @return {Promise<void>} |
| 233 | + */ |
| 234 | +async function extractAndSaveURLsWithBrowser(input, targetDirectory, browser) { |
| 235 | + for (const reference of extractURLsFromText(input)) { |
| 236 | + try { |
| 237 | + await saveReference(reference, targetDirectory, browser); |
| 238 | + } catch (e) { |
| 239 | + console.log(`[${reference.id}]: error: ${e}`); |
| 240 | + } |
| 241 | + } |
| 242 | +} |
| 243 | + |
| 244 | +/** |
| 245 | + * Extract IEEE-style references with URLs from the given string, and download the URL content |
| 246 | + * to a file in the specified directory, with the filename equal to the reference number and |
| 247 | + * the file extension matching the mime type of the file. |
| 248 | + * |
| 249 | + * @example |
| 250 | + * await extractAndSaveAllURLs( |
| 251 | + * '[1] First reference. [Online]. Available: https://jfhr.de/reference-archive/example.html (Accessed: 2023-07-23)\n' + |
| 252 | + * '[2] Second reference. [Online]. Available: https://jfhr.de/reference-archive/example.pdf (Accessed: 2023-07-23)', |
| 253 | + * './archive/' |
| 254 | + * ); |
| 255 | + * // Will create the following files: |
| 256 | + * // ./archive/1.mhtml |
| 257 | + * // ./archive/2.pdf |
| 258 | + * |
| 259 | + * @param input {string} |
| 260 | + * @param targetDirectory {string} |
| 261 | + * @return {Promise<void>} |
| 262 | + */ |
| 263 | +export async function extractAndSaveAllURLs(input, targetDirectory) { |
| 264 | + const browser = await chromium.launch(); |
| 265 | + try { |
| 266 | + await extractAndSaveURLsWithBrowser(input, targetDirectory, browser); |
| 267 | + } finally { |
| 268 | + await browser.close(); |
| 269 | + } |
| 270 | +} |
0 commit comments