node-single-file-core-1.0.47+ds1/0000775000175000017500000000000015271143347015364 5ustar emfoxemfoxnode-single-file-core-1.0.47+ds1/package.json0000644000175000017500000000077703560116604017655 0ustar emfoxemfox{ "name": "single-file-core", "version": "1.0.47", "description": "SingleFile Core", "author": "Gildas Lormeau", "license": "AGPL-3.0-or-later", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { "type": "git", "url": "git+https://github.com/gildas-lormeau/single-file-core.git" }, "bugs": { "url": "https://github.com/gildas-lormeau/single-file-core/issues" }, "homepage": "https://github.com/gildas-lormeau/single-file-core#readme" } node-single-file-core-1.0.47+ds1/single-file.js0000644000175000017500000000573303560116604020120 0ustar emfoxemfox/* * Copyright 2010-2022 Gildas Lormeau * contact : gildas.lormeau gmail.com * * This file is part of SingleFile. * * The code in this file is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License * (GNU AGPL) as published by the Free Software Foundation, either version 3 * of the License, or (at your option) any later version. * * The code in this file is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero * General Public License for more details. * * As additional permission under GNU AGPL version 3 section 7, you may * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU * AGPL normally required by section 4, provided you include this license * notice and a URL through which recipients can access the Corresponding * Source. */ /* global globalThis */ import * as processors from "./processors/index.js"; import * as vendor from "./vendor/index.js"; import * as modules from "./modules/index.js"; import * as core from "./single-file-core.js"; import * as helper from "./single-file-helper.js"; import * as util from "./single-file-util.js"; let SingleFile; export { init, getPageData, processors, vendor, modules, helper, SingleFile }; function init(initOptions) { if (typeof SingleFile == "undefined") { SingleFile = core.getClass(util.getInstance(initOptions), vendor.cssTree); } } async function getPageData(options = {}, initOptions, doc = globalThis.document, win = globalThis) { const frames = processors.frameTree; let framesSessionId; init(initOptions); if (doc && win) { helper.initDoc(doc); const preInitializationPromises = []; if (!options.saveRawPage) { if (!options.removeFrames && frames && globalThis.frames && globalThis.frames.length) { let frameTreePromise; if (options.loadDeferredImages) { frameTreePromise = new Promise(resolve => globalThis.setTimeout(() => resolve(frames.getAsync(options)), options.loadDeferredImagesMaxIdleTime - frames.TIMEOUT_INIT_REQUEST_MESSAGE)); } else { frameTreePromise = frames.getAsync(options); } preInitializationPromises.push(frameTreePromise); } if (options.loadDeferredImages) { preInitializationPromises.push(processors.lazy.process(options)); } } [options.frames] = await Promise.all(preInitializationPromises); framesSessionId = options.frames && options.frames.sessionId; } options.doc = doc; options.win = win; options.insertCanonicalLink = true; options.onprogress = event => { if (event.type === event.RESOURCES_INITIALIZED && doc && win && options.loadDeferredImages) { processors.lazy.resetZoomLevel(options); } }; const processor = new SingleFile(options); await processor.run(); if (framesSessionId) { frames.cleanup(framesSessionId); } return await processor.getPageData(); }node-single-file-core-1.0.47+ds1/single-file-util.js0000644000175000017500000003444703560116604021077 0ustar emfoxemfox/* * Copyright 2010-2022 Gildas Lormeau * contact : gildas.lormeau gmail.com * * This file is part of SingleFile. * * The code in this file is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License * (GNU AGPL) as published by the Free Software Foundation, either version 3 * of the License, or (at your option) any later version. * * The code in this file is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero * General Public License for more details. * * As additional permission under GNU AGPL version 3 section 7, you may * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU * AGPL normally required by section 4, provided you include this license * notice and a URL through which recipients can access the Corresponding * Source. */ /* global globalThis, URLSearchParams */ import * as vendor from "./vendor/index.js"; import * as modules from "./modules/index.js"; import * as helper from "./single-file-helper.js"; const DEBUG = false; const ONE_MB = 1024 * 1024; const PREFIX_CONTENT_TYPE_TEXT = "text/"; const DEFAULT_REPLACED_CHARACTERS = ["~", "+", "\\\\", "?", "%", "*", ":", "|", "\"", "<", ">", "\x00-\x1f", "\x7F"]; const DEFAULT_REPLACEMENT_CHARACTER = "_"; const URL = globalThis.URL; const DOMParser = globalThis.DOMParser; const Blob = globalThis.Blob; const FileReader = globalThis.FileReader; const fetch = (url, options) => globalThis.fetch(url, options); const crypto = globalThis.crypto; const TextDecoder = globalThis.TextDecoder; const TextEncoder = globalThis.TextEncoder; export { getInstance }; function getInstance(utilOptions) { utilOptions = utilOptions || {}; utilOptions.fetch = utilOptions.fetch || fetch; utilOptions.frameFetch = utilOptions.frameFetch || utilOptions.fetch || fetch; return { getContent, parseURL(resourceURL, baseURI) { if (baseURI === undefined) { return new URL(resourceURL); } else { return new URL(resourceURL, baseURI); } }, resolveURL(resourceURL, baseURI) { return this.parseURL(resourceURL, baseURI).href; }, getSearchParams(searchParams) { return Array.from(new URLSearchParams(searchParams)); }, getValidFilename(filename, replacedCharacters = DEFAULT_REPLACED_CHARACTERS, replacementCharacter = DEFAULT_REPLACEMENT_CHARACTER) { replacedCharacters.forEach(replacedCharacter => filename = filename.replace(new RegExp("[" + replacedCharacter + "]+", "g"), replacementCharacter)); filename = filename .replace(/\.\.\//g, "") .replace(/^\/+/, "") .replace(/\/+/g, "/") .replace(/\/$/, "") .replace(/\.$/, "") .replace(/\.\//g, "." + replacementCharacter) .replace(/\/\./g, "/" + replacementCharacter); return filename; }, parseDocContent(content, baseURI) { const doc = (new DOMParser()).parseFromString(content, "text/html"); if (!doc.head) { doc.documentElement.insertBefore(doc.createElement("HEAD"), doc.body); } let baseElement = doc.querySelector("base"); if (!baseElement || !baseElement.getAttribute("href")) { if (baseElement) { baseElement.remove(); } baseElement = doc.createElement("base"); baseElement.setAttribute("href", baseURI); doc.head.insertBefore(baseElement, doc.head.firstChild); } return doc; }, parseXMLContent(content) { return (new DOMParser()).parseFromString(content, "text/xml"); }, parseSVGContent(content) { const doc = (new DOMParser()).parseFromString(content, "image/svg+xml"); if (doc.querySelector("parsererror")) { return (new DOMParser()).parseFromString(content, "text/html"); } else { return doc; } }, async digest(algo, text) { try { const hash = await crypto.subtle.digest(algo, new TextEncoder("utf-8").encode(text)); return hex(hash); } catch (error) { return ""; } }, getContentSize(content) { return new Blob([content]).size; }, formatFilename(content, options) { return modules.templateFormatter.formatFilename(content, options, this); }, evalTemplate(template, options, content, dontReplaceSlash) { return modules.templateFormatter.evalTemplate(template, options, this, content, dontReplaceSlash); }, minifyHTML(doc, options) { return modules.htmlMinifier.process(doc, options); }, minifyCSSRules(stylesheets, styles, mediaAllInfo) { return modules.cssRulesMinifier.process(stylesheets, styles, mediaAllInfo); }, removeUnusedFonts(doc, stylesheets, styles, options) { return modules.fontsMinifier.process(doc, stylesheets, styles, options); }, removeAlternativeFonts(doc, stylesheets, fontDeclarations, fontTests) { return modules.fontsAltMinifier.process(doc, stylesheets, fontDeclarations, fontTests); }, getMediaAllInfo(doc, stylesheets, styles) { return modules.matchedRules.getMediaAllInfo(doc, stylesheets, styles); }, compressCSS(content, options) { return vendor.cssMinifier.processString(content, options); }, minifyMedias(stylesheets) { return modules.mediasAltMinifier.process(stylesheets); }, removeAlternativeImages(doc) { return modules.imagesAltMinifier.process(doc); }, parseSrcset(srcset) { return vendor.srcsetParser.process(srcset); }, preProcessDoc(doc, win, options) { return helper.preProcessDoc(doc, win, options); }, postProcessDoc(doc, markedElements, invalidElements) { helper.postProcessDoc(doc, markedElements, invalidElements); }, serialize(doc, compressHTML) { return modules.serializer.process(doc, compressHTML); }, removeQuotes(string) { return helper.removeQuotes(string); }, ON_BEFORE_CAPTURE_EVENT_NAME: helper.ON_BEFORE_CAPTURE_EVENT_NAME, ON_AFTER_CAPTURE_EVENT_NAME: helper.ON_AFTER_CAPTURE_EVENT_NAME, WIN_ID_ATTRIBUTE_NAME: helper.WIN_ID_ATTRIBUTE_NAME, REMOVED_CONTENT_ATTRIBUTE_NAME: helper.REMOVED_CONTENT_ATTRIBUTE_NAME, HIDDEN_CONTENT_ATTRIBUTE_NAME: helper.HIDDEN_CONTENT_ATTRIBUTE_NAME, HIDDEN_FRAME_ATTRIBUTE_NAME: helper.HIDDEN_FRAME_ATTRIBUTE_NAME, IMAGE_ATTRIBUTE_NAME: helper.IMAGE_ATTRIBUTE_NAME, POSTER_ATTRIBUTE_NAME: helper.POSTER_ATTRIBUTE_NAME, VIDEO_ATTRIBUTE_NAME: helper.VIDEO_ATTRIBUTE_NAME, CANVAS_ATTRIBUTE_NAME: helper.CANVAS_ATTRIBUTE_NAME, STYLE_ATTRIBUTE_NAME: helper.STYLE_ATTRIBUTE_NAME, INPUT_VALUE_ATTRIBUTE_NAME: helper.INPUT_VALUE_ATTRIBUTE_NAME, SHADOW_ROOT_ATTRIBUTE_NAME: helper.SHADOW_ROOT_ATTRIBUTE_NAME, PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME: helper.PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME, STYLESHEET_ATTRIBUTE_NAME: helper.STYLESHEET_ATTRIBUTE_NAME, SELECTED_CONTENT_ATTRIBUTE_NAME: helper.SELECTED_CONTENT_ATTRIBUTE_NAME, INVALID_ELEMENT_ATTRIBUTE_NAME: helper.INVALID_ELEMENT_ATTRIBUTE_NAME, COMMENT_HEADER: helper.COMMENT_HEADER, COMMENT_HEADER_LEGACY: helper.COMMENT_HEADER_LEGACY, SINGLE_FILE_UI_ELEMENT_CLASS: helper.SINGLE_FILE_UI_ELEMENT_CLASS, EMPTY_RESOURCE: helper.EMPTY_RESOURCE }; async function getContent(resourceURL, options) { let response, startTime, networkTimeoutId, networkTimeoutPromise, resolveNetworkTimeoutPromise; const fetchResource = utilOptions.fetch; const fetchFrameResource = utilOptions.frameFetch; if (DEBUG) { startTime = Date.now(); log(" // STARTED download url =", resourceURL, "asBinary =", options.asBinary); } if (options.blockMixedContent && /^https:/i.test(options.baseURI) && !/^https:/i.test(resourceURL)) { return getFetchResponse(resourceURL, options); } if (options.networkTimeout) { networkTimeoutPromise = new Promise((resolve, reject) => { resolveNetworkTimeoutPromise = resolve; networkTimeoutId = globalThis.setTimeout(() => reject(new Error("network timeout")), options.networkTimeout); }); } else { networkTimeoutPromise = new Promise(resolve => { resolveNetworkTimeoutPromise = resolve; }); } try { const accept = options.acceptHeaders ? options.acceptHeaders[options.expectedType] : "*/*"; if (options.frameId) { try { response = await Promise.race([ fetchFrameResource(resourceURL, { frameId: options.frameId, referrer: options.resourceReferrer, headers: { accept } }), networkTimeoutPromise ]); } catch (error) { response = await Promise.race([ fetchResource(resourceURL, { headers: { accept } }), networkTimeoutPromise ]); } } else { response = await Promise.race([ fetchResource(resourceURL, { referrer: options.resourceReferrer, headers: { accept } }), networkTimeoutPromise ]); } } catch (error) { return getFetchResponse(resourceURL, options); } finally { resolveNetworkTimeoutPromise(); if (options.networkTimeout) { globalThis.clearTimeout(networkTimeoutId); } } let buffer; try { buffer = await response.arrayBuffer(); } catch (error) { return { data: options.asBinary ? helper.EMPTY_RESOURCE : "", resourceURL }; } resourceURL = response.url || resourceURL; let contentType = "", charset; try { const mimeType = new vendor.MIMEType(response.headers.get("content-type")); contentType = mimeType.type + "/" + mimeType.subtype; charset = mimeType.parameters.get("charset"); } catch (error) { // ignored } if (!contentType) { contentType = guessMIMEType(options.expectedType, buffer); } if (!charset && options.charset) { charset = options.charset; } if (options.asBinary) { if (response.status >= 400) { return getFetchResponse(resourceURL, options); } try { if (DEBUG) { log(" // ENDED download url =", resourceURL, "delay =", Date.now() - startTime); } if (options.maxResourceSizeEnabled && buffer.byteLength > options.maxResourceSize * ONE_MB) { return getFetchResponse(resourceURL, options); } else { return getFetchResponse(resourceURL, options, buffer, null, contentType); } } catch (error) { return getFetchResponse(resourceURL, options); } } else { if (response.status >= 400 || (options.validateTextContentType && contentType && !contentType.startsWith(PREFIX_CONTENT_TYPE_TEXT))) { return getFetchResponse(resourceURL, options); } if (!charset) { charset = "utf-8"; } if (DEBUG) { log(" // ENDED download url =", resourceURL, "delay =", Date.now() - startTime); } if (options.maxResourceSizeEnabled && buffer.byteLength > options.maxResourceSize * ONE_MB) { return getFetchResponse(resourceURL, options, null, charset); } else { try { return getFetchResponse(resourceURL, options, buffer, charset, contentType); } catch (error) { return getFetchResponse(resourceURL, options, null, charset); } } } } } async function getFetchResponse(resourceURL, options, data, charset, contentType) { if (data) { if (options.asBinary) { const reader = new FileReader(); reader.readAsDataURL(new Blob([data], { type: contentType + (options.charset ? ";charset=" + options.charset : "") })); data = await new Promise((resolve, reject) => { reader.addEventListener("load", () => resolve(reader.result), false); reader.addEventListener("error", reject, false); }); } else { const firstBytes = new Uint8Array(data.slice(0, 4)); if (firstBytes[0] == 132 && firstBytes[1] == 49 && firstBytes[2] == 149 && firstBytes[3] == 51) { charset = "gb18030"; } else if (firstBytes[0] == 255 && firstBytes[1] == 254) { charset = "utf-16le"; } else if (firstBytes[0] == 254 && firstBytes[1] == 255) { charset = "utf-16be"; } try { data = new TextDecoder(charset).decode(data); } catch (error) { charset = "utf-8"; data = new TextDecoder(charset).decode(data); } data = data.replace(/\ufeff/gi, ""); } } else { data = options.asBinary ? helper.EMPTY_RESOURCE : ""; } return { data, resourceURL, charset }; } function guessMIMEType(expectedType, buffer) { if (expectedType == "image") { if (compareBytes([255, 255, 255, 255], [0, 0, 1, 0])) { return "image/x-icon"; } if (compareBytes([255, 255, 255, 255], [0, 0, 2, 0])) { return "image/x-icon"; } if (compareBytes([255, 255], [78, 77])) { return "image/bmp"; } if (compareBytes([255, 255, 255, 255, 255, 255], [71, 73, 70, 56, 57, 97])) { return "image/gif"; } if (compareBytes([255, 255, 255, 255, 255, 255], [71, 73, 70, 56, 59, 97])) { return "image/gif"; } if (compareBytes([255, 255, 255, 255, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255], [82, 73, 70, 70, 0, 0, 0, 0, 87, 69, 66, 80, 86, 80])) { return "image/webp"; } if (compareBytes([255, 255, 255, 255, 255, 255, 255, 255], [137, 80, 78, 71, 13, 10, 26, 10])) { return "image/png"; } if (compareBytes([255, 255, 255], [255, 216, 255])) { return "image/jpeg"; } } if (expectedType == "font") { if (compareBytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 80])) { return "application/vnd.ms-fontobject"; } if (compareBytes([255, 255, 255, 255], [0, 1, 0, 0])) { return "font/ttf"; } if (compareBytes([255, 255, 255, 255], [79, 84, 84, 79])) { return "font/otf"; } if (compareBytes([255, 255, 255, 255], [116, 116, 99, 102])) { return "font/collection"; } if (compareBytes([255, 255, 255, 255], [119, 79, 70, 70])) { return "font/woff"; } if (compareBytes([255, 255, 255, 255], [119, 79, 70, 50])) { return "font/woff2"; } } function compareBytes(mask, pattern) { let patternMatch = true, index = 0; if (buffer.byteLength >= pattern.length) { const value = new Uint8Array(buffer, 0, mask.length); for (index = 0; index < mask.length && patternMatch; index++) { patternMatch = patternMatch && ((value[index] & mask[index]) == pattern[index]); } return patternMatch; } } } // https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest function hex(buffer) { const hexCodes = []; const view = new DataView(buffer); for (let i = 0; i < view.byteLength; i += 4) { const value = view.getUint32(i); const stringValue = value.toString(16); const padding = "00000000"; const paddedValue = (padding + stringValue).slice(-padding.length); hexCodes.push(paddedValue); } return hexCodes.join(""); } function log(...args) { console.log("S-File ", ...args); // eslint-disable-line no-console }node-single-file-core-1.0.47+ds1/single-file-infobar.js0000644000175000017500000000204503560116604021527 0ustar emfoxemfox/* * Copyright 2010-2022 Gildas Lormeau * contact : gildas.lormeau gmail.com * * This file is part of SingleFile. * * The code in this file is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License * (GNU AGPL) as published by the Free Software Foundation, either version 3 * of the License, or (at your option) any later version. * * The code in this file is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero * General Public License for more details. * * As additional permission under GNU AGPL version 3 section 7, you may * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU * AGPL normally required by section 4, provided you include this license * notice and a URL through which recipients can access the Corresponding * Source. */ import "./common/content-infobar-web.js";node-single-file-core-1.0.47+ds1/single-file-hooks-frames.js0000644000175000017500000000207403560116604022507 0ustar emfoxemfox/* * Copyright 2010-2022 Gildas Lormeau * contact : gildas.lormeau gmail.com * * This file is part of SingleFile. * * The code in this file is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License * (GNU AGPL) as published by the Free Software Foundation, either version 3 * of the License, or (at your option) any later version. * * The code in this file is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero * General Public License for more details. * * As additional permission under GNU AGPL version 3 section 7, you may * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU * AGPL normally required by section 4, provided you include this license * notice and a URL through which recipients can access the Corresponding * Source. */ import "./processors/hooks/content/content-hooks-frames-web.js";node-single-file-core-1.0.47+ds1/single-file-helper.js0000644000175000017500000005447203560116604021401 0ustar emfoxemfox/* * Copyright 2010-2022 Gildas Lormeau * contact : gildas.lormeau gmail.com * * This file is part of SingleFile. * * The code in this file is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License * (GNU AGPL) as published by the Free Software Foundation, either version 3 * of the License, or (at your option) any later version. * * The code in this file is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero * General Public License for more details. * * As additional permission under GNU AGPL version 3 section 7, you may * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU * AGPL normally required by section 4, provided you include this license * notice and a URL through which recipients can access the Corresponding * Source. */ /* global globalThis, CustomEvent */ import * as cssUnescape from "./vendor/css-unescape.js"; import * as hooksFrames from "./processors/hooks/content/content-hooks-frames.js"; const ON_BEFORE_CAPTURE_EVENT_NAME = "single-file-on-before-capture"; const ON_AFTER_CAPTURE_EVENT_NAME = "single-file-on-after-capture"; const REMOVED_CONTENT_ATTRIBUTE_NAME = "data-single-file-removed-content"; const HIDDEN_CONTENT_ATTRIBUTE_NAME = "data-single-file-hidden-content"; const KEPT_CONTENT_ATTRIBUTE_NAME = "data-single-file-kept-content"; const HIDDEN_FRAME_ATTRIBUTE_NAME = "data-single-file-hidden-frame"; const PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME = "data-single-file-preserved-space-element"; const SHADOW_ROOT_ATTRIBUTE_NAME = "data-single-file-shadow-root-element"; const WIN_ID_ATTRIBUTE_NAME = "data-single-file-win-id"; const IMAGE_ATTRIBUTE_NAME = "data-single-file-image"; const POSTER_ATTRIBUTE_NAME = "data-single-file-poster"; const VIDEO_ATTRIBUTE_NAME = "data-single-file-video"; const CANVAS_ATTRIBUTE_NAME = "data-single-file-canvas"; const STYLE_ATTRIBUTE_NAME = "data-single-file-movable-style"; const INPUT_VALUE_ATTRIBUTE_NAME = "data-single-file-input-value"; const LAZY_SRC_ATTRIBUTE_NAME = "data-single-file-lazy-loaded-src"; const STYLESHEET_ATTRIBUTE_NAME = "data-single-file-stylesheet"; const DISABLED_NOSCRIPT_ATTRIBUTE_NAME = "data-single-file-disabled-noscript"; const SELECTED_CONTENT_ATTRIBUTE_NAME = "data-single-file-selected-content"; const INVALID_ELEMENT_ATTRIBUTE_NAME = "data-single-file-invalid-element"; const ASYNC_SCRIPT_ATTRIBUTE_NAME = "data-single-file-async-script"; const FLOW_ELEMENTS_SELECTOR = "*:not(base):not(link):not(meta):not(noscript):not(script):not(style):not(template):not(title)"; const KEPT_TAG_NAMES = ["NOSCRIPT", "DISABLED-NOSCRIPT", "META", "LINK", "STYLE", "TITLE", "TEMPLATE", "SOURCE", "OBJECT", "SCRIPT", "HEAD", "BODY"]; const REGEXP_SIMPLE_QUOTES_STRING = /^'(.*?)'$/; const REGEXP_DOUBLE_QUOTES_STRING = /^"(.*?)"$/; const FONT_WEIGHTS = { regular: "400", normal: "400", bold: "700", bolder: "700", lighter: "100" }; const COMMENT_HEADER = "Page saved with SingleFile"; const COMMENT_HEADER_LEGACY = "Archive processed by SingleFile"; const SINGLE_FILE_UI_ELEMENT_CLASS = "single-file-ui-element"; const EMPTY_RESOURCE = "data:,"; const addEventListener = (type, listener, options) => globalThis.addEventListener(type, listener, options); const dispatchEvent = event => { try { globalThis.dispatchEvent(event); } catch (error) { /* ignored */ } }; const JSON = globalThis.JSON; export { initUserScriptHandler, initDoc, preProcessDoc, postProcessDoc, serialize, removeQuotes, flatten, getFontWeight, normalizeFontFamily, getShadowRoot, ON_BEFORE_CAPTURE_EVENT_NAME, ON_AFTER_CAPTURE_EVENT_NAME, WIN_ID_ATTRIBUTE_NAME, PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME, REMOVED_CONTENT_ATTRIBUTE_NAME, HIDDEN_CONTENT_ATTRIBUTE_NAME, HIDDEN_FRAME_ATTRIBUTE_NAME, IMAGE_ATTRIBUTE_NAME, POSTER_ATTRIBUTE_NAME, VIDEO_ATTRIBUTE_NAME, CANVAS_ATTRIBUTE_NAME, INPUT_VALUE_ATTRIBUTE_NAME, SHADOW_ROOT_ATTRIBUTE_NAME, STYLE_ATTRIBUTE_NAME, LAZY_SRC_ATTRIBUTE_NAME, STYLESHEET_ATTRIBUTE_NAME, SELECTED_CONTENT_ATTRIBUTE_NAME, INVALID_ELEMENT_ATTRIBUTE_NAME, ASYNC_SCRIPT_ATTRIBUTE_NAME, COMMENT_HEADER, COMMENT_HEADER_LEGACY, SINGLE_FILE_UI_ELEMENT_CLASS, EMPTY_RESOURCE }; function initUserScriptHandler() { addEventListener("single-file-user-script-init", () => globalThis._singleFile_waitForUserScript = async eventPrefixName => { const event = new CustomEvent(eventPrefixName + "-request", { cancelable: true }); const promiseResponse = new Promise(resolve => addEventListener(eventPrefixName + "-response", resolve)); dispatchEvent(event); if (event.defaultPrevented) { await promiseResponse; } }); } function initDoc(doc) { doc.querySelectorAll("meta[http-equiv=refresh]").forEach(element => { element.removeAttribute("http-equiv"); element.setAttribute("disabled-http-equiv", "refresh"); }); } function preProcessDoc(doc, win, options) { doc.querySelectorAll("noscript:not([" + DISABLED_NOSCRIPT_ATTRIBUTE_NAME + "])").forEach(element => { element.setAttribute(DISABLED_NOSCRIPT_ATTRIBUTE_NAME, element.textContent); element.textContent = ""; }); initDoc(doc); if (doc.head) { doc.head.querySelectorAll(FLOW_ELEMENTS_SELECTOR).forEach(element => element.hidden = true); } doc.querySelectorAll("svg foreignObject").forEach(element => { const flowElements = element.querySelectorAll("html > head > " + FLOW_ELEMENTS_SELECTOR + ", html > body > " + FLOW_ELEMENTS_SELECTOR); if (flowElements.length) { Array.from(element.childNodes).forEach(node => node.remove()); flowElements.forEach(flowElement => element.appendChild(flowElement)); } }); const invalidElements = new Map(); let elementsInfo; if (win && doc.documentElement) { doc.querySelectorAll("button button, a a").forEach(element => { const placeHolderElement = doc.createElement("template"); placeHolderElement.setAttribute(INVALID_ELEMENT_ATTRIBUTE_NAME, ""); placeHolderElement.content.appendChild(element.cloneNode(true)); invalidElements.set(element, placeHolderElement); element.replaceWith(placeHolderElement); }); elementsInfo = getElementsInfo(win, doc, doc.documentElement, options); if (options.moveStylesInHead) { doc.querySelectorAll("body style, body ~ style").forEach(element => { const computedStyle = getComputedStyle(win, element); if (computedStyle && testHiddenElement(element, computedStyle)) { element.setAttribute(STYLE_ATTRIBUTE_NAME, ""); elementsInfo.markedElements.push(element); } }); } } else { elementsInfo = { canvases: [], images: [], posters: [], videos: [], usedFonts: [], shadowRoots: [], markedElements: [] }; } return { canvases: elementsInfo.canvases, fonts: getFontsData(doc), stylesheets: getStylesheetsData(doc), images: elementsInfo.images, posters: elementsInfo.posters, videos: elementsInfo.videos, usedFonts: Array.from(elementsInfo.usedFonts.values()), shadowRoots: elementsInfo.shadowRoots, referrer: doc.referrer, markedElements: elementsInfo.markedElements, invalidElements }; } function getElementsInfo(win, doc, element, options, data = { usedFonts: new Map(), canvases: [], images: [], posters: [], videos: [], shadowRoots: [], markedElements: [] }, ascendantHidden) { if (element.childNodes) { const elements = Array.from(element.childNodes).filter(node => (node instanceof win.HTMLElement) || (node instanceof win.SVGElement)); elements.forEach(element => { let elementHidden, elementKept, computedStyle; if (!options.autoSaveExternalSave && (options.removeHiddenElements || options.removeUnusedFonts || options.compressHTML)) { computedStyle = getComputedStyle(win, element); if (element instanceof win.HTMLElement) { if (options.removeHiddenElements) { elementKept = ((ascendantHidden || element.closest("html > head")) && KEPT_TAG_NAMES.includes(element.tagName)) || element.closest("details"); if (!elementKept) { elementHidden = ascendantHidden || testHiddenElement(element, computedStyle); if (elementHidden) { element.setAttribute(HIDDEN_CONTENT_ATTRIBUTE_NAME, ""); data.markedElements.push(element); } } } } if (!elementHidden) { if (options.compressHTML && computedStyle) { const whiteSpace = computedStyle.getPropertyValue("white-space"); if (whiteSpace && whiteSpace.startsWith("pre")) { element.setAttribute(PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME, ""); data.markedElements.push(element); } } if (options.removeUnusedFonts) { getUsedFont(computedStyle, options, data.usedFonts); getUsedFont(getComputedStyle(win, element, ":first-letter"), options, data.usedFonts); getUsedFont(getComputedStyle(win, element, ":before"), options, data.usedFonts); getUsedFont(getComputedStyle(win, element, ":after"), options, data.usedFonts); } } } getResourcesInfo(win, doc, element, options, data, elementHidden, computedStyle); const shadowRoot = !(element instanceof win.SVGElement) && getShadowRoot(element); if (shadowRoot && !element.classList.contains(SINGLE_FILE_UI_ELEMENT_CLASS)) { const shadowRootInfo = {}; element.setAttribute(SHADOW_ROOT_ATTRIBUTE_NAME, data.shadowRoots.length); data.markedElements.push(element); data.shadowRoots.push(shadowRootInfo); getElementsInfo(win, doc, shadowRoot, options, data, elementHidden); shadowRootInfo.content = shadowRoot.innerHTML; shadowRootInfo.mode = shadowRoot.mode; try { if (shadowRoot.adoptedStyleSheets && shadowRoot.adoptedStyleSheets.length) { shadowRootInfo.adoptedStyleSheets = Array.from(shadowRoot.adoptedStyleSheets).map(stylesheet => Array.from(stylesheet.cssRules).map(cssRule => cssRule.cssText).join("\n")); } } catch (error) { // ignored } } getElementsInfo(win, doc, element, options, data, elementHidden); if (!options.autoSaveExternalSave && options.removeHiddenElements && ascendantHidden) { if (elementKept || element.getAttribute(KEPT_CONTENT_ATTRIBUTE_NAME) == "") { if (element.parentElement) { element.parentElement.setAttribute(KEPT_CONTENT_ATTRIBUTE_NAME, ""); data.markedElements.push(element.parentElement); } } else if (elementHidden) { element.setAttribute(REMOVED_CONTENT_ATTRIBUTE_NAME, ""); data.markedElements.push(element); } } }); } return data; } function getResourcesInfo(win, doc, element, options, data, elementHidden, computedStyle) { if (element.tagName == "CANVAS") { try { data.canvases.push({ dataURI: element.toDataURL("image/png", "") }); element.setAttribute(CANVAS_ATTRIBUTE_NAME, data.canvases.length - 1); data.markedElements.push(element); } catch (error) { // ignored } } if (element.tagName == "IMG") { const imageData = { currentSrc: elementHidden ? EMPTY_RESOURCE : (options.loadDeferredImages && element.getAttribute(LAZY_SRC_ATTRIBUTE_NAME)) || element.currentSrc }; data.images.push(imageData); element.setAttribute(IMAGE_ATTRIBUTE_NAME, data.images.length - 1); data.markedElements.push(element); element.removeAttribute(LAZY_SRC_ATTRIBUTE_NAME); computedStyle = computedStyle || getComputedStyle(win, element); if (computedStyle) { imageData.size = getSize(win, element, computedStyle); const boxShadow = computedStyle.getPropertyValue("box-shadow"); const backgroundImage = computedStyle.getPropertyValue("background-image"); if ((!boxShadow || boxShadow == "none") && (!backgroundImage || backgroundImage == "none") && (imageData.size.pxWidth > 1 || imageData.size.pxHeight > 1)) { imageData.replaceable = true; imageData.backgroundColor = computedStyle.getPropertyValue("background-color"); imageData.objectFit = computedStyle.getPropertyValue("object-fit"); imageData.boxSizing = computedStyle.getPropertyValue("box-sizing"); imageData.objectPosition = computedStyle.getPropertyValue("object-position"); } } } if (element.tagName == "VIDEO") { const src = element.currentSrc; if (src && !src.startsWith("blob:") && !src.startsWith("data:")) { const computedStyle = getComputedStyle(win, element.parentNode); data.videos.push({ positionParent: computedStyle && computedStyle.getPropertyValue("position"), src, size: { pxWidth: element.clientWidth, pxHeight: element.clientHeight }, currentTime: element.currentTime }); element.setAttribute(VIDEO_ATTRIBUTE_NAME, data.videos.length - 1); } if (!element.getAttribute("poster")) { const canvasElement = doc.createElement("canvas"); const context = canvasElement.getContext("2d"); canvasElement.width = element.clientWidth; canvasElement.height = element.clientHeight; try { context.drawImage(element, 0, 0, canvasElement.width, canvasElement.height); data.posters.push(canvasElement.toDataURL("image/png", "")); element.setAttribute(POSTER_ATTRIBUTE_NAME, data.posters.length - 1); data.markedElements.push(element); } catch (error) { // ignored } } } if (element.tagName == "IFRAME") { if (elementHidden && options.removeHiddenElements) { element.setAttribute(HIDDEN_FRAME_ATTRIBUTE_NAME, ""); data.markedElements.push(element); } } if (element.tagName == "INPUT") { if (element.type != "password") { element.setAttribute(INPUT_VALUE_ATTRIBUTE_NAME, element.value); data.markedElements.push(element); } if (element.type == "radio" || element.type == "checkbox") { element.setAttribute(INPUT_VALUE_ATTRIBUTE_NAME, element.checked); data.markedElements.push(element); } } if (element.tagName == "TEXTAREA") { element.setAttribute(INPUT_VALUE_ATTRIBUTE_NAME, element.value); data.markedElements.push(element); } if (element.tagName == "SELECT") { element.querySelectorAll("option").forEach(option => { if (option.selected) { option.setAttribute(INPUT_VALUE_ATTRIBUTE_NAME, ""); data.markedElements.push(option); } }); } if (element.tagName == "SCRIPT") { if (element.async && element.getAttribute("async") != "" && element.getAttribute("async") != "async") { element.setAttribute(ASYNC_SCRIPT_ATTRIBUTE_NAME, ""); data.markedElements.push(element); } element.textContent = element.textContent.replace(/<\/script>/gi, "<\\/script>"); } } function getUsedFont(computedStyle, options, usedFonts) { if (computedStyle) { const fontStyle = computedStyle.getPropertyValue("font-style") || "normal"; computedStyle.getPropertyValue("font-family").split(",").forEach(fontFamilyName => { fontFamilyName = normalizeFontFamily(fontFamilyName); if (!options.loadedFonts || options.loadedFonts.find(font => normalizeFontFamily(font.family) == fontFamilyName && font.style == fontStyle)) { const fontWeight = getFontWeight(computedStyle.getPropertyValue("font-weight")); const fontVariant = computedStyle.getPropertyValue("font-variant") || "normal"; const value = [fontFamilyName, fontWeight, fontStyle, fontVariant]; usedFonts.set(JSON.stringify(value), [fontFamilyName, fontWeight, fontStyle, fontVariant]); } }); } } function getShadowRoot(element) { const chrome = globalThis.chrome; if (element.openOrClosedShadowRoot) { return element.openOrClosedShadowRoot; } else if (chrome && chrome.dom && chrome.dom.openOrClosedShadowRoot) { try { return chrome.dom.openOrClosedShadowRoot(element); } catch (error) { return element.shadowRoot; } } else { return element.shadowRoot; } } function normalizeFontFamily(fontFamilyName = "") { return removeQuotes(cssUnescape.process(fontFamilyName.trim())).toLowerCase(); } function testHiddenElement(element, computedStyle) { let hidden = false; if (computedStyle) { const display = computedStyle.getPropertyValue("display"); const opacity = computedStyle.getPropertyValue("opacity"); const visibility = computedStyle.getPropertyValue("visibility"); hidden = display == "none"; if (!hidden && (opacity == "0" || visibility == "hidden") && element.getBoundingClientRect) { const boundingRect = element.getBoundingClientRect(); hidden = !boundingRect.width && !boundingRect.height; } } return Boolean(hidden); } function postProcessDoc(doc, markedElements, invalidElements) { doc.querySelectorAll("[" + DISABLED_NOSCRIPT_ATTRIBUTE_NAME + "]").forEach(element => { element.textContent = element.getAttribute(DISABLED_NOSCRIPT_ATTRIBUTE_NAME); element.removeAttribute(DISABLED_NOSCRIPT_ATTRIBUTE_NAME); }); doc.querySelectorAll("meta[disabled-http-equiv]").forEach(element => { element.setAttribute("http-equiv", element.getAttribute("disabled-http-equiv")); element.removeAttribute("disabled-http-equiv"); }); if (doc.head) { doc.head.querySelectorAll("*:not(base):not(link):not(meta):not(noscript):not(script):not(style):not(template):not(title)").forEach(element => element.removeAttribute("hidden")); } if (!markedElements) { const singleFileAttributes = [REMOVED_CONTENT_ATTRIBUTE_NAME, HIDDEN_FRAME_ATTRIBUTE_NAME, HIDDEN_CONTENT_ATTRIBUTE_NAME, PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME, IMAGE_ATTRIBUTE_NAME, POSTER_ATTRIBUTE_NAME, VIDEO_ATTRIBUTE_NAME, CANVAS_ATTRIBUTE_NAME, INPUT_VALUE_ATTRIBUTE_NAME, SHADOW_ROOT_ATTRIBUTE_NAME, STYLESHEET_ATTRIBUTE_NAME, ASYNC_SCRIPT_ATTRIBUTE_NAME]; markedElements = doc.querySelectorAll(singleFileAttributes.map(name => "[" + name + "]").join(",")); } markedElements.forEach(element => { element.removeAttribute(REMOVED_CONTENT_ATTRIBUTE_NAME); element.removeAttribute(HIDDEN_CONTENT_ATTRIBUTE_NAME); element.removeAttribute(KEPT_CONTENT_ATTRIBUTE_NAME); element.removeAttribute(HIDDEN_FRAME_ATTRIBUTE_NAME); element.removeAttribute(PRESERVED_SPACE_ELEMENT_ATTRIBUTE_NAME); element.removeAttribute(IMAGE_ATTRIBUTE_NAME); element.removeAttribute(POSTER_ATTRIBUTE_NAME); element.removeAttribute(VIDEO_ATTRIBUTE_NAME); element.removeAttribute(CANVAS_ATTRIBUTE_NAME); element.removeAttribute(INPUT_VALUE_ATTRIBUTE_NAME); element.removeAttribute(SHADOW_ROOT_ATTRIBUTE_NAME); element.removeAttribute(STYLESHEET_ATTRIBUTE_NAME); element.removeAttribute(ASYNC_SCRIPT_ATTRIBUTE_NAME); element.removeAttribute(STYLE_ATTRIBUTE_NAME); }); if (invalidElements) { Array.from(invalidElements.entries()).forEach(([element, placeholderElement]) => placeholderElement.replaceWith(element)); } } function getStylesheetsData(doc) { if (doc) { const contents = []; doc.querySelectorAll("style").forEach((styleElement, styleIndex) => { try { const tempStyleElement = doc.createElement("style"); tempStyleElement.textContent = styleElement.textContent; doc.body.appendChild(tempStyleElement); const stylesheet = tempStyleElement.sheet; tempStyleElement.remove(); if (!stylesheet || stylesheet.cssRules.length != styleElement.sheet.cssRules.length) { styleElement.setAttribute(STYLESHEET_ATTRIBUTE_NAME, styleIndex); contents[styleIndex] = Array.from(styleElement.sheet.cssRules).map(cssRule => cssRule.cssText).join("\n"); } } catch (error) { // ignored } }); return contents; } } function getSize(win, imageElement, computedStyle) { let pxWidth = imageElement.naturalWidth; let pxHeight = imageElement.naturalHeight; if (!pxWidth && !pxHeight) { const noStyleAttribute = imageElement.getAttribute("style") == null; computedStyle = computedStyle || getComputedStyle(win, imageElement); if (computedStyle) { let removeBorderWidth = false; if (computedStyle.getPropertyValue("box-sizing") == "content-box") { const boxSizingValue = imageElement.style.getPropertyValue("box-sizing"); const boxSizingPriority = imageElement.style.getPropertyPriority("box-sizing"); const clientWidth = imageElement.clientWidth; imageElement.style.setProperty("box-sizing", "border-box", "important"); removeBorderWidth = imageElement.clientWidth != clientWidth; if (boxSizingValue) { imageElement.style.setProperty("box-sizing", boxSizingValue, boxSizingPriority); } else { imageElement.style.removeProperty("box-sizing"); } } let paddingLeft, paddingRight, paddingTop, paddingBottom, borderLeft, borderRight, borderTop, borderBottom; paddingLeft = getWidth("padding-left", computedStyle); paddingRight = getWidth("padding-right", computedStyle); paddingTop = getWidth("padding-top", computedStyle); paddingBottom = getWidth("padding-bottom", computedStyle); if (removeBorderWidth) { borderLeft = getWidth("border-left-width", computedStyle); borderRight = getWidth("border-right-width", computedStyle); borderTop = getWidth("border-top-width", computedStyle); borderBottom = getWidth("border-bottom-width", computedStyle); } else { borderLeft = borderRight = borderTop = borderBottom = 0; } pxWidth = Math.max(0, imageElement.clientWidth - paddingLeft - paddingRight - borderLeft - borderRight); pxHeight = Math.max(0, imageElement.clientHeight - paddingTop - paddingBottom - borderTop - borderBottom); if (noStyleAttribute) { imageElement.removeAttribute("style"); } } } return { pxWidth, pxHeight }; } function getWidth(styleName, computedStyle) { if (computedStyle.getPropertyValue(styleName).endsWith("px")) { return parseFloat(computedStyle.getPropertyValue(styleName)); } } function getFontsData() { return hooksFrames.getFontsData(); } function serialize(doc) { const docType = doc.doctype; let docTypeString = ""; if (docType) { docTypeString = " "; } return docTypeString + doc.documentElement.outerHTML; } function removeQuotes(string) { if (string.match(REGEXP_SIMPLE_QUOTES_STRING)) { string = string.replace(REGEXP_SIMPLE_QUOTES_STRING, "$1"); } else { string = string.replace(REGEXP_DOUBLE_QUOTES_STRING, "$1"); } return string.trim(); } function getFontWeight(weight) { return FONT_WEIGHTS[weight.toLowerCase().trim()] || weight; } function flatten(array) { return array.flat ? array.flat() : array.reduce((a, b) => a.concat(Array.isArray(b) ? flatten(b) : b), []); } function getComputedStyle(win, element, pseudoElement) { try { return win.getComputedStyle(element, pseudoElement); } catch (error) { // ignored } }node-single-file-core-1.0.47+ds1/single-file-frames.js0000644000175000017500000000210203560116604021356 0ustar emfoxemfox/* * Copyright 2010-2022 Gildas Lormeau * contact : gildas.lormeau gmail.com * * This file is part of SingleFile. * * The code in this file is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License * (GNU AGPL) as published by the Free Software Foundation, either version 3 * of the License, or (at your option) any later version. * * The code in this file is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero * General Public License for more details. * * As additional permission under GNU AGPL version 3 section 7, you may * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU * AGPL normally required by section 4, provided you include this license * notice and a URL through which recipients can access the Corresponding * Source. */ export * from "./processors/frame-tree/content/content-frame-tree.js";node-single-file-core-1.0.47+ds1/single-file-core.js0000644000175000017500000024567503560116604021061 0ustar emfoxemfox/* * Copyright 2010-2022 Gildas Lormeau * contact : gildas.lormeau gmail.com * * This file is part of SingleFile. * * The code in this file is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License * (GNU AGPL) as published by the Free Software Foundation, either version 3 * of the License, or (at your option) any later version. * * The code in this file is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero * General Public License for more details. * * As additional permission under GNU AGPL version 3 section 7, you may * distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU * AGPL normally required by section 4, provided you include this license * notice and a URL through which recipients can access the Corresponding * Source. */ /* global globalThis */ const DEBUG = false; const Set = globalThis.Set; const Map = globalThis.Map; const JSON = globalThis.JSON; let util, cssTree; function getClass(...args) { [util, cssTree] = args; return SingleFileClass; } class SingleFileClass { constructor(options) { this.options = options; } async run() { const waitForUserScript = globalThis._singleFile_waitForUserScript; if (this.options.userScriptEnabled && waitForUserScript) { await waitForUserScript(util.ON_BEFORE_CAPTURE_EVENT_NAME); } this.runner = new Runner(this.options, true); await this.runner.loadPage(); await this.runner.initialize(); if (this.options.userScriptEnabled && waitForUserScript) { await waitForUserScript(util.ON_AFTER_CAPTURE_EVENT_NAME); } await this.runner.run(); } cancel() { this.cancelled = true; if (this.runner) { this.runner.cancel(); } } getPageData() { return this.runner.getPageData(); } } // ------------- // ProgressEvent // ------------- const PAGE_LOADING = "page-loading"; const PAGE_LOADED = "page-loaded"; const RESOURCES_INITIALIZING = "resource-initializing"; const RESOURCES_INITIALIZED = "resources-initialized"; const RESOURCE_LOADED = "resource-loaded"; const PAGE_ENDED = "page-ended"; const STAGE_STARTED = "stage-started"; const STAGE_ENDED = "stage-ended"; const STAGE_TASK_STARTED = "stage-task-started"; const STAGE_TASK_ENDED = "stage-task-ended"; class ProgressEvent { constructor(type, detail) { return { type, detail, PAGE_LOADING, PAGE_LOADED, RESOURCES_INITIALIZING, RESOURCES_INITIALIZED, RESOURCE_LOADED, PAGE_ENDED, STAGE_STARTED, STAGE_ENDED, STAGE_TASK_STARTED, STAGE_TASK_ENDED }; } } // ------ // Runner // ------ const RESOLVE_URLS_STAGE = 0; const REPLACE_DATA_STAGE = 1; const REPLACE_DOCS_STAGE = 2; const POST_PROCESS_STAGE = 3; const STAGES = [ { sequential: [ { action: "preProcessPage" }, { option: "loadDeferredImagesKeepZoomLevel", action: "resetZoomLevel" }, { action: "replaceStyleContents" }, { action: "replaceInvalidElements" }, { action: "resetCharsetMeta" }, { option: "saveFavicon", action: "saveFavicon" }, { action: "replaceCanvasElements" }, { action: "insertFonts" }, { action: "insertShadowRootContents" }, { action: "setInputValues" }, { option: "moveStylesInHead", action: "moveStylesInHead" }, { option: "blockScripts", action: "removeEmbedScripts" }, { option: "selected", action: "removeUnselectedElements" }, { option: "blockVideos", action: "insertVideoPosters" }, { option: "blockVideos", action: "insertVideoLinks" }, { option: "removeFrames", action: "removeFrames" }, { action: "removeDiscardedResources" }, { option: "removeHiddenElements", action: "removeHiddenElements" }, { action: "resolveHrefs" }, { action: "resolveStyleAttributeURLs" } ], parallel: [{ option: "blockVideos", action: "insertMissingVideoPosters" }, { action: "resolveStylesheetURLs" }, { option: "!removeFrames", action: "resolveFrameURLs" }] }, { sequential: [ { option: "removeUnusedStyles", action: "removeUnusedStyles" }, { option: "removeAlternativeMedias", action: "removeAlternativeMedias" }, { option: "removeUnusedFonts", action: "removeUnusedFonts" } ], parallel: [{ action: "processStylesheets" }, { action: "processStyleAttributes" }, { action: "processPageResources" }, { action: "processScripts" }] }, { sequential: [{ option: "removeAlternativeImages", action: "removeAlternativeImages" }], parallel: [ { option: "removeAlternativeFonts", action: "removeAlternativeFonts" }, { option: "!removeFrames", action: "processFrames" } ] }, { sequential: [{ action: "replaceStylesheets" }, { action: "replaceStyleAttributes" }, { action: "insertVariables" }, { option: "compressHTML", action: "compressHTML" }, { action: "cleanupPage" }], parallel: [{ option: "enableMaff", action: "insertMAFFMetaData" }, { action: "setDocInfo" }] } ]; class Runner { constructor(options, root) { const rootDocDefined = root && options.doc; this.root = root; this.options = options; this.options.url = this.options.url || (rootDocDefined && this.options.doc.location.href); const matchResourceReferrer = this.options.url.match(/^.*\//); this.options.resourceReferrer = this.options.passReferrerOnError && matchResourceReferrer && matchResourceReferrer[0]; this.options.baseURI = rootDocDefined && this.options.doc.baseURI; this.options.rootDocument = root; this.options.updatedResources = this.options.updatedResources || {}; this.options.fontTests = new Map(); this.batchRequest = new BatchRequest(); this.processor = new Processor(options, this.batchRequest); if (rootDocDefined) { const docData = util.preProcessDoc(this.options.doc, this.options.win, this.options); this.options.canvases = docData.canvases; this.options.fonts = docData.fonts; this.options.stylesheets = docData.stylesheets; this.options.images = docData.images; this.options.posters = docData.posters; this.options.videos = docData.videos; this.options.usedFonts = docData.usedFonts; this.options.shadowRoots = docData.shadowRoots; this.options.referrer = docData.referrer; this.markedElements = docData.markedElements; this.invalidElements = docData.invalidElements; } if (this.options.saveRawPage) { this.options.removeFrames = true; } this.options.content = this.options.content || (rootDocDefined ? util.serialize(this.options.doc) : null); this.onprogress = options.onprogress || (() => {}); } async loadPage() { this.onprogress(new ProgressEvent(PAGE_LOADING, { pageURL: this.options.url, frame: !this.root })); await this.processor.loadPage(this.options.content); this.onprogress(new ProgressEvent(PAGE_LOADED, { pageURL: this.options.url, frame: !this.root })); } async initialize() { this.onprogress(new ProgressEvent(RESOURCES_INITIALIZING, { pageURL: this.options.url })); await this.executeStage(RESOLVE_URLS_STAGE); this.pendingPromises = this.executeStage(REPLACE_DATA_STAGE); if (this.root && this.options.doc) { util.postProcessDoc(this.options.doc, this.markedElements, this.invalidElements); } } cancel() { this.cancelled = true; this.batchRequest.cancel(); if (this.root) { if (this.options.frames) { this.options.frames.forEach(cancelRunner); } } function cancelRunner(resourceData) { if (resourceData.runner) { resourceData.runner.cancel(); } } } async run() { if (this.root) { this.processor.initialize(this.batchRequest); this.onprogress(new ProgressEvent(RESOURCES_INITIALIZED, { pageURL: this.options.url, max: this.processor.maxResources })); } await this.batchRequest.run(detail => { detail.pageURL = this.options.url; this.onprogress(new ProgressEvent(RESOURCE_LOADED, detail)); }, this.options); await this.pendingPromises; this.options.doc = null; this.options.win = null; await this.executeStage(REPLACE_DOCS_STAGE); await this.executeStage(POST_PROCESS_STAGE); this.processor.finalize(); } getDocument() { return this.processor.doc; } getStyleSheets() { return this.processor.stylesheets; } getPageData() { if (this.root) { this.onprogress(new ProgressEvent(PAGE_ENDED, { pageURL: this.options.url })); } return this.processor.getPageData(); } async executeStage(step) { if (DEBUG) { log("**** STARTED STAGE", step, "****"); } const frame = !this.root; this.onprogress(new ProgressEvent(STAGE_STARTED, { pageURL: this.options.url, step, frame })); STAGES[step].sequential.forEach(task => { let startTime; if (DEBUG) { startTime = Date.now(); log(" -- STARTED task =", task.action); } this.onprogress(new ProgressEvent(STAGE_TASK_STARTED, { pageURL: this.options.url, step, task: task.action, frame })); if (!this.cancelled) { this.executeTask(task); } this.onprogress(new ProgressEvent(STAGE_TASK_ENDED, { pageURL: this.options.url, step, task: task.action, frame })); if (DEBUG) { log(" -- ENDED task =", task.action, "delay =", Date.now() - startTime); } }); let parallelTasksPromise; if (STAGES[step].parallel) { parallelTasksPromise = await Promise.all( STAGES[step].parallel.map(async task => { let startTime; if (DEBUG) { startTime = Date.now(); log(" // STARTED task =", task.action); } this.onprogress(new ProgressEvent(STAGE_TASK_STARTED, { pageURL: this.options.url, step, task: task.action, frame })); if (!this.cancelled) { await this.executeTask(task); } this.onprogress(new ProgressEvent(STAGE_TASK_ENDED, { pageURL: this.options.url, step, task: task.action, frame })); if (DEBUG) { log(" // ENDED task =", task.action, "delay =", Date.now() - startTime); } }) ); } else { parallelTasksPromise = Promise.resolve(); } this.onprogress(new ProgressEvent(STAGE_ENDED, { pageURL: this.options.url, step, frame })); if (DEBUG) { log("**** ENDED STAGE", step, "****"); } return parallelTasksPromise; } executeTask(task) { if (!task.option || (task.option.startsWith("!") && !this.options[task.option]) || this.options[task.option]) { return this.processor[task.action](); } } } // ------------ // BatchRequest // ------------ class BatchRequest { constructor() { this.requests = new Map(); this.duplicates = new Map(); } addURL(resourceURL, { asBinary, expectedType, groupDuplicates, baseURI, blockMixedContent } = {}) { return new Promise((resolve, reject) => { const requestKey = JSON.stringify([resourceURL, asBinary, expectedType, baseURI, blockMixedContent]); let resourceRequests = this.requests.get(requestKey); if (!resourceRequests) { resourceRequests = []; this.requests.set(requestKey, resourceRequests); } const callbacks = { resolve, reject }; resourceRequests.push(callbacks); if (groupDuplicates) { let duplicateRequests = this.duplicates.get(requestKey); if (!duplicateRequests) { duplicateRequests = []; this.duplicates.set(requestKey, duplicateRequests); } duplicateRequests.push(callbacks); } }); } getMaxResources() { return this.requests.size; } run(onloadListener, options) { const resourceURLs = [...this.requests.keys()]; let indexResource = 0; return Promise.all( resourceURLs.map(async requestKey => { const [resourceURL, asBinary, expectedType, baseURI, blockMixedContent] = JSON.parse(requestKey); const resourceRequests = this.requests.get(requestKey); try { const currentIndexResource = indexResource; indexResource = indexResource + 1; const content = await util.getContent(resourceURL, { asBinary, expectedType, maxResourceSize: options.maxResourceSize, maxResourceSizeEnabled: options.maxResourceSizeEnabled, frameId: options.windowId, resourceReferrer: options.resourceReferrer, baseURI, blockMixedContent, acceptHeaders: options.acceptHeaders, networkTimeout: options.networkTimeout }); onloadListener({ url: resourceURL }); if (!this.cancelled) { resourceRequests.forEach(callbacks => { const duplicateCallbacks = this.duplicates.get(requestKey); const duplicate = duplicateCallbacks && duplicateCallbacks.length > 1 && duplicateCallbacks.includes(callbacks); callbacks.resolve({ content: content.data, indexResource: currentIndexResource, duplicate }); }); } } catch (error) { indexResource = indexResource + 1; onloadListener({ url: resourceURL }); resourceRequests.forEach(resourceRequest => resourceRequest.reject(error)); } this.requests.delete(requestKey); }) ); } cancel() { this.cancelled = true; const resourceURLs = [...this.requests.keys()]; resourceURLs.forEach(requestKey => { const resourceRequests = this.requests.get(requestKey); resourceRequests.forEach(callbacks => callbacks.reject()); this.requests.delete(requestKey); }); } } // --------- // Processor // --------- const PREFIXES_FORBIDDEN_DATA_URI = ["data:text/"]; const PREFIX_DATA_URI_IMAGE_SVG = "data:image/svg+xml"; const SCRIPT_TAG_FOUND = /