@cliqz/autoconsent
Version:
This is a library of rules for navigating through common consent popups on the web. These rules can be run in a Firefox webextension, or in a puppeteer orchestrated headless browser. Using these rules, opt-in and opt-out options can be selected automatica
811 lines (796 loc) • 28.8 kB
JavaScript
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
/* eslint-disable no-restricted-syntax,no-await-in-loop,no-underscore-dangle */
async function waitFor(predicate, maxTimes, interval) {
let result = await predicate();
if (!result && maxTimes > 0) {
return new Promise((resolve) => {
setTimeout(async () => {
resolve(waitFor(predicate, maxTimes - 1, interval));
}, interval);
});
}
return Promise.resolve(result);
}
async function success(action) {
const result = await action;
if (!result) {
throw new Error(`Action failed: ${action}`);
}
return result;
}
class AutoConsentBase {
constructor(name) {
this.hasSelfTest = true;
this.name = name;
}
detectCmp(tab) {
throw new Error('Not Implemented');
}
async detectPopup(tab) {
return false;
}
detectFrame(tab, frame) {
return false;
}
optOut(tab) {
throw new Error('Not Implemented');
}
optIn(tab) {
throw new Error('Not Implemented');
}
openCmp(tab) {
throw new Error('Not Implemented');
}
async test(tab) {
// try IAB by default
return Promise.resolve(true);
}
}
async function evaluateRule(rule, tab) {
if (rule.frame && !tab.frame) {
await waitFor(() => Promise.resolve(!!tab.frame), 10, 500);
}
const frameId = rule.frame && tab.frame ? tab.frame.id : undefined;
const results = [];
if (rule.exists) {
results.push(tab.elementExists(rule.exists, frameId));
}
if (rule.visible) {
results.push(tab.elementsAreVisible(rule.visible, rule.check, frameId));
}
if (rule.eval) {
results.push(new Promise(async (resolve) => {
// catch eval error silently
try {
resolve(await tab.eval(rule.eval, frameId));
}
catch (e) {
resolve(false);
}
}));
}
if (rule.waitFor) {
results.push(tab.waitForElement(rule.waitFor, rule.timeout || 10000, frameId));
}
if (rule.click) {
if (rule.all === true) {
results.push(tab.clickElements(rule.click, frameId));
}
else {
results.push(tab.clickElement(rule.click, frameId));
}
}
if (rule.waitForThenClick) {
results.push(tab.waitForElement(rule.waitForThenClick, rule.timeout || 10000, frameId)
.then(() => tab.clickElement(rule.waitForThenClick, frameId)));
}
if (rule.wait) {
results.push(tab.wait(rule.wait));
}
if (rule.goto) {
results.push(tab.goto(rule.goto));
}
if (rule.hide) {
results.push(tab.hideElements(rule.hide, frameId));
}
if (rule.waitForFrame) {
results.push(waitFor(() => !!tab.frame, 40, 500));
}
// boolean and of results
return (await Promise.all(results)).reduce((a, b) => a && b, true);
}
class AutoConsent extends AutoConsentBase {
constructor(config) {
super(config.name);
this.config = config;
}
async _runRulesParallel(tab, rules) {
const detections = await Promise.all(rules.map(rule => evaluateRule(rule, tab)));
return detections.every(r => !!r);
}
async _runRulesSequentially(tab, rules) {
for (const rule of rules) {
const result = await evaluateRule(rule, tab);
if (!result && !rule.optional) {
return false;
}
}
return true;
}
async detectCmp(tab) {
if (this.config.detectCmp) {
return this._runRulesParallel(tab, this.config.detectCmp);
}
return false;
}
async detectPopup(tab) {
if (this.config.detectPopup) {
return this._runRulesParallel(tab, this.config.detectPopup);
}
return false;
}
detectFrame(tab, frame) {
if (this.config.frame) {
return frame.url.startsWith(this.config.frame);
}
return false;
}
async optOut(tab) {
if (this.config.optOut) {
return this._runRulesSequentially(tab, this.config.optOut);
}
return false;
}
async optIn(tab) {
if (this.config.optIn) {
return this._runRulesSequentially(tab, this.config.optIn);
}
return false;
}
async openCmp(tab) {
if (this.config.openCmp) {
return this._runRulesSequentially(tab, this.config.openCmp);
}
return false;
}
async test(tab) {
if (this.config.test) {
return this._runRulesSequentially(tab, this.config.test);
}
return super.test(tab);
}
}
/**
* This code is in most parts copied from https://github.com/cavi-au/Consent-O-Matic/blob/master/Extension/Tools.js
* which is licened under the MIT.
*/
class Tools {
static setBase(base) {
Tools.base = base;
}
static findElement(options, parent = null, multiple = false) {
let possibleTargets = null;
if (parent != null) {
possibleTargets = Array.from(parent.querySelectorAll(options.selector));
}
else {
if (Tools.base != null) {
possibleTargets = Array.from(Tools.base.querySelectorAll(options.selector));
}
else {
possibleTargets = Array.from(document.querySelectorAll(options.selector));
}
}
if (options.textFilter != null) {
possibleTargets = possibleTargets.filter(possibleTarget => {
let textContent = possibleTarget.textContent.toLowerCase();
if (Array.isArray(options.textFilter)) {
let foundText = false;
for (let text of options.textFilter) {
if (textContent.indexOf(text.toLowerCase()) !== -1) {
foundText = true;
break;
}
}
return foundText;
}
else if (options.textFilter != null) {
return textContent.indexOf(options.textFilter.toLowerCase()) !== -1;
}
});
}
if (options.styleFilters != null) {
possibleTargets = possibleTargets.filter(possibleTarget => {
let styles = window.getComputedStyle(possibleTarget);
let keep = true;
for (let styleFilter of options.styleFilters) {
let option = styles[styleFilter.option];
if (styleFilter.negated) {
keep = keep && option !== styleFilter.value;
}
else {
keep = keep && option === styleFilter.value;
}
}
return keep;
});
}
if (options.displayFilter != null) {
possibleTargets = possibleTargets.filter(possibleTarget => {
if (options.displayFilter) {
//We should be displayed
return possibleTarget.offsetHeight !== 0;
}
else {
//We should not be displayed
return possibleTarget.offsetHeight === 0;
}
});
}
if (options.iframeFilter != null) {
possibleTargets = possibleTargets.filter(possibleTarget => {
if (options.iframeFilter) {
//We should be inside an iframe
return window.location !== window.parent.location;
}
else {
//We should not be inside an iframe
return window.location === window.parent.location;
}
});
}
if (options.childFilter != null) {
possibleTargets = possibleTargets.filter(possibleTarget => {
let oldBase = Tools.base;
Tools.setBase(possibleTarget);
let childResults = Tools.find(options.childFilter);
Tools.setBase(oldBase);
return childResults.target != null;
});
}
if (multiple) {
return possibleTargets;
}
else {
if (possibleTargets.length > 1) {
console.warn("Multiple possible targets: ", possibleTargets, options, parent);
}
return possibleTargets[0];
}
}
static find(options, multiple = false) {
let results = [];
if (options.parent != null) {
let parent = Tools.findElement(options.parent, null, multiple);
if (parent != null) {
if (parent instanceof Array) {
parent.forEach(p => {
let targets = Tools.findElement(options.target, p, multiple);
if (targets instanceof Array) {
targets.forEach(target => {
results.push({
parent: p,
target: target
});
});
}
else {
results.push({
parent: p,
target: targets
});
}
});
return results;
}
else {
let targets = Tools.findElement(options.target, parent, multiple);
if (targets instanceof Array) {
targets.forEach(target => {
results.push({
parent: parent,
target: target
});
});
}
else {
results.push({
parent: parent,
target: targets
});
}
}
}
}
else {
let targets = Tools.findElement(options.target, null, multiple);
if (targets instanceof Array) {
targets.forEach(target => {
results.push({
parent: null,
target: target
});
});
}
else {
results.push({
parent: null,
target: targets
});
}
}
if (results.length === 0) {
results.push({
parent: null,
target: null
});
}
if (multiple) {
return results;
}
else {
if (results.length !== 1) {
console.warn("Multiple results found, even though multiple false", results);
}
return results[0];
}
}
}
Tools.base = null;
function matches(config) {
const result = Tools.find(config);
if (config.type === "css") {
return !!result.target;
}
else if (config.type === "checkbox") {
return !!result.target && result.target.checked;
}
}
const DEBUG = false;
class Tab {
constructor(page, url, frames) {
// puppeteer doesn't have tab IDs
this.id = 1;
this.page = page;
this.url = url;
this.frames = frames;
}
async elementExists(selector, frameId = 0) {
try {
const elements = await this.frames[frameId].$$(selector);
DEBUG && console.log('[exists]', selector, elements.length > 0);
return elements.length > 0;
}
catch (e) {
console.warn(e);
return false;
}
}
async clickElement(selector, frameId = 0) {
if (await this.elementExists(selector, frameId)) {
try {
const result = await this.frames[frameId].evaluate((s) => {
try {
document.querySelector(s).click();
return true;
}
catch (e) {
return e.toString();
}
}, selector);
DEBUG && console.log('[click]', selector, result);
return result;
}
catch (e) {
console.warn(e);
return false;
}
}
return false;
}
async clickElements(selector, frameId = 0) {
const elements = await this.frames[frameId].$$(selector);
try {
DEBUG && console.log('[click all]', selector);
await this.frames[frameId].evaluate((s) => {
const elem = document.querySelectorAll(s);
elem.forEach(e => e.click());
}, selector);
return true;
}
catch (e) {
console.warn(e);
return false;
}
}
async elementsAreVisible(selector, check, frameId = 0) {
if (!await this.elementExists(selector, frameId)) {
return false;
}
const visible = await this.frames[frameId].$$eval(selector, (nodes) => nodes.map((n) => n.offsetParent !== null || window.getComputedStyle(n).display !== "none"));
if (visible.length === 0) {
return false;
}
else if (check === 'any') {
return visible.some(r => r);
}
else if (check === 'none') {
return visible.every(r => !r);
}
return visible.every(r => r);
}
async getAttribute(selector, attribute, frameId = 0) {
const elem = await this.frames[frameId].$(selector);
if (elem) {
return (await elem.getProperty(attribute)).jsonValue();
}
}
async eval(script, frameId = 0) {
return await this.frames[frameId].evaluate(script);
}
async waitForElement(selector, timeout, frameId = 0) {
const interval = 200;
const times = Math.ceil(timeout / interval);
return waitFor(() => this.elementExists(selector, frameId), times, interval);
}
async waitForThenClick(selector, timeout, frameId = 0) {
await this.waitForElement(selector, timeout, frameId);
await this.clickElement(selector, frameId);
return true;
}
async hideElements(selectors, frameId = 0) {
// TODO implement this
return Promise.resolve(true);
}
async goto(url) {
return this.page.goto(url);
}
wait(ms) {
return new Promise((resolve) => {
setTimeout(() => resolve(true), ms);
});
}
matches(options) {
const script = `(() => {
const Tools = ${Tools.toString()};
const matches = ${matches.toString()};
return matches(${JSON.stringify(options)})
})();
`;
return this.frames[0].evaluate(script);
}
executeAction(config, param) {
throw new Error("Method not implemented.");
}
}
class TrustArc extends AutoConsentBase {
constructor() {
super("TrustArc");
}
detectFrame(_, frame) {
return frame.url.startsWith("https://consent-pref.trustarc.com/?");
}
async detectCmp(tab) {
if (tab.frame &&
tab.frame.url.startsWith("https://consent-pref.trustarc.com/?")) {
return true;
}
return tab.elementExists("#truste-show-consent");
}
async detectPopup(tab) {
return ((await tab.elementsAreVisible("#truste-consent-content")) ||
(tab.frame &&
(await tab.waitForElement("#defaultpreferencemanager", 5000, tab.frame.id))));
}
async openFrame(tab) {
if (await tab.elementExists("#truste-show-consent")) {
await tab.clickElement("#truste-show-consent");
}
}
async navigateToSettings(tab, frameId) {
// wait for it to load
await waitFor(async () => {
return ((await tab.elementExists(".shp", frameId)) ||
(await tab.elementsAreVisible(".advance", "any", frameId)) ||
tab.elementExists(".switch span:first-child", frameId));
}, 10, 500);
// splash screen -> hit more information
if (await tab.elementExists(".shp", frameId)) {
await tab.clickElement(".shp", frameId);
}
await tab.waitForElement(".prefPanel", 5000, frameId);
// go to advanced settings if not yet shown
if (await tab.elementsAreVisible(".advance", "any", frameId)) {
await tab.clickElement(".advance", frameId);
}
// takes a while to load the opt-in/opt-out buttons
return await waitFor(() => tab.elementsAreVisible(".switch span:first-child", "any", frameId), 5, 1000);
}
async optOut(tab) {
// await tab.hideElements(['.truste_overlay', '.truste_box_overlay', '.trustarc-banner', '.truste-banner']);
if (await tab.elementExists("#truste-consent-required")) {
return tab.clickElement("#truste-consent-required");
}
if (!tab.frame) {
await tab.clickElement("#truste-show-consent");
await waitFor(async () => !!tab.frame &&
(await tab.elementsAreVisible(".mainContent", "any", tab.frame.id)), 50, 100);
}
const frameId = tab.frame.id;
tab.hideElements([".truste_popframe", ".truste_overlay", ".truste_box_overlay", "#truste-consent-track"]);
if (await tab.elementExists(".required", frameId)) {
await tab.clickElement(".required", frameId);
}
else {
await this.navigateToSettings(tab, frameId);
await tab.clickElements(".switch span:nth-child(1):not(.active)", frameId);
await tab.clickElement(".submit", frameId);
}
await tab.waitForThenClick("#gwt-debug-close_id", 20000, tab.frame.id);
return true;
}
async optIn(tab) {
if (!tab.frame) {
await this.openFrame(tab);
await waitFor(() => !!tab.frame, 10, 200);
}
const frameId = tab.frame.id;
await this.navigateToSettings(tab, frameId);
await tab.clickElements(".switch span:nth-child(2)", frameId);
await tab.clickElement(".submit", frameId);
await waitFor(() => tab.elementExists("#gwt-debug-close_id", frameId), 300, 1000);
await tab.clickElement("#gwt-debug-close_id", frameId);
return true;
}
async openCmp(tab) {
await tab.eval("truste.eu.clickListener()");
return true;
}
async test() {
// TODO: find out how to test TrustArc
return true;
}
}
class Cookiebot extends AutoConsentBase {
constructor() {
super('Cybotcookiebot');
}
async detectCmp(tab) {
try {
return await tab.eval('typeof window.CookieConsent === "object" && typeof window.CookieConsent.name === "string"');
}
catch (e) {
return false;
}
}
detectPopup(tab) {
return tab.elementExists('#CybotCookiebotDialog,#dtcookie-container,#cookiebanner');
}
async optOut(tab) {
if (await tab.elementExists('#dtcookie-container')) {
return tab.clickElement('.h-dtcookie-decline');
}
if (await tab.elementExists('.cookiebot__button--settings')) {
await tab.clickElement('.cookiebot__button--settings');
}
if (await tab.elementsAreVisible('#CybotCookiebotDialogBodyButtonDecline', 'all')) {
return await tab.clickElement('#CybotCookiebotDialogBodyButtonDecline');
}
if (await tab.elementExists('.cookiebanner__link--details')) {
await tab.clickElement('.cookiebanner__link--details');
}
await tab.clickElements('.CybotCookiebotDialogBodyLevelButton:checked:enabled,input[id*="CybotCookiebotDialogBodyLevelButton"]:checked:enabled');
if (await tab.elementExists('#CybotCookiebotDialogBodyButtonDecline')) {
await tab.clickElement('#CybotCookiebotDialogBodyButtonDecline');
}
if (await tab.elementExists('#CybotCookiebotDialogBodyButtonAcceptSelected')) {
await tab.clickElement('#CybotCookiebotDialogBodyButtonAcceptSelected');
}
else {
await tab.clickElements('#CybotCookiebotDialogBodyLevelButtonAccept,#CybotCookiebotDialogBodyButtonAccept,#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowallSelection');
}
return true;
}
async optIn(tab) {
if (await tab.elementExists('#dtcookie-container')) {
return tab.clickElement('.h-dtcookie-accept');
}
await tab.clickElements('.CybotCookiebotDialogBodyLevelButton:not(:checked):enabled');
await tab.clickElement('#CybotCookiebotDialogBodyLevelButtonAccept');
await tab.clickElement('#CybotCookiebotDialogBodyButtonAccept');
return true;
}
async openCmp(tab) {
await tab.eval('CookieConsent.renew() || true');
return tab.waitForElement('#CybotCookiebotDialog', 10000);
}
async test(tab) {
return tab.eval('CookieConsent.declined === true');
}
}
class SourcePoint extends AutoConsentBase {
constructor() {
super("Sourcepoint");
}
detectFrame(_, frame) {
const url = new URL(frame.url);
return (url.pathname === '/index.html' || url.pathname === '/privacy-manager/index.html')
&& url.searchParams.has('message_id') && url.searchParams.has('requestUUID');
}
async detectCmp(tab) {
return await tab.elementExists("div[id^='sp_message_container_']") || !!tab.frame;
}
async detectPopup(tab) {
return await tab.elementsAreVisible("div[id^='sp_message_container_']") || !!tab.frame;
}
async optIn(tab) {
return tab.clickElement(".sp_choice_type_11", tab.frame.id);
}
isManagerOpen(tab) {
return tab.frame && new URL(tab.frame.url).pathname === "/privacy-manager/index.html";
}
async optOut(tab) {
tab.hideElements(["div[id^='sp_message_container_']"]);
if (!this.isManagerOpen(tab)) {
await waitFor(() => !!tab.frame, 30, 100);
if (!await tab.elementExists("button.sp_choice_type_12", tab.frame.id)) {
// do not sell button
return tab.clickElement('button.sp_choice_type_13', tab.frame.id);
}
await success(tab.clickElement("button.sp_choice_type_12", tab.frame.id));
await waitFor(() => new URL(tab.frame.url).pathname === "/privacy-manager/index.html", 200, 100);
}
await tab.waitForElement('.type-modal', 20000, tab.frame.id);
// reject all button is offered by some sites
try {
const path = await Promise.race([
tab.waitForElement('.sp_choice_type_REJECT_ALL', 2000, tab.frame.id).then(r => 0),
tab.waitForElement('.reject-toggle', 2000, tab.frame.id).then(() => 1),
tab.waitForElement('.pm-features', 2000, tab.frame.id).then(r => 2),
]);
if (path === 0) {
await tab.wait(1000);
return await success(tab.clickElement('.sp_choice_type_REJECT_ALL', tab.frame.id));
}
else if (path === 1) {
await tab.clickElement('.reject-toggle', tab.frame.id);
}
else {
await tab.waitForElement('.pm-features', 10000, tab.frame.id);
await tab.clickElements('.checked > span', tab.frame.id);
if (await tab.elementExists('.chevron', tab.frame.id)) {
await tab.clickElement('.chevron', tab.frame.id);
}
}
}
catch (e) { }
return await tab.clickElement('.sp_choice_type_SAVE_AND_EXIT', tab.frame.id);
}
async test(tab) {
await tab.eval("__tcfapi('getTCData', 2, r => window.__rcsResult = r)");
return tab.eval("Object.values(window.__rcsResult.purpose.consents).every(c => !c)");
}
}
// Note: JS API is also available:
// https://help.consentmanager.net/books/cmp/page/javascript-api
class ConsentManager extends AutoConsentBase {
constructor() {
super("consentmanager.net");
}
detectCmp(tab) {
return tab.elementExists("#cmpbox");
}
detectPopup(tab) {
return tab.elementsAreVisible("#cmpbox .cmpmore", "any");
}
async optOut(tab) {
if (await tab.elementExists(".cmpboxbtnno")) {
return tab.clickElement(".cmpboxbtnno");
}
if (await tab.elementExists(".cmpwelcomeprpsbtn")) {
await tab.clickElements(".cmpwelcomeprpsbtn > a[aria-checked=true]");
return await tab.clickElement(".cmpboxbtnsave");
}
await tab.clickElement(".cmpboxbtncustom");
await tab.waitForElement(".cmptblbox", 2000);
await tab.clickElements(".cmptdchoice > a[aria-checked=true]");
return tab.clickElement(".cmpboxbtnyescustomchoices");
}
async optIn(tab) {
return tab.clickElement(".cmpboxbtnyes");
}
}
// Note: JS API is also available:
// https://help.consentmanager.net/books/cmp/page/javascript-api
class Evidon extends AutoConsentBase {
constructor() {
super("Evidon");
}
detectCmp(tab) {
return tab.elementExists("#_evidon_banner");
}
detectPopup(tab) {
return tab.elementsAreVisible("#_evidon_banner");
}
async optOut(tab) {
if (await tab.elementExists("#_evidon-decline-button")) {
return tab.clickElement("#_evidon-decline-button");
}
tab.hideElements(["#evidon-prefdiag-overlay", "#evidon-prefdiag-background"]);
await tab.clickElement("#_evidon-option-button");
await tab.waitForElement("#evidon-prefdiag-overlay", 5000);
return tab.clickElement("#evidon-prefdiag-decline");
}
async optIn(tab) {
return tab.clickElement("#_evidon-accept-button");
}
}
const rules = [
new TrustArc(),
new Cookiebot(),
new SourcePoint(),
new ConsentManager(),
new Evidon(),
];
function createAutoCMP(config) {
return new AutoConsent(config);
}
const rules$1 = rules;
class ConsentOMaticCMP {
constructor(name, config) {
this.name = name;
this.config = config;
this.methods = new Map();
config.methods.forEach(methodConfig => {
if (methodConfig.action) {
this.methods.set(methodConfig.name, methodConfig.action);
}
});
this.hasSelfTest = this.methods.has("TEST_CONSENT");
}
async detectCmp(tab) {
return (await Promise.all(this.config.detectors.map(detectorConfig => tab.matches(detectorConfig.presentMatcher)))).some(matched => matched);
}
async detectPopup(tab) {
return (await Promise.all(this.config.detectors.map(detectorConfig => tab.matches(detectorConfig.showingMatcher)))).some(matched => matched);
}
async executeAction(tab, method, param) {
if (this.methods.has(method)) {
return tab.executeAction(this.methods.get(method), param);
}
return true;
}
async optOut(tab) {
await this.executeAction(tab, "HIDE_CMP");
await this.executeAction(tab, "OPEN_OPTIONS");
await this.executeAction(tab, "HIDE_CMP");
await this.executeAction(tab, "DO_CONSENT", []);
await this.executeAction(tab, "SAVE_CONSENT");
return true;
}
async optIn(tab) {
await this.executeAction(tab, "HIDE_CMP");
await this.executeAction(tab, "OPEN_OPTIONS");
await this.executeAction(tab, "HIDE_CMP");
await this.executeAction(tab, "DO_CONSENT", ['D', 'A', 'B', 'E', 'F', 'X']);
await this.executeAction(tab, "SAVE_CONSENT");
return true;
}
async openCmp(tab) {
await this.executeAction(tab, "HIDE_CMP");
await this.executeAction(tab, "OPEN_OPTIONS");
return true;
}
test(tab) {
return this.executeAction(tab, "TEST_CONSENT");
}
detectFrame(tab, frame) {
return false;
}
}
exports.ConsentOMaticCMP = ConsentOMaticCMP;
exports.Tab = Tab;
exports.createAutoCMP = createAutoCMP;
exports.rules = rules$1;
exports.waitFor = waitFor;