Skip to content

Clarify filters and versions for processing #291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aselbie wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Clarify filters and versions for processing #291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 13 additions & 19 deletions src/js/__tests__/contentUtils-test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {
scanForScripts,
FOUND_SCRIPTS,
storeFoundJS,
UNINITIALIZED,
UNKNOWN,
} from '../contentUtils';
import {setCurrentOrigin} from '../content/updateCurrentState';

Expand All @@ -23,7 +23,7 @@ describe('contentUtils', () => {
window.chrome.runtime.sendMessage = jest.fn(() => {});
setCurrentOrigin('FACEBOOK');
FOUND_SCRIPTS.clear();
FOUND_SCRIPTS.set(UNINITIALIZED, []);
FOUND_SCRIPTS.set(UNKNOWN, []);
});
describe('storeFoundJS', () => {
it('should handle scripts with src correctly', () => {
Expand All @@ -33,8 +33,8 @@ describe('contentUtils', () => {
getAttribute: () => {},
};
storeFoundJS(fakeScriptNode);
expect(FOUND_SCRIPTS.get(UNINITIALIZED).length).toEqual(1);
expect(FOUND_SCRIPTS.get(UNINITIALIZED)[0].src).toEqual(fakeUrl);
expect(FOUND_SCRIPTS.get(UNKNOWN).length).toEqual(1);
expect(FOUND_SCRIPTS.get(UNKNOWN)[0].src).toEqual(fakeUrl);
expect(window.chrome.runtime.sendMessage.mock.calls.length).toBe(1);
});
it('should handle inline scripts correctly', () => {
Expand All @@ -49,11 +49,9 @@ describe('contentUtils', () => {
src: '',
};
storeFoundJS(fakeScriptNode);
expect(FOUND_SCRIPTS.get(UNINITIALIZED).length).toEqual(1);
expect(FOUND_SCRIPTS.get(UNINITIALIZED)[0].rawjs).toEqual(fakeInnerHtml);
expect(FOUND_SCRIPTS.get(UNINITIALIZED)[0].lookupKey).toEqual(
fakeLookupKey,
);
expect(FOUND_SCRIPTS.get(UNKNOWN).length).toEqual(1);
expect(FOUND_SCRIPTS.get(UNKNOWN)[0].rawjs).toEqual(fakeInnerHtml);
expect(FOUND_SCRIPTS.get(UNKNOWN)[0].lookupKey).toEqual(fakeLookupKey);
expect(window.chrome.runtime.sendMessage.mock.calls.length).toBe(1);
});
it('should send update icon message if valid', () => {
Expand Down Expand Up @@ -102,10 +100,8 @@ describe('contentUtils', () => {
src: '',
};
hasInvalidScripts(fakeElement);
expect(FOUND_SCRIPTS.get(UNINITIALIZED).length).toBe(1);
expect(FOUND_SCRIPTS.get(UNINITIALIZED)[0].type).toBe(
MESSAGE_TYPE.RAW_JS,
);
expect(FOUND_SCRIPTS.get(UNKNOWN).length).toBe(1);
expect(FOUND_SCRIPTS.get(UNKNOWN)[0].type).toBe(MESSAGE_TYPE.RAW_JS);
expect(window.chrome.runtime.sendMessage.mock.calls.length).toBe(1);
expect(window.chrome.runtime.sendMessage.mock.calls[0][0].type).toBe(
MESSAGE_TYPE.UPDATE_STATE,
Expand Down Expand Up @@ -149,7 +145,7 @@ describe('contentUtils', () => {
tagName: 'tagName',
};
hasInvalidScripts(fakeElement);
expect(FOUND_SCRIPTS.get(UNINITIALIZED).length).toBe(0);
expect(FOUND_SCRIPTS.get(UNKNOWN).length).toBe(0);
expect(window.chrome.runtime.sendMessage.mock.calls.length).toBe(0);
});
it('should store any script element direct children', () => {
Expand Down Expand Up @@ -192,10 +188,8 @@ describe('contentUtils', () => {
tagName: 'tagName',
};
hasInvalidScripts(fakeElement);
expect(FOUND_SCRIPTS.get(UNINITIALIZED).length).toBe(1);
expect(FOUND_SCRIPTS.get(UNINITIALIZED)[0].type).toBe(
MESSAGE_TYPE.RAW_JS,
);
expect(FOUND_SCRIPTS.get(UNKNOWN).length).toBe(1);
expect(FOUND_SCRIPTS.get(UNKNOWN)[0].type).toBe(MESSAGE_TYPE.RAW_JS);
expect(window.chrome.runtime.sendMessage.mock.calls.length).toBe(1);
expect(window.chrome.runtime.sendMessage.mock.calls[0][0].type).toBe(
MESSAGE_TYPE.UPDATE_STATE,
Expand Down Expand Up @@ -271,7 +265,7 @@ describe('contentUtils', () => {
tagName: 'tagName',
};
hasInvalidScripts(fakeElement);
expect(FOUND_SCRIPTS.get(UNINITIALIZED).length).toBe(2);
expect(FOUND_SCRIPTS.get(UNKNOWN).length).toBe(2);
expect(window.chrome.runtime.sendMessage.mock.calls.length).toBe(2);
});
});
Expand Down
120 changes: 65 additions & 55 deletions src/js/contentUtils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,24 +61,31 @@ const SOURCE_SCRIPTS = new Map();
* */
const INLINE_SCRIPTS: Array<Map<string, string>> = [];

// Map<version, Array<ScriptDetails>>
export const UNINITIALIZED = 'UNINITIALIZED';
const BOTH = 'BOTH';
let currentFilterType = UNINITIALIZED;
export const UNKNOWN = 'UNKNOWN';

// Filter types
const MANIFEST_NOT_LOADED = 'MANIFEST_NOT_LOADED';
const BOTH_MANIFESTS_LOADED = 'BOTH_MANIFESTS_LOADED';
const MANIFEST_LOADED = 'MANIFEST_LOADED';
const ANY_FILTER = 'ANY_FILTER';

const filterTypeByVersion: Map<string, string> = new Map([
[UNKNOWN, MANIFEST_NOT_LOADED],
]);
export const FOUND_SCRIPTS = new Map<string, Array<ScriptDetails>>([
[UNINITIALIZED, []],
[UNKNOWN, []],
]);
const ALL_FOUND_SCRIPT_TAGS = new Set();

type ScriptDetailsWithSrc = {
otherType: string;
filterTypeRequiredToProcess: string;
src: string;
};
type ScriptDetailsRaw = {
type: typeof MESSAGE_TYPE.RAW_JS;
rawjs: string;
lookupKey: string;
otherType: string;
filterTypeRequiredToProcess: string;
};
type ScriptDetails = ScriptDetailsRaw | ScriptDetailsWithSrc;
let manifestTimeoutID: string | number = '';
Expand Down Expand Up @@ -130,42 +137,35 @@ function handleManifestNode(manifestNode: HTMLScriptElement): void {
}

let leaves = rawManifest.leaves;
let otherType = '';
let filterType: string;
let roothash = rawManifest.root;
let version = rawManifest.version;
let version: string;
let messagePayload: MessagePayload;

if (originConfig.longTailIsLoadedConditionally) {
leaves = rawManifest.manifest;
const otherHashes = rawManifest.manifest_hashes;
roothash = otherHashes.combined_hash;

const maybeManifestType = manifestNode.getAttribute('data-manifest-type');
if (maybeManifestType === null) {
updateCurrentState(
STATES.INVALID,
'manifest is missing `data-manifest-type` prop',
);
} else {
otherType = maybeManifestType;
const manifestType = manifestNode.getAttribute('data-manifest-type');
if (manifestType === null) {
invalidateAndThrow('manifest is missing `data-manifest-type` prop');
}

const maybeManifestRev = manifestNode.getAttribute('data-manifest-rev');
if (maybeManifestRev === null) {
updateCurrentState(
STATES.INVALID,
'manifest is missing `data-manifest-rev` prop',
);
invalidateAndThrow('manifest is missing `data-manifest-rev` prop');
} else {
version = maybeManifestRev;
}

// If this is the first manifest we've found, start processing scripts for
// that type. If we have encountered a second manifest, we can assume both
// main and longtail manifests are present.
if (currentFilterType === UNINITIALIZED) {
currentFilterType = otherType;
if (filterTypeByVersion.has(version)) {
filterType = BOTH_MANIFESTS_LOADED;
} else {
currentFilterType = BOTH;
filterType = manifestType;
}

messagePayload = {
Expand All @@ -179,7 +179,8 @@ function handleManifestNode(manifestNode: HTMLScriptElement): void {
};
} else {
// for whatsapp
currentFilterType = BOTH;
version = rawManifest.version;
filterType = MANIFEST_LOADED;

messagePayload = {
type: MESSAGE_TYPE.LOAD_MANIFEST,
Expand All @@ -191,23 +192,18 @@ function handleManifestNode(manifestNode: HTMLScriptElement): void {
};
}

// now that we know the actual version of the scripts, transfer the ones we know about.
// also set the correct manifest type, "otherType" for already collected scripts
const foundScriptsWithoutVersion = FOUND_SCRIPTS.get(UNINITIALIZED);
filterTypeByVersion.set(version, filterType);

// Initialize this version in FOUND_SCRIPTS if it doesn't already exist.
const scriptsForVersion = FOUND_SCRIPTS.get(version) ?? [];
FOUND_SCRIPTS.set(version, scriptsForVersion);

// Now that we definitely have a version, we can grab any scripts with an
// unknown version and move them into that version's queue.
const foundScriptsWithoutVersion = FOUND_SCRIPTS.get(UNKNOWN);
if (foundScriptsWithoutVersion) {
const scriptsWithUpdatedType = foundScriptsWithoutVersion.map(script => ({
...script,
otherType: currentFilterType,
}));

FOUND_SCRIPTS.set(version, [
...scriptsWithUpdatedType,
...(FOUND_SCRIPTS.get(version) ?? []),
]);
FOUND_SCRIPTS.delete(UNINITIALIZED);
} else if (!FOUND_SCRIPTS.has(version)) {
// New version is being loaded in
FOUND_SCRIPTS.set(version, []);
scriptsForVersion.push(...foundScriptsWithoutVersion);
FOUND_SCRIPTS.delete(UNKNOWN);
}

sendMessageToBackground(messagePayload, response => {
Expand Down Expand Up @@ -244,7 +240,9 @@ function handleScriptNode(scriptNode: HTMLScriptElement): void {
// If this scripts contains packages from both main and longtail manifests
// then require both manifests to be loaded before processing this script,
// otherwise use the single type specified.
const otherType = manifest2 ? BOTH : manifest1.split('_')[1];
const filterTypeRequiredToProcess = manifest2
? BOTH_MANIFESTS_LOADED
: manifest1.split('_')[1];

// It is safe to assume a script will not contain packages from different
// versions, so we can use the first manifest version as the script version.
Expand All @@ -258,7 +256,7 @@ function handleScriptNode(scriptNode: HTMLScriptElement): void {

const scriptDetails = {
src: scriptNode.src,
otherType,
filterTypeRequiredToProcess,
};

ALL_FOUND_SCRIPT_TAGS.add(scriptNode.src);
Expand All @@ -269,7 +267,7 @@ function handleScriptNode(scriptNode: HTMLScriptElement): void {
if (scriptNode.src !== '') {
scriptDetails = {
src: scriptNode.src,
otherType: currentFilterType,
filterTypeRequiredToProcess: MANIFEST_LOADED,
};
ALL_FOUND_SCRIPT_TAGS.add(scriptNode.src);
} else {
Expand All @@ -282,11 +280,14 @@ function handleScriptNode(scriptNode: HTMLScriptElement): void {
type: MESSAGE_TYPE.RAW_JS,
rawjs: scriptNode.innerHTML,
lookupKey: hashLookupKey,
otherType: currentFilterType,
filterTypeRequiredToProcess: MANIFEST_LOADED,
};
}

FOUND_SCRIPTS.get(FOUND_SCRIPTS.keys().next().value)?.push(scriptDetails);
const latestVersionInMap = Array.from(FOUND_SCRIPTS.values()).pop();
if (latestVersionInMap) {
latestVersionInMap.push(scriptDetails);
}
}

updateCurrentState(STATES.PROCESSING);
Expand Down Expand Up @@ -450,15 +451,20 @@ async function processJSWithSrc(

export const processFoundJS = async (version: string): Promise<void> => {
const scriptsForVersion = FOUND_SCRIPTS.get(version);
if (!scriptsForVersion) {
const filterTypeForVersion = filterTypeByVersion.get(version);
if (!scriptsForVersion || !filterTypeForVersion) {
invalidateAndThrow(
`attempting to process scripts for nonexistent version ${version}`,
);
}
const scripts = scriptsForVersion.splice(0).filter(script => {
if (
script.otherType === currentFilterType ||
[BOTH, UNINITIALIZED].includes(currentFilterType)
(script.filterTypeRequiredToProcess === ANY_FILTER,
[
script.filterTypeRequiredToProcess,
BOTH_MANIFESTS_LOADED,
MANIFEST_LOADED,
].includes(filterTypeForVersion))
) {
return true;
} else {
Expand Down Expand Up @@ -639,13 +645,17 @@ chrome.runtime.onMessage.addListener(request => {
log: `Tab is processing ${request.response.url}`,
});
ALL_FOUND_SCRIPT_TAGS.add(request.response.url);
const uninitializedScripts = FOUND_SCRIPTS.get(
FOUND_SCRIPTS.keys().next().value,
);
if (uninitializedScripts) {
uninitializedScripts.push({

// Normally we use data attributes to get the version and type for a script.
// We cannot do that in this case because we're intercepting a request and thus
// have no script tag. Instead we will assume that this belongs to the latest
// version, and will start processing it as soon as we have loaded any manifest
// for that version.
const latestVersionInMap = Array.from(FOUND_SCRIPTS.values()).pop();
if (latestVersionInMap) {
latestVersionInMap.push({
src: request.response.url,
otherType: currentFilterType,
filterTypeRequiredToProcess: ANY_FILTER,
});
}
updateCurrentState(STATES.PROCESSING);
Expand Down