From 7c3b7dc1837953f844ed557b16526dc6fc29c027 Mon Sep 17 00:00:00 2001 From: Dobes Vandermeer Date: Mon, 17 Mar 2025 12:31:35 -0700 Subject: [PATCH 1/4] Include filename and dep names in hash calculations If the name of a dependency or file changes, but the calculated hash does not, it should still result in a different hash. --- packages/hasher/src/TargetHasher.ts | 12 ++++++----- .../hasher/src/__tests__/TargetHasher.test.ts | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/packages/hasher/src/TargetHasher.ts b/packages/hasher/src/TargetHasher.ts index cb3f6b7f0..d2e4bb686 100644 --- a/packages/hasher/src/TargetHasher.ts +++ b/packages/hasher/src/TargetHasher.ts @@ -174,9 +174,9 @@ export class TargetHasher { } const files = await globAsync(target.inputs, { cwd: root }); - const fileFashes = hash(files, { cwd: root }) ?? {}; + const fileHashes = hash(files, { cwd: root }) ?? {}; - const hashes = Object.values(fileFashes) as string[]; + const hashes = Object.entries(fileHashes).map(p => p.join(' ')); return hashStrings(hashes); } @@ -201,18 +201,20 @@ export class TargetHasher { const fileHashes = this.fileHasher.hash(files) ?? {}; // this list is sorted by file name // get target hashes - const targetDepHashes = target.dependencies?.sort().map((targetDep) => this.targetHashes[targetDep]); + const targetDepHashes = target.dependencies?.sort().map( + (targetDep) => [targetDep, this.targetHashes[targetDep]].join(' ') + ); const globalFileHashes = await this.getEnvironmentGlobHashes(root, target); const combinedHashes = [ // Environmental hashes - ...Object.values(globalFileHashes), + ...Object.entries(globalFileHashes).map(p => p.join(' ')), `${target.id}|${JSON.stringify(this.options.cliArgs)}`, this.options.cacheKey || "", // File content hashes based on target.inputs - ...Object.values(fileHashes), + ...Object.entries(fileHashes).map(p => p.join(' ')), // Dependency hashes ...resolvedDependencies, diff --git a/packages/hasher/src/__tests__/TargetHasher.test.ts b/packages/hasher/src/__tests__/TargetHasher.test.ts index f8411792d..a4d665b08 100644 --- a/packages/hasher/src/__tests__/TargetHasher.test.ts +++ b/packages/hasher/src/__tests__/TargetHasher.test.ts @@ -172,4 +172,24 @@ describe("The main Hasher class", () => { monorepo1.cleanup(); }); + + it("creates different hashes when file path is different but file content is the same", async () => { + const content = "THIS IS CONTENT"; + const monorepo1 = await setupFixture("monorepo-with-global-files"); + const hasher = new TargetHasher({ root: monorepo1.root, environmentGlob: [] }); + const target = createTarget(monorepo1.root, "package-a", "build"); + target.inputs = ["file1.txt"]; + fs.writeFileSync(path.join(monorepo1.root, "packages", "package-a", "file1.txt"), content); + const target2 = createTarget(monorepo1.root, "package-a", "build"); + target2.inputs = ["file2.txt"]; + fs.writeFileSync(path.join(monorepo1.root, "packages", "package-a", "file2.txt"), content); + + const hash = await getHash(hasher, target); + const hash2 = await getHash(hasher, target2); + + expect(hash).not.toEqual(hash2); + + monorepo1.cleanup(); + }); + }); From e0b90a275816767317b281c0af40c1047804bbc6 Mon Sep 17 00:00:00 2001 From: Dobes Vandermeer Date: Mon, 17 Mar 2025 13:18:04 -0700 Subject: [PATCH 2/4] Also change the hash if a non-existent file is referenced (not using a pattern) --- packages/hasher/src/FileHasher.ts | 18 ++++++++----- packages/hasher/src/PackageTree.ts | 26 ++++++++++++++----- .../hasher/src/__tests__/TargetHasher.test.ts | 16 ++++++++++++ 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/packages/hasher/src/FileHasher.ts b/packages/hasher/src/FileHasher.ts index 1f7ebc684..9aa8e41a2 100644 --- a/packages/hasher/src/FileHasher.ts +++ b/packages/hasher/src/FileHasher.ts @@ -107,12 +107,18 @@ export class FileHasher { const updatedHashes = fastHash(updatedFiles, { cwd: this.options.root, concurrency: 4 }) ?? {}; for (const [file, hash] of Object.entries(updatedHashes)) { - const stat = fs.statSync(path.join(this.options.root, file), { bigint: true }); - this.#store[file] = { - mtime: stat.mtimeMs, - size: Number(stat.size), - hash: hash ?? "", - }; + try { + const stat = fs.statSync(path.join(this.options.root, file), { bigint: true }); + this.#store[file] = { + mtime: stat.mtimeMs, + size: Number(stat.size), + hash: hash ?? "", + }; + } catch(e) { + if(e.code !== "ENOENT") { + throw e; + } + } hashes[file] = hash ?? ""; } diff --git a/packages/hasher/src/PackageTree.ts b/packages/hasher/src/PackageTree.ts index e770093bc..00319117b 100644 --- a/packages/hasher/src/PackageTree.ts +++ b/packages/hasher/src/PackageTree.ts @@ -121,14 +121,26 @@ export class PackageTree { const key = `${packageName}\0${patterns.join("\0")}`; if (!this.#memoizedPackageFiles[key]) { - const packagePatterns = patterns.map((pattern) => { - if (pattern.startsWith("!")) { - return `!${path.join(packagePath, pattern.slice(1)).replace(/\\/g, "/")}`; + const packagePatterns: string[] = []; + const simplePaths: string[] = []; + for(const pattern of patterns) { + // If the input is a pattern, we have to run micromatch to convert that into a list of files + if(/[{}*?\[\]!+()]|@\(/.test(pattern)) { + if (pattern.startsWith("!")) { + packagePatterns.push(`!${path.join(packagePath, pattern.slice(1)).replace(/\\/g, "/")}`); + } else { + packagePatterns.push(path.join(packagePath, pattern).replace(/\\/g, "/")); + } + } else { + // No special characters, so no need to do pattern matching, just take the file exactly as is, and + // assume it could/should exist + simplePaths.push(pattern); } - - return path.join(packagePath, pattern).replace(/\\/g, "/"); - }); - this.#memoizedPackageFiles[key] = micromatch(packageFiles, packagePatterns, { dot: true }); + } + this.#memoizedPackageFiles[key] = [ + ...simplePaths, + ...(packagePatterns.length ? micromatch(packageFiles, packagePatterns, { dot: true }) : []), + ].sort(); } return this.#memoizedPackageFiles[key]; diff --git a/packages/hasher/src/__tests__/TargetHasher.test.ts b/packages/hasher/src/__tests__/TargetHasher.test.ts index a4d665b08..ad87cba6a 100644 --- a/packages/hasher/src/__tests__/TargetHasher.test.ts +++ b/packages/hasher/src/__tests__/TargetHasher.test.ts @@ -173,6 +173,22 @@ describe("The main Hasher class", () => { monorepo1.cleanup(); }); + it("creates different hashes when file path is different but files do not exist", async () => { + const monorepo1 = await setupFixture("monorepo-with-global-files"); + const hasher = new TargetHasher({ root: monorepo1.root, environmentGlob: [] }); + const target = createTarget(monorepo1.root, "package-a", "build"); + target.inputs = ["file1.txt"]; + const target2 = createTarget(monorepo1.root, "package-a", "build"); + target2.inputs = ["file2.txt"]; + + const hash = await getHash(hasher, target); + const hash2 = await getHash(hasher, target2); + + expect(hash).not.toEqual(hash2); + + monorepo1.cleanup(); + }); + it("creates different hashes when file path is different but file content is the same", async () => { const content = "THIS IS CONTENT"; const monorepo1 = await setupFixture("monorepo-with-global-files"); From 3c3cb697143a46f256b8b9a9deca88b07cbf0971 Mon Sep 17 00:00:00 2001 From: Dobes Vandermeer Date: Tue, 15 Apr 2025 16:16:05 -0700 Subject: [PATCH 3/4] Fix simple path calculation --- packages/hasher/src/PackageTree.ts | 2 +- .../hasher/src/__tests__/TargetHasher.test.ts | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/hasher/src/PackageTree.ts b/packages/hasher/src/PackageTree.ts index 00319117b..75ee26202 100644 --- a/packages/hasher/src/PackageTree.ts +++ b/packages/hasher/src/PackageTree.ts @@ -134,7 +134,7 @@ export class PackageTree { } else { // No special characters, so no need to do pattern matching, just take the file exactly as is, and // assume it could/should exist - simplePaths.push(pattern); + simplePaths.push(path.join(packagePath, pattern)); } } this.#memoizedPackageFiles[key] = [ diff --git a/packages/hasher/src/__tests__/TargetHasher.test.ts b/packages/hasher/src/__tests__/TargetHasher.test.ts index ad87cba6a..75c4b090f 100644 --- a/packages/hasher/src/__tests__/TargetHasher.test.ts +++ b/packages/hasher/src/__tests__/TargetHasher.test.ts @@ -103,6 +103,29 @@ describe("The main Hasher class", () => { monorepo2.cleanup(); }); + it("creates different hashes when a src file identified without any wildcard is changed", async () => { + const monorepo1 = await setupFixture("monorepo"); + const hasher = new TargetHasher({ root: monorepo1.root, environmentGlob: [] }); + const target = createTarget(monorepo1.root, "package-a", "build"); + target.inputs = ["package.json", "src/index.ts"]; + const hash = await getHash(hasher, target); + + const monorepo2 = await setupFixture("monorepo"); + const hasher2 = new TargetHasher({ root: monorepo2.root, environmentGlob: [] }); + const target2 = createTarget(monorepo2.root, "package-a", "build"); + target2.inputs = ["package.json", "src/index.ts"]; + + await monorepo2.commitFiles({ "packages/package-a/src/index.ts": "console.log('hello world');" }); + + const hash2 = await getHash(hasher2, target2); + + expect(hash).not.toEqual(hash2); + + monorepo1.cleanup(); + monorepo2.cleanup(); + }); + + it("creates different hashes when a src file has changed for a dependency", async () => { const monorepo1 = await setupFixture("monorepo-with-deps"); const hasher = new TargetHasher({ root: monorepo1.root, environmentGlob: [] }); From ce8941e73273c028694418b1ca9eeada4d110868 Mon Sep 17 00:00:00 2001 From: Dobes Vandermeer Date: Fri, 18 Apr 2025 21:19:47 -0700 Subject: [PATCH 4/4] Change files --- .../change-8f44ede0-283e-4c95-bd01-a6e5d6c22499.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 change/change-8f44ede0-283e-4c95-bd01-a6e5d6c22499.json diff --git a/change/change-8f44ede0-283e-4c95-bd01-a6e5d6c22499.json b/change/change-8f44ede0-283e-4c95-bd01-a6e5d6c22499.json new file mode 100644 index 000000000..9f711befd --- /dev/null +++ b/change/change-8f44ede0-283e-4c95-bd01-a6e5d6c22499.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "type": "patch", + "comment": "Include filename and dep names in hash calculations", + "packageName": "@lage-run/hasher", + "email": "dobes@formative.com", + "dependentChangeType": "patch" + } + ] +} \ No newline at end of file