From b468ff1a143f91671bbe899f5a3662fe83b7f75c Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Sun, 7 Apr 2024 23:22:23 +0200 Subject: [PATCH 1/3] fix: use fast-glob for resolving projects --- package.json | 2 +- src/check-project/index.js | 35 ++++----- src/release.js | 21 ++--- src/test-dependant/index.js | 9 +-- src/utils.js | 149 ++++++++---------------------------- 5 files changed, 61 insertions(+), 155 deletions(-) diff --git a/package.json b/package.json index 90db3c506..78589a472 100644 --- a/package.json +++ b/package.json @@ -283,7 +283,7 @@ "micromark-extension-gfm-strikethrough": "^2.0.0", "micromark-extension-gfm-table": "^2.0.0", "micromark-extension-gfm-task-list-item": "^2.0.1", - "minimatch": "^9.0.0", + "fast-glob": "^3.3.2", "mocha": "^10.0.0", "npm-package-json-lint": "^7.0.0", "nyc": "^15.1.0", diff --git a/src/check-project/index.js b/src/check-project/index.js index b7eecc885..143caad5a 100755 --- a/src/check-project/index.js +++ b/src/check-project/index.js @@ -10,8 +10,8 @@ import prompt from 'prompt' import semver from 'semver' import yargsParser from 'yargs-parser' import { + getSubprojectDirectories, isMonorepoProject, - glob, usesReleasePlease } from '../utils.js' import { checkBuildFiles } from './check-build-files.js' @@ -114,32 +114,27 @@ async function processMonorepo (projectDir, manifest, branchName, repoUrl, ciFil const projectDirs = [] - for (const workspace of workspaces) { - for await (const subProjectDir of glob('.', workspace, { - cwd: projectDir, - absolute: true - })) { - const stat = await fs.stat(subProjectDir) + for (const subProjectDir of await getSubprojectDirectories(projectDir, workspaces)) { + const stat = await fs.stat(subProjectDir) - if (!stat.isDirectory()) { - continue - } + if (!stat.isDirectory()) { + continue + } - const manfest = path.join(subProjectDir, 'package.json') + const manfest = path.join(subProjectDir, 'package.json') - if (!fs.existsSync(manfest)) { - continue - } + if (!fs.existsSync(manfest)) { + continue + } - const pkg = fs.readJSONSync(manfest) - const homePage = `${repoUrl}/tree/${branchName}${subProjectDir.substring(projectDir.length)}` + const pkg = fs.readJSONSync(manfest) + const homePage = `${repoUrl}/tree/${branchName}${subProjectDir.substring(projectDir.length)}` - console.info('Found monorepo project', pkg.name) + console.info('Found monorepo project', pkg.name) - await processModule(subProjectDir, pkg, branchName, repoUrl, homePage, ciFile, manifest) + await processModule(subProjectDir, pkg, branchName, repoUrl, homePage, ciFile, manifest) - projectDirs.push(subProjectDir) - } + projectDirs.push(subProjectDir) } await alignMonorepoProjectDependencies(projectDirs) diff --git a/src/release.js b/src/release.js index 39ff7af85..10ee1a4f4 100644 --- a/src/release.js +++ b/src/release.js @@ -5,7 +5,7 @@ import { execa } from 'execa' import fs from 'fs-extra' import Listr from 'listr' import { calculateSiblingVersion } from './check-project/utils.js' -import { isMonorepoProject, isMonorepoRoot, hasDocs, glob } from './utils.js' +import { isMonorepoProject, isMonorepoRoot, hasDocs, getSubprojectDirectories } from './utils.js' /** * @typedef {import("./types").GlobalOptions} GlobalOptions @@ -151,18 +151,13 @@ async function calculateSiblingVersions (rootDir, workspaces) { /** @type {Record} */ const siblingVersions = {} - for (const workspace of workspaces) { - for await (const subProjectDir of glob(rootDir, workspace, { - cwd: rootDir, - absolute: true - })) { - const pkg = JSON.parse(fs.readFileSync(path.join(subProjectDir, 'package.json'), { - encoding: 'utf-8' - })) - - siblingVersions[pkg.name] = calculateSiblingVersion(pkg.version) - packageDirs.push(subProjectDir) - } + for (const subProjectDir of await getSubprojectDirectories(rootDir, workspaces)) { + const pkg = JSON.parse(fs.readFileSync(path.join(subProjectDir, 'package.json'), { + encoding: 'utf-8' + })) + + siblingVersions[pkg.name] = calculateSiblingVersion(pkg.version) + packageDirs.push(subProjectDir) } return { diff --git a/src/test-dependant/index.js b/src/test-dependant/index.js index 115d0f212..0e6a2117b 100644 --- a/src/test-dependant/index.js +++ b/src/test-dependant/index.js @@ -4,8 +4,7 @@ import os from 'os' import path from 'path' import fs from 'fs-extra' import { - exec, - glob + exec, getSubprojectDirectories } from '../utils.js' /** @@ -197,10 +196,8 @@ const testMonoRepo = async (targetDir, deps, scriptName) => { } // test each package that depends on passed deps - for (const pattern of config.workspaces) { - for await (const match of glob(targetDir, pattern)) { - await testModule(path.join(targetDir, match), deps, scriptName) - } + for (const match of await getSubprojectDirectories(targetDir, config.workspaces)) { + await testModule(path.join(targetDir, match), deps, scriptName) } } diff --git a/src/utils.js b/src/utils.js index 803a96b0c..aba03f42e 100644 --- a/src/utils.js +++ b/src/utils.js @@ -14,10 +14,10 @@ import { download } from '@electron/get' import envPaths from 'env-paths' import { execa } from 'execa' import extract from 'extract-zip' +import fg from 'fast-glob' import fs from 'fs-extra' import kleur from 'kleur' import Listr from 'listr' -import { minimatch } from 'minimatch' import PQueue from 'p-queue' import lockfile from 'proper-lockfile' import { readPackageUpSync } from 'read-pkg-up' @@ -355,7 +355,7 @@ export async function everyMonorepoProject (projectDir, fn, opts) { } /** @type {Record} */ - const projects = parseProjects(projectDir, workspaces) + const projects = await parseProjects(projectDir, workspaces) checkForCircularDependencies(projects) @@ -402,40 +402,45 @@ export async function everyMonorepoProject (projectDir, fn, opts) { * @param {string} projectDir * @param {string[]} workspaces */ -export function parseProjects (projectDir, workspaces) { +export const getSubprojectDirectories = async (projectDir, workspaces) => fg.glob(workspaces, { + cwd: projectDir, + onlyFiles: false +}) + +/** + * + * @param {string} projectDir + * @param {string[]} workspaces + */ +export async function parseProjects (projectDir, workspaces) { /** @type {Record} */ const projects = {} - for (const workspace of workspaces) { - for (const subProjectDir of glob('.', workspace, { - cwd: projectDir, - absolute: true - })) { - const stat = fs.statSync(subProjectDir) + for (const subProjectDir of await getSubprojectDirectories(projectDir, workspaces)) { + const stat = fs.statSync(subProjectDir) - if (!stat.isDirectory()) { - continue - } + if (!stat.isDirectory()) { + continue + } - const manfest = path.join(subProjectDir, 'package.json') + const manfest = path.join(subProjectDir, 'package.json') - if (!fs.existsSync(manfest)) { - continue - } + if (!fs.existsSync(manfest)) { + continue + } - const pkg = fs.readJSONSync(manfest) - - projects[pkg.name] = { - manifest: pkg, - dir: subProjectDir, - siblingDependencies: [], - dependencies: [ - ...Object.keys(pkg.dependencies ?? {}), - ...Object.keys(pkg.devDependencies ?? {}), - ...Object.keys(pkg.optionalDependencies ?? {}), - ...Object.keys(pkg.peerDependencies ?? {}) - ] - } + const pkg = fs.readJSONSync(manfest) + + projects[pkg.name] = { + manifest: pkg, + dir: subProjectDir, + siblingDependencies: [], + dependencies: [ + ...Object.keys(pkg.dependencies ?? {}), + ...Object.keys(pkg.devDependencies ?? {}), + ...Object.keys(pkg.optionalDependencies ?? {}), + ...Object.keys(pkg.peerDependencies ?? {}) + ] } } @@ -504,92 +509,6 @@ function checkForCircularDependencies (projects) { } } -/** - * @typedef {object} GlobOptions - * @property {string} [cwd] The current working directory - * @property {boolean} [absolute] If true produces absolute paths (default: false) - * @property {boolean} [nodir] If true yields file paths and skip directories (default: false) - * - * Iterable filename pattern matcher - * - * @param {string} dir - * @param {string} pattern - * @param {GlobOptions & import('minimatch').MinimatchOptions} [options] - * @returns {Generator} - */ -export function * glob (dir, pattern, options = {}) { - const absoluteDir = path.resolve(dir) - const relativeDir = path.relative(options.cwd ?? process.cwd(), dir) - - const stats = fs.statSync(absoluteDir) - - if (stats.isDirectory()) { - for (const entry of _glob(absoluteDir, '', pattern, options)) { - yield entry - } - - return - } - - if (minimatch(relativeDir, pattern, options)) { - yield options.absolute === true ? absoluteDir : relativeDir - } -} - -/** - * @param {string} base - * @param {string} dir - * @param {string} pattern - * @param {GlobOptions & import('minimatch').MinimatchOptions} options - * @returns {Generator} - */ -function * _glob (base, dir, pattern, options) { - const p = path.join(base, dir) - - if (!fs.existsSync(p)) { - return - } - - const stats = fs.statSync(p) - - if (!stats.isDirectory()) { - return - } - - const d = fs.opendirSync(p) - - try { - while (true) { - const entry = d.readSync() - - if (entry == null) { - break - } - - const relativeEntryPath = path.join(dir, entry.name) - const absoluteEntryPath = path.join(base, dir, entry.name) - - let match = minimatch(relativeEntryPath, pattern, options) - - const isDirectory = entry.isDirectory() - - if (isDirectory && options.nodir === true) { - match = false - } - - if (match) { - yield options.absolute === true ? absoluteEntryPath : relativeEntryPath - } - - if (isDirectory) { - yield * _glob(base, relativeEntryPath, pattern, options) - } - } - } finally { - d.closeSync() - } -} - /** * * @param {Error} error From 1bd0bfb65af41c3a3fe941a7054f027d2e8ba992 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Mon, 8 Apr 2024 00:42:36 +0200 Subject: [PATCH 2/3] feat: workspace runtime argument --- src/check-project/index.js | 2 +- src/cmds/run.js | 9 +++++++++ src/docs/readme-updater-plugin.js | 2 +- src/exec.js | 2 +- src/release-rc.js | 4 ++-- src/release.js | 8 ++++---- src/run.js | 3 ++- src/test-dependant/index.js | 2 +- src/types.ts | 9 +++++++++ src/utils.js | 33 ++++++++++++++++++------------- test/run.js | 14 +++++++++++++ 11 files changed, 63 insertions(+), 25 deletions(-) diff --git a/src/check-project/index.js b/src/check-project/index.js index 143caad5a..83dc114bc 100755 --- a/src/check-project/index.js +++ b/src/check-project/index.js @@ -114,7 +114,7 @@ async function processMonorepo (projectDir, manifest, branchName, repoUrl, ciFil const projectDirs = [] - for (const subProjectDir of await getSubprojectDirectories(projectDir, workspaces)) { + for (const subProjectDir of await getSubprojectDirectories(workspaces, projectDir)) { const stat = await fs.stat(subProjectDir) if (!stat.isDirectory()) { diff --git a/src/cmds/run.js b/src/cmds/run.js index 33b0ff5ed..1697c108a 100644 --- a/src/cmds/run.js +++ b/src/cmds/run.js @@ -1,5 +1,6 @@ import { loadUserConfig } from '../config/user.js' import runCmd from '../run.js' +import { listWorkspaces } from '../utils.js' /** * @typedef {import("yargs").Argv} Argv @@ -39,6 +40,14 @@ export default { type: 'number', describe: 'How many scripts to run at the same time', default: userConfig.run.concurrency + }, + + workspaces: { + // an array of strings + array: true, + describe: 'Run the script in a specific workspace', + default: await listWorkspaces(process.cwd()), + alias: ['workspace', 'roots'] } }) .positional('script', { diff --git a/src/docs/readme-updater-plugin.js b/src/docs/readme-updater-plugin.js index 4f5f16032..6958e408a 100644 --- a/src/docs/readme-updater-plugin.js +++ b/src/docs/readme-updater-plugin.js @@ -15,7 +15,7 @@ export function load (app) { let projects = {} if (isMonorepoParent) { - projects = parseProjects(process.cwd(), pkg.workspaces) + projects = parseProjects(pkg.workspaces) } // when rendering has finished, work out which UrlMappings refer to the index diff --git a/src/exec.js b/src/exec.js index e785143ff..6a419a854 100644 --- a/src/exec.js +++ b/src/exec.js @@ -14,7 +14,7 @@ export default { async run (ctx) { const forwardArgs = ctx['--'] ? ctx['--'] : [] - await everyMonorepoProject(process.cwd(), async (project) => { + await everyMonorepoProject(async (project) => { console.info('') // eslint-disable-line no-console console.info(kleur.grey(`${project.manifest.name}:`), `> ${ctx.command}${forwardArgs.length > 0 ? ` ${forwardArgs.join(' ')}` : ''}`) // eslint-disable-line no-console diff --git a/src/release-rc.js b/src/release-rc.js index cfb39aafe..d86177d17 100644 --- a/src/release-rc.js +++ b/src/release-rc.js @@ -22,7 +22,7 @@ async function releaseMonorepoRcs (commit, ctx) { /** @type {Record} */ const versions = {} - await everyMonorepoProject(process.cwd(), async (project) => { + await everyMonorepoProject(async (project) => { if (project.manifest.private === true) { console.info(`Skipping private package ${project.manifest.name}`) return @@ -43,7 +43,7 @@ async function releaseMonorepoRcs (commit, ctx) { console.info('') // publish packages - await everyMonorepoProject(process.cwd(), async (project) => { + await everyMonorepoProject(async (project) => { if (project.manifest.private === true) { console.info(`Skipping private package ${project.manifest.name}`) return diff --git a/src/release.js b/src/release.js index 10ee1a4f4..d73733363 100644 --- a/src/release.js +++ b/src/release.js @@ -69,7 +69,7 @@ const tasks = new Listr([ const { siblingVersions, packageDirs - } = await calculateSiblingVersions(rootDir, workspaces) + } = await calculateSiblingVersions(workspaces, rootDir) // check these dependency types for monorepo siblings const dependencyTypes = [ @@ -142,16 +142,16 @@ const tasks = new Listr([ ], { renderer: 'verbose' }) /** - * @param {string} rootDir * @param {string[]} workspaces + * @param {string} rootDir */ -async function calculateSiblingVersions (rootDir, workspaces) { +async function calculateSiblingVersions (workspaces, rootDir) { const packageDirs = [] /** @type {Record} */ const siblingVersions = {} - for (const subProjectDir of await getSubprojectDirectories(rootDir, workspaces)) { + for (const subProjectDir of await getSubprojectDirectories(workspaces, rootDir)) { const pkg = JSON.parse(fs.readFileSync(path.join(subProjectDir, 'package.json'), { encoding: 'utf-8' })) diff --git a/src/run.js b/src/run.js index b39462e68..c83826667 100644 --- a/src/run.js +++ b/src/run.js @@ -20,7 +20,7 @@ export default { const forwardArgs = ctx['--'] == null ? [] : ['--', ...ctx['--']] - await everyMonorepoProject(process.cwd(), async (project) => { + await everyMonorepoProject(async (project) => { for (const script of scripts) { if (project.manifest.scripts[script] == null) { continue @@ -45,6 +45,7 @@ export default { } } }, { + workspaces: ctx.workspaces, concurrency: ctx.concurrency }) } diff --git a/src/test-dependant/index.js b/src/test-dependant/index.js index 0e6a2117b..fcc226076 100644 --- a/src/test-dependant/index.js +++ b/src/test-dependant/index.js @@ -196,7 +196,7 @@ const testMonoRepo = async (targetDir, deps, scriptName) => { } // test each package that depends on passed deps - for (const match of await getSubprojectDirectories(targetDir, config.workspaces)) { + for (const match of await getSubprojectDirectories(config.workspaces, targetDir)) { await testModule(path.join(targetDir, match), deps, scriptName) } } diff --git a/src/types.ts b/src/types.ts index 27c5356ce..9254e6eb3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -109,6 +109,7 @@ interface GlobalOptions { * Full config from configuration file */ fileConfig: Options + } interface BuildOptions { @@ -391,9 +392,11 @@ interface ExecOptions { * Run commands in parallel up to this limit */ concurrency?: number + } interface RunOptions { + /** * If false, the script will continue to be run in other packages */ @@ -408,6 +411,12 @@ interface RunOptions { * Run scripts in parallel up to this limit */ concurrency?: number + + /** + * Workspaces to run the command in + */ + workspaces?: string[] + } export type { diff --git a/src/utils.js b/src/utils.js index aba03f42e..566132f2b 100644 --- a/src/utils.js +++ b/src/utils.js @@ -332,6 +332,14 @@ export function findBinary (bin) { return bin } +/** + * @param {string} projectDir + */ +export const listWorkspaces = async (projectDir) => { + const manifest = fs.readJSONSync(path.join(projectDir, 'package.json')) + return manifest.workspaces ?? undefined +} + /** * @typedef {object} Project * @property {any} manifest @@ -341,21 +349,20 @@ export function findBinary (bin) { */ /** - * @param {string} projectDir * @param {(project: Project) => Promise} fn * @param {object} [opts] + * @param {string[]?} [opts.workspaces] * @param {number} [opts.concurrency] */ -export async function everyMonorepoProject (projectDir, fn, opts) { - const manifest = fs.readJSONSync(path.join(projectDir, 'package.json')) - const workspaces = manifest.workspaces - - if (!workspaces || !Array.isArray(workspaces)) { +export async function everyMonorepoProject (fn, opts) { + const workspaces = (opts?.workspaces ?? await listWorkspaces(process.cwd())).filter((/** @type {string | null} */ workspace) => workspace != null) + if (!workspaces || !Array.isArray(workspaces) || workspaces.length === 0) { throw new Error('No monorepo workspaces found') } /** @type {Record} */ - const projects = await parseProjects(projectDir, workspaces) + + const projects = await parseProjects(workspaces) checkForCircularDependencies(projects) @@ -398,25 +405,23 @@ export async function everyMonorepoProject (projectDir, fn, opts) { } /** - * - * @param {string} projectDir * @param {string[]} workspaces + * @param {string | undefined} cwd */ -export const getSubprojectDirectories = async (projectDir, workspaces) => fg.glob(workspaces, { - cwd: projectDir, +export const getSubprojectDirectories = async (workspaces, cwd = process.cwd()) => fg.glob(workspaces, { + cwd, onlyFiles: false }) /** * - * @param {string} projectDir * @param {string[]} workspaces */ -export async function parseProjects (projectDir, workspaces) { +export async function parseProjects (workspaces) { /** @type {Record} */ const projects = {} - for (const subProjectDir of await getSubprojectDirectories(projectDir, workspaces)) { + for (const subProjectDir of await getSubprojectDirectories(workspaces)) { const stat = fs.statSync(subProjectDir) if (!stat.isDirectory()) { diff --git a/test/run.js b/test/run.js index 355bc1033..08ebb03af 100644 --- a/test/run.js +++ b/test/run.js @@ -117,4 +117,18 @@ very test`) expect(out.indexOf('b: very test')).to.be.lt(out.indexOf('d: npm run test')) expect(out.indexOf('c: very test')).to.be.lt(out.indexOf('d: npm run test')) }) + + it('can run in specifc workspaces', async function () { + this.timeout(120 * 1000) // slow ci is slow + + const result = await execa(bin, ['run', 'test', '--workspaces=**/a-workspace-project'], { + cwd: projectDir + }) + + expect(result.stdout).to.equal(` +a-workspace-project: npm run test +a-workspace-project: > a-workspace-project@1.0.0 test +a-workspace-project: > echo very test +a-workspace-project: very test`) + }) }) From 8c3bc3700c60e866e76e6b4d1a92d4a69db2c650 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Mon, 8 Apr 2024 09:44:49 +0200 Subject: [PATCH 3/3] fix: dont follow symbolic links --- src/utils.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index 566132f2b..25b84954c 100644 --- a/src/utils.js +++ b/src/utils.js @@ -410,7 +410,8 @@ export async function everyMonorepoProject (fn, opts) { */ export const getSubprojectDirectories = async (workspaces, cwd = process.cwd()) => fg.glob(workspaces, { cwd, - onlyFiles: false + onlyFiles: false, + followSymbolicLinks: false }) /**