Skip to content

Commit 2bd294e

Browse files
webprowebpro
committed
Refactor globbin' to improve debug output
1 parent 7464371 commit 2bd294e

File tree

5 files changed

+79
-58
lines changed

5 files changed

+79
-58
lines changed

‎packages/knip/src/WorkspaceWorker.ts‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -137,12 +137,7 @@ export class WorkspaceWorker {
137137
}
138138
}
139139

140-
const enabledPlugins = getKeysByValue(this.enabledPluginsMap, true);
141-
142-
const enabledPluginTitles = enabledPlugins.map(name => Plugins[name].title);
143-
debugLogObject(this.name, 'Enabled plugins', enabledPluginTitles);
144-
145-
return enabledPlugins;
140+
return getKeysByValue(this.enabledPluginsMap, true);
146141
}
147142

148143
private getConfigForPlugin(pluginName: PluginName): EnsuredPluginConfiguration {
@@ -300,13 +295,12 @@ export class WorkspaceWorker {
300295

301296
if (!config) return;
302297

303-
const configFilePaths = await _glob({ patterns, cwd: baseScriptOptions.rootCwd, dir: cwd, gitignore: false });
298+
const label = 'config file';
299+
const configFilePaths = await _glob({ patterns, cwd: rootCwd, dir: cwd, gitignore: false, label });
304300

305301
const remainingConfigFilePaths = configFilePaths.filter(filePath => !this.allConfigFilePaths.has(filePath));
306302
for (const f of remainingConfigFilePaths) if (basename(f) !== 'package.json') this.allConfigFilePaths.add(f);
307303

308-
if (configFilePaths.length > 0) debugLogArray([name, plugin.title], 'config file paths', configFilePaths);
309-
310304
const options = {
311305
...baseScriptOptions,
312306
config,
@@ -366,6 +360,9 @@ export class WorkspaceWorker {
366360
}
367361
};
368362

363+
const enabledPluginTitles = this.enabledPlugins.map(name => Plugins[name].title);
364+
debugLogObject(this.name, 'Enabled plugins', enabledPluginTitles);
365+
369366
for (const pluginName of this.enabledPlugins) {
370367
const patterns = [...this.getConfigurationFilePatterns(pluginName), ...(configFiles.get(pluginName) ?? [])];
371368
configFiles.delete(pluginName);

‎packages/knip/src/index.ts‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -162,8 +162,10 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {
162162

163163
const deps = new Set<Input>();
164164

165-
debugLogArray(name, 'Definition paths', definitionPaths);
166-
for (const id of definitionPaths) deps.add(toProductionEntry(id, { containingFilePath: tsConfigFilePath }));
165+
if (definitionPaths.length > 0) {
166+
debugLogArray(name, 'Definition paths', definitionPaths);
167+
for (const id of definitionPaths) deps.add(toProductionEntry(id, { containingFilePath: tsConfigFilePath }));
168+
}
167169

168170
const ignore = worker.getIgnorePatterns();
169171
const sharedGlobOptions = { cwd, dir, gitignore };
@@ -172,7 +174,6 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {
172174

173175
// Add entry paths from package.json#main, #bin, #exports
174176
const entryPathsFromManifest = await getEntryPathsFromManifest(manifest, { ...sharedGlobOptions, ignore });
175-
debugLogArray(name, 'Entry paths in package.json', entryPathsFromManifest);
176177
for (const id of entryPathsFromManifest.map(id => toProductionEntry(id))) deps.add(id);
177178

178179
// Get dependencies from plugins
@@ -218,58 +219,58 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {
218219
const negatedEntryPatterns: string[] = Array.from(entryFilePatterns).map(negate);
219220

220221
{
222+
const label = 'entry';
221223
const patterns = worker.getProductionEntryFilePatterns(negatedEntryPatterns);
222-
const workspaceEntryPaths = await _glob({ ...sharedGlobOptions, patterns, gitignore: false });
223-
debugLogArray(name, 'Entry paths', workspaceEntryPaths);
224+
const workspaceEntryPaths = await _glob({ ...sharedGlobOptions, patterns, gitignore: false, label });
224225
principal.addEntryPaths(workspaceEntryPaths);
225226
}
226227

227228
{
229+
const label = 'production plugin entry';
228230
const patterns = Array.from(productionEntryFilePatterns);
229-
const pluginWorkspaceEntryPaths = await _glob({ ...sharedGlobOptions, patterns });
230-
debugLogArray(name, 'Production plugin entry paths', pluginWorkspaceEntryPaths);
231+
const pluginWorkspaceEntryPaths = await _glob({ ...sharedGlobOptions, patterns, label });
231232
principal.addEntryPaths(pluginWorkspaceEntryPaths, { skipExportsAnalysis: true });
232233
}
233234

234235
{
236+
const label = 'project';
235237
const patterns = worker.getProductionProjectFilePatterns(negatedEntryPatterns);
236-
const workspaceProjectPaths = await _glob({ ...sharedGlobOptions, patterns });
237-
debugLogArray(name, 'Project paths', workspaceProjectPaths);
238+
const workspaceProjectPaths = await _glob({ ...sharedGlobOptions, patterns, label });
238239
for (const projectPath of workspaceProjectPaths) principal.addProjectPath(projectPath);
239240
}
240241
} else {
241242
{
243+
const label = 'entry';
242244
const patterns = worker.getEntryFilePatterns();
243-
const workspaceEntryPaths = await _glob({ ...sharedGlobOptions, patterns, gitignore: false });
244-
debugLogArray(name, 'Entry paths', workspaceEntryPaths);
245+
const workspaceEntryPaths = await _glob({ ...sharedGlobOptions, patterns, gitignore: false, label });
245246
principal.addEntryPaths(workspaceEntryPaths);
246247
}
247248

248249
{
250+
const label = 'project';
249251
const patterns = worker.getProjectFilePatterns([...productionEntryFilePatterns]);
250-
const workspaceProjectPaths = await _glob({ ...sharedGlobOptions, patterns });
251-
debugLogArray(name, 'Project paths', workspaceProjectPaths);
252+
const workspaceProjectPaths = await _glob({ ...sharedGlobOptions, patterns, label });
252253
for (const projectPath of workspaceProjectPaths) principal.addProjectPath(projectPath);
253254
}
254255

255256
{
257+
const label = 'plugin entry';
256258
const patterns = worker.getPluginEntryFilePatterns([...entryFilePatterns, ...productionEntryFilePatterns]);
257-
const pluginWorkspaceEntryPaths = await _glob({ ...sharedGlobOptions, patterns });
258-
debugLogArray(name, 'Plugin entry paths', pluginWorkspaceEntryPaths);
259+
const pluginWorkspaceEntryPaths = await _glob({ ...sharedGlobOptions, patterns, label });
259260
principal.addEntryPaths(pluginWorkspaceEntryPaths, { skipExportsAnalysis: true });
260261
}
261262

262263
{
264+
const label = 'plugin project';
263265
const patterns = worker.getPluginProjectFilePatterns();
264-
const pluginWorkspaceProjectPaths = await _glob({ ...sharedGlobOptions, patterns });
265-
debugLogArray(name, 'Plugin project paths', pluginWorkspaceProjectPaths);
266+
const pluginWorkspaceProjectPaths = await _glob({ ...sharedGlobOptions, patterns, label });
266267
for (const projectPath of pluginWorkspaceProjectPaths) principal.addProjectPath(projectPath);
267268
}
268269

269270
{
271+
const label = 'plugin configuration';
270272
const patterns = worker.getPluginConfigPatterns();
271-
const configurationEntryPaths = await _glob({ ...sharedGlobOptions, patterns });
272-
debugLogArray(name, 'Plugin configuration paths', configurationEntryPaths);
273+
const configurationEntryPaths = await _glob({ ...sharedGlobOptions, patterns, label });
273274
principal.addEntryPaths(configurationEntryPaths, { skipExportsAnalysis: true });
274275
}
275276
}

‎packages/knip/src/util/glob-core.ts‎

Lines changed: 40 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import fg, { type Options as FastGlobOptions } from 'fast-glob';
55
import picomatch from 'picomatch';
66
import { GLOBAL_IGNORE_PATTERNS, ROOT_WORKSPACE_NAME } from '../constants.js';
77
import { timerify } from './Performance.js';
8+
import { compact } from './array.js';
89
import { debugLogObject } from './debug.js';
910
import { isFile } from './fs.js';
1011
import { dirname, join, relative, toPosix } from './path.js';
@@ -19,13 +20,15 @@ type GlobOptions = {
1920
readonly gitignore: boolean;
2021
readonly cwd: string;
2122
readonly dir: string;
23+
label?: string;
2224
} & FastGlobOptionsWithoutCwd;
2325

2426
type FastGlobOptionsWithoutCwd = Pick<FastGlobOptions, 'onlyDirectories' | 'ignore' | 'absolute' | 'dot'>;
2527

2628
type Gitignores = { ignores: Set<string>; unignores: string[] };
2729

28-
const cachedIgnores = new Map<string, Gitignores>();
30+
const cachedGitIgnores = new Map<string, Gitignores>();
31+
const cachedGlobIgnores = new Map<string, string[]>();
2932

3033
/** @internal */
3134
export const convertGitignoreToPicomatchIgnorePatterns = (pattern: string) => {
@@ -146,13 +149,13 @@ export const findAndParseGitignores = async (cwd: string) => {
146149
}
147150

148151
const cacheDir = ancestor ? cwd : dir;
149-
const cacheForDir = cachedIgnores.get(cwd);
152+
const cacheForDir = cachedGitIgnores.get(cwd);
150153

151154
if (ancestor && cacheForDir) {
152155
for (const pattern of dirIgnores) cacheForDir?.ignores.add(pattern);
153156
cacheForDir.unignores = Array.from(new Set([...cacheForDir.unignores, ...dirUnignores]));
154157
} else {
155-
cachedIgnores.set(cacheDir, { ignores: dirIgnores, unignores: Array.from(dirUnignores) });
158+
cachedGitIgnores.set(cacheDir, { ignores: dirIgnores, unignores: Array.from(dirUnignores) });
156159
}
157160

158161
for (const pattern of dirIgnores) matchers.add(_picomatch(pattern, pmOptions));
@@ -177,44 +180,60 @@ export const findAndParseGitignores = async (cwd: string) => {
177180
deepFilter: timerify(deepFilter),
178181
});
179182

180-
debugLogObject('*', 'Parsed gitignore files', { gitignoreFiles, ignores, unignores });
183+
debugLogObject('*', 'Parsed gitignore files', { gitignoreFiles });
181184

182185
return { gitignoreFiles, ignores, unignores };
183186
};
184187

185188
const _parseFindGitignores = timerify(findAndParseGitignores);
186189

187-
export async function globby(patterns: string | string[], options: GlobOptions): Promise<string[]> {
190+
export async function glob(patterns: string | string[], options: GlobOptions): Promise<string[]> {
188191
if (Array.isArray(patterns) && patterns.length === 0) return [];
189192

190-
const ignore = options.gitignore && Array.isArray(options.ignore) ? [...options.ignore] : [];
193+
const canCache = options.label && options.gitignore;
194+
const willCache = canCache && !cachedGlobIgnores.has(options.dir);
195+
const cachedIgnores = canCache && cachedGlobIgnores.get(options.dir);
196+
197+
const _ignore = options.gitignore && Array.isArray(options.ignore) ? [...options.ignore] : [];
191198

192199
if (options.gitignore) {
193-
let dir = options.dir;
194-
let prev: string;
195-
while (dir) {
196-
const cacheForDir = cachedIgnores.get(dir);
197-
if (cacheForDir) {
198-
// fast-glob doesn't support negated patterns in `ignore` (i.e. unignores are.. ignored): https://github.com/mrmlnc/fast-glob/issues/86
199-
ignore.push(...cacheForDir.ignores);
200+
if (willCache) {
201+
let dir = options.dir;
202+
let prev: string;
203+
while (dir) {
204+
const cacheForDir = cachedGitIgnores.get(dir);
205+
if (cacheForDir) {
206+
// fast-glob doesn't support negated patterns in `ignore` (i.e. unignores are.. ignored): https://github.com/mrmlnc/fast-glob/issues/86
207+
_ignore.push(...cacheForDir.ignores);
208+
}
209+
// biome-ignore lint/suspicious/noAssignInExpressions: deal with it
210+
dir = dirname((prev = dir));
211+
if (prev === dir || dir === '.') break;
200212
}
201-
// biome-ignore lint/suspicious/noAssignInExpressions: deal with it
202-
dir = dirname((prev = dir));
203-
if (prev === dir || dir === '.') break;
204213
}
205214
} else {
206-
ignore.push(...GLOBAL_IGNORE_PATTERNS);
215+
_ignore.push(...GLOBAL_IGNORE_PATTERNS);
207216
}
208217

209-
const fgOptions = Object.assign(options, { ignore });
218+
const ignore = cachedIgnores || compact(_ignore);
219+
220+
const { dir, label, ...fgOptions } = { ...options, ignore };
221+
222+
const paths = await fg.glob(patterns, fgOptions);
223+
224+
debugLogObject(
225+
relative(options.cwd, options.dir) || ROOT_WORKSPACE_NAME,
226+
label ? `Finding ${label} paths` : 'Finding paths',
227+
() => ({ patterns, ...fgOptions, ignore: cachedIgnores ? `identical to ${dir} ↑` : ignore, paths })
228+
);
210229

211-
debugLogObject(relative(options.cwd, options.dir) || ROOT_WORKSPACE_NAME, 'Glob options', { patterns, ...fgOptions });
230+
if (willCache) cachedGlobIgnores.set(options.dir, ignore);
212231

213-
return fg.glob(patterns, fgOptions);
232+
return paths;
214233
}
215234

216235
export async function getGitIgnoredHandler(options: Options): Promise<(path: string) => boolean> {
217-
cachedIgnores.clear();
236+
cachedGitIgnores.clear();
218237

219238
if (options.gitignore === false) return () => false;
220239

‎packages/knip/src/util/glob.ts‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,25 @@ import fg from 'fast-glob';
22
import { GLOBAL_IGNORE_PATTERNS } from '../constants.js';
33
import { timerify } from './Performance.js';
44
import { compact } from './array.js';
5-
import { globby } from './glob-core.js';
5+
import { glob } from './glob-core.js';
66
import { isAbsolute, join, relative } from './path.js';
77

88
interface GlobOptions {
99
cwd: string;
1010
dir?: string;
1111
patterns: string[];
1212
gitignore?: boolean;
13+
name?: boolean;
14+
label?: string;
1315
}
1416

17+
const prepend = (pattern: string, relativePath: string) =>
18+
isAbsolute(pattern.replace(/^!/, '')) ? pattern : prependDirToPattern(relativePath, pattern);
19+
1520
// Globbing from root as cwd to include all gitignore files and ignore patterns, so we need to prepend dirs to patterns
1621
const prependDirToPatterns = (cwd: string, dir: string, patterns: string[]) => {
1722
const relativePath = relative(cwd, dir);
18-
const prepend = (pattern: string) =>
19-
isAbsolute(pattern.replace(/^!/, '')) ? pattern : prependDirToPattern(relativePath, pattern);
20-
return compact([patterns].flat().map(prepend).map(removeProductionSuffix)).sort(negatedLast);
23+
return compact([patterns].flat().map(p => removeProductionSuffix(prepend(p, relativePath)))).sort(negatedLast);
2124
};
2225

2326
const removeProductionSuffix = (pattern: string) => pattern.replace(/!$/, '');
@@ -33,20 +36,21 @@ export const negate = (pattern: string) => pattern.replace(/^!?/, '!');
3336
export const hasProductionSuffix = (pattern: string) => pattern.endsWith('!');
3437
export const hasNoProductionSuffix = (pattern: string) => !pattern.endsWith('!');
3538

36-
const glob = async ({ cwd, dir = cwd, patterns, gitignore = true }: GlobOptions) => {
39+
const defaultGlob = async ({ cwd, dir = cwd, patterns, gitignore = true, label }: GlobOptions) => {
3740
if (patterns.length === 0) return [];
3841

3942
const globPatterns = prependDirToPatterns(cwd, dir, patterns);
4043

4144
// Only negated patterns? Bail out.
4245
if (globPatterns[0].startsWith('!')) return [];
4346

44-
return globby(globPatterns, {
47+
return glob(globPatterns, {
4548
cwd,
4649
dir,
4750
gitignore,
4851
absolute: true,
4952
dot: true,
53+
label,
5054
});
5155
};
5256

@@ -58,14 +62,14 @@ const firstGlob = async ({ cwd, patterns }: GlobOptions) => {
5862
};
5963

6064
const dirGlob = async ({ cwd, patterns, gitignore = true }: GlobOptions) =>
61-
globby(patterns, {
65+
glob(patterns, {
6266
cwd,
6367
dir: cwd,
6468
onlyDirectories: true,
6569
gitignore,
6670
});
6771

68-
export const _glob = timerify(glob);
72+
export const _glob = timerify(defaultGlob);
6973

7074
export const _firstGlob = timerify(firstGlob);
7175

‎packages/knip/src/util/package-json.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,5 +73,5 @@ export const getEntryPathsFromManifest = (
7373
// - exist
7474
// - are not (generated) files that are .gitignore'd
7575
// - do not match configured `ignore` patterns
76-
return _glob({ ...sharedGlobOptions, patterns: Array.from(entryPaths) });
76+
return _glob({ ...sharedGlobOptions, patterns: Array.from(entryPaths), label: 'package.json entry' });
7777
};

0 commit comments

Comments
 (0)