Skip to content

Commit 12e2906

Browse files
msukkarimsukkari
committed
add back support for multiple workspaces and add exclude logic
1 parent 4f86085 commit 12e2906

File tree

9 files changed

+101
-178
lines changed

9 files changed

+101
-178
lines changed

‎packages/backend/src/bitbucket.ts

Lines changed: 77 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,11 @@ interface BitbucketClient {
3333
baseUrl: string;
3434
gitUrl: string;
3535
getPaginated: <T, V extends CloudGetRequestPath>(path: V, get: (url: V) => Promise<PaginatedResponse<T>>) => Promise<T[]>;
36-
getReposForWorkspace: (client: BitbucketClient, workspace: string) => Promise<{validRepos: BitbucketRepository[], notFoundWorkspaces: string[]}>;
36+
getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundWorkspaces: string[]}>;
3737
getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundProjects: string[]}>;
3838
getRepos: (client: BitbucketClient, repos: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundRepos: string[]}>;
39+
shouldExcludeRepo: (repo: BitbucketRepository, config: BitbucketConnectionConfig) => boolean;
3940
/*
40-
countForks: (client: BitbucketClient, repo: Repo) => Promise<number>;
41-
countWatchers: (client: BitbucketClient, repo: Repo) => Promise<number>;
4241
getBranches: (client: BitbucketClient, repo: string) => Promise<string[]>;
4342
getTags: (client: BitbucketClient, repo: string) => Promise<string[]>;
4443
*/
@@ -77,8 +76,8 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon
7776
repos: [],
7877
};
7978

80-
if (config.workspace) {
81-
const { validRepos, notFoundWorkspaces } = await client.getReposForWorkspace(client, config.workspace);
79+
if (config.workspaces) {
80+
const { validRepos, notFoundWorkspaces } = await client.getReposForWorkspace(client, config.workspaces);
8281
allRepos = allRepos.concat(validRepos);
8382
notFound.orgs = notFoundWorkspaces;
8483
}
@@ -95,8 +94,12 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon
9594
notFound.repos = notFoundRepos;
9695
}
9796

97+
const filteredRepos = allRepos.filter((repo) => {
98+
return !client.shouldExcludeRepo(repo, config);
99+
});
100+
98101
return {
99-
validRepos: allRepos,
102+
validRepos: filteredRepos,
100103
notFound,
101104
};
102105
}
@@ -123,10 +126,8 @@ function cloudClient(user: string | undefined, token: string | undefined): Bitbu
123126
getReposForWorkspace: cloudGetReposForWorkspace,
124127
getReposForProjects: cloudGetReposForProjects,
125128
getRepos: cloudGetRepos,
129+
shouldExcludeRepo: cloudShouldExcludeRepo,
126130
/*
127-
getRepos: cloudGetRepos,
128-
countForks: cloudCountForks,
129-
countWatchers: cloudCountWatchers,
130131
getBranches: cloudGetBranches,
131132
getTags: cloudGetTags,
132133
*/
@@ -163,39 +164,57 @@ const getPaginatedCloud = async <T, V extends CloudGetRequestPath>(path: V, get:
163164
}
164165

165166

166-
async function cloudGetReposForWorkspace(client: BitbucketClient, workspace: string): Promise<{validRepos: CloudRepository[], notFoundWorkspaces: string[]}> {
167-
try {
168-
logger.debug(`Fetching all repos for workspace ${workspace}...`);
169-
170-
const path = `/repositories/${workspace}` as CloudGetRequestPath;
171-
const { durationMs, data } = await measure(async () => {
172-
const fetchFn = () => client.getPaginated<CloudRepository, typeof path>(path, async (url) => {
173-
const response = await client.apiClient.GET(url, {
174-
params: {
175-
path: {
176-
workspace,
167+
async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{validRepos: CloudRepository[], notFoundWorkspaces: string[]}> {
168+
const results = await Promise.allSettled(workspaces.map(async (workspace) => {
169+
try {
170+
logger.debug(`Fetching all repos for workspace ${workspace}...`);
171+
172+
const path = `/repositories/${workspace}` as CloudGetRequestPath;
173+
const { durationMs, data } = await measure(async () => {
174+
const fetchFn = () => client.getPaginated<CloudRepository, typeof path>(path, async (url) => {
175+
const response = await client.apiClient.GET(url, {
176+
params: {
177+
path: {
178+
workspace,
179+
}
177180
}
181+
});
182+
const { data, error } = response;
183+
if (error) {
184+
throw new Error(`Failed to fetch projects for workspace ${workspace}: ${JSON.stringify(error)}`);
178185
}
186+
return data;
179187
});
180-
const { data, error } = response;
181-
if (error) {
182-
throw new Error(`Failed to fetch projects for workspace ${workspace}: ${JSON.stringify(error)}`);
183-
}
184-
return data;
188+
return fetchWithRetry(fetchFn, `workspace ${workspace}`, logger);
185189
});
186-
return fetchWithRetry(fetchFn, `workspace ${workspace}`, logger);
187-
});
188-
logger.debug(`Found ${data.length} repos for workspace ${workspace} in ${durationMs}ms.`);
189-
190-
return {
191-
validRepos: data,
192-
notFoundWorkspaces: [],
193-
};
194-
} catch (e) {
195-
Sentry.captureException(e);
196-
logger.error(`Failed to get repos for workspace ${workspace}: ${e}`);
197-
throw e;
198-
}
190+
logger.debug(`Found ${data.length} repos for workspace ${workspace} in ${durationMs}ms.`);
191+
192+
return {
193+
type: 'valid' as const,
194+
data: data,
195+
};
196+
} catch (e: any) {
197+
Sentry.captureException(e);
198+
logger.error(`Failed to get repos for workspace ${workspace}: ${e}`);
199+
200+
const status = e?.cause?.response?.status;
201+
if (status == 404) {
202+
logger.error(`Workspace ${workspace} not found or invalid access`)
203+
return {
204+
type: 'notFound' as const,
205+
value: workspace
206+
}
207+
}
208+
throw e;
209+
}
210+
}));
211+
212+
throwIfAnyFailed(results);
213+
const { validItems: validRepos, notFoundItems: notFoundWorkspaces } = processPromiseResults(results);
214+
return {
215+
validRepos,
216+
notFoundWorkspaces,
217+
};
199218
}
200219

201220
async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{validRepos: CloudRepository[], notFoundProjects: string[]}> {
@@ -295,4 +314,24 @@ async function cloudGetRepos(client: BitbucketClient, repos: string[]): Promise<
295314
validRepos,
296315
notFoundRepos
297316
};
317+
}
318+
319+
function cloudShouldExcludeRepo(repo: BitbucketRepository, config: BitbucketConnectionConfig): boolean {
320+
const cloudRepo = repo as CloudRepository;
321+
322+
const shouldExclude = (() => {
323+
if (config.exclude?.repos && config.exclude.repos.includes(cloudRepo.full_name!)) {
324+
return true;
325+
}
326+
327+
if (!!config.exclude?.forks && cloudRepo.parent !== undefined) {
328+
return true;
329+
}
330+
})();
331+
332+
if (shouldExclude) {
333+
logger.debug(`Excluding repo ${cloudRepo.full_name} because it matches the exclude pattern`);
334+
return true;
335+
}
336+
return false;
298337
}

‎packages/backend/src/repoManager.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,7 @@ export class RepoManager implements IRepoManager {
462462
}
463463

464464
private async runGarbageCollectionJob(job: Job<RepoGarbageCollectionPayload>) {
465-
this.logger.info(`Running garbage collection job (id: ${job.id}) for repo ${job.data.repo.id}`);
465+
this.logger.info(`Running garbage collection job (id: ${job.id}) for repo ${job.data.repo.displayName} (id: ${job.data.repo.id})`);
466466
this.promClient.activeRepoGarbageCollectionJobs.inc();
467467

468468
const repo = job.data.repo as Repo;

‎packages/schemas/src/v3/bitbucket.schema.ts

Lines changed: 5 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,11 @@ const schema = {
6767
"default": "cloud",
6868
"description": "The type of Bitbucket deployment"
6969
},
70-
"workspace": {
71-
"type": "string",
70+
"workspaces": {
71+
"type": "array",
72+
"items": {
73+
"type": "string"
74+
},
7275
"description": "List of workspaces to sync. Ignored if deploymentType is server."
7376
},
7477
"projects": {
@@ -98,32 +101,6 @@ const schema = {
98101
"default": false,
99102
"description": "Exclude forked repositories from syncing."
100103
},
101-
"workspaces": {
102-
"type": "array",
103-
"items": {
104-
"type": "string"
105-
},
106-
"examples": [
107-
[
108-
"workspace1",
109-
"workspace2"
110-
]
111-
],
112-
"description": "List of specific workspaces to exclude from syncing. Ignored if deploymentType is server."
113-
},
114-
"projects": {
115-
"type": "array",
116-
"items": {
117-
"type": "string"
118-
},
119-
"examples": [
120-
[
121-
"project1",
122-
"project2"
123-
]
124-
],
125-
"description": "List of specific projects to exclude from syncing."
126-
},
127104
"repos": {
128105
"type": "array",
129106
"items": {

‎packages/schemas/src/v3/bitbucket.type.ts

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ export interface BitbucketConnectionConfig {
3636
/**
3737
* List of workspaces to sync. Ignored if deploymentType is server.
3838
*/
39-
workspace?: string;
39+
workspaces?: string[];
4040
/**
4141
* List of projects to sync
4242
*/
@@ -54,14 +54,6 @@ export interface BitbucketConnectionConfig {
5454
* Exclude forked repositories from syncing.
5555
*/
5656
forks?: boolean;
57-
/**
58-
* List of specific workspaces to exclude from syncing. Ignored if deploymentType is server.
59-
*/
60-
workspaces?: string[];
61-
/**
62-
* List of specific projects to exclude from syncing.
63-
*/
64-
projects?: string[];
6557
/**
6658
* List of specific repos to exclude from syncing.
6759
*/

‎packages/schemas/src/v3/connection.schema.ts

Lines changed: 5 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -546,8 +546,11 @@ const schema = {
546546
"default": "cloud",
547547
"description": "The type of Bitbucket deployment"
548548
},
549-
"workspace": {
550-
"type": "string",
549+
"workspaces": {
550+
"type": "array",
551+
"items": {
552+
"type": "string"
553+
},
551554
"description": "List of workspaces to sync. Ignored if deploymentType is server."
552555
},
553556
"projects": {
@@ -577,32 +580,6 @@ const schema = {
577580
"default": false,
578581
"description": "Exclude forked repositories from syncing."
579582
},
580-
"workspaces": {
581-
"type": "array",
582-
"items": {
583-
"type": "string"
584-
},
585-
"examples": [
586-
[
587-
"workspace1",
588-
"workspace2"
589-
]
590-
],
591-
"description": "List of specific workspaces to exclude from syncing. Ignored if deploymentType is server."
592-
},
593-
"projects": {
594-
"type": "array",
595-
"items": {
596-
"type": "string"
597-
},
598-
"examples": [
599-
[
600-
"project1",
601-
"project2"
602-
]
603-
],
604-
"description": "List of specific projects to exclude from syncing."
605-
},
606583
"repos": {
607584
"type": "array",
608585
"items": {

‎packages/schemas/src/v3/connection.type.ts

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ export interface BitbucketConnectionConfig {
272272
/**
273273
* List of workspaces to sync. Ignored if deploymentType is server.
274274
*/
275-
workspace?: string;
275+
workspaces?: string[];
276276
/**
277277
* List of projects to sync
278278
*/
@@ -290,14 +290,6 @@ export interface BitbucketConnectionConfig {
290290
* Exclude forked repositories from syncing.
291291
*/
292292
forks?: boolean;
293-
/**
294-
* List of specific workspaces to exclude from syncing. Ignored if deploymentType is server.
295-
*/
296-
workspaces?: string[];
297-
/**
298-
* List of specific projects to exclude from syncing.
299-
*/
300-
projects?: string[];
301293
/**
302294
* List of specific repos to exclude from syncing.
303295
*/

‎packages/schemas/src/v3/index.schema.ts

Lines changed: 5 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -625,8 +625,11 @@ const schema = {
625625
"default": "cloud",
626626
"description": "The type of Bitbucket deployment"
627627
},
628-
"workspace": {
629-
"type": "string",
628+
"workspaces": {
629+
"type": "array",
630+
"items": {
631+
"type": "string"
632+
},
630633
"description": "List of workspaces to sync. Ignored if deploymentType is server."
631634
},
632635
"projects": {
@@ -656,32 +659,6 @@ const schema = {
656659
"default": false,
657660
"description": "Exclude forked repositories from syncing."
658661
},
659-
"workspaces": {
660-
"type": "array",
661-
"items": {
662-
"type": "string"
663-
},
664-
"examples": [
665-
[
666-
"workspace1",
667-
"workspace2"
668-
]
669-
],
670-
"description": "List of specific workspaces to exclude from syncing. Ignored if deploymentType is server."
671-
},
672-
"projects": {
673-
"type": "array",
674-
"items": {
675-
"type": "string"
676-
},
677-
"examples": [
678-
[
679-
"project1",
680-
"project2"
681-
]
682-
],
683-
"description": "List of specific projects to exclude from syncing."
684-
},
685662
"repos": {
686663
"type": "array",
687664
"items": {

‎packages/schemas/src/v3/index.type.ts

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,7 @@ export interface BitbucketConnectionConfig {
338338
/**
339339
* List of workspaces to sync. Ignored if deploymentType is server.
340340
*/
341-
workspace?: string;
341+
workspaces?: string[];
342342
/**
343343
* List of projects to sync
344344
*/
@@ -356,14 +356,6 @@ export interface BitbucketConnectionConfig {
356356
* Exclude forked repositories from syncing.
357357
*/
358358
forks?: boolean;
359-
/**
360-
* List of specific workspaces to exclude from syncing. Ignored if deploymentType is server.
361-
*/
362-
workspaces?: string[];
363-
/**
364-
* List of specific projects to exclude from syncing.
365-
*/
366-
projects?: string[];
367359
/**
368360
* List of specific repos to exclude from syncing.
369361
*/

0 commit comments

Comments
 (0)