Skip to content

Commit 9fb9973

Browse files
msukkarimsukkari
committed
wip bitbucket support
1 parent ff43d10 commit 9fb9973

14 files changed

+1119
-49
lines changed

‎package.json

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,8 @@
2020
"dotenv-cli": "^8.0.0",
2121
"npm-run-all": "^4.1.5"
2222
},
23-
"packageManager": "yarn@4.7.0"
23+
"packageManager": "yarn@4.7.0",
24+
"dependencies": {
25+
"@coderabbitai/bitbucket": "^1.1.3"
26+
}
2427
}

‎packages/backend/src/bitbucket.ts

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
import { createBitbucketCloudClient } from "@coderabbitai/bitbucket/cloud";
2+
import { paths } from "@coderabbitai/bitbucket/cloud";
3+
import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/bitbucket.type";
4+
import type { ClientOptions, Client, ClientPathsWithMethod } from "openapi-fetch";
5+
import { createLogger } from "./logger.js";
6+
import { PrismaClient, Repo } from "@sourcebot/db";
7+
import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js";
8+
import { env } from "./env.js";
9+
import * as Sentry from "@sentry/node";
10+
import {
11+
SchemaBranch as CloudBranch,
12+
SchemaProject as CloudProject,
13+
SchemaRepository as CloudRepository,
14+
SchemaTag as CloudTag,
15+
SchemaWorkspace as CloudWorkspace
16+
} from "@coderabbitai/bitbucket/cloud/openapi";
17+
import { SchemaRepository as ServerRepository } from "@coderabbitai/bitbucket/server/openapi";
18+
import { processPromiseResults } from "./connectionUtils.js";
19+
import { throwIfAnyFailed } from "./connectionUtils.js";
20+
21+
const logger = createLogger("Bitbucket");
22+
const BITBUCKET_CLOUD_GIT = 'https://bitbucket.org';
23+
const BITBUCKET_CLOUD_API = 'https://api.bitbucket.org/2.0';
24+
const BITBUCKET_CLOUD = "cloud";
25+
const BITBUCKET_SERVER = "server";
26+
27+
export type BitbucketRepository = CloudRepository | ServerRepository;
28+
29+
interface BitbucketClient {
30+
deploymentType: string;
31+
token: string | undefined;
32+
apiClient: any;
33+
baseUrl: string;
34+
gitUrl: string;
35+
getPaginated: <T, V extends CloudGetRequestPath>(path: V, get: (url: V) => Promise<PaginatedResponse<T>>) => Promise<T[]>;
36+
getReposForWorkspace: (client: BitbucketClient, workspace: string) => Promise<{validRepos: BitbucketRepository[], notFoundWorkspaces: string[]}>;
37+
getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundProjects: string[]}>;
38+
/*
39+
getRepos: (client: BitbucketClient, workspace: string, project: string) => Promise<Repo[]>;
40+
countForks: (client: BitbucketClient, repo: Repo) => Promise<number>;
41+
countWatchers: (client: BitbucketClient, repo: Repo) => Promise<number>;
42+
getBranches: (client: BitbucketClient, repo: string) => Promise<string[]>;
43+
getTags: (client: BitbucketClient, repo: string) => Promise<string[]>;
44+
*/
45+
}
46+
47+
// afaik, this is the only way of extracting the client API type
48+
type CloudAPI = ReturnType<typeof createBitbucketCloudClient>;
49+
50+
// Defines a type that is a union of all API paths that have a GET method in the
51+
// client api.
52+
type CloudGetRequestPath = ClientPathsWithMethod<CloudAPI, "get">;
53+
54+
type PaginatedResponse<T> = {
55+
readonly next?: string;
56+
readonly page?: number;
57+
readonly pagelen?: number;
58+
readonly previous?: string;
59+
readonly size?: number;
60+
readonly values?: readonly T[];
61+
}
62+
63+
export const getBitbucketReposFromConfig = async (config: BitbucketConnectionConfig, orgId: number, db: PrismaClient) => {
64+
const token = await getTokenFromConfig(config.token, orgId, db, logger);
65+
66+
//const deploymentType = config.deploymentType;
67+
const client = cloudClient(token);
68+
69+
let allRepos: BitbucketRepository[] = [];
70+
let notFound: {
71+
orgs: string[],
72+
users: string[],
73+
repos: string[],
74+
} = {
75+
orgs: [],
76+
users: [],
77+
repos: [],
78+
};
79+
80+
if (config.workspace) {
81+
const { validRepos, notFoundWorkspaces } = await client.getReposForWorkspace(client, config.workspace);
82+
allRepos = allRepos.concat(validRepos);
83+
notFound.orgs = notFoundWorkspaces;
84+
}
85+
86+
if (config.projects) {
87+
const { validRepos, notFoundProjects } = await client.getReposForProjects(client, config.projects);
88+
allRepos = allRepos.concat(validRepos);
89+
notFound.repos = notFoundProjects;
90+
}
91+
92+
return {
93+
validRepos: allRepos,
94+
notFound,
95+
};
96+
}
97+
98+
function cloudClient(token: string | undefined): BitbucketClient {
99+
const clientOptions: ClientOptions = {
100+
baseUrl: BITBUCKET_CLOUD_API,
101+
headers: {
102+
Accept: "application/json",
103+
Authorization: `Bearer ${token}`,
104+
},
105+
};
106+
107+
const apiClient = createBitbucketCloudClient(clientOptions);
108+
var client: BitbucketClient = {
109+
deploymentType: BITBUCKET_CLOUD,
110+
token: token,
111+
apiClient: apiClient,
112+
baseUrl: BITBUCKET_CLOUD_API,
113+
gitUrl: BITBUCKET_CLOUD_GIT,
114+
getPaginated: getPaginatedCloud,
115+
getReposForWorkspace: cloudGetReposForWorkspace,
116+
getReposForProjects: cloudGetReposForProjects,
117+
/*
118+
getRepos: cloudGetRepos,
119+
countForks: cloudCountForks,
120+
countWatchers: cloudCountWatchers,
121+
getBranches: cloudGetBranches,
122+
getTags: cloudGetTags,
123+
*/
124+
}
125+
126+
return client;
127+
}
128+
129+
/**
130+
* We need to do `V extends CloudGetRequestPath` since we will need to call `apiClient.GET(url, ...)`, which
131+
* expects `url` to be of type `CloudGetRequestPath`. See example.
132+
**/
133+
const getPaginatedCloud = async <T, V extends CloudGetRequestPath>(path: V, get: (url: V) => Promise<PaginatedResponse<T>>) => {
134+
const results: T[] = [];
135+
let url = path;
136+
137+
while (true) {
138+
const response = await get(url);
139+
140+
if (!response.values || response.values.length === 0) {
141+
break;
142+
}
143+
144+
results.push(...response.values);
145+
146+
if (!response.next) {
147+
break;
148+
}
149+
150+
// cast required here since response.next is a string.
151+
url = response.next as V;
152+
}
153+
return results;
154+
}
155+
156+
157+
async function cloudGetReposForWorkspace(client: BitbucketClient, workspace: string): Promise<{validRepos: CloudRepository[], notFoundWorkspaces: string[]}> {
158+
try {
159+
logger.debug(`Fetching all repos for workspace ${workspace}...`);
160+
161+
const path = `/repositories/${workspace}` as CloudGetRequestPath;
162+
const { durationMs, data } = await measure(async () => {
163+
const fetchFn = () => client.getPaginated<CloudRepository, typeof path>(path, async (url) => {
164+
const response = await client.apiClient.GET(url, {
165+
params: {
166+
path: {
167+
workspace,
168+
}
169+
}
170+
});
171+
const { data, error } = response;
172+
if (error) {
173+
throw new Error (`Failed to fetch projects for workspace ${workspace}: ${error.type}`);
174+
}
175+
return data;
176+
});
177+
return fetchWithRetry(fetchFn, `workspace ${workspace}`, logger);
178+
});
179+
logger.debug(`Found ${data.length} repos for workspace ${workspace} in ${durationMs}ms.`);
180+
181+
return {
182+
validRepos: data,
183+
notFoundWorkspaces: [],
184+
};
185+
} catch (e) {
186+
Sentry.captureException(e);
187+
logger.error(`Failed to get repos for workspace ${workspace}: ${e}`);
188+
throw e;
189+
}
190+
}
191+
192+
async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{validRepos: CloudRepository[], notFoundProjects: string[]}> {
193+
const results = await Promise.allSettled(projects.map(async (project) => {
194+
const [workspace, project_name] = project.split('/');
195+
logger.debug(`Fetching all repos for project ${project} for workspace ${workspace}...`);
196+
197+
try {
198+
const path = `/repositories/${workspace}?q=project.key="${project_name}"` as CloudGetRequestPath;
199+
const repos = await client.getPaginated<CloudRepository, typeof path>(path, async (url) => {
200+
const response = await client.apiClient.GET(url);
201+
const { data, error } = response;
202+
if (error) {
203+
throw new Error (`Failed to fetch projects for workspace ${workspace}: ${error.type}`);
204+
}
205+
return data;
206+
});
207+
208+
logger.debug(`Found ${repos.length} repos for project ${project_name} for workspace ${workspace}.`);
209+
return {
210+
type: 'valid' as const,
211+
data: repos
212+
}
213+
} catch (e: any) {
214+
Sentry.captureException(e);
215+
logger.error(`Failed to fetch repos for project ${project_name}: ${e}`);
216+
217+
const status = e?.cause?.response?.status;
218+
if (status == 404) {
219+
logger.error(`Project ${project_name} not found in ${workspace} or invalid access`)
220+
return {
221+
type: 'notFound' as const,
222+
value: project
223+
}
224+
}
225+
throw e;
226+
}
227+
}));
228+
229+
throwIfAnyFailed(results);
230+
const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults(results);
231+
return {
232+
validRepos,
233+
notFoundProjects
234+
}
235+
}

‎packages/backend/src/connectionManager.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { Settings } from "./types.js";
44
import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
55
import { createLogger } from "./logger.js";
66
import { Redis } from 'ioredis';
7-
import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig } from "./repoCompileUtils.js";
7+
import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig, compileBitbucketConfig } from "./repoCompileUtils.js";
88
import { BackendError, BackendException } from "@sourcebot/error";
99
import { captureEvent } from "./posthog.js";
1010
import { env } from "./env.js";
@@ -170,6 +170,9 @@ export class ConnectionManager implements IConnectionManager {
170170
case 'gerrit': {
171171
return await compileGerritConfig(config, job.data.connectionId, orgId);
172172
}
173+
case 'bitbucket': {
174+
return await compileBitbucketConfig(config, job.data.connectionId, orgId, this.db);
175+
}
173176
}
174177
})();
175178
} catch (err) {

‎packages/backend/src/repoCompileUtils.ts

Lines changed: 116 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,14 @@ import { getGitHubReposFromConfig } from "./github.js";
33
import { getGitLabReposFromConfig } from "./gitlab.js";
44
import { getGiteaReposFromConfig } from "./gitea.js";
55
import { getGerritReposFromConfig } from "./gerrit.js";
6+
import { getBitbucketReposFromConfig } from "./bitbucket.js";
7+
import { SchemaRepository as BitbucketServerRepository } from "@coderabbitai/bitbucket/server/openapi";
8+
import { SchemaRepository as BitbucketCloudRepository } from "@coderabbitai/bitbucket/cloud/openapi";
69
import { Prisma, PrismaClient } from '@sourcebot/db';
710
import { WithRequired } from "./types.js"
811
import { marshalBool } from "./utils.js";
912
import { createLogger } from './logger.js';
10-
import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
13+
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
1114
import { RepoMetadata } from './types.js';
1215
import path from 'path';
1316

@@ -312,4 +315,116 @@ export const compileGerritConfig = async (
312315
repos: [],
313316
}
314317
};
318+
}
319+
320+
export const compileBitbucketConfig = async (
321+
config: BitbucketConnectionConfig,
322+
connectionId: number,
323+
orgId: number,
324+
db: PrismaClient) => {
325+
326+
const bitbucketReposResult = await getBitbucketReposFromConfig(config, orgId, db);
327+
const bitbucketRepos = bitbucketReposResult.validRepos;
328+
const notFound = bitbucketReposResult.notFound;
329+
330+
const hostUrl = config.url ?? 'https://bitbucket.org';
331+
const repoNameRoot = new URL(hostUrl)
332+
.toString()
333+
.replace(/^https?:\/\//, '');
334+
335+
const repos = bitbucketRepos.map((repo) => {
336+
const deploymentType = config.deploymentType;
337+
let record: RepoData;
338+
if (deploymentType === 'server') {
339+
const serverRepo = repo as BitbucketServerRepository;
340+
341+
const repoDisplayName = serverRepo.name!;
342+
const repoName = path.join(repoNameRoot, repoDisplayName);
343+
const cloneUrl = `${hostUrl}/${serverRepo.slug}`;
344+
const webUrl = `${hostUrl}/${serverRepo.slug}`;
345+
346+
record = {
347+
external_id: serverRepo.id!.toString(),
348+
external_codeHostType: 'bitbucket-server',
349+
external_codeHostUrl: hostUrl,
350+
cloneUrl: cloneUrl.toString(),
351+
webUrl: webUrl.toString(),
352+
name: repoName,
353+
displayName: repoDisplayName,
354+
isFork: false,
355+
isArchived: false,
356+
org: {
357+
connect: {
358+
id: orgId,
359+
},
360+
},
361+
connections: {
362+
create: {
363+
connectionId: connectionId,
364+
}
365+
},
366+
metadata: {
367+
gitConfig: {
368+
'zoekt.web-url-type': 'bitbucket-server',
369+
'zoekt.web-url': webUrl ?? '',
370+
'zoekt.name': repoName,
371+
'zoekt.archived': marshalBool(false),
372+
'zoekt.fork': marshalBool(false),
373+
'zoekt.public': marshalBool(serverRepo.public),
374+
'zoekt.display-name': repoDisplayName,
375+
},
376+
},
377+
}
378+
} else {
379+
const cloudRepo = repo as BitbucketCloudRepository;
380+
381+
const repoDisplayName = cloudRepo.full_name!;
382+
const repoName = path.join(repoNameRoot, repoDisplayName);
383+
384+
const cloneInfo = cloudRepo.links!.clone![0];
385+
const webInfo = cloudRepo.links!.self!;
386+
const cloneUrl = new URL(cloneInfo.href!);
387+
const webUrl = new URL(webInfo.href!);
388+
389+
record = {
390+
external_id: cloudRepo.uuid!,
391+
external_codeHostType: 'bitbucket-cloud',
392+
external_codeHostUrl: hostUrl,
393+
cloneUrl: cloneUrl.toString(),
394+
webUrl: webUrl.toString(),
395+
name: repoName,
396+
displayName: repoDisplayName,
397+
isFork: false,
398+
isArchived: false,
399+
org: {
400+
connect: {
401+
id: orgId,
402+
},
403+
},
404+
connections: {
405+
create: {
406+
connectionId: connectionId,
407+
}
408+
},
409+
metadata: {
410+
gitConfig: {
411+
'zoekt.web-url-type': 'bitbucket-cloud',
412+
'zoekt.web-url': webUrl.toString(),
413+
'zoekt.name': repoName,
414+
'zoekt.archived': marshalBool(false),
415+
'zoekt.fork': marshalBool(false),
416+
'zoekt.public': marshalBool(cloudRepo.is_private === false),
417+
'zoekt.display-name': repoDisplayName,
418+
},
419+
},
420+
}
421+
}
422+
423+
return record;
424+
})
425+
426+
return {
427+
repoData: repos,
428+
notFound,
429+
};
315430
}

0 commit comments

Comments
 (0)