1 | import * as wmillclient from "windmill-client"; |
2 | import wmill from "windmill-cli"; |
3 | import { basename } from "node:path" |
4 | const util = require('util'); |
5 | const exec = util.promisify(require('child_process').exec); |
6 | import process from "process"; |
7 |
|
8 | type GpgKey = { |
9 | email: string, |
10 | private_key: string, |
11 | passphrase: string |
12 | } |
13 |
|
14 | type PathType = |
15 | | "script" |
16 | | "flow" |
17 | | "app" |
18 | | "folder" |
19 | | "resource" |
20 | | "variable" |
21 | | "resourcetype" |
22 | | "schedule" |
23 | | "user" |
24 | | "group" |
25 | | "httptrigger" |
26 | | "websockettrigger" |
27 | | "kafkatrigger" |
28 | | "natstrigger" |
29 | | "postgrestrigger" |
30 | | "mqtttrigger" |
31 | | "sqstrigger" |
32 | | "gcptrigger"; |
33 |
|
34 | let gpgFingerprint: string | undefined = undefined; |
35 |
|
36 | export async function main( |
37 | workspace_id: string, |
38 | repo_url_resource_path: string, |
39 | path_type: PathType, |
40 | skip_secret = true, |
41 | path: string | undefined, |
42 | parent_path: string | undefined, |
43 | commit_msg: string, |
44 | use_individual_branch = false, |
45 | group_by_folder = false |
46 | ) { |
47 | const repo_resource = await wmillclient.getResource(repo_url_resource_path); |
48 | const cwd = process.cwd(); |
49 | process.env["HOME"] = "." |
50 | console.log( |
51 | `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}` |
52 | ); |
53 |
|
54 | if (repo_resource.is_github_app) { |
55 | const token = await get_gh_app_token() |
56 | const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token); |
57 | repo_resource.url = authRepoUrl; |
58 | } |
59 |
|
60 | const repo_name = await git_clone(cwd, repo_resource, use_individual_branch); |
61 | await move_to_git_branch( |
62 | workspace_id, |
63 | path_type, |
64 | path, |
65 | parent_path, |
66 | use_individual_branch, |
67 | group_by_folder |
68 | ); |
69 | const subfolder = repo_resource.folder ?? ""; |
70 | const branch_or_default = repo_resource.branch ?? "<DEFAULT>"; |
71 | console.log( |
72 | `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}` |
73 | ); |
74 | await wmill_sync_pull( |
75 | path_type, |
76 | workspace_id, |
77 | path, |
78 | parent_path, |
79 | skip_secret |
80 | ); |
81 | try { |
82 | await git_push(path, parent_path, commit_msg, repo_resource); |
83 | } catch (e) { |
84 | throw (e) |
85 | } finally { |
86 | await delete_pgp_keys() |
87 | } |
88 | console.log("Finished syncing"); |
89 | process.chdir(`${cwd}`); |
90 | } |
91 | async function git_clone( |
92 | cwd: string, |
93 | repo_resource: any, |
94 | use_individual_branch: boolean |
95 | ): Promise<string> { |
96 | |
97 | let repo_url = repo_resource.url; |
98 | const subfolder = repo_resource.folder ?? ""; |
99 | const branch = repo_resource.branch ?? ""; |
100 | const repo_name = basename(repo_url, ".git"); |
101 | const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/); |
102 | if (azureMatch) { |
103 | console.log( |
104 | "Requires Azure DevOps service account access token, requesting..." |
105 | ); |
106 | const azureResource = await wmillclient.getResource(azureMatch.groups.url); |
107 | const response = await fetch( |
108 | `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, |
109 | { |
110 | method: "POST", |
111 | body: new URLSearchParams({ |
112 | client_id: azureResource.azureClientId, |
113 | client_secret: azureResource.azureClientSecret, |
114 | grant_type: "client_credentials", |
115 | resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", |
116 | }), |
117 | } |
118 | ); |
119 | const { access_token } = await response.json(); |
120 | repo_url = repo_url.replace(azureMatch[0], access_token); |
121 | } |
122 | const args = ["clone", "--quiet", "--depth", "1"]; |
123 | if (use_individual_branch) { |
124 | args.push("--no-single-branch"); |
125 | } |
126 | if (subfolder !== "") { |
127 | args.push("--sparse"); |
128 | } |
129 | if (branch !== "") { |
130 | args.push("--branch"); |
131 | args.push(branch); |
132 | } |
133 | args.push(repo_url); |
134 | args.push(repo_name); |
135 | await sh_run(-1, "git", ...args); |
136 | try { |
137 | process.chdir(`${cwd}/${repo_name}`); |
138 | } catch (err) { |
139 | console.log( |
140 | `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}` |
141 | ); |
142 | throw err; |
143 | } |
144 | process.chdir(`${cwd}/${repo_name}`); |
145 | if (subfolder !== "") { |
146 | await sh_run(undefined, "git", "sparse-checkout", "add", subfolder); |
147 | } |
148 | try { |
149 | process.chdir(`${cwd}/${repo_name}/${subfolder}`); |
150 | } catch (err) { |
151 | console.log( |
152 | `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}` |
153 | ); |
154 | throw err; |
155 | } |
156 | return repo_name; |
157 | } |
158 | async function move_to_git_branch( |
159 | workspace_id: string, |
160 | path_type: PathType, |
161 | path: string | undefined, |
162 | parent_path: string | undefined, |
163 | use_individual_branch: boolean, |
164 | group_by_folder: boolean |
165 | ) { |
166 | if (!use_individual_branch || path_type === "user" || path_type === "group") { |
167 | return; |
168 | } |
169 | const branchName = group_by_folder |
170 | ? `wm_deploy/${workspace_id}/${(path ?? parent_path) |
171 | ?.split("/") |
172 | .slice(0, 2) |
173 | .join("__")}` |
174 | : `wm_deploy/${workspace_id}/${path_type}/${( |
175 | path ?? parent_path |
176 | )?.replaceAll("/", "__")}`; |
177 | try { |
178 | await sh_run(undefined, "git", "checkout", branchName); |
179 | } catch (err) { |
180 | console.log( |
181 | `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}` |
182 | ); |
183 | try { |
184 | await sh_run(undefined, "git", "checkout", "-b", branchName); |
185 | await sh_run( |
186 | undefined, |
187 | "git", |
188 | "config", |
189 | "--add", |
190 | "--bool", |
191 | "push.autoSetupRemote", |
192 | "true" |
193 | ); |
194 | } catch (err) { |
195 | console.log( |
196 | `Error checking out branch '${branchName}'. Error was:\n${err}` |
197 | ); |
198 | throw err; |
199 | } |
200 | } |
201 | console.log(`Successfully switched to branch ${branchName}`); |
202 | } |
203 | async function git_push( |
204 | path: string | undefined, |
205 | parent_path: string | undefined, |
206 | commit_msg: string, |
207 | repo_resource: any |
208 | ) { |
209 | let user_email = process.env["WM_EMAIL"] ?? "" |
210 | let user_name = process.env["WM_USERNAME"] ?? "" |
211 |
|
212 | if (repo_resource.gpg_key) { |
213 | await set_gpg_signing_secret(repo_resource.gpg_key); |
214 | user_email = repo_resource.gpg_key.email |
215 | } |
216 |
|
217 | await sh_run( |
218 | undefined, |
219 | "git", |
220 | "config", |
221 | "user.email", |
222 | user_email |
223 | ); |
224 | await sh_run( |
225 | undefined, |
226 | "git", |
227 | "config", |
228 | "user.name", |
229 | user_name |
230 | ); |
231 | if (path !== undefined && path !== null && path !== "") { |
232 | try { |
233 | await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`); |
234 | } catch (e) { |
235 | console.log(`Unable to stage files matching ${path}**, ${e}`); |
236 | } |
237 | } |
238 | if (parent_path !== undefined && parent_path !== null && parent_path !== "") { |
239 | try { |
240 | await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`); |
241 | } catch (e) { |
242 | console.log(`Unable to stage files matching ${parent_path}, ${e}`); |
243 | } |
244 | } |
245 | try { |
246 | await sh_run(undefined, "git", "diff", "--cached", "--quiet"); |
247 | } catch { |
248 | |
249 | await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`); |
250 | try { |
251 | await sh_run(undefined, "git", "push", "--porcelain"); |
252 | } catch (e) { |
253 | console.log(`Could not push, trying to rebase first: ${e}`); |
254 | await sh_run(undefined, "git", "pull", "--rebase"); |
255 | await sh_run(undefined, "git", "push", "--porcelain"); |
256 | } |
257 | return; |
258 | } |
259 | console.log("No changes detected, nothing to commit. Returning..."); |
260 | } |
261 | async function sh_run( |
262 | secret_position: number | undefined, |
263 | cmd: string, |
264 | ...args: string[] |
265 | ) { |
266 | const nargs = secret_position != undefined ? args.slice() : args; |
267 | if (secret_position && secret_position < 0) { |
268 | secret_position = nargs.length - 1 + secret_position; |
269 | } |
270 | let secret: string | undefined = undefined |
271 | if (secret_position != undefined) { |
272 | nargs[secret_position] = "***"; |
273 | secret = args[secret_position] |
274 | } |
275 |
|
276 | console.log(`Running '${cmd} ${nargs.join(" ")} ...'`); |
277 | const command = exec(`${cmd} ${args.join(" ")}`) |
278 | |
279 | |
280 | |
281 | try { |
282 | const { stdout, stderr } = await command |
283 | if (stdout.length > 0) { |
284 | console.log(stdout); |
285 | } |
286 | if (stderr.length > 0) { |
287 | console.log(stderr); |
288 | } |
289 | console.log("Command successfully executed"); |
290 | return stdout; |
291 |
|
292 | } catch (error) { |
293 | let errorString = error.toString(); |
294 | if (secret) { |
295 | errorString = errorString.replace(secret, "***"); |
296 | } |
297 | const err = `SH command '${cmd} ${nargs.join( |
298 | " " |
299 | )}' returned with error ${errorString}`; |
300 | throw Error(err); |
301 | } |
302 | } |
303 |
|
304 | function regexFromPath( |
305 | path_type: PathType, |
306 | path: string |
307 | ) { |
308 | if (path_type == "flow") { |
309 | return `${path}.flow/*`; |
310 | } if (path_type == "app") { |
311 | return `${path}.app/*`; |
312 | } else if (path_type == "folder") { |
313 | return `${path}/folder.meta.*`; |
314 | } else if (path_type == "resourcetype") { |
315 | return `${path}.resource-type.*`; |
316 | } else if (path_type == "resource") { |
317 | return `${path}.resource.*`; |
318 | } else if (path_type == "variable") { |
319 | return `${path}.variable.*`; |
320 | } else if (path_type == "schedule") { |
321 | return `${path}.schedule.*`; |
322 | } else if (path_type == "user") { |
323 | return `${path}.user.*`; |
324 | } else if (path_type == "group") { |
325 | return `${path}.group.*`; |
326 | } else if (path_type == "httptrigger") { |
327 | return `${path}.http_trigger.*` |
328 | } else if (path_type == "websockettrigger") { |
329 | return `${path}.websocket_trigger.*` |
330 | } else if (path_type == "kafkatrigger") { |
331 | return `${path}.kafka_trigger.*` |
332 | } else if (path_type == "natstrigger") { |
333 | return `${path}.nats_trigger.*` |
334 | } else if (path_type == "postgrestrigger") { |
335 | return `${path}.postgres_trigger.*` |
336 | } else if (path_type == "mqtttrigger") { |
337 | return `${path}.mqtt_trigger.*` |
338 | } else if (path_type == "sqstrigger") { |
339 | return `${path}.sqs_trigger.*` |
340 | } else if (path_type == "gcptrigger") { |
341 | return `${path}.gcp_trigger.*` |
342 | } else { |
343 | return `${path}.*`; |
344 | } |
345 | } |
346 |
|
347 | async function wmill_sync_pull( |
348 | path_type: PathType, |
349 | workspace_id: string, |
350 | path: string | undefined, |
351 | parent_path: string | undefined, |
352 | skip_secret: boolean |
353 | ) { |
354 | const includes = []; |
355 | if (path !== undefined && path !== null && path !== "") { |
356 | includes.push(regexFromPath(path_type, path)); |
357 | } |
358 | if (parent_path !== undefined && parent_path !== null && parent_path !== "") { |
359 | includes.push(regexFromPath(path_type, parent_path)); |
360 | } |
361 | await wmill_run( |
362 | 6, |
363 | "workspace", |
364 | "add", |
365 | workspace_id, |
366 | workspace_id, |
367 | process.env["BASE_URL"] + "/", |
368 | "--token", |
369 | process.env["WM_TOKEN"] ?? "" |
370 | ); |
371 | console.log("Pulling workspace into git repo"); |
372 | await wmill_run( |
373 | 3, |
374 | "sync", |
375 | "pull", |
376 | "--token", |
377 | process.env["WM_TOKEN"] ?? "", |
378 | "--workspace", |
379 | workspace_id, |
380 | "--yes", |
381 | skip_secret ? "--skip-secrets" : "", |
382 | "--include-schedules", |
383 | "--include-users", |
384 | "--include-groups", |
385 | "--include-triggers", |
386 | "--extra-includes", |
387 | includes.join(",") |
388 | ); |
389 | } |
390 |
|
391 | async function wmill_run(secret_position: number, ...cmd: string[]) { |
392 | cmd = cmd.filter((elt) => elt !== ""); |
393 | const cmd2 = cmd.slice(); |
394 | cmd2[secret_position] = "***"; |
395 | console.log(`Running 'wmill ${cmd2.join(" ")} ...'`); |
396 | await wmill.parse(cmd); |
397 | console.log("Command successfully executed"); |
398 | } |
399 |
|
400 |
|
401 | async function set_gpg_signing_secret(gpg_key: GpgKey) { |
402 | try { |
403 | console.log("Setting GPG private key for git commits"); |
404 |
|
405 | const formattedGpgContent = gpg_key.private_key |
406 | .replace( |
407 | /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/, |
408 | (_: string, header: string, body: string, footer: string) => |
409 | header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer |
410 | ); |
411 |
|
412 | const gpg_path = `/tmp/gpg`; |
413 | await sh_run(undefined, "mkdir", "-p", gpg_path); |
414 | await sh_run(undefined, "chmod", "700", gpg_path) |
415 | process.env.GNUPGHOME = gpg_path; |
416 | |
417 |
|
418 |
|
419 | try { |
420 | await sh_run( |
421 | 1, |
422 | "bash", |
423 | "-c", |
424 | `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF` |
425 | ); |
426 | } catch (e) { |
427 | |
428 | throw new Error('Failed to import GPG key!') |
429 | } |
430 |
|
431 | const listKeysOutput = await sh_run( |
432 | undefined, |
433 | "gpg", |
434 | "--list-secret-keys", |
435 | "--with-colons", |
436 | "--keyid-format=long" |
437 | ); |
438 |
|
439 | const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/); |
440 |
|
441 | if (!keyInfoMatch) { |
442 | throw new Error("Failed to extract GPG Key ID and Fingerprint"); |
443 | } |
444 |
|
445 | const keyId = keyInfoMatch[1]; |
446 | gpgFingerprint = keyInfoMatch[2]; |
447 |
|
448 | if (gpg_key.passphrase) { |
449 | |
450 | |
451 | await sh_run( |
452 | 1, |
453 | "bash", |
454 | "-c", |
455 | `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}` |
456 | ); |
457 | } |
458 |
|
459 | |
460 | await sh_run(undefined, "git", "config", "user.signingkey", keyId); |
461 | await sh_run(undefined, "git", "config", "commit.gpgsign", "true"); |
462 | console.log(`GPG signing configured with key ID: ${keyId} `); |
463 |
|
464 | } catch (e) { |
465 | console.error(`Failure while setting GPG key: ${e} `); |
466 | await delete_pgp_keys(); |
467 | } |
468 | } |
469 |
|
470 | async function delete_pgp_keys() { |
471 | console.log("deleting gpg keys") |
472 | if (gpgFingerprint) { |
473 | await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint) |
474 | await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint) |
475 | } |
476 | } |
477 |
|
478 | async function get_gh_app_token() { |
479 | const workspace = process.env["WM_WORKSPACE"]; |
480 | const jobToken = process.env["WM_TOKEN"]; |
481 |
|
482 | const baseUrl = |
483 | process.env["BASE_INTERNAL_URL"] ?? |
484 | process.env["BASE_URL"] ?? |
485 | "http://localhost:8000"; |
486 |
|
487 | const url = `${baseUrl}/api/w/${workspace}/github_app/token`; |
488 |
|
489 | const response = await fetch(url, { |
490 | method: 'POST', |
491 | headers: { |
492 | 'Content-Type': 'application/json', |
493 | 'Authorization': `Bearer ${jobToken}`, |
494 | }, |
495 | body: JSON.stringify({ |
496 | job_token: jobToken, |
497 | }), |
498 | }); |
499 |
|
500 | if (!response.ok) { |
501 | throw new Error(`Error: ${response.statusText}`); |
502 | } |
503 |
|
504 | const data = await response.json(); |
505 |
|
506 | return data.token; |
507 | } |
508 |
|
509 | function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) { |
510 | if (!gitHubUrl || !installationToken) { |
511 | throw new Error("Both GitHub URL and Installation Token are required."); |
512 | } |
513 |
|
514 | try { |
515 | const url = new URL(gitHubUrl); |
516 |
|
517 | |
518 | if (url.hostname !== "github.com") { |
519 | throw new Error("Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."); |
520 | } |
521 |
|
522 | |
523 | return `https://x-access-token:${installationToken}@github.com${url.pathname}`; |
524 | } catch (e) { |
525 | const error = e as Error |
526 | throw new Error(`Invalid URL: ${error.message}`) |
527 | } |
528 | } |