1 | import * as wmillclient from "windmill-client"; |
2 | import wmill from "windmill-cli"; |
3 | import { basename } from "node:path"; |
4 | const util = require("util"); |
5 | const exec = util.promisify(require("child_process").exec); |
6 | import process from "process"; |
7 |
|
8 | type GpgKey = { |
9 | email: string; |
10 | private_key: string; |
11 | passphrase: string; |
12 | }; |
13 |
|
14 | const FORKED_WORKSPACE_PREFIX = "wm-fork-"; |
15 | const FORKED_BRANCH_PREFIX = "wm-fork"; |
16 |
|
17 | type PathType = |
18 | | "script" |
19 | | "flow" |
20 | | "app" |
21 | | "folder" |
22 | | "resource" |
23 | | "variable" |
24 | | "resourcetype" |
25 | | "schedule" |
26 | | "user" |
27 | | "group" |
28 | | "httptrigger" |
29 | | "websockettrigger" |
30 | | "kafkatrigger" |
31 | | "natstrigger" |
32 | | "postgrestrigger" |
33 | | "mqtttrigger" |
34 | | "sqstrigger" |
35 | | "gcptrigger" |
36 | | "emailtrigger"; |
37 |
|
38 | let gpgFingerprint: string | undefined = undefined; |
39 |
|
40 | export async function main( |
41 | workspace_id: string, |
42 | repo_url_resource_path: string, |
43 | path_type: PathType, |
44 | skip_secret = true, |
45 | path: string | undefined, |
46 | parent_path: string | undefined, |
47 | commit_msg: string, |
48 | parent_workspace_id?: string, |
49 | use_individual_branch = false, |
50 | group_by_folder = false |
51 | ) { |
52 | let safeDirectoryPath: string | undefined; |
53 | const repo_resource = await wmillclient.getResource(repo_url_resource_path); |
54 | const cwd = process.cwd(); |
55 | process.env["HOME"] = "."; |
56 | console.log( |
57 | `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}` |
58 | ); |
59 |
|
60 | if (repo_resource.is_github_app) { |
61 | const token = await get_gh_app_token(); |
62 | const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token); |
63 | repo_resource.url = authRepoUrl; |
64 | } |
65 |
|
66 | const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)); |
67 | safeDirectoryPath = cloneSafeDirectoryPath; |
68 |
|
69 |
|
70 | |
71 | |
72 | |
73 |
|
74 | if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) { |
75 | if (use_individual_branch) { |
76 | console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`"); |
77 | use_individual_branch = false; |
78 | } |
79 | if (group_by_folder) { |
80 | console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`"); |
81 | group_by_folder = false; |
82 | } |
83 | } |
84 |
|
85 | if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) { |
86 | const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName); |
87 | console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`); |
88 | await move_to_git_branch( |
89 | parent_workspace_id, |
90 | path_type, |
91 | path, |
92 | parent_path, |
93 | use_individual_branch, |
94 | group_by_folder, |
95 | clonedBranchName |
96 | ); |
97 | } |
98 |
|
99 | await move_to_git_branch( |
100 | workspace_id, |
101 | path_type, |
102 | path, |
103 | parent_path, |
104 | use_individual_branch, |
105 | group_by_folder, |
106 | clonedBranchName |
107 | ); |
108 | const subfolder = repo_resource.folder ?? ""; |
109 | const branch_or_default = repo_resource.branch ?? "<DEFAULT>"; |
110 | console.log( |
111 | `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}` |
112 | ); |
113 | await wmill_sync_pull( |
114 | path_type, |
115 | workspace_id, |
116 | path, |
117 | parent_path, |
118 | skip_secret, |
119 | repo_url_resource_path, |
120 | use_individual_branch, |
121 | repo_resource.branch |
122 | ); |
123 | try { |
124 | await git_push(path, parent_path, commit_msg, repo_resource); |
125 | } catch (e) { |
126 | throw e; |
127 | } finally { |
128 | await delete_pgp_keys(); |
129 | |
130 | if (safeDirectoryPath) { |
131 | try { |
132 | await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath); |
133 | } catch (e) { |
134 | console.log(`Warning: Could not unset safe.directory config: ${e}`); |
135 | } |
136 | } |
137 | } |
138 | console.log("Finished syncing"); |
139 | process.chdir(`${cwd}`); |
140 | } |
141 |
|
142 | function get_fork_branch_name(w_id: string, originalBranch: string): string { |
143 | if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) { |
144 | return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`); |
145 | } |
146 | return w_id; |
147 | } |
148 |
|
149 | async function git_clone( |
150 | cwd: string, |
151 | repo_resource: any, |
152 | no_single_branch: boolean, |
153 | ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> { |
154 | |
155 | let repo_url = repo_resource.url; |
156 | const subfolder = repo_resource.folder ?? ""; |
157 | const branch = repo_resource.branch ?? ""; |
158 | const repo_name = basename(repo_url, ".git"); |
159 | const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/); |
160 | if (azureMatch) { |
161 | console.log( |
162 | "Requires Azure DevOps service account access token, requesting..." |
163 | ); |
164 | const azureResource = await wmillclient.getResource(azureMatch.groups.url); |
165 | const response = await fetch( |
166 | `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, |
167 | { |
168 | method: "POST", |
169 | body: new URLSearchParams({ |
170 | client_id: azureResource.azureClientId, |
171 | client_secret: azureResource.azureClientSecret, |
172 | grant_type: "client_credentials", |
173 | resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", |
174 | }), |
175 | } |
176 | ); |
177 | const { access_token } = await response.json(); |
178 | repo_url = repo_url.replace(azureMatch[0], access_token); |
179 | } |
180 | const args = ["clone", "--quiet", "--depth", "1"]; |
181 | if (no_single_branch) { |
182 | args.push("--no-single-branch"); |
183 | } |
184 | if (subfolder !== "") { |
185 | args.push("--sparse"); |
186 | } |
187 | if (branch !== "") { |
188 | args.push("--branch"); |
189 | args.push(branch); |
190 | } |
191 | args.push(repo_url); |
192 | args.push(repo_name); |
193 | await sh_run(-1, "git", ...args); |
194 | try { |
195 | process.chdir(`${cwd}/${repo_name}`); |
196 | const safeDirectoryPath = process.cwd(); |
197 | |
198 | try { |
199 | await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd()); |
200 | } catch (e) { |
201 | console.log(`Warning: Could not add safe.directory config: ${e}`); |
202 | } |
203 |
|
204 | if (subfolder !== "") { |
205 | await sh_run(undefined, "git", "sparse-checkout", "add", subfolder); |
206 | try { |
207 | process.chdir(`${cwd}/${repo_name}/${subfolder}`); |
208 | } catch (err) { |
209 | console.log( |
210 | `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}` |
211 | ); |
212 | throw err; |
213 | } |
214 | } |
215 | const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim(); |
216 | return { repo_name, safeDirectoryPath, clonedBranchName }; |
217 |
|
218 | } catch (err) { |
219 | console.log( |
220 | `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}` |
221 | ); |
222 | throw err; |
223 | } |
224 | } |
225 | async function move_to_git_branch( |
226 | workspace_id: string, |
227 | path_type: PathType, |
228 | path: string | undefined, |
229 | parent_path: string | undefined, |
230 | use_individual_branch: boolean, |
231 | group_by_folder: boolean, |
232 | originalBranchName: string |
233 | ) { |
234 | let branchName; |
235 | if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) { |
236 | branchName = get_fork_branch_name(workspace_id, originalBranchName); |
237 | } else { |
238 | if (!use_individual_branch || path_type === "user" || path_type === "group") { |
239 | return; |
240 | } |
241 | branchName = group_by_folder |
242 | ? `wm_deploy/${workspace_id}/${(path ?? parent_path) |
243 | ?.split("/") |
244 | .slice(0, 2) |
245 | .join("__")}` |
246 | : `wm_deploy/${workspace_id}/${path_type}/${( |
247 | path ?? parent_path |
248 | )?.replaceAll("/", "__")}`; |
249 | } |
250 |
|
251 | try { |
252 | await sh_run(undefined, "git", "checkout", branchName); |
253 | } catch (err) { |
254 | console.log( |
255 | `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}` |
256 | ); |
257 | try { |
258 | await sh_run(undefined, "git", "checkout", "-b", branchName); |
259 | await sh_run( |
260 | undefined, |
261 | "git", |
262 | "config", |
263 | "--add", |
264 | "--bool", |
265 | "push.autoSetupRemote", |
266 | "true" |
267 | ); |
268 | } catch (err) { |
269 | console.log( |
270 | `Error checking out branch '${branchName}'. Error was:\n${err}` |
271 | ); |
272 | throw err; |
273 | } |
274 | } |
275 | console.log(`Successfully switched to branch ${branchName}`); |
276 | } |
277 | async function git_push( |
278 | path: string | undefined, |
279 | parent_path: string | undefined, |
280 | commit_msg: string, |
281 | repo_resource: any |
282 | ) { |
283 | let user_email = process.env["WM_EMAIL"] ?? ""; |
284 | let user_name = process.env["WM_USERNAME"] ?? ""; |
285 |
|
286 | if (repo_resource.gpg_key) { |
287 | await set_gpg_signing_secret(repo_resource.gpg_key); |
288 | |
289 | await sh_run( |
290 | undefined, |
291 | "git", |
292 | "config", |
293 | "user.email", |
294 | repo_resource.gpg_key.email |
295 | ); |
296 | await sh_run(undefined, "git", "config", "user.name", user_name); |
297 | } else { |
298 | await sh_run(undefined, "git", "config", "user.email", user_email); |
299 | await sh_run(undefined, "git", "config", "user.name", user_name); |
300 | } |
301 |
|
302 | if (path !== undefined && path !== null && path !== "") { |
303 | try { |
304 | await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`); |
305 | } catch (e) { |
306 | console.log(`Unable to stage files matching ${path}**, ${e}`); |
307 | } |
308 | } |
309 | if (parent_path !== undefined && parent_path !== null && parent_path !== "") { |
310 | try { |
311 | await sh_run( |
312 | undefined, |
313 | "git", |
314 | "add", |
315 | "wmill-lock.yaml", |
316 | `${parent_path}**` |
317 | ); |
318 | } catch (e) { |
319 | console.log(`Unable to stage files matching ${parent_path}, ${e}`); |
320 | } |
321 | } |
322 | try { |
323 | await sh_run(undefined, "git", "diff", "--cached", "--quiet"); |
324 | } catch { |
325 | |
326 | const commitArgs = ["git", "commit"]; |
327 |
|
328 | |
329 | commitArgs.push("--author", `"${user_name} <${user_email}>"`); |
330 | commitArgs.push( |
331 | "-m", |
332 | `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"` |
333 | ); |
334 |
|
335 | await sh_run(undefined, ...commitArgs); |
336 | try { |
337 | await sh_run(undefined, "git", "push", "--porcelain"); |
338 | } catch (e) { |
339 | console.log(`Could not push, trying to rebase first: ${e}`); |
340 | await sh_run(undefined, "git", "pull", "--rebase"); |
341 | await sh_run(undefined, "git", "push", "--porcelain"); |
342 | } |
343 | return; |
344 | } |
345 | console.log("No changes detected, nothing to commit. Returning..."); |
346 | } |
347 | async function sh_run( |
348 | secret_position: number | undefined, |
349 | cmd: string, |
350 | ...args: string[] |
351 | ) { |
352 | const nargs = secret_position != undefined ? args.slice() : args; |
353 | if (secret_position && secret_position < 0) { |
354 | secret_position = nargs.length - 1 + secret_position; |
355 | } |
356 | let secret: string | undefined = undefined; |
357 | if (secret_position != undefined) { |
358 | nargs[secret_position] = "***"; |
359 | secret = args[secret_position]; |
360 | } |
361 |
|
362 | console.log(`Running '${cmd} ${nargs.join(" ")} ...'`); |
363 | const command = exec(`${cmd} ${args.join(" ")}`); |
364 | |
365 | |
366 | |
367 | try { |
368 | const { stdout, stderr } = await command; |
369 | if (stdout.length > 0) { |
370 | console.log(stdout); |
371 | } |
372 | if (stderr.length > 0) { |
373 | console.log(stderr); |
374 | } |
375 | console.log("Command successfully executed"); |
376 | return stdout; |
377 | } catch (error) { |
378 | let errorString = error.toString(); |
379 | if (secret) { |
380 | errorString = errorString.replace(secret, "***"); |
381 | } |
382 | const err = `SH command '${cmd} ${nargs.join( |
383 | " " |
384 | )}' returned with error ${errorString}`; |
385 | throw Error(err); |
386 | } |
387 | } |
388 |
|
389 | function regexFromPath(path_type: PathType, path: string) { |
390 | if (path_type == "flow") { |
391 | return `${path}.flow/*`; |
392 | } |
393 | if (path_type == "app") { |
394 | return `${path}.app/*`; |
395 | } else if (path_type == "folder") { |
396 | return `${path}/folder.meta.*`; |
397 | } else if (path_type == "resourcetype") { |
398 | return `${path}.resource-type.*`; |
399 | } else if (path_type == "resource") { |
400 | return `${path}.resource.*`; |
401 | } else if (path_type == "variable") { |
402 | return `${path}.variable.*`; |
403 | } else if (path_type == "schedule") { |
404 | return `${path}.schedule.*`; |
405 | } else if (path_type == "user") { |
406 | return `${path}.user.*`; |
407 | } else if (path_type == "group") { |
408 | return `${path}.group.*`; |
409 | } else if (path_type == "httptrigger") { |
410 | return `${path}.http_trigger.*`; |
411 | } else if (path_type == "websockettrigger") { |
412 | return `${path}.websocket_trigger.*`; |
413 | } else if (path_type == "kafkatrigger") { |
414 | return `${path}.kafka_trigger.*`; |
415 | } else if (path_type == "natstrigger") { |
416 | return `${path}.nats_trigger.*`; |
417 | } else if (path_type == "postgrestrigger") { |
418 | return `${path}.postgres_trigger.*`; |
419 | } else if (path_type == "mqtttrigger") { |
420 | return `${path}.mqtt_trigger.*`; |
421 | } else if (path_type == "sqstrigger") { |
422 | return `${path}.sqs_trigger.*`; |
423 | } else if (path_type == "gcptrigger") { |
424 | return `${path}.gcp_trigger.*`; |
425 | } else if (path_type == "emailtrigger") { |
426 | return `${path}.email_trigger.*`; |
427 | } else { |
428 | return `${path}.*`; |
429 | } |
430 | } |
431 |
|
432 | async function wmill_sync_pull( |
433 | path_type: PathType, |
434 | workspace_id: string, |
435 | path: string | undefined, |
436 | parent_path: string | undefined, |
437 | skip_secret: boolean, |
438 | repo_url_resource_path: string, |
439 | use_individual_branch: boolean, |
440 | original_branch?: string |
441 | ) { |
442 | const includes = []; |
443 | if (path !== undefined && path !== null && path !== "") { |
444 | includes.push(regexFromPath(path_type, path)); |
445 | } |
446 | if (parent_path !== undefined && parent_path !== null && parent_path !== "") { |
447 | includes.push(regexFromPath(path_type, parent_path)); |
448 | } |
449 | await wmill_run( |
450 | 6, |
451 | "workspace", |
452 | "add", |
453 | workspace_id, |
454 | workspace_id, |
455 | process.env["BASE_URL"] + "/", |
456 | "--token", |
457 | process.env["WM_TOKEN"] ?? "" |
458 | ); |
459 | console.log("Pulling workspace into git repo"); |
460 | const args = [ |
461 | "sync", |
462 | "pull", |
463 | "--token", |
464 | process.env["WM_TOKEN"] ?? "", |
465 | "--workspace", |
466 | workspace_id, |
467 | "--repository", |
468 | repo_url_resource_path, |
469 | "--yes", |
470 | skip_secret ? "--skip-secrets" : "", |
471 | "--include-schedules", |
472 | "--include-users", |
473 | "--include-groups", |
474 | "--include-triggers", |
475 | ]; |
476 |
|
477 | |
478 | if (path_type === "settings" && !use_individual_branch) { |
479 | args.push("--include-settings"); |
480 | } |
481 |
|
482 | |
483 | if (path_type === "key" && !use_individual_branch) { |
484 | args.push("--include-key"); |
485 | } |
486 |
|
487 | args.push("--extra-includes", includes.join(",")); |
488 |
|
489 | |
490 | if (use_individual_branch && original_branch) { |
491 | console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`); |
492 | args.push("--promotion", original_branch); |
493 | } |
494 |
|
495 | await wmill_run(3, ...args); |
496 | } |
497 |
|
498 | async function wmill_run(secret_position: number, ...cmd: string[]) { |
499 | cmd = cmd.filter((elt) => elt !== ""); |
500 | const cmd2 = cmd.slice(); |
501 | cmd2[secret_position] = "***"; |
502 | console.log(`Running 'wmill ${cmd2.join(" ")} ...'`); |
503 | await wmill.parse(cmd); |
504 | console.log("Command successfully executed"); |
505 | } |
506 |
|
507 |
|
508 | async function set_gpg_signing_secret(gpg_key: GpgKey) { |
509 | try { |
510 | console.log("Setting GPG private key for git commits"); |
511 |
|
512 | const formattedGpgContent = gpg_key.private_key.replace( |
513 | /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/, |
514 | (_: string, header: string, body: string, footer: string) => |
515 | header + |
516 | "\n" + |
517 | "\n" + |
518 | body.replace(/ ([^\s])/g, "\n$1").trim() + |
519 | "\n" + |
520 | footer |
521 | ); |
522 |
|
523 | const gpg_path = `/tmp/gpg`; |
524 | await sh_run(undefined, "mkdir", "-p", gpg_path); |
525 | await sh_run(undefined, "chmod", "700", gpg_path); |
526 | process.env.GNUPGHOME = gpg_path; |
527 | |
528 |
|
529 | try { |
530 | await sh_run( |
531 | 1, |
532 | "bash", |
533 | "-c", |
534 | `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF` |
535 | ); |
536 | } catch (e) { |
537 | |
538 | throw new Error("Failed to import GPG key!"); |
539 | } |
540 |
|
541 | const listKeysOutput = await sh_run( |
542 | undefined, |
543 | "gpg", |
544 | "--list-secret-keys", |
545 | "--with-colons", |
546 | "--keyid-format=long" |
547 | ); |
548 |
|
549 | const keyInfoMatch = listKeysOutput.match( |
550 | /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/ |
551 | ); |
552 |
|
553 | if (!keyInfoMatch) { |
554 | throw new Error("Failed to extract GPG Key ID and Fingerprint"); |
555 | } |
556 |
|
557 | const keyId = keyInfoMatch[1]; |
558 | gpgFingerprint = keyInfoMatch[2]; |
559 |
|
560 | if (gpg_key.passphrase) { |
561 | |
562 | |
563 | await sh_run( |
564 | 1, |
565 | "bash", |
566 | "-c", |
567 | `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}` |
568 | ); |
569 | } |
570 |
|
571 | |
572 | await sh_run(undefined, "git", "config", "user.signingkey", keyId); |
573 | await sh_run(undefined, "git", "config", "commit.gpgsign", "true"); |
574 | console.log(`GPG signing configured with key ID: ${keyId} `); |
575 | } catch (e) { |
576 | console.error(`Failure while setting GPG key: ${e} `); |
577 | await delete_pgp_keys(); |
578 | } |
579 | } |
580 |
|
581 | async function delete_pgp_keys() { |
582 | console.log("deleting gpg keys"); |
583 | if (gpgFingerprint) { |
584 | await sh_run( |
585 | undefined, |
586 | "gpg", |
587 | "--batch", |
588 | "--yes", |
589 | "--pinentry-mode", |
590 | "loopback", |
591 | "--delete-secret-key", |
592 | gpgFingerprint |
593 | ); |
594 | await sh_run( |
595 | undefined, |
596 | "gpg", |
597 | "--batch", |
598 | "--yes", |
599 | "--delete-key", |
600 | "--pinentry-mode", |
601 | "loopback", |
602 | gpgFingerprint |
603 | ); |
604 | } |
605 | } |
606 |
|
607 | async function get_gh_app_token() { |
608 | const workspace = process.env["WM_WORKSPACE"]; |
609 | const jobToken = process.env["WM_TOKEN"]; |
610 |
|
611 | const baseUrl = |
612 | process.env["BASE_INTERNAL_URL"] ?? |
613 | process.env["BASE_URL"] ?? |
614 | "http://localhost:8000"; |
615 |
|
616 | const url = `${baseUrl}/api/w/${workspace}/github_app/token`; |
617 |
|
618 | const response = await fetch(url, { |
619 | method: "POST", |
620 | headers: { |
621 | "Content-Type": "application/json", |
622 | Authorization: `Bearer ${jobToken}`, |
623 | }, |
624 | body: JSON.stringify({ |
625 | job_token: jobToken, |
626 | }), |
627 | }); |
628 |
|
629 | if (!response.ok) { |
630 | throw new Error(`Error: ${response.statusText}`); |
631 | } |
632 |
|
633 | const data = await response.json(); |
634 |
|
635 | return data.token; |
636 | } |
637 |
|
638 | function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) { |
639 | if (!gitHubUrl || !installationToken) { |
640 | throw new Error("Both GitHub URL and Installation Token are required."); |
641 | } |
642 |
|
643 | try { |
644 | const url = new URL(gitHubUrl); |
645 |
|
646 | |
647 | if (url.hostname !== "github.com") { |
648 | throw new Error( |
649 | "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'." |
650 | ); |
651 | } |
652 |
|
653 | |
654 | return `https://x-access-token:${installationToken}@github.com${url.pathname}`; |
655 | } catch (e) { |
656 | const error = e as Error; |
657 | throw new Error(`Invalid URL: ${error.message}`); |
658 | } |
659 | } |
660 |
|