1 | import * as wmillclient from "windmill-client"; |
2 | import wmill from "windmill-cli"; |
3 | import { basename } from "node:path"; |
4 | const util = require("util"); |
5 | const exec = util.promisify(require("child_process").exec); |
6 | import process from "process"; |
7 |
|
8 | type GpgKey = { |
9 | email: string; |
10 | private_key: string; |
11 | passphrase: string; |
12 | }; |
13 |
|
14 | type PathType = |
15 | | "script" |
16 | | "flow" |
17 | | "app" |
18 | | "folder" |
19 | | "resource" |
20 | | "variable" |
21 | | "resourcetype" |
22 | | "schedule" |
23 | | "user" |
24 | | "group" |
25 | | "httptrigger" |
26 | | "websockettrigger" |
27 | | "kafkatrigger" |
28 | | "natstrigger" |
29 | | "postgrestrigger" |
30 | | "mqtttrigger" |
31 | | "sqstrigger" |
32 | | "gcptrigger"; |
33 |
|
34 | let gpgFingerprint: string | undefined = undefined; |
35 |
|
36 | export async function main( |
37 | workspace_id: string, |
38 | repo_url_resource_path: string, |
39 | path_type: PathType, |
40 | skip_secret = true, |
41 | path: string | undefined, |
42 | parent_path: string | undefined, |
43 | commit_msg: string, |
44 | use_individual_branch = false, |
45 | group_by_folder = false |
46 | ) { |
47 | const repo_resource = await wmillclient.getResource(repo_url_resource_path); |
48 | const cwd = process.cwd(); |
49 | process.env["HOME"] = "."; |
50 | console.log( |
51 | `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}` |
52 | ); |
53 |
|
54 | if (repo_resource.is_github_app) { |
55 | const token = await get_gh_app_token(); |
56 | const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token); |
57 | repo_resource.url = authRepoUrl; |
58 | } |
59 |
|
60 | const repo_name = await git_clone(cwd, repo_resource, use_individual_branch); |
61 | await move_to_git_branch( |
62 | workspace_id, |
63 | path_type, |
64 | path, |
65 | parent_path, |
66 | use_individual_branch, |
67 | group_by_folder |
68 | ); |
69 | const subfolder = repo_resource.folder ?? ""; |
70 | const branch_or_default = repo_resource.branch ?? "<DEFAULT>"; |
71 | console.log( |
72 | `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}` |
73 | ); |
74 | await wmill_sync_pull( |
75 | path_type, |
76 | workspace_id, |
77 | path, |
78 | parent_path, |
79 | skip_secret |
80 | ); |
81 | try { |
82 | await git_push(path, parent_path, commit_msg, repo_resource); |
83 | } catch (e) { |
84 | throw e; |
85 | } finally { |
86 | await delete_pgp_keys(); |
87 | } |
88 | console.log("Finished syncing"); |
89 | process.chdir(`${cwd}`); |
90 | } |
91 | async function git_clone( |
92 | cwd: string, |
93 | repo_resource: any, |
94 | use_individual_branch: boolean |
95 | ): Promise<string> { |
96 | |
97 | let repo_url = repo_resource.url; |
98 | const subfolder = repo_resource.folder ?? ""; |
99 | const branch = repo_resource.branch ?? ""; |
100 | const repo_name = basename(repo_url, ".git"); |
101 | const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/); |
102 | if (azureMatch) { |
103 | console.log( |
104 | "Requires Azure DevOps service account access token, requesting..." |
105 | ); |
106 | const azureResource = await wmillclient.getResource(azureMatch.groups.url); |
107 | const response = await fetch( |
108 | `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, |
109 | { |
110 | method: "POST", |
111 | body: new URLSearchParams({ |
112 | client_id: azureResource.azureClientId, |
113 | client_secret: azureResource.azureClientSecret, |
114 | grant_type: "client_credentials", |
115 | resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", |
116 | }), |
117 | } |
118 | ); |
119 | const { access_token } = await response.json(); |
120 | repo_url = repo_url.replace(azureMatch[0], access_token); |
121 | } |
122 | const args = ["clone", "--quiet", "--depth", "1"]; |
123 | if (use_individual_branch) { |
124 | args.push("--no-single-branch"); |
125 | } |
126 | if (subfolder !== "") { |
127 | args.push("--sparse"); |
128 | } |
129 | if (branch !== "") { |
130 | args.push("--branch"); |
131 | args.push(branch); |
132 | } |
133 | args.push(repo_url); |
134 | args.push(repo_name); |
135 | await sh_run(-1, "git", ...args); |
136 | try { |
137 | process.chdir(`${cwd}/${repo_name}`); |
138 | } catch (err) { |
139 | console.log( |
140 | `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}` |
141 | ); |
142 | throw err; |
143 | } |
144 | process.chdir(`${cwd}/${repo_name}`); |
145 | if (subfolder !== "") { |
146 | await sh_run(undefined, "git", "sparse-checkout", "add", subfolder); |
147 | } |
148 | try { |
149 | process.chdir(`${cwd}/${repo_name}/${subfolder}`); |
150 | } catch (err) { |
151 | console.log( |
152 | `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}` |
153 | ); |
154 | throw err; |
155 | } |
156 | return repo_name; |
157 | } |
158 | async function move_to_git_branch( |
159 | workspace_id: string, |
160 | path_type: PathType, |
161 | path: string | undefined, |
162 | parent_path: string | undefined, |
163 | use_individual_branch: boolean, |
164 | group_by_folder: boolean |
165 | ) { |
166 | if (!use_individual_branch || path_type === "user" || path_type === "group") { |
167 | return; |
168 | } |
169 | const branchName = group_by_folder |
170 | ? `wm_deploy/${workspace_id}/${(path ?? parent_path) |
171 | ?.split("/") |
172 | .slice(0, 2) |
173 | .join("__")}` |
174 | : `wm_deploy/${workspace_id}/${path_type}/${( |
175 | path ?? parent_path |
176 | )?.replaceAll("/", "__")}`; |
177 | try { |
178 | await sh_run(undefined, "git", "checkout", branchName); |
179 | } catch (err) { |
180 | console.log( |
181 | `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}` |
182 | ); |
183 | try { |
184 | await sh_run(undefined, "git", "checkout", "-b", branchName); |
185 | await sh_run( |
186 | undefined, |
187 | "git", |
188 | "config", |
189 | "--add", |
190 | "--bool", |
191 | "push.autoSetupRemote", |
192 | "true" |
193 | ); |
194 | } catch (err) { |
195 | console.log( |
196 | `Error checking out branch '${branchName}'. Error was:\n${err}` |
197 | ); |
198 | throw err; |
199 | } |
200 | } |
201 | console.log(`Successfully switched to branch ${branchName}`); |
202 | } |
203 | async function git_push( |
204 | path: string | undefined, |
205 | parent_path: string | undefined, |
206 | commit_msg: string, |
207 | repo_resource: any |
208 | ) { |
209 | let user_email = process.env["WM_EMAIL"] ?? ""; |
210 | let user_name = process.env["WM_USERNAME"] ?? ""; |
211 |
|
212 | if (repo_resource.gpg_key) { |
213 | await set_gpg_signing_secret(repo_resource.gpg_key); |
214 | |
215 | await sh_run( |
216 | undefined, |
217 | "git", |
218 | "config", |
219 | "user.email", |
220 | repo_resource.gpg_key.email |
221 | ); |
222 | await sh_run(undefined, "git", "config", "user.name", user_name); |
223 | } else { |
224 | await sh_run(undefined, "git", "config", "user.email", user_email); |
225 | await sh_run(undefined, "git", "config", "user.name", user_name); |
226 | } |
227 |
|
228 | if (path !== undefined && path !== null && path !== "") { |
229 | try { |
230 | await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`); |
231 | } catch (e) { |
232 | console.log(`Unable to stage files matching ${path}**, ${e}`); |
233 | } |
234 | } |
235 | if (parent_path !== undefined && parent_path !== null && parent_path !== "") { |
236 | try { |
237 | await sh_run( |
238 | undefined, |
239 | "git", |
240 | "add", |
241 | "wmill-lock.yaml", |
242 | `${parent_path}**` |
243 | ); |
244 | } catch (e) { |
245 | console.log(`Unable to stage files matching ${parent_path}, ${e}`); |
246 | } |
247 | } |
248 | try { |
249 | await sh_run(undefined, "git", "diff", "--cached", "--quiet"); |
250 | } catch { |
251 | |
252 | const commitArgs = ["git", "commit"]; |
253 |
|
254 | |
255 | commitArgs.push("--author", `"${user_name} <${user_email}>"`); |
256 | commitArgs.push( |
257 | "-m", |
258 | `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"` |
259 | ); |
260 |
|
261 | await sh_run(undefined, ...commitArgs); |
262 | try { |
263 | await sh_run(undefined, "git", "push", "--porcelain"); |
264 | } catch (e) { |
265 | console.log(`Could not push, trying to rebase first: ${e}`); |
266 | await sh_run(undefined, "git", "pull", "--rebase"); |
267 | await sh_run(undefined, "git", "push", "--porcelain"); |
268 | } |
269 | return; |
270 | } |
271 | console.log("No changes detected, nothing to commit. Returning..."); |
272 | } |
273 | async function sh_run( |
274 | secret_position: number | undefined, |
275 | cmd: string, |
276 | ...args: string[] |
277 | ) { |
278 | const nargs = secret_position != undefined ? args.slice() : args; |
279 | if (secret_position && secret_position < 0) { |
280 | secret_position = nargs.length - 1 + secret_position; |
281 | } |
282 | let secret: string | undefined = undefined; |
283 | if (secret_position != undefined) { |
284 | nargs[secret_position] = "***"; |
285 | secret = args[secret_position]; |
286 | } |
287 |
|
288 | console.log(`Running '${cmd} ${nargs.join(" ")} ...'`); |
289 | const command = exec(`${cmd} ${args.join(" ")}`); |
290 | |
291 | |
292 | |
293 | try { |
294 | const { stdout, stderr } = await command; |
295 | if (stdout.length > 0) { |
296 | console.log(stdout); |
297 | } |
298 | if (stderr.length > 0) { |
299 | console.log(stderr); |
300 | } |
301 | console.log("Command successfully executed"); |
302 | return stdout; |
303 | } catch (error) { |
304 | let errorString = error.toString(); |
305 | if (secret) { |
306 | errorString = errorString.replace(secret, "***"); |
307 | } |
308 | const err = `SH command '${cmd} ${nargs.join( |
309 | " " |
310 | )}' returned with error ${errorString}`; |
311 | throw Error(err); |
312 | } |
313 | } |
314 |
|
315 | function regexFromPath(path_type: PathType, path: string) { |
316 | if (path_type == "flow") { |
317 | return `${path}.flow/*`; |
318 | } |
319 | if (path_type == "app") { |
320 | return `${path}.app/*`; |
321 | } else if (path_type == "folder") { |
322 | return `${path}/folder.meta.*`; |
323 | } else if (path_type == "resourcetype") { |
324 | return `${path}.resource-type.*`; |
325 | } else if (path_type == "resource") { |
326 | return `${path}.resource.*`; |
327 | } else if (path_type == "variable") { |
328 | return `${path}.variable.*`; |
329 | } else if (path_type == "schedule") { |
330 | return `${path}.schedule.*`; |
331 | } else if (path_type == "user") { |
332 | return `${path}.user.*`; |
333 | } else if (path_type == "group") { |
334 | return `${path}.group.*`; |
335 | } else if (path_type == "httptrigger") { |
336 | return `${path}.http_trigger.*`; |
337 | } else if (path_type == "websockettrigger") { |
338 | return `${path}.websocket_trigger.*`; |
339 | } else if (path_type == "kafkatrigger") { |
340 | return `${path}.kafka_trigger.*`; |
341 | } else if (path_type == "natstrigger") { |
342 | return `${path}.nats_trigger.*`; |
343 | } else if (path_type == "postgrestrigger") { |
344 | return `${path}.postgres_trigger.*`; |
345 | } else if (path_type == "mqtttrigger") { |
346 | return `${path}.mqtt_trigger.*`; |
347 | } else if (path_type == "sqstrigger") { |
348 | return `${path}.sqs_trigger.*`; |
349 | } else if (path_type == "gcptrigger") { |
350 | return `${path}.gcp_trigger.*`; |
351 | } else { |
352 | return `${path}.*`; |
353 | } |
354 | } |
355 |
|
356 | async function wmill_sync_pull( |
357 | path_type: PathType, |
358 | workspace_id: string, |
359 | path: string | undefined, |
360 | parent_path: string | undefined, |
361 | skip_secret: boolean |
362 | ) { |
363 | const includes = []; |
364 | if (path !== undefined && path !== null && path !== "") { |
365 | includes.push(regexFromPath(path_type, path)); |
366 | } |
367 | if (parent_path !== undefined && parent_path !== null && parent_path !== "") { |
368 | includes.push(regexFromPath(path_type, parent_path)); |
369 | } |
370 | await wmill_run( |
371 | 6, |
372 | "workspace", |
373 | "add", |
374 | workspace_id, |
375 | workspace_id, |
376 | process.env["BASE_URL"] + "/", |
377 | "--token", |
378 | process.env["WM_TOKEN"] ?? "" |
379 | ); |
380 | console.log("Pulling workspace into git repo"); |
381 | await wmill_run( |
382 | 3, |
383 | "sync", |
384 | "pull", |
385 | "--token", |
386 | process.env["WM_TOKEN"] ?? "", |
387 | "--workspace", |
388 | workspace_id, |
389 | "--yes", |
390 | skip_secret ? "--skip-secrets" : "", |
391 | "--include-schedules", |
392 | "--include-users", |
393 | "--include-groups", |
394 | "--include-triggers", |
395 | "--extra-includes", |
396 | includes.join(",") |
397 | ); |
398 | } |
399 |
|
400 | async function wmill_run(secret_position: number, ...cmd: string[]) { |
401 | cmd = cmd.filter((elt) => elt !== ""); |
402 | const cmd2 = cmd.slice(); |
403 | cmd2[secret_position] = "***"; |
404 | console.log(`Running 'wmill ${cmd2.join(" ")} ...'`); |
405 | await wmill.parse(cmd); |
406 | console.log("Command successfully executed"); |
407 | } |
408 |
|
409 |
|
410 | async function set_gpg_signing_secret(gpg_key: GpgKey) { |
411 | try { |
412 | console.log("Setting GPG private key for git commits"); |
413 |
|
414 | const formattedGpgContent = gpg_key.private_key.replace( |
415 | /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/, |
416 | (_: string, header: string, body: string, footer: string) => |
417 | header + |
418 | "\n" + |
419 | "\n" + |
420 | body.replace(/ ([^\s])/g, "\n$1").trim() + |
421 | "\n" + |
422 | footer |
423 | ); |
424 |
|
425 | const gpg_path = `/tmp/gpg`; |
426 | await sh_run(undefined, "mkdir", "-p", gpg_path); |
427 | await sh_run(undefined, "chmod", "700", gpg_path); |
428 | process.env.GNUPGHOME = gpg_path; |
429 | |
430 |
|
431 | try { |
432 | await sh_run( |
433 | 1, |
434 | "bash", |
435 | "-c", |
436 | `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF` |
437 | ); |
438 | } catch (e) { |
439 | |
440 | throw new Error("Failed to import GPG key!"); |
441 | } |
442 |
|
443 | const listKeysOutput = await sh_run( |
444 | undefined, |
445 | "gpg", |
446 | "--list-secret-keys", |
447 | "--with-colons", |
448 | "--keyid-format=long" |
449 | ); |
450 |
|
451 | const keyInfoMatch = listKeysOutput.match( |
452 | /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/ |
453 | ); |
454 |
|
455 | if (!keyInfoMatch) { |
456 | throw new Error("Failed to extract GPG Key ID and Fingerprint"); |
457 | } |
458 |
|
459 | const keyId = keyInfoMatch[1]; |
460 | gpgFingerprint = keyInfoMatch[2]; |
461 |
|
462 | if (gpg_key.passphrase) { |
463 | |
464 | |
465 | await sh_run( |
466 | 1, |
467 | "bash", |
468 | "-c", |
469 | `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}` |
470 | ); |
471 | } |
472 |
|
473 | |
474 | await sh_run(undefined, "git", "config", "user.signingkey", keyId); |
475 | await sh_run(undefined, "git", "config", "commit.gpgsign", "true"); |
476 | console.log(`GPG signing configured with key ID: ${keyId} `); |
477 | } catch (e) { |
478 | console.error(`Failure while setting GPG key: ${e} `); |
479 | await delete_pgp_keys(); |
480 | } |
481 | } |
482 |
|
483 | async function delete_pgp_keys() { |
484 | console.log("deleting gpg keys"); |
485 | if (gpgFingerprint) { |
486 | await sh_run( |
487 | undefined, |
488 | "gpg", |
489 | "--batch", |
490 | "--yes", |
491 | "--pinentry-mode", |
492 | "loopback", |
493 | "--delete-secret-key", |
494 | gpgFingerprint |
495 | ); |
496 | await sh_run( |
497 | undefined, |
498 | "gpg", |
499 | "--batch", |
500 | "--yes", |
501 | "--delete-key", |
502 | "--pinentry-mode", |
503 | "loopback", |
504 | gpgFingerprint |
505 | ); |
506 | } |
507 | } |
508 |
|
509 | async function get_gh_app_token() { |
510 | const workspace = process.env["WM_WORKSPACE"]; |
511 | const jobToken = process.env["WM_TOKEN"]; |
512 |
|
513 | const baseUrl = |
514 | process.env["BASE_INTERNAL_URL"] ?? |
515 | process.env["BASE_URL"] ?? |
516 | "http://localhost:8000"; |
517 |
|
518 | const url = `${baseUrl}/api/w/${workspace}/github_app/token`; |
519 |
|
520 | const response = await fetch(url, { |
521 | method: "POST", |
522 | headers: { |
523 | "Content-Type": "application/json", |
524 | Authorization: `Bearer ${jobToken}`, |
525 | }, |
526 | body: JSON.stringify({ |
527 | job_token: jobToken, |
528 | }), |
529 | }); |
530 |
|
531 | if (!response.ok) { |
532 | throw new Error(`Error: ${response.statusText}`); |
533 | } |
534 |
|
535 | const data = await response.json(); |
536 |
|
537 | return data.token; |
538 | } |
539 |
|
540 | function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) { |
541 | if (!gitHubUrl || !installationToken) { |
542 | throw new Error("Both GitHub URL and Installation Token are required."); |
543 | } |
544 |
|
545 | try { |
546 | const url = new URL(gitHubUrl); |
547 |
|
548 | |
549 | if (url.hostname !== "github.com") { |
550 | throw new Error( |
551 | "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'." |
552 | ); |
553 | } |
554 |
|
555 | |
556 | return `https://x-access-token:${installationToken}@github.com${url.pathname}`; |
557 | } catch (e) { |
558 | const error = e as Error; |
559 | throw new Error(`Invalid URL: ${error.message}`); |
560 | } |
561 | } |