logo Windmill Hub
Login
All items
1

Sync script to Git repo

by
Published Dec 1, 2023

This script will pull the script from the current workspace in a temporary folder, then commit it to the remote Git repository and push it. Only the script-related file will be pushed, nothing else. It takes as input the `git_repository` resource containing the repository URL, the script path, and the commit message to use for the commit. All params are mandatory.

Script windmill Verified
Use in Windmill

The script

Submitted by pyranota275 Bun
Verified 9 days ago
All edits
Permalink
1 
import * as wmillclient from "windmill-client";
2
3 
import wmill from "[email protected]";
4 
import { basename } from "node:path";
5 
const util = require("util");
6 
const exec = util.promisify(require("child_process").exec);
7 
import process from "process";
8
9 
type GpgKey = {
10 
  email: string;
11 
  private_key: string;
12 
  passphrase: string;
13 
};
14
15 
const FORKED_WORKSPACE_PREFIX = "wm-fork-";
16
17 
type PathType =
18 
  | "script"
19 
  | "flow"
20 
  | "app"
21 
  | "raw_app"
22 
  | "folder"
23 
  | "resource"
24 
  | "variable"
25 
  | "resourcetype"
26 
  | "schedule"
27 
  | "user"
28 
  | "group"
29 
  | "httptrigger"
30 
  | "websockettrigger"
31 
  | "kafkatrigger"
32 
  | "natstrigger"
33 
  | "postgrestrigger"
34 
  | "mqtttrigger"
35 
  | "sqstrigger"
36 
  | "gcptrigger"
37 
  | "azuretrigger"
38 
  | "emailtrigger";
39
40 
type SyncObject = {
41 
  path_type: PathType;
42 
  path: string | undefined;
43 
  parent_path: string | undefined;
44 
  commit_msg: string;
45 
};
46
47 
let gpgFingerprint: string | undefined = undefined;
48
49 
export async function main(
50 
  items: SyncObject[],
51 
  // Compat, do not use in code, rely on `items` instead
52 
  path_type: PathType | undefined,
53 
  path: string | undefined,
54 
  parent_path: string | undefined,
55 
  commit_msg: string | undefined,
56 
  //
57 
  workspace_id: string,
58 
  repo_url_resource_path: string,
59 
  skip_secret: boolean = true,
60 
  use_individual_branch: boolean = false,
61 
  group_by_folder: boolean = false,
62 
  only_create_branch: boolean = false,
63 
  parent_workspace_id?: string,
64 
) {
65 
  if (path_type !== undefined && commit_msg !== undefined) {
66 
    items = [
67 
      {
68 
        path_type,
69 
        path,
70 
        parent_path,
71 
        commit_msg,
72 
      },
73 
    ];
74 
  }
75 
  await inner(
76 
    items,
77 
    workspace_id,
78 
    repo_url_resource_path,
79 
    skip_secret,
80 
    use_individual_branch,
81 
    group_by_folder,
82 
    only_create_branch,
83 
    parent_workspace_id,
84 
  );
85 
}
86
87 
async function inner(
88 
  items: SyncObject[],
89 
  workspace_id: string,
90 
  repo_url_resource_path: string,
91 
  skip_secret: boolean = true,
92 
  use_individual_branch: boolean = false,
93 
  group_by_folder: boolean = false,
94 
  only_create_branch: boolean = false,
95 
  parent_workspace_id?: string,
96 
) {
97 
  let safeDirectoryPath: string | undefined;
98 
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
99 
  const cwd = process.cwd();
100 
  process.env["HOME"] = ".";
101 
  if (!only_create_branch) {
102 
    for (const item of items) {
103 
      console.log(
104 
        `Syncing ${item.path_type} ${item.path ?? ""} with parent ${item.parent_path ?? ""}`,
105 
      );
106 
    }
107 
  }
108
109 
  if (repo_resource.is_github_app) {
110 
    const token = await get_gh_app_token();
111 
    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
112 
    repo_resource.url = authRepoUrl;
113 
  }
114
115 
  const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath } =
116 
    await git_clone(
117 
      cwd,
118 
      repo_resource,
119 
      use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX),
120 
    );
121 
  safeDirectoryPath = cloneSafeDirectoryPath;
122
123 
  // GPG signing must be set up BEFORE git_push (which is in-script and runs
124 
  // back-to-back with the agent pre-warm — that's the whole point of doing
125 
  // commit/push in the script instead of the CLI).
126 
  if (repo_resource.gpg_key) {
127 
    await set_gpg_signing_secret(repo_resource.gpg_key);
128 
  }
129
130 
  const subfolder = repo_resource.folder ?? "";
131 
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
132 
  console.log(
133 
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`,
134 
  );
135
136 
  try {
137 
    // CLI owns: branch checkout (wm_deploy/<workspace>/... or wm-fork/...),
138 
    // include-set derivation from items, promotion-overrides resolution,
139 
    // workspace zip pull. With --skip-commit, the CLI returns without
140 
    // committing — the script does that next.
141 
    const args = [
142 
      "sync",
143 
      "git-deploy",
144 
      "--token",
145 
      process.env["WM_TOKEN"] ?? "",
146 
      "--workspace",
147 
      workspace_id,
148 
      "--base-url",
149 
      process.env["BASE_URL"] + "/",
150 
      "--repository",
151 
      repo_url_resource_path,
152 
      "--git-deploy-items",
153 
      JSON.stringify(items),
154 
    ];
155 
    if (use_individual_branch) args.push("--use-individual-branch");
156 
    if (group_by_folder) args.push("--group-by-folder");
157 
    if (only_create_branch) args.push("--only-create-branch");
158 
    if (parent_workspace_id) {
159 
      args.push("--parent-workspace-id", parent_workspace_id);
160 
    }
161 
    if (skip_secret) args.push("--skip-secrets");
162 
    await wmill_run(3, ...args);
163
164 
    // In-script commit + push. Stays in the SAME process as the dummy gpg
165 
    // sign in set_gpg_signing_secret — agent cache is warm, signing works.
166 
    if (!only_create_branch) {
167 
      await git_push(items, repo_resource, only_create_branch);
168 
    }
169 
  } catch (e) {
170 
    throw e;
171 
  } finally {
172 
    await delete_pgp_keys();
173 
    if (safeDirectoryPath) {
174 
      try {
175 
        await sh_run(
176 
          undefined,
177 
          "git",
178 
          "config",
179 
          "--global",
180 
          "--unset",
181 
          "safe.directory",
182 
          safeDirectoryPath,
183 
        );
184 
      } catch (e) {
185 
        console.log(`Warning: Could not unset safe.directory config: ${e}`);
186 
      }
187 
    }
188 
  }
189 
  console.log("Finished syncing");
190 
  process.chdir(`${cwd}`);
191 
}
192
193 
async function git_clone(
194 
  cwd: string,
195 
  repo_resource: any,
196 
  no_single_branch: boolean,
197 
): Promise<{
198 
  repo_name: string;
199 
  safeDirectoryPath: string;
200 
  clonedBranchName: string;
201 
}> {
202 
  let repo_url = repo_resource.url;
203 
  const subfolder = repo_resource.folder ?? "";
204 
  const branch = repo_resource.branch ?? "";
205 
  const repo_name = basename(repo_url, ".git");
206 
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
207 
  if (azureMatch) {
208 
    console.log(
209 
      "Requires Azure DevOps service account access token, requesting...",
210 
    );
211 
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
212 
    const response = await fetch(
213 
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
214 
      {
215 
        method: "POST",
216 
        body: new URLSearchParams({
217 
          client_id: azureResource.azureClientId,
218 
          client_secret: azureResource.azureClientSecret,
219 
          grant_type: "client_credentials",
220 
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
221 
        }),
222 
      },
223 
    );
224 
    const { access_token } = await response.json();
225 
    repo_url = repo_url.replace(azureMatch[0], access_token);
226 
  }
227 
  const args = ["clone", "--quiet", "--depth", "1"];
228 
  if (no_single_branch) {
229 
    args.push("--no-single-branch");
230 
  }
231 
  if (subfolder !== "") {
232 
    args.push("--sparse");
233 
  }
234 
  if (branch !== "") {
235 
    args.push("--branch");
236 
    args.push(branch);
237 
  }
238 
  args.push(repo_url);
239 
  args.push(repo_name);
240 
  await sh_run(-1, "git", ...args);
241 
  try {
242 
    process.chdir(`${cwd}/${repo_name}`);
243 
    const safeDirectoryPath = process.cwd();
244 
    try {
245 
      await sh_run(
246 
        undefined,
247 
        "git",
248 
        "config",
249 
        "--global",
250 
        "--add",
251 
        "safe.directory",
252 
        process.cwd(),
253 
      );
254 
    } catch (e) {
255 
      console.log(`Warning: Could not add safe.directory config: ${e}`);
256 
    }
257
258 
    if (subfolder !== "") {
259 
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
260 
      try {
261 
        process.chdir(`${cwd}/${repo_name}/${subfolder}`);
262 
      } catch (err) {
263 
        console.log(
264 
          `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`,
265 
        );
266 
        throw err;
267 
      }
268 
    }
269 
    const clonedBranchName = (
270 
      await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")
271 
    ).trim();
272 
    return { repo_name, safeDirectoryPath, clonedBranchName };
273 
  } catch (err) {
274 
    console.log(
275 
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`,
276 
    );
277 
    throw err;
278 
  }
279 
}
280
281 
function composeCommitHeader(items: SyncObject[]): string {
282 
  const typeCounts = new Map<PathType, number>();
283 
  for (const item of items) {
284 
    typeCounts.set(item.path_type, (typeCounts.get(item.path_type) ?? 0) + 1);
285 
  }
286
287 
  const sortedTypes = Array.from(typeCounts.entries()).sort(
288 
    (a, b) => b[1] - a[1],
289 
  );
290
291 
  const parts: string[] = [];
292 
  let othersCount = 0;
293
294 
  for (let i = 0; i < sortedTypes.length; i++) {
295 
    const [pathType, count] = sortedTypes[i];
296 
    if (i < 3) {
297 
      const label = count > 1 ? `${pathType}s` : pathType;
298 
      if (i == 2 && sortedTypes.length == 3) {
299 
        parts.push(`and ${count} ${label}`);
300 
      } else {
301 
        parts.push(`${count} ${label}`);
302 
      }
303 
    } else {
304 
      othersCount += count;
305 
    }
306 
  }
307
308 
  let header = `[WM]: Deployed ${parts.join(", ")}`;
309 
  if (othersCount > 0) {
310 
    header += ` and ${othersCount} other object${othersCount > 1 ? "s" : ""}`;
311 
  }
312 
  return header;
313 
}
314
315 
// Stage / commit / push the deploy. Mirrors hub/28217's git_push EXACTLY
316 
// (same staging globs, same commit-message construction quirks, same
317 
// rebase-retry fallback). Keeping this in-script — instead of letting the
318 
// CLI's gitSyncDeployPush handle it — is what restores GPG correctness:
319 
// set_gpg_signing_secret already pre-warmed the agent in THIS process
320 
// milliseconds ago, so signing inherits a warm cache via plain exec.
321 
async function git_push(
322 
  items: SyncObject[],
323 
  repo_resource: any,
324 
  only_create_branch: boolean,
325 
) {
326 
  const user_email = process.env["WM_EMAIL"] ?? "";
327 
  const user_name = process.env["WM_USERNAME"] ?? "";
328
329 
  if (repo_resource.gpg_key) {
330 
    // GPG path: committer identity = the GPG key's email (else git rejects
331 
    // the signed commit's committer/key mismatch).
332 
    await sh_run(
333 
      undefined,
334 
      "git",
335 
      "config",
336 
      "user.email",
337 
      repo_resource.gpg_key.email,
338 
    );
339 
    await sh_run(undefined, "git", "config", "user.name", user_name);
340 
  } else {
341 
    await sh_run(undefined, "git", "config", "user.email", user_email);
342 
    await sh_run(undefined, "git", "config", "user.name", user_name);
343 
  }
344 
  if (only_create_branch) {
345 
    await sh_run(undefined, "git", "push", "--porcelain");
346 
  }
347
348 
  const commit_description: string[] = [];
349 
  for (const { path, parent_path, commit_msg } of items) {
350 
    if (path !== undefined && path !== null && path !== "") {
351 
      try {
352 
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `'${path}**'`);
353 
      } catch (e) {
354 
        console.log(`Unable to stage files matching ${path}**, ${e}`);
355 
      }
356 
    }
357 
    if (
358 
      parent_path !== undefined &&
359 
      parent_path !== null &&
360 
      parent_path !== ""
361 
    ) {
362 
      try {
363 
        await sh_run(
364 
          undefined,
365 
          "git",
366 
          "add",
367 
          "wmill-lock.yaml",
368 
          `'${parent_path}**'`,
369 
        );
370 
      } catch (e) {
371 
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
372 
      }
373 
    }
374 
    commit_description.push(commit_msg);
375 
  }
376
377 
  try {
378 
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
379 
  } catch {
380 
    // git diff --cached --quiet exits 1 iff there is something staged.
381 
    const commitArgs = ["git", "commit"];
382 
    commitArgs.push("--author", `"${user_name} <${user_email}>"`);
383
384 
    const [header, description] =
385 
      commit_description.length == 1
386 
        ? [commit_description[0], ""]
387 
        : [composeCommitHeader(items), commit_description.join("\n")];
388
389 
    commitArgs.push(
390 
      "-m",
391 
      `"${header == undefined || header == "" ? "no commit msg" : header}"`,
392 
      "-m",
393 
      `"${description}"`,
394 
    );
395
396 
    await sh_run(undefined, ...commitArgs);
397 
    try {
398 
      await sh_run(undefined, "git", "push", "--porcelain");
399 
    } catch (e) {
400 
      console.log(`Could not push, trying to rebase first: ${e}`);
401 
      await sh_run(undefined, "git", "pull", "--rebase");
402 
      await sh_run(undefined, "git", "push", "--porcelain");
403 
    }
404 
    return;
405 
  }
406
407 
  console.log("No changes detected, nothing to commit. Returning...");
408 
}
409
410 
async function sh_run(
411 
  secret_position: number | undefined,
412 
  cmd: string,
413 
  ...args: string[]
414 
) {
415 
  const nargs = secret_position != undefined ? args.slice() : args;
416 
  if (secret_position && secret_position < 0) {
417 
    secret_position = nargs.length - 1 + secret_position;
418 
  }
419 
  let secret: string | undefined = undefined;
420 
  if (secret_position != undefined) {
421 
    nargs[secret_position] = "***";
422 
    secret = args[secret_position];
423 
  }
424
425 
  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
426 
  const command = exec(`${cmd} ${args.join(" ")}`);
427 
  try {
428 
    const { stdout, stderr } = await command;
429 
    if (stdout.length > 0) {
430 
      console.log(stdout);
431 
    }
432 
    if (stderr.length > 0) {
433 
      console.log(stderr);
434 
    }
435 
    console.log("Command successfully executed");
436 
    return stdout;
437 
  } catch (error) {
438 
    let errorString = error.toString();
439 
    if (secret) {
440 
      errorString = errorString.replace(secret, "***");
441 
    }
442 
    const err = `SH command '${cmd} ${nargs.join(
443 
      " ",
444 
    )}' returned with error ${errorString}`;
445 
    throw Error(err);
446 
  }
447 
}
448
449 
async function wmill_run(secret_position: number, ...cmd: string[]) {
450 
  cmd = cmd.filter((elt) => elt !== "");
451 
  const cmd2 = cmd.slice();
452 
  cmd2[secret_position] = "***";
453 
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
454 
  await wmill.parse(cmd);
455 
  console.log("Command successfully executed");
456 
}
457
458 
// Identical to hub/28217's set_gpg_signing_secret. Sets up GNUPGHOME,
459 
// imports the key, pre-warms the agent's passphrase cache with a dummy
460 
// `gpg -bsau`, and configures git's local user.signingkey + commit.gpgsign.
461 
// Because git_push runs immediately after in the SAME process (no CLI
462 
// boundary), the agent cache is still warm at sign time.
463 
async function set_gpg_signing_secret(gpg_key: GpgKey) {
464 
  try {
465 
    console.log("Setting GPG private key for git commits");
466
467 
    const formattedGpgContent = gpg_key.private_key.replace(
468 
      /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
469 
      (_: string, header: string, body: string, footer: string) =>
470 
        header +
471 
        "\n" +
472 
        "\n" +
473 
        body.replace(/ ([^\s])/g, "\n$1").trim() +
474 
        "\n" +
475 
        footer,
476 
    );
477
478 
    const gpg_path = `/tmp/gpg`;
479 
    await sh_run(undefined, "mkdir", "-p", gpg_path);
480 
    await sh_run(undefined, "chmod", "700", gpg_path);
481 
    process.env.GNUPGHOME = gpg_path;
482
483 
    try {
484 
      await sh_run(
485 
        1,
486 
        "bash",
487 
        "-c",
488 
        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`,
489 
      );
490 
    } catch (e) {
491 
      throw new Error("Failed to import GPG key!");
492 
    }
493
494 
    const listKeysOutput = await sh_run(
495 
      undefined,
496 
      "gpg",
497 
      "--list-secret-keys",
498 
      "--with-colons",
499 
      "--keyid-format=long",
500 
    );
501
502 
    const keyInfoMatch = listKeysOutput.match(
503 
      /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/,
504 
    );
505
506 
    if (!keyInfoMatch) {
507 
      throw new Error("Failed to extract GPG Key ID and Fingerprint");
508 
    }
509
510 
    const keyId = keyInfoMatch[1];
511 
    gpgFingerprint = keyInfoMatch[2];
512
513 
    if (gpg_key.passphrase) {
514 
      // Pre-warm the agent: this dummy sign loads the passphrase into
515 
      // gpg-agent's cache so the actual `git commit` later in this same
516 
      // process doesn't need to prompt.
517 
      await sh_run(
518 
        1,
519 
        "bash",
520 
        "-c",
521 
        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`,
522 
      );
523 
    }
524
525 
    await sh_run(undefined, "git", "config", "user.signingkey", keyId);
526 
    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
527 
    console.log(`GPG signing configured with key ID: ${keyId} `);
528 
  } catch (e) {
529 
    console.error(`Failure while setting GPG key: ${e} `);
530 
    await delete_pgp_keys();
531 
  }
532 
}
533
534 
async function delete_pgp_keys() {
535 
  console.log("deleting gpg keys");
536 
  if (gpgFingerprint) {
537 
    await sh_run(
538 
      undefined,
539 
      "gpg",
540 
      "--batch",
541 
      "--yes",
542 
      "--pinentry-mode",
543 
      "loopback",
544 
      "--delete-secret-key",
545 
      gpgFingerprint,
546 
    );
547 
    await sh_run(
548 
      undefined,
549 
      "gpg",
550 
      "--batch",
551 
      "--yes",
552 
      "--delete-key",
553 
      "--pinentry-mode",
554 
      "loopback",
555 
      gpgFingerprint,
556 
    );
557 
  }
558 
}
559
560 
async function get_gh_app_token() {
561 
  const workspace = process.env["WM_WORKSPACE"];
562 
  const jobToken = process.env["WM_TOKEN"];
563
564 
  const baseUrl =
565 
    process.env["BASE_INTERNAL_URL"] ??
566 
    process.env["BASE_URL"] ??
567 
    "http://localhost:8000";
568
569 
  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
570
571 
  const response = await fetch(url, {
572 
    method: "POST",
573 
    headers: {
574 
      "Content-Type": "application/json",
575 
      Authorization: `Bearer ${jobToken}`,
576 
    },
577 
    body: JSON.stringify({
578 
      job_token: jobToken,
579 
    }),
580 
  });
581
582 
  if (!response.ok) {
583 
    const errorBody = await response.text().catch(() => "");
584 
    throw new Error(
585 
      `GitHub App token error (${response.status}): ${errorBody || response.statusText}`,
586 
    );
587 
  }
588
589 
  const data = await response.json();
590
591 
  return data.token;
592 
}
593
594 
function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
595 
  if (!gitHubUrl || !installationToken) {
596 
    throw new Error("Both GitHub URL and Installation Token are required.");
597 
  }
598
599 
  const url = new URL(gitHubUrl);
600 
  return `https://x-access-token:${installationToken}@${url.hostname}${url.pathname}`;
601 
}
602
Other submissions
  • Submitted by hugo697 Deno
    Created 689 days ago
    All edits
    Permalink
    1 
    import * as wmillclient from "npm:[email protected]";
    2 
    import wmill from "https://deno.land/x/[email protected]/main.ts";
    3 
    import { basename } from "https://deno.land/[email protected]/path/mod.ts";
    4
    5 
    export async function main(
    6 
      workspace_id: string,
    7 
      repo_url_resource_path: string,
    8 
      path_type:
    9 
        | "script"
    10 
        | "flow"
    11 
        | "app"
    12 
        | "folder"
    13 
        | "resource"
    14 
        | "variable"
    15 
        | "resourcetype"
    16 
        | "schedule"
    17 
        | "user"
    18 
        | "group",
    19 
      skip_secret = true,
    20 
      path: string | undefined,
    21 
      parent_path: string | undefined,
    22 
      commit_msg: string,
    23 
      use_individual_branch = false,
    24 
      group_by_folder = false
    25 
    ) {
    26 
      const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    27
    28 
      const cwd = Deno.cwd();
    29 
      Deno.env.set("HOME", ".");
    30 
      console.log(
    31 
        `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    32 
      );
    33
    34 
      const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    35 
      await move_to_git_branch(
    36 
        workspace_id,
    37 
        path_type,
    38 
        path,
    39 
        parent_path,
    40 
        use_individual_branch,
    41 
        group_by_folder
    42 
      );
    43
    44 
      const subfolder = repo_resource.folder ?? "";
    45 
      const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    46 
      console.log(
    47 
        `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    48 
      );
    49
    50 
      await wmill_sync_pull(
    51 
        path_type,
    52 
        workspace_id,
    53 
        path,
    54 
        parent_path,
    55 
        skip_secret
    56 
      );
    57
    58 
      await git_push(path, parent_path, commit_msg);
    59
    60 
      console.log("Finished syncing");
    61 
      Deno.chdir(`${cwd}`);
    62 
    }
    63
    64 
    async function git_clone(
    65 
      cwd: string,
    66 
      repo_resource: any,
    67 
      use_individual_branch: boolean
    68 
    ): Promise<string> {
    69 
      // TODO: handle private SSH keys as well
    70 
      let repo_url = repo_resource.url;
    71 
      const subfolder = repo_resource.folder ?? "";
    72 
      const branch = repo_resource.branch ?? "";
    73 
      const repo_name = basename(repo_url, ".git");
    74
    75 
      const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    76
    77 
      if (azureMatch) {
    78 
        console.log(
    79 
          "Requires Azure DevOps service account access token, requesting..."
    80 
        );
    81 
        const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    82
    83 
        const response = await fetch(
    84 
          `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
    85 
          {
    86 
            method: "POST",
    87 
            body: new URLSearchParams({
    88 
              client_id: azureResource.azureClientId,
    89 
              client_secret: azureResource.azureClientSecret,
    90 
              grant_type: "client_credentials",
    91 
              resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
    92 
            }),
    93 
          }
    94 
        );
    95
    96 
        const { access_token } = await response.json();
    97
    98 
        repo_url = repo_url.replace(azureMatch[0], access_token);
    99 
      }
    100
    101 
      const args = ["clone", "--quiet", "--depth", "1"];
    102 
      if (use_individual_branch) {
    103 
        args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    104 
      }
    105 
      if (subfolder !== "") {
    106 
        args.push("--sparse");
    107 
      }
    108 
      if (branch !== "") {
    109 
        args.push("--branch");
    110 
        args.push(branch);
    111 
      }
    112 
      args.push(repo_url);
    113 
      args.push(repo_name);
    114 
      await sh_run(-1, "git", ...args);
    115
    116 
      try {
    117 
        Deno.chdir(`${cwd}/${repo_name}`);
    118 
      } catch (err) {
    119 
        console.log(
    120 
          `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    121 
        );
    122 
        throw err;
    123 
      }
    124 
      Deno.chdir(`${cwd}/${repo_name}`);
    125 
      if (subfolder !== "") {
    126 
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    127 
      }
    128 
      try {
    129 
        Deno.chdir(`${cwd}/${repo_name}/${subfolder}`);
    130 
      } catch (err) {
    131 
        console.log(
    132 
          `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
    133 
        );
    134 
        throw err;
    135 
      }
    136
    137 
      return repo_name;
    138 
    }
    139
    140 
    async function move_to_git_branch(
    141 
      workspace_id: string,
    142 
      path_type:
    143 
        | "script"
    144 
        | "flow"
    145 
        | "app"
    146 
        | "folder"
    147 
        | "resource"
    148 
        | "variable"
    149 
        | "resourcetype"
    150 
        | "schedule"
    151 
        | "user"
    152 
        | "group",
    153 
      path: string | undefined,
    154 
      parent_path: string | undefined,
    155 
      use_individual_branch: boolean,
    156 
      group_by_folder: boolean
    157 
    ) {
    158 
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
    159 
        return;
    160 
      }
    161
    162 
      const branchName = group_by_folder
    163 
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
    164 
            ?.split("/")
    165 
            .slice(0, 2)
    166 
            .join("__")}`
    167 
        : `wm_deploy/${workspace_id}/${path_type}/${(
    168 
            path ?? parent_path
    169 
          )?.replaceAll("/", "__")}`;
    170
    171 
      try {
    172 
        await sh_run(undefined, "git", "checkout", branchName);
    173 
      } catch (err) {
    174 
        console.log(
    175 
          `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    176 
        );
    177 
        try {
    178 
          await sh_run(undefined, "git", "checkout", "-b", branchName);
    179 
          await sh_run(
    180 
            undefined,
    181 
            "git",
    182 
            "config",
    183 
            "--add",
    184 
            "--bool",
    185 
            "push.autoSetupRemote",
    186 
            "true"
    187 
          );
    188 
        } catch (err) {
    189 
          console.log(
    190 
            `Error checking out branch '${branchName}'. Error was:\n${err}`
    191 
          );
    192 
          throw err;
    193 
        }
    194 
      }
    195 
      console.log(`Successfully switched to branch ${branchName}`);
    196 
    }
    197
    198 
    async function git_push(
    199 
      path: string | undefined,
    200 
      parent_path: string | undefined,
    201 
      commit_msg: string
    202 
    ) {
    203 
      await sh_run(
    204 
        undefined,
    205 
        "git",
    206 
        "config",
    207 
        "user.email",
    208 
        Deno.env.get("WM_EMAIL") ?? ""
    209 
      );
    210 
      await sh_run(
    211 
        undefined,
    212 
        "git",
    213 
        "config",
    214 
        "user.name",
    215 
        Deno.env.get("WM_USERNAME") ?? ""
    216 
      );
    217 
      if (path !== undefined && path !== null && path !== "") {
    218 
        try {
    219 
          await sh_run(undefined, "git", "add", `${path}**`);
    220 
        } catch (e) {
    221 
          console.log(`Unable to stage files matching ${path}**, ${e}`);
    222 
        }
    223 
      }
    224 
      if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    225 
        try {
    226 
          await sh_run(undefined, "git", "add", `${parent_path}**`);
    227 
        } catch (e) {
    228 
          console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    229 
        }
    230 
      }
    231 
      try {
    232 
        await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    233 
      } catch {
    234 
        // git diff returns exit-code = 1 when there's at least one staged changes
    235 
        await sh_run(undefined, "git", "commit", "-m", commit_msg);
    236 
        try {
    237 
            await sh_run(undefined, "git", "push", "--porcelain");
    238 
        } catch {
    239 
            console.log("Could not push, trying to rebase first");
    240 
            await sh_run(undefined, "git", "pull", "--rebase");
    241 
            await sh_run(undefined, "git", "push", "--porcelain");
    242 
        }
    243 
        return;
    244 
      }
    245 
      console.log("No changes detected, nothing to commit. Returning...");
    246 
    }
    247
    248 
    async function sh_run(
    249 
      secret_position: number | undefined,
    250 
      cmd: string,
    251 
      ...args: string[]
    252 
    ) {
    253 
      const nargs = secret_position != undefined ? args.slice() : args;
    254 
      if (secret_position < 0) {
    255 
        secret_position = nargs.length - 1 + secret_position;
    256 
      }
    257 
      if (secret_position != undefined) {
    258 
        nargs[secret_position] = "***";
    259 
      }
    260 
      console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    261 
      const command = new Deno.Command(cmd, {
    262 
        args: args,
    263 
      });
    264
    265 
      const { code, stdout, stderr } = await command.output();
    266 
      if (stdout.length > 0) {
    267 
        console.log(new TextDecoder().decode(stdout));
    268 
      }
    269 
      if (stderr.length > 0) {
    270 
        console.log(new TextDecoder().decode(stderr));
    271 
      }
    272 
      if (code !== 0) {
    273 
        const err = `SH command '${cmd} ${nargs.join(
    274 
          " "
    275 
        )}' returned with a non-zero status ${code}.`;
    276 
        throw Error(err);
    277 
      }
    278 
      console.log("Command successfully executed");
    279 
    }
    280
    281 
    function regexFromPath(
    282 
      path_type:
    283 
        | "script"
    284 
        | "flow"
    285 
        | "app"
    286 
        | "folder"
    287 
        | "resource"
    288 
        | "variable"
    289 
        | "resourcetype"
    290 
        | "schedule"
    291 
        | "user"
    292 
        | "group",
    293 
      path: string
    294 
    ) {
    295 
      if (path_type == "flow") {
    296 
        return `${path}.flow/*`;
    297 
      } if (path_type == "app") {
    298 
        return `${path}.app/*`;
    299 
      } else if (path_type == "folder") {
    300 
        return `${path}/folder.meta.*`;
    301 
      } else if (path_type == "resourcetype") {
    302 
        return `${path}.resource-type.*`;
    303 
      } else if (path_type == "resource") {
    304 
        return `${path}.resource.*`;
    305 
      } else if (path_type == "variable") {
    306 
        return `${path}.variable.*`;
    307 
      } else if (path_type == "schedule") {
    308 
        return `${path}.schedule.*`;
    309 
      } else if (path_type == "user") {
    310 
        return `${path}.user.*`;
    311 
      } else if (path_type == "group") {
    312 
        return `${path}.group.*`;
    313 
      } else {
    314 
        return `${path}.*`;
    315 
      }
    316 
    }
    317 
    async function wmill_sync_pull(
    318 
      path_type:
    319 
        | "script"
    320 
        | "flow"
    321 
        | "app"
    322 
        | "folder"
    323 
        | "resource"
    324 
        | "variable"
    325 
        | "resourcetype"
    326 
        | "schedule"
    327 
        | "user"
    328 
        | "group",
    329 
      workspace_id: string,
    330 
      path: string | undefined,
    331 
      parent_path: string | undefined,
    332 
      skip_secret: boolean
    333 
    ) {
    334 
      const includes = [];
    335 
      if (path !== undefined && path !== null && path !== "") {
    336 
        includes.push(regexFromPath(path_type, path));
    337 
      }
    338 
      if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    339 
        includes.push(regexFromPath(path_type, parent_path));
    340 
      }
    341
    342 
      await wmill_run(
    343 
        6,
    344 
        "workspace",
    345 
        "add",
    346 
        workspace_id,
    347 
        workspace_id,
    348 
        Deno.env.get("BASE_INTERNAL_URL") + "/",
    349 
        "--token",
    350 
        Deno.env.get("WM_TOKEN") ?? ""
    351 
      );
    352 
      console.log("Pulling workspace into git repo");
    353 
      await wmill_run(
    354 
        3,
    355 
        "sync",
    356 
        "pull",
    357 
        "--token",
    358 
        Deno.env.get("WM_TOKEN") ?? "",
    359 
        "--workspace",
    360 
        workspace_id,
    361 
        "--yes",
    362 
        "--raw",
    363 
        skip_secret ? "--skip-secrets" : "",
    364 
        "--include-schedules",
    365 
        "--include-users",
    366 
        "--include-groups",
    367 
        "--includes",
    368 
        includes.join(",")
    369 
      );
    370 
    }
    371
    372 
    async function wmill_run(secret_position: number, ...cmd: string[]) {
    373 
      cmd = cmd.filter((elt) => elt !== "");
    374 
      const cmd2 = cmd.slice();
    375 
      cmd2[secret_position] = "***";
    376 
      console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    377 
      await wmill.parse(cmd);
    378 
      console.log("Command successfully executed");
    379 
    }
    380
  • Submitted by rubenfiszel Bun
    Created 182 days ago
    All edits
    Permalink
    1 
    import * as wmillclient from "windmill-client";
    2 
    import wmill from "[email protected]";
    3 
    import { basename } from "node:path";
    4 
    const util = require("util");
    5 
    const exec = util.promisify(require("child_process").exec);
    6 
    import process from "process";
    7
    8 
    type GpgKey = {
    9 
      email: string;
    10 
      private_key: string;
    11 
      passphrase: string;
    12 
    };
    13
    14 
    const FORKED_WORKSPACE_PREFIX = "wm-fork-";
    15 
    const FORKED_BRANCH_PREFIX = "wm-fork";
    16
    17 
    type PathType =
    18 
      | "script"
    19 
      | "flow"
    20 
      | "app"
    21 
      | "folder"
    22 
      | "resource"
    23 
      | "variable"
    24 
      | "resourcetype"
    25 
      | "schedule"
    26 
      | "user"
    27 
      | "group"
    28 
      | "httptrigger"
    29 
      | "websockettrigger"
    30 
      | "kafkatrigger"
    31 
      | "natstrigger"
    32 
      | "postgrestrigger"
    33 
      | "mqtttrigger"
    34 
      | "sqstrigger"
    35 
      | "gcptrigger"
    36 
      | "emailtrigger";
    37
    38 
    let gpgFingerprint: string | undefined = undefined;
    39
    40 
    export async function main(
    41 
      workspace_id: string,
    42 
      repo_url_resource_path: string,
    43 
      path_type: PathType,
    44 
      skip_secret = true,
    45 
      path: string | undefined,
    46 
      parent_path: string | undefined,
    47 
      commit_msg: string,
    48 
      parent_workspace_id?: string,
    49 
      use_individual_branch = false,
    50 
      group_by_folder = false,
    51 
      only_create_branch: boolean = false
    52 
    ) {
    53 
      let safeDirectoryPath: string | undefined;
    54 
      const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    55 
      const cwd = process.cwd();
    56 
      process.env["HOME"] = ".";
    57 
      if (!only_create_branch) {
    58 
        console.log(
    59 
          `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    60 
        );
    61 
      }
    62
    63 
      if (repo_resource.is_github_app) {
    64 
        const token = await get_gh_app_token();
    65 
        const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
    66 
        repo_resource.url = authRepoUrl;
    67 
      }
    68
    69 
      const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    70 
      safeDirectoryPath = cloneSafeDirectoryPath;
    71
    72
    73 
      // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    74 
      // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    75 
      // settings, but we need to infer it from the workspace id
    76
    77 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    78 
        if (use_individual_branch) {
    79 
          console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
    80 
          use_individual_branch = false;
    81 
        }
    82 
        if (group_by_folder) {
    83 
          console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
    84 
          group_by_folder = false;
    85 
        }
    86 
      }
    87
    88 
      if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    89 
        const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
    90 
        console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
    91 
        await move_to_git_branch(
    92 
          parent_workspace_id,
    93 
          path_type,
    94 
          path,
    95 
          parent_path,
    96 
          use_individual_branch,
    97 
          group_by_folder,
    98 
          clonedBranchName
    99 
        );
    100 
      }
    101
    102 
      await move_to_git_branch(
    103 
        workspace_id,
    104 
        path_type,
    105 
        path,
    106 
        parent_path,
    107 
        use_individual_branch,
    108 
        group_by_folder,
    109 
        clonedBranchName
    110 
      );
    111
    112
    113 
      const subfolder = repo_resource.folder ?? "";
    114 
      const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    115 
      console.log(
    116 
        `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    117 
      );
    118
    119 
      // If we want to just create the branch, we can skip pulling the changes.
    120 
      if (!only_create_branch) {
    121 
        await wmill_sync_pull(
    122 
          path_type,
    123 
          workspace_id,
    124 
          path,
    125 
          parent_path,
    126 
          skip_secret,
    127 
          repo_url_resource_path,
    128 
          use_individual_branch,
    129 
          repo_resource.branch
    130 
        );
    131 
      }
    132 
      try {
    133 
        await git_push(path, parent_path, commit_msg, repo_resource, only_create_branch);
    134 
      } catch (e) {
    135 
        throw e;
    136 
      } finally {
    137 
        await delete_pgp_keys();
    138 
        // Cleanup: remove safe.directory config
    139 
        if (safeDirectoryPath) {
    140 
          try {
    141 
            await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
    142 
          } catch (e) {
    143 
            console.log(`Warning: Could not unset safe.directory config: ${e}`);
    144 
          }
    145 
        }
    146 
      }
    147 
      console.log("Finished syncing");
    148 
      process.chdir(`${cwd}`);
    149 
    }
    150
    151 
    function get_fork_branch_name(w_id: string, originalBranch: string): string {
    152 
      if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    153 
        return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    154 
      }
    155 
      return w_id;
    156 
    }
    157
    158 
    async function git_clone(
    159 
      cwd: string,
    160 
      repo_resource: any,
    161 
      no_single_branch: boolean,
    162 
    ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    163 
      // TODO: handle private SSH keys as well
    164 
      let repo_url = repo_resource.url;
    165 
      const subfolder = repo_resource.folder ?? "";
    166 
      const branch = repo_resource.branch ?? "";
    167 
      const repo_name = basename(repo_url, ".git");
    168 
      const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    169 
      if (azureMatch) {
    170 
        console.log(
    171 
          "Requires Azure DevOps service account access token, requesting..."
    172 
        );
    173 
        const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    174 
        const response = await fetch(
    175 
          `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
    176 
          {
    177 
            method: "POST",
    178 
            body: new URLSearchParams({
    179 
              client_id: azureResource.azureClientId,
    180 
              client_secret: azureResource.azureClientSecret,
    181 
              grant_type: "client_credentials",
    182 
              resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
    183 
            }),
    184 
          }
    185 
        );
    186 
        const { access_token } = await response.json();
    187 
        repo_url = repo_url.replace(azureMatch[0], access_token);
    188 
      }
    189 
      const args = ["clone", "--quiet", "--depth", "1"];
    190 
      if (no_single_branch) {
    191 
        args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    192 
      }
    193 
      if (subfolder !== "") {
    194 
        args.push("--sparse");
    195 
      }
    196 
      if (branch !== "") {
    197 
        args.push("--branch");
    198 
        args.push(branch);
    199 
      }
    200 
      args.push(repo_url);
    201 
      args.push(repo_name);
    202 
      await sh_run(-1, "git", ...args);
    203 
      try {
    204 
        process.chdir(`${cwd}/${repo_name}`);
    205 
        const safeDirectoryPath = process.cwd();
    206 
        // Add safe.directory to handle dubious ownership in cloned repo
    207 
        try {
    208 
          await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
    209 
        } catch (e) {
    210 
          console.log(`Warning: Could not add safe.directory config: ${e}`);
    211 
        }
    212
    213 
        if (subfolder !== "") {
    214 
          await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    215 
          try {
    216 
            process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    217 
          } catch (err) {
    218 
            console.log(
    219 
              `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
    220 
            );
    221 
            throw err;
    222 
          }
    223 
        }
    224 
        const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
    225 
        return { repo_name, safeDirectoryPath, clonedBranchName };
    226
    227 
      } catch (err) {
    228 
        console.log(
    229 
          `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    230 
        );
    231 
        throw err;
    232 
      }
    233 
    }
    234 
    async function move_to_git_branch(
    235 
      workspace_id: string,
    236 
      path_type: PathType,
    237 
      path: string | undefined,
    238 
      parent_path: string | undefined,
    239 
      use_individual_branch: boolean,
    240 
      group_by_folder: boolean,
    241 
      originalBranchName: string
    242 
    ) {
    243 
      let branchName;
    244 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    245 
        branchName = get_fork_branch_name(workspace_id, originalBranchName);
    246 
      } else {
    247 
        if (!use_individual_branch || path_type === "user" || path_type === "group") {
    248 
          return;
    249 
        }
    250 
        branchName = group_by_folder
    251 
          ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
    252 
            ?.split("/")
    253 
            .slice(0, 2)
    254 
            .join("__")}`
    255 
          : `wm_deploy/${workspace_id}/${path_type}/${(
    256 
            path ?? parent_path
    257 
          )?.replaceAll("/", "__")}`;
    258 
      }
    259
    260 
      try {
    261 
        await sh_run(undefined, "git", "checkout", branchName);
    262 
      } catch (err) {
    263 
        console.log(
    264 
          `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    265 
        );
    266 
        try {
    267 
          await sh_run(undefined, "git", "checkout", "-b", branchName);
    268 
          await sh_run(
    269 
            undefined,
    270 
            "git",
    271 
            "config",
    272 
            "--add",
    273 
            "--bool",
    274 
            "push.autoSetupRemote",
    275 
            "true"
    276 
          );
    277 
        } catch (err) {
    278 
          console.log(
    279 
            `Error checking out branch '${branchName}'. Error was:\n${err}`
    280 
          );
    281 
          throw err;
    282 
        }
    283 
      }
    284 
      console.log(`Successfully switched to branch ${branchName}`);
    285 
    }
    286 
    async function git_push(
    287 
      path: string | undefined,
    288 
      parent_path: string | undefined,
    289 
      commit_msg: string,
    290 
      repo_resource: any,
    291 
      only_create_branch: boolean,
    292 
    ) {
    293 
      let user_email = process.env["WM_EMAIL"] ?? "";
    294 
      let user_name = process.env["WM_USERNAME"] ?? "";
    295
    296 
      if (repo_resource.gpg_key) {
    297 
        await set_gpg_signing_secret(repo_resource.gpg_key);
    298 
        // Configure git with GPG key email for signing
    299 
        await sh_run(
    300 
          undefined,
    301 
          "git",
    302 
          "config",
    303 
          "user.email",
    304 
          repo_resource.gpg_key.email
    305 
        );
    306 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    307 
      } else {
    308 
        await sh_run(undefined, "git", "config", "user.email", user_email);
    309 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    310 
      }
    311 
      if (only_create_branch) {
    312 
        await sh_run(undefined, "git", "push", "--porcelain");
    313 
      }
    314
    315 
      if (path !== undefined && path !== null && path !== "") {
    316 
        try {
    317 
          await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
    318 
        } catch (e) {
    319 
          console.log(`Unable to stage files matching ${path}**, ${e}`);
    320 
        }
    321 
      }
    322 
      if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    323 
        try {
    324 
          await sh_run(
    325 
            undefined,
    326 
            "git",
    327 
            "add",
    328 
            "wmill-lock.yaml",
    329 
            `${parent_path}**`
    330 
          );
    331 
        } catch (e) {
    332 
          console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    333 
        }
    334 
      }
    335 
      try {
    336 
        await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    337 
      } catch {
    338 
        // git diff returns exit-code = 1 when there's at least one staged changes
    339 
        const commitArgs = ["git", "commit"];
    340
    341 
        // Always use --author to set consistent authorship
    342 
        commitArgs.push("--author", `"${user_name} <${user_email}>"`);
    343 
        commitArgs.push(
    344 
          "-m",
    345 
          `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
    346 
        );
    347
    348 
        await sh_run(undefined, ...commitArgs);
    349 
        try {
    350 
          await sh_run(undefined, "git", "push", "--porcelain");
    351 
        } catch (e) {
    352 
          console.log(`Could not push, trying to rebase first: ${e}`);
    353 
          await sh_run(undefined, "git", "pull", "--rebase");
    354 
          await sh_run(undefined, "git", "push", "--porcelain");
    355 
        }
    356 
        return;
    357 
      }
    358 
      console.log("No changes detected, nothing to commit. Returning...");
    359 
    }
    360 
    async function sh_run(
    361 
      secret_position: number | undefined,
    362 
      cmd: string,
    363 
      ...args: string[]
    364 
    ) {
    365 
      const nargs = secret_position != undefined ? args.slice() : args;
    366 
      if (secret_position && secret_position < 0) {
    367 
        secret_position = nargs.length - 1 + secret_position;
    368 
      }
    369 
      let secret: string | undefined = undefined;
    370 
      if (secret_position != undefined) {
    371 
        nargs[secret_position] = "***";
    372 
        secret = args[secret_position];
    373 
      }
    374
    375 
      console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    376 
      const command = exec(`${cmd} ${args.join(" ")}`);
    377 
      // new Deno.Command(cmd, {
    378 
      //   args: args,
    379 
      // });
    380 
      try {
    381 
        const { stdout, stderr } = await command;
    382 
        if (stdout.length > 0) {
    383 
          console.log(stdout);
    384 
        }
    385 
        if (stderr.length > 0) {
    386 
          console.log(stderr);
    387 
        }
    388 
        console.log("Command successfully executed");
    389 
        return stdout;
    390 
      } catch (error) {
    391 
        let errorString = error.toString();
    392 
        if (secret) {
    393 
          errorString = errorString.replace(secret, "***");
    394 
        }
    395 
        const err = `SH command '${cmd} ${nargs.join(
    396 
          " "
    397 
        )}' returned with error ${errorString}`;
    398 
        throw Error(err);
    399 
      }
    400 
    }
    401
    402 
    function regexFromPath(path_type: PathType, path: string) {
    403 
      if (path_type == "flow") {
    404 
        return `${path}.flow/*`;
    405 
      }
    406 
      if (path_type == "app") {
    407 
        return `${path}.app/*`;
    408 
      } else if (path_type == "folder") {
    409 
        return `${path}/folder.meta.*`;
    410 
      } else if (path_type == "resourcetype") {
    411 
        return `${path}.resource-type.*`;
    412 
      } else if (path_type == "resource") {
    413 
        return `${path}.resource.*`;
    414 
      } else if (path_type == "variable") {
    415 
        return `${path}.variable.*`;
    416 
      } else if (path_type == "schedule") {
    417 
        return `${path}.schedule.*`;
    418 
      } else if (path_type == "user") {
    419 
        return `${path}.user.*`;
    420 
      } else if (path_type == "group") {
    421 
        return `${path}.group.*`;
    422 
      } else if (path_type == "httptrigger") {
    423 
        return `${path}.http_trigger.*`;
    424 
      } else if (path_type == "websockettrigger") {
    425 
        return `${path}.websocket_trigger.*`;
    426 
      } else if (path_type == "kafkatrigger") {
    427 
        return `${path}.kafka_trigger.*`;
    428 
      } else if (path_type == "natstrigger") {
    429 
        return `${path}.nats_trigger.*`;
    430 
      } else if (path_type == "postgrestrigger") {
    431 
        return `${path}.postgres_trigger.*`;
    432 
      } else if (path_type == "mqtttrigger") {
    433 
        return `${path}.mqtt_trigger.*`;
    434 
      } else if (path_type == "sqstrigger") {
    435 
        return `${path}.sqs_trigger.*`;
    436 
      } else if (path_type == "gcptrigger") {
    437 
        return `${path}.gcp_trigger.*`;
    438 
      } else if (path_type == "emailtrigger") {
    439 
        return `${path}.email_trigger.*`;
    440 
      } else {
    441 
        return `${path}.*`;
    442 
      }
    443 
    }
    444
    445 
    async function wmill_sync_pull(
    446 
      path_type: PathType,
    447 
      workspace_id: string,
    448 
      path: string | undefined,
    449 
      parent_path: string | undefined,
    450 
      skip_secret: boolean,
    451 
      repo_url_resource_path: string,
    452 
      use_individual_branch: boolean,
    453 
      original_branch?: string
    454 
    ) {
    455 
      const includes = [];
    456 
      if (path !== undefined && path !== null && path !== "") {
    457 
        includes.push(regexFromPath(path_type, path));
    458 
      }
    459 
      if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    460 
        includes.push(regexFromPath(path_type, parent_path));
    461 
      }
    462
    463 
      console.log("Pulling workspace into git repo");
    464
    465 
      const args = [
    466 
        "sync",
    467 
        "pull",
    468 
        "--token",
    469 
        process.env["WM_TOKEN"] ?? "",
    470 
        "--workspace",
    471 
        workspace_id,
    472 
        "--base-url",
    473 
        process.env["BASE_URL"] + "/",
    474 
        "--repository",
    475 
        repo_url_resource_path,
    476 
        "--yes",
    477 
        skip_secret ? "--skip-secrets" : "",
    478 
      ];
    479
    480 
      if (path_type === "schedule" && !use_individual_branch) {
    481 
        args.push("--include-schedules");
    482 
      }
    483
    484 
      if (path_type === "group" && !use_individual_branch) {
    485 
        args.push("--include-groups");
    486 
      }
    487
    488 
      if (path_type === "user" && !use_individual_branch) {
    489 
        args.push("--include-users");
    490 
      }
    491
    492 
      if (path_type.includes("trigger") && !use_individual_branch) {
    493 
        args.push("--include-triggers");
    494 
      }
    495 
      // Only include settings when specifically deploying settings
    496 
      if (path_type === "settings" && !use_individual_branch) {
    497 
        args.push("--include-settings");
    498 
      }
    499
    500 
      // Only include key when specifically deploying keys
    501 
      if (path_type === "key" && !use_individual_branch) {
    502 
        args.push("--include-key");
    503 
      }
    504
    505 
      args.push("--extra-includes", includes.join(","));
    506
    507 
      // If using individual branches, apply promotion settings from original branch
    508 
      if (use_individual_branch && original_branch) {
    509 
        console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
    510 
        args.push("--promotion", original_branch);
    511 
      }
    512
    513 
      await wmill_run(3, ...args);
    514 
    }
    515
    516 
    async function wmill_run(secret_position: number, ...cmd: string[]) {
    517 
      cmd = cmd.filter((elt) => elt !== "");
    518 
      const cmd2 = cmd.slice();
    519 
      cmd2[secret_position] = "***";
    520 
      console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    521 
      await wmill.parse(cmd);
    522 
      console.log("Command successfully executed");
    523 
    }
    524
    525 
    // Function to set up GPG signing
    526 
    async function set_gpg_signing_secret(gpg_key: GpgKey) {
    527 
      try {
    528 
        console.log("Setting GPG private key for git commits");
    529
    530 
        const formattedGpgContent = gpg_key.private_key.replace(
    531 
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
    532 
          (_: string, header: string, body: string, footer: string) =>
    533 
            header +
    534 
            "\n" +
    535 
            "\n" +
    536 
            body.replace(/ ([^\s])/g, "\n$1").trim() +
    537 
            "\n" +
    538 
            footer
    539 
        );
    540
    541 
        const gpg_path = `/tmp/gpg`;
    542 
        await sh_run(undefined, "mkdir", "-p", gpg_path);
    543 
        await sh_run(undefined, "chmod", "700", gpg_path);
    544 
        process.env.GNUPGHOME = gpg_path;
    545 
        // process.env.GIT_TRACE = 1;
    546
    547 
        try {
    548 
          await sh_run(
    549 
            1,
    550 
            "bash",
    551 
            "-c",
    552 
            `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
    553 
          );
    554 
        } catch (e) {
    555 
          // Original error would contain sensitive data
    556 
          throw new Error("Failed to import GPG key!");
    557 
        }
    558
    559 
        const listKeysOutput = await sh_run(
    560 
          undefined,
    561 
          "gpg",
    562 
          "--list-secret-keys",
    563 
          "--with-colons",
    564 
          "--keyid-format=long"
    565 
        );
    566
    567 
        const keyInfoMatch = listKeysOutput.match(
    568 
          /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
    569 
        );
    570
    571 
        if (!keyInfoMatch) {
    572 
          throw new Error("Failed to extract GPG Key ID and Fingerprint");
    573 
        }
    574
    575 
        const keyId = keyInfoMatch[1];
    576 
        gpgFingerprint = keyInfoMatch[2];
    577
    578 
        if (gpg_key.passphrase) {
    579 
          // This is adummy command to unlock the key
    580 
          // with passphrase to load it into agent
    581 
          await sh_run(
    582 
            1,
    583 
            "bash",
    584 
            "-c",
    585 
            `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
    586 
          );
    587 
        }
    588
    589 
        // Configure Git to use the extracted key
    590 
        await sh_run(undefined, "git", "config", "user.signingkey", keyId);
    591 
        await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
    592 
        console.log(`GPG signing configured with key ID: ${keyId} `);
    593 
      } catch (e) {
    594 
        console.error(`Failure while setting GPG key: ${e} `);
    595 
        await delete_pgp_keys();
    596 
      }
    597 
    }
    598
    599 
    async function delete_pgp_keys() {
    600 
      console.log("deleting gpg keys");
    601 
      if (gpgFingerprint) {
    602 
        await sh_run(
    603 
          undefined,
    604 
          "gpg",
    605 
          "--batch",
    606 
          "--yes",
    607 
          "--pinentry-mode",
    608 
          "loopback",
    609 
          "--delete-secret-key",
    610 
          gpgFingerprint
    611 
        );
    612 
        await sh_run(
    613 
          undefined,
    614 
          "gpg",
    615 
          "--batch",
    616 
          "--yes",
    617 
          "--delete-key",
    618 
          "--pinentry-mode",
    619 
          "loopback",
    620 
          gpgFingerprint
    621 
        );
    622 
      }
    623 
    }
    624
    625 
    async function get_gh_app_token() {
    626 
      const workspace = process.env["WM_WORKSPACE"];
    627 
      const jobToken = process.env["WM_TOKEN"];
    628
    629 
      const baseUrl =
    630 
        process.env["BASE_INTERNAL_URL"] ??
    631 
        process.env["BASE_URL"] ??
    632 
        "http://localhost:8000";
    633
    634 
      const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
    635
    636 
      const response = await fetch(url, {
    637 
        method: "POST",
    638 
        headers: {
    639 
          "Content-Type": "application/json",
    640 
          Authorization: `Bearer ${jobToken}`,
    641 
        },
    642 
        body: JSON.stringify({
    643 
          job_token: jobToken,
    644 
        }),
    645 
      });
    646
    647 
      if (!response.ok) {
    648 
        throw new Error(`Error: ${response.statusText}`);
    649 
      }
    650
    651 
      const data = await response.json();
    652
    653 
      return data.token;
    654 
    }
    655
    656 
    function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    657 
      if (!gitHubUrl || !installationToken) {
    658 
        throw new Error("Both GitHub URL and Installation Token are required.");
    659 
      }
    660
    661 
      try {
    662 
        const url = new URL(gitHubUrl);
    663
    664 
        // GitHub repository URL should be in the format: https://github.com/owner/repo.git
    665 
        if (url.hostname !== "github.com") {
    666 
          throw new Error(
    667 
            "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
    668 
          );
    669 
        }
    670
    671 
        // Convert URL to include the installation token
    672 
        return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    673 
      } catch (e) {
    674 
        const error = e as Error;
    675 
        throw new Error(`Invalid URL: ${error.message}`);
    676 
      }
    677 
    }
    678
  • Submitted by pyranota275 Bun
    Created 155 days ago
    All edits
    Permalink
    1 
    import * as wmillclient from "windmill-client";
    2 
    import wmill from "[email protected]";
    3 
    import { basename } from "node:path";
    4 
    const util = require("util");
    5 
    const exec = util.promisify(require("child_process").exec);
    6 
    import process from "process";
    7
    8 
    type GpgKey = {
    9 
      email: string;
    10 
      private_key: string;
    11 
      passphrase: string;
    12 
    };
    13
    14 
    const FORKED_WORKSPACE_PREFIX = "wm-fork-";
    15 
    const FORKED_BRANCH_PREFIX = "wm-fork";
    16
    17 
    type PathType =
    18 
      | "script"
    19 
      | "flow"
    20 
      | "app"
    21 
      | "folder"
    22 
      | "resource"
    23 
      | "variable"
    24 
      | "resourcetype"
    25 
      | "schedule"
    26 
      | "user"
    27 
      | "group"
    28 
      | "httptrigger"
    29 
      | "websockettrigger"
    30 
      | "kafkatrigger"
    31 
      | "natstrigger"
    32 
      | "postgrestrigger"
    33 
      | "mqtttrigger"
    34 
      | "sqstrigger"
    35 
      | "gcptrigger"
    36 
      | "emailtrigger";
    37
    38 
    type SyncObject = {
    39 
      path_type: PathType;
    40 
      path: string | undefined;
    41 
      parent_path: string | undefined;
    42 
      commit_msg: string;
    43 
    };
    44
    45 
    let gpgFingerprint: string | undefined = undefined;
    46
    47 
    export async function main(
    48 
      items: SyncObject[],
    49 
      // Compat, do not use in code, rely on `items` instead
    50 
      path_type: PathType | undefined,
    51 
      path: string | undefined,
    52 
      parent_path: string | undefined,
    53 
      commit_msg: string | undefined,
    54 
      //
    55 
      workspace_id: string,
    56 
      repo_url_resource_path: string,
    57 
      skip_secret: boolean = true,
    58 
      use_individual_branch: boolean = false,
    59 
      group_by_folder: boolean = false,
    60 
      only_create_branch: boolean = false,
    61 
      parent_workspace_id?: string,
    62 
    ) {
    63
    64 
      if (path_type !== undefined && commit_msg !== undefined) {
    65 
        items = [{
    66 
          path_type,
    67 
          path,
    68 
          parent_path,
    69 
          commit_msg,
    70 
        }];
    71 
      }
    72 
      await inner(items, workspace_id, repo_url_resource_path, skip_secret, use_individual_branch, group_by_folder, only_create_branch, parent_workspace_id);
    73 
    }
    74
    75 
    async function inner(
    76 
      items: SyncObject[],
    77 
      workspace_id: string,
    78 
      repo_url_resource_path: string,
    79 
      skip_secret: boolean = true,
    80 
      use_individual_branch: boolean = false,
    81 
      group_by_folder: boolean = false,
    82 
      only_create_branch: boolean = false,
    83 
      parent_workspace_id?: string,
    84 
    ) {
    85
    86 
      let safeDirectoryPath: string | undefined;
    87 
      const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    88 
      const cwd = process.cwd();
    89 
      process.env["HOME"] = ".";
    90 
      if (!only_create_branch) {
    91 
        for (const item of items) {
    92 
          console.log(
    93 
            `Syncing ${item.path_type} ${item.path ?? ""} with parent ${item.parent_path ?? ""}`
    94 
          );
    95 
        }
    96 
      }
    97
    98 
      if (repo_resource.is_github_app) {
    99 
        const token = await get_gh_app_token();
    100 
        const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
    101 
        repo_resource.url = authRepoUrl;
    102 
      }
    103
    104 
      const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    105 
      safeDirectoryPath = cloneSafeDirectoryPath;
    106
    107
    108 
      // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    109 
      // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    110 
      // settings, but we need to infer it from the workspace id
    111
    112 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    113 
        if (use_individual_branch) {
    114 
          console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
    115 
          use_individual_branch = false;
    116 
        }
    117 
        if (group_by_folder) {
    118 
          console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
    119 
          group_by_folder = false;
    120 
        }
    121 
      }
    122
    123 
      if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    124 
        const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
    125 
        console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
    126 
        await git_checkout_branch(
    127 
          items,
    128 
          parent_workspace_id,
    129 
          use_individual_branch,
    130 
          group_by_folder,
    131 
          clonedBranchName
    132 
        );
    133 
      }
    134
    135 
      await git_checkout_branch(
    136 
        items,
    137 
        workspace_id,
    138 
        use_individual_branch,
    139 
        group_by_folder,
    140 
        clonedBranchName
    141 
      );
    142
    143
    144 
      const subfolder = repo_resource.folder ?? "";
    145 
      const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    146 
      console.log(
    147 
        `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    148 
      );
    149
    150 
      // If we want to just create the branch, we can skip pulling the changes.
    151 
      if (!only_create_branch) {
    152 
        await wmill_sync_pull(
    153 
          items,
    154 
          workspace_id,
    155 
          skip_secret,
    156 
          repo_url_resource_path,
    157 
          use_individual_branch,
    158 
          repo_resource.branch
    159 
        );
    160 
      }
    161 
      try {
    162 
        await git_push(items, repo_resource, only_create_branch);
    163 
      } catch (e) {
    164 
        throw e;
    165 
      } finally {
    166 
        await delete_pgp_keys();
    167 
        // Cleanup: remove safe.directory config
    168 
        if (safeDirectoryPath) {
    169 
          try {
    170 
            await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
    171 
          } catch (e) {
    172 
            console.log(`Warning: Could not unset safe.directory config: ${e}`);
    173 
          }
    174 
        }
    175 
      }
    176 
      console.log("Finished syncing");
    177 
      process.chdir(`${cwd}`);
    178 
    }
    179
    180
    181 
    function get_fork_branch_name(w_id: string, originalBranch: string): string {
    182 
      if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    183 
        return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    184 
      }
    185 
      return w_id;
    186 
    }
    187
    188 
    async function git_clone(
    189 
      cwd: string,
    190 
      repo_resource: any,
    191 
      no_single_branch: boolean,
    192 
    ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    193 
      // TODO: handle private SSH keys as well
    194 
      let repo_url = repo_resource.url;
    195 
      const subfolder = repo_resource.folder ?? "";
    196 
      const branch = repo_resource.branch ?? "";
    197 
      const repo_name = basename(repo_url, ".git");
    198 
      const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    199 
      if (azureMatch) {
    200 
        console.log(
    201 
          "Requires Azure DevOps service account access token, requesting..."
    202 
        );
    203 
        const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    204 
        const response = await fetch(
    205 
          `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
    206 
          {
    207 
            method: "POST",
    208 
            body: new URLSearchParams({
    209 
              client_id: azureResource.azureClientId,
    210 
              client_secret: azureResource.azureClientSecret,
    211 
              grant_type: "client_credentials",
    212 
              resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
    213 
            }),
    214 
          }
    215 
        );
    216 
        const { access_token } = await response.json();
    217 
        repo_url = repo_url.replace(azureMatch[0], access_token);
    218 
      }
    219 
      const args = ["clone", "--quiet", "--depth", "1"];
    220 
      if (no_single_branch) {
    221 
        args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    222 
      }
    223 
      if (subfolder !== "") {
    224 
        args.push("--sparse");
    225 
      }
    226 
      if (branch !== "") {
    227 
        args.push("--branch");
    228 
        args.push(branch);
    229 
      }
    230 
      args.push(repo_url);
    231 
      args.push(repo_name);
    232 
      await sh_run(-1, "git", ...args);
    233 
      try {
    234 
        process.chdir(`${cwd}/${repo_name}`);
    235 
        const safeDirectoryPath = process.cwd();
    236 
        // Add safe.directory to handle dubious ownership in cloned repo
    237 
        try {
    238 
          await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
    239 
        } catch (e) {
    240 
          console.log(`Warning: Could not add safe.directory config: ${e}`);
    241 
        }
    242
    243 
        if (subfolder !== "") {
    244 
          await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    245 
          try {
    246 
            process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    247 
          } catch (err) {
    248 
            console.log(
    249 
              `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
    250 
            );
    251 
            throw err;
    252 
          }
    253 
        }
    254 
        const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
    255 
        return { repo_name, safeDirectoryPath, clonedBranchName };
    256
    257 
      } catch (err) {
    258 
        console.log(
    259 
          `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    260 
        );
    261 
        throw err;
    262 
      }
    263 
    }
    264 
    async function git_checkout_branch(
    265 
      items: SyncObject[],
    266 
      workspace_id: string,
    267 
      use_individual_branch: boolean,
    268 
      group_by_folder: boolean,
    269 
      originalBranchName: string
    270 
    ) {
    271 
      let branchName;
    272 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    273 
        branchName = get_fork_branch_name(workspace_id, originalBranchName);
    274 
      } else {
    275
    276 
        if (!use_individual_branch
    277 
          // If individual branch is true, we can assume items is of length 1, as debouncing is disabled for jobs with this flag
    278 
          || items[0].path_type === "user" || items[0].path_type === "group") {
    279 
          return;
    280 
        }
    281
    282 
        // as mentioned above, it is safe to assume that items.len is 1
    283 
        const [path, parent_path] = [items[0].path, items[0].parent_path];
    284 
        branchName = group_by_folder
    285 
          ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
    286 
            ?.split("/")
    287 
            .slice(0, 2)
    288 
            .join("__")}`
    289 
          : `wm_deploy/${workspace_id}/${items[0].path_type}/${(
    290 
            path ?? parent_path
    291 
          )?.replaceAll("/", "__")}`;
    292 
      }
    293
    294 
      try {
    295 
        await sh_run(undefined, "git", "checkout", branchName);
    296 
      } catch (err) {
    297 
        console.log(
    298 
          `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    299 
        );
    300 
        try {
    301 
          await sh_run(undefined, "git", "checkout", "-b", branchName);
    302 
          await sh_run(
    303 
            undefined,
    304 
            "git",
    305 
            "config",
    306 
            "--add",
    307 
            "--bool",
    308 
            "push.autoSetupRemote",
    309 
            "true"
    310 
          );
    311 
        } catch (err) {
    312 
          console.log(
    313 
            `Error checking out branch '${branchName}'. Error was:\n${err}`
    314 
          );
    315 
          throw err;
    316 
        }
    317 
      }
    318 
      console.log(`Successfully switched to branch ${branchName}`);
    319 
    }
    320
    321 
    function composeCommitHeader(items: SyncObject[]): string {
    322 
      // Count occurrences of each path_type
    323 
      const typeCounts = new Map<PathType, number>();
    324 
      for (const item of items) {
    325 
        typeCounts.set(item.path_type, (typeCounts.get(item.path_type) ?? 0) + 1);
    326 
      }
    327
    328 
      // Sort by count descending to get the top 2
    329 
      const sortedTypes = Array.from(typeCounts.entries()).sort((a, b) => b[1] - a[1]);
    330
    331 
      const parts: string[] = [];
    332 
      let othersCount = 0;
    333
    334 
      for (let i = 0; i < sortedTypes.length; i++) {
    335 
        const [pathType, count] = sortedTypes[i];
    336 
        if (i < 3) {
    337 
          // Pluralize the path type if count > 1
    338 
          const label = count > 1 ? `${pathType}s` : pathType;
    339
    340 
          if (i == 2 && sortedTypes.length == 3) {
    341 
            parts.push(`and ${count} ${label}`);
    342 
          } else {
    343 
            parts.push(`${count} ${label}`);
    344 
          }
    345 
        } else {
    346 
          othersCount += count;
    347 
        }
    348 
      }
    349
    350 
      let header = `[WM] Deployed ${parts.join(", ")}`;
    351 
      if (othersCount > 0) {
    352 
        header += ` and ${othersCount} other object${othersCount > 1 ? "s" : ""}`;
    353 
      }
    354
    355 
      return header;
    356 
    }
    357
    358 
    async function git_push(
    359 
      items: SyncObject[],
    360 
      repo_resource: any,
    361 
      only_create_branch: boolean,
    362 
    ) {
    363 
      let user_email = process.env["WM_EMAIL"] ?? "";
    364 
      let user_name = process.env["WM_USERNAME"] ?? "";
    365
    366 
      if (repo_resource.gpg_key) {
    367 
        await set_gpg_signing_secret(repo_resource.gpg_key);
    368 
        // Configure git with GPG key email for signing
    369 
        await sh_run(
    370 
          undefined,
    371 
          "git",
    372 
          "config",
    373 
          "user.email",
    374 
          repo_resource.gpg_key.email
    375 
        );
    376 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    377 
      } else {
    378 
        await sh_run(undefined, "git", "config", "user.email", user_email);
    379 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    380 
      }
    381 
      if (only_create_branch) {
    382 
        await sh_run(undefined, "git", "push", "--porcelain");
    383 
      }
    384
    385 
      let commit_description: string[] = [];
    386 
      for (const { path, parent_path, commit_msg } of items) {
    387 
        if (path !== undefined && path !== null && path !== "") {
    388 
          try {
    389 
            await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
    390 
          } catch (e) {
    391 
            console.log(`Unable to stage files matching ${path}**, ${e}`);
    392 
          }
    393 
        }
    394 
        if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    395 
          try {
    396 
            await sh_run(
    397 
              undefined,
    398 
              "git",
    399 
              "add",
    400 
              "wmill-lock.yaml",
    401 
              `${parent_path}**`
    402 
            );
    403 
          } catch (e) {
    404 
            console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    405 
          }
    406 
        }
    407
    408 
        commit_description.push(commit_msg);
    409 
      }
    410
    411 
      try {
    412 
        await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    413 
      } catch {
    414 
        // git diff returns exit-code = 1 when there's at least one staged changes
    415 
        const commitArgs = ["git", "commit"];
    416
    417 
        // Always use --author to set consistent authorship
    418 
        commitArgs.push("--author", `"${user_name} <${user_email}>"`);
    419
    420 
        const [header, description] = (commit_description.length == 1)
    421 
          ? [commit_description[0], ""]
    422 
          : [composeCommitHeader(items), commit_description.join("\n")];
    423
    424 
        commitArgs.push(
    425 
          "-m",
    426 
          `"${header == undefined || header == "" ? "no commit msg" : header}"`,
    427 
          "-m",
    428 
          `"${description}"`
    429 
        );
    430
    431 
        await sh_run(undefined, ...commitArgs);
    432 
        try {
    433 
          await sh_run(undefined, "git", "push", "--porcelain");
    434 
        } catch (e) {
    435 
          console.log(`Could not push, trying to rebase first: ${e}`);
    436 
          await sh_run(undefined, "git", "pull", "--rebase");
    437 
          await sh_run(undefined, "git", "push", "--porcelain");
    438 
        }
    439 
        return;
    440 
      }
    441
    442 
      console.log("No changes detected, nothing to commit. Returning...");
    443 
    }
    444 
    async function sh_run(
    445 
      secret_position: number | undefined,
    446 
      cmd: string,
    447 
      ...args: string[]
    448 
    ) {
    449 
      const nargs = secret_position != undefined ? args.slice() : args;
    450 
      if (secret_position && secret_position < 0) {
    451 
        secret_position = nargs.length - 1 + secret_position;
    452 
      }
    453 
      let secret: string | undefined = undefined;
    454 
      if (secret_position != undefined) {
    455 
        nargs[secret_position] = "***";
    456 
        secret = args[secret_position];
    457 
      }
    458
    459 
      console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    460 
      const command = exec(`${cmd} ${args.join(" ")}`);
    461 
      // new Deno.Command(cmd, {
    462 
      //   args: args,
    463 
      // });
    464 
      try {
    465 
        const { stdout, stderr } = await command;
    466 
        if (stdout.length > 0) {
    467 
          console.log(stdout);
    468 
        }
    469 
        if (stderr.length > 0) {
    470 
          console.log(stderr);
    471 
        }
    472 
        console.log("Command successfully executed");
    473 
        return stdout;
    474 
      } catch (error) {
    475 
        let errorString = error.toString();
    476 
        if (secret) {
    477 
          errorString = errorString.replace(secret, "***");
    478 
        }
    479 
        const err = `SH command '${cmd} ${nargs.join(
    480 
          " "
    481 
        )}' returned with error ${errorString}`;
    482 
        throw Error(err);
    483 
      }
    484 
    }
    485
    486 
    function regexFromPath(path_type: PathType, path: string) {
    487 
      if (path_type == "flow") {
    488 
        return `${path}.flow/*`;
    489 
      }
    490 
      if (path_type == "app") {
    491 
        return `${path}.app/*`;
    492 
      } else if (path_type == "folder") {
    493 
        return `${path}/folder.meta.*`;
    494 
      } else if (path_type == "resourcetype") {
    495 
        return `${path}.resource-type.*`;
    496 
      } else if (path_type == "resource") {
    497 
        return `${path}.resource.*`;
    498 
      } else if (path_type == "variable") {
    499 
        return `${path}.variable.*`;
    500 
      } else if (path_type == "schedule") {
    501 
        return `${path}.schedule.*`;
    502 
      } else if (path_type == "user") {
    503 
        return `${path}.user.*`;
    504 
      } else if (path_type == "group") {
    505 
        return `${path}.group.*`;
    506 
      } else if (path_type == "httptrigger") {
    507 
        return `${path}.http_trigger.*`;
    508 
      } else if (path_type == "websockettrigger") {
    509 
        return `${path}.websocket_trigger.*`;
    510 
      } else if (path_type == "kafkatrigger") {
    511 
        return `${path}.kafka_trigger.*`;
    512 
      } else if (path_type == "natstrigger") {
    513 
        return `${path}.nats_trigger.*`;
    514 
      } else if (path_type == "postgrestrigger") {
    515 
        return `${path}.postgres_trigger.*`;
    516 
      } else if (path_type == "mqtttrigger") {
    517 
        return `${path}.mqtt_trigger.*`;
    518 
      } else if (path_type == "sqstrigger") {
    519 
        return `${path}.sqs_trigger.*`;
    520 
      } else if (path_type == "gcptrigger") {
    521 
        return `${path}.gcp_trigger.*`;
    522 
      } else if (path_type == "emailtrigger") {
    523 
        return `${path}.email_trigger.*`;
    524 
      } else {
    525 
        return `${path}.*`;
    526 
      }
    527 
    }
    528
    529 
    async function wmill_sync_pull(
    530 
      items: SyncObject[],
    531 
      workspace_id: string,
    532 
      skip_secret: boolean,
    533 
      repo_url_resource_path: string,
    534 
      use_individual_branch: boolean,
    535 
      original_branch?: string
    536 
    ) {
    537 
      const includes = [];
    538 
      for (const item of items) {
    539 
        const { path_type, path, parent_path } = item;
    540 
        if (path !== undefined && path !== null && path !== "") {
    541 
          includes.push(regexFromPath(path_type, path));
    542 
        }
    543 
        if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    544 
          includes.push(regexFromPath(path_type, parent_path));
    545 
        }
    546 
      }
    547
    548 
      console.log("Pulling workspace into git repo");
    549
    550 
      const args = [
    551 
        "sync",
    552 
        "pull",
    553 
        "--token",
    554 
        process.env["WM_TOKEN"] ?? "",
    555 
        "--workspace",
    556 
        workspace_id,
    557 
        "--base-url",
    558 
        process.env["BASE_URL"] + "/",
    559 
        "--repository",
    560 
        repo_url_resource_path,
    561 
        "--yes",
    562 
        skip_secret ? "--skip-secrets" : "",
    563 
      ];
    564
    565 
      if (items.some(item => item.path_type === "schedule") && !use_individual_branch) {
    566 
        args.push("--include-schedules");
    567 
      }
    568
    569 
      if (items.some(item => item.path_type === "group") && !use_individual_branch) {
    570 
        args.push("--include-groups");
    571 
      }
    572
    573 
      if (items.some(item => item.path_type === "user") && !use_individual_branch) {
    574 
        args.push("--include-users");
    575 
      }
    576
    577 
      if (items.some(item => item.path_type.includes("trigger")) && !use_individual_branch) {
    578 
        args.push("--include-triggers");
    579 
      }
    580 
      // Only include settings when specifically deploying settings
    581 
      if (items.some(item => item.path_type === "settings") && !use_individual_branch) {
    582 
        args.push("--include-settings");
    583 
      }
    584
    585 
      // Only include key when specifically deploying keys
    586 
      if (items.some(item => item.path_type === "key") && !use_individual_branch) {
    587 
        args.push("--include-key");
    588 
      }
    589
    590 
      args.push("--extra-includes", includes.join(","));
    591
    592 
      // If using individual branches, apply promotion settings from original branch
    593 
      if (use_individual_branch && original_branch) {
    594 
        console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
    595 
        args.push("--promotion", original_branch);
    596 
      }
    597
    598 
      await wmill_run(3, ...args);
    599 
    }
    600
    601 
    async function wmill_run(secret_position: number, ...cmd: string[]) {
    602 
      cmd = cmd.filter((elt) => elt !== "");
    603 
      const cmd2 = cmd.slice();
    604 
      cmd2[secret_position] = "***";
    605 
      console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    606 
      await wmill.parse(cmd);
    607 
      console.log("Command successfully executed");
    608 
    }
    609
    610 
    // Function to set up GPG signing
    611 
    async function set_gpg_signing_secret(gpg_key: GpgKey) {
    612 
      try {
    613 
        console.log("Setting GPG private key for git commits");
    614
    615 
        const formattedGpgContent = gpg_key.private_key.replace(
    616 
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
    617 
          (_: string, header: string, body: string, footer: string) =>
    618 
            header +
    619 
            "\n" +
    620 
            "\n" +
    621 
            body.replace(/ ([^\s])/g, "\n$1").trim() +
    622 
            "\n" +
    623 
            footer
    624 
        );
    625
    626 
        const gpg_path = `/tmp/gpg`;
    627 
        await sh_run(undefined, "mkdir", "-p", gpg_path);
    628 
        await sh_run(undefined, "chmod", "700", gpg_path);
    629 
        process.env.GNUPGHOME = gpg_path;
    630 
        // process.env.GIT_TRACE = 1;
    631
    632 
        try {
    633 
          await sh_run(
    634 
            1,
    635 
            "bash",
    636 
            "-c",
    637 
            `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
    638 
          );
    639 
        } catch (e) {
    640 
          // Original error would contain sensitive data
    641 
          throw new Error("Failed to import GPG key!");
    642 
        }
    643
    644 
        const listKeysOutput = await sh_run(
    645 
          undefined,
    646 
          "gpg",
    647 
          "--list-secret-keys",
    648 
          "--with-colons",
    649 
          "--keyid-format=long"
    650 
        );
    651
    652 
        const keyInfoMatch = listKeysOutput.match(
    653 
          /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
    654 
        );
    655
    656 
        if (!keyInfoMatch) {
    657 
          throw new Error("Failed to extract GPG Key ID and Fingerprint");
    658 
        }
    659
    660 
        const keyId = keyInfoMatch[1];
    661 
        gpgFingerprint = keyInfoMatch[2];
    662
    663 
        if (gpg_key.passphrase) {
    664 
          // This is adummy command to unlock the key
    665 
          // with passphrase to load it into agent
    666 
          await sh_run(
    667 
            1,
    668 
            "bash",
    669 
            "-c",
    670 
            `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
    671 
          );
    672 
        }
    673
    674 
        // Configure Git to use the extracted key
    675 
        await sh_run(undefined, "git", "config", "user.signingkey", keyId);
    676 
        await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
    677 
        console.log(`GPG signing configured with key ID: ${keyId} `);
    678 
      } catch (e) {
    679 
        console.error(`Failure while setting GPG key: ${e} `);
    680 
        await delete_pgp_keys();
    681 
      }
    682 
    }
    683
    684 
    async function delete_pgp_keys() {
    685 
      console.log("deleting gpg keys");
    686 
      if (gpgFingerprint) {
    687 
        await sh_run(
    688 
          undefined,
    689 
          "gpg",
    690 
          "--batch",
    691 
          "--yes",
    692 
          "--pinentry-mode",
    693 
          "loopback",
    694 
          "--delete-secret-key",
    695 
          gpgFingerprint
    696 
        );
    697 
        await sh_run(
    698 
          undefined,
    699 
          "gpg",
    700 
          "--batch",
    701 
          "--yes",
    702 
          "--delete-key",
    703 
          "--pinentry-mode",
    704 
          "loopback",
    705 
          gpgFingerprint
    706 
        );
    707 
      }
    708 
    }
    709
    710 
    async function get_gh_app_token() {
    711 
      const workspace = process.env["WM_WORKSPACE"];
    712 
      const jobToken = process.env["WM_TOKEN"];
    713
    714 
      const baseUrl =
    715 
        process.env["BASE_INTERNAL_URL"] ??
    716 
        process.env["BASE_URL"] ??
    717 
        "http://localhost:8000";
    718
    719 
      const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
    720
    721 
      const response = await fetch(url, {
    722 
        method: "POST",
    723 
        headers: {
    724 
          "Content-Type": "application/json",
    725 
          Authorization: `Bearer ${jobToken}`,
    726 
        },
    727 
        body: JSON.stringify({
    728 
          job_token: jobToken,
    729 
        }),
    730 
      });
    731
    732 
      if (!response.ok) {
    733 
        throw new Error(`Error: ${response.statusText}`);
    734 
      }
    735
    736 
      const data = await response.json();
    737
    738 
      return data.token;
    739 
    }
    740
    741 
    function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    742 
      if (!gitHubUrl || !installationToken) {
    743 
        throw new Error("Both GitHub URL and Installation Token are required.");
    744 
      }
    745
    746 
      try {
    747 
        const url = new URL(gitHubUrl);
    748
    749 
        // GitHub repository URL should be in the format: https://github.com/owner/repo.git
    750 
        if (url.hostname !== "github.com") {
    751 
          throw new Error(
    752 
            "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
    753 
          );
    754 
        }
    755
    756 
        // Convert URL to include the installation token
    757 
        return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    758 
      } catch (e) {
    759 
        const error = e as Error;
    760 
        throw new Error(`Invalid URL: ${error.message}`);
    761 
      }
    762 
    }
  • Submitted by pyranota275 Bun
    Created 156 days ago
    All edits
    Permalink
    1 
    import * as wmillclient from "windmill-client";
    2 
    import wmill from "[email protected]";
    3 
    import { basename } from "node:path";
    4 
    const util = require("util");
    5 
    const exec = util.promisify(require("child_process").exec);
    6 
    import process from "process";
    7
    8 
    type GpgKey = {
    9 
      email: string;
    10 
      private_key: string;
    11 
      passphrase: string;
    12 
    };
    13
    14 
    const FORKED_WORKSPACE_PREFIX = "wm-fork-";
    15 
    const FORKED_BRANCH_PREFIX = "wm-fork";
    16
    17 
    type PathType =
    18 
      | "script"
    19 
      | "flow"
    20 
      | "app"
    21 
      | "folder"
    22 
      | "resource"
    23 
      | "variable"
    24 
      | "resourcetype"
    25 
      | "schedule"
    26 
      | "user"
    27 
      | "group"
    28 
      | "httptrigger"
    29 
      | "websockettrigger"
    30 
      | "kafkatrigger"
    31 
      | "natstrigger"
    32 
      | "postgrestrigger"
    33 
      | "mqtttrigger"
    34 
      | "sqstrigger"
    35 
      | "gcptrigger"
    36 
      | "emailtrigger";
    37
    38 
    type SyncObject = {
    39 
      path_type: PathType;
    40 
      path: string | undefined;
    41 
      parent_path: string | undefined;
    42 
      commit_msg: string;
    43 
    };
    44
    45 
    let gpgFingerprint: string | undefined = undefined;
    46
    47 
    export async function main(
    48 
      items: SyncObject[],
    49 
      // Compat, do not use in code, rely on `items` instead
    50 
      path_type: PathType | undefined,
    51 
      path: string | undefined,
    52 
      parent_path: string | undefined,
    53 
      commit_msg: string | undefined,
    54 
      //
    55 
      workspace_id: string,
    56 
      repo_url_resource_path: string,
    57 
      skip_secret: boolean = true,
    58 
      use_individual_branch: boolean = false,
    59 
      group_by_folder: boolean = false,
    60 
      only_create_branch: boolean = false,
    61 
      parent_workspace_id?: string,
    62 
    ) {
    63
    64 
      if (path_type !== undefined && commit_msg !== undefined) {
    65 
        items = [{
    66 
          path_type,
    67 
          path,
    68 
          parent_path,
    69 
          commit_msg,
    70 
        }];
    71 
      }
    72 
      await inner(items, workspace_id, repo_url_resource_path, skip_secret, use_individual_branch, group_by_folder, only_create_branch, parent_workspace_id);
    73 
    }
    74
    75 
    async function inner(
    76 
      items: SyncObject[],
    77 
      workspace_id: string,
    78 
      repo_url_resource_path: string,
    79 
      skip_secret: boolean = true,
    80 
      use_individual_branch: boolean = false,
    81 
      group_by_folder: boolean = false,
    82 
      only_create_branch: boolean = false,
    83 
      parent_workspace_id?: string,
    84 
    ) {
    85
    86 
      let safeDirectoryPath: string | undefined;
    87 
      const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    88 
      const cwd = process.cwd();
    89 
      process.env["HOME"] = ".";
    90 
      if (!only_create_branch) {
    91 
        for (const item of items) {
    92 
          console.log(
    93 
            `Syncing ${item.path_type} ${item.path ?? ""} with parent ${item.parent_path ?? ""}`
    94 
          );
    95 
        }
    96 
      }
    97
    98 
      if (repo_resource.is_github_app) {
    99 
        const token = await get_gh_app_token();
    100 
        const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
    101 
        repo_resource.url = authRepoUrl;
    102 
      }
    103
    104 
      const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    105 
      safeDirectoryPath = cloneSafeDirectoryPath;
    106
    107
    108 
      // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    109 
      // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    110 
      // settings, but we need to infer it from the workspace id
    111
    112 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    113 
        if (use_individual_branch) {
    114 
          console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
    115 
          use_individual_branch = false;
    116 
        }
    117 
        if (group_by_folder) {
    118 
          console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
    119 
          group_by_folder = false;
    120 
        }
    121 
      }
    122
    123 
      if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    124 
        const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
    125 
        console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
    126 
        await git_checkout_branch(
    127 
          items,
    128 
          parent_workspace_id,
    129 
          use_individual_branch,
    130 
          group_by_folder,
    131 
          clonedBranchName
    132 
        );
    133 
      }
    134
    135 
      await git_checkout_branch(
    136 
        items,
    137 
        workspace_id,
    138 
        use_individual_branch,
    139 
        group_by_folder,
    140 
        clonedBranchName
    141 
      );
    142
    143
    144 
      const subfolder = repo_resource.folder ?? "";
    145 
      const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    146 
      console.log(
    147 
        `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    148 
      );
    149
    150 
      // If we want to just create the branch, we can skip pulling the changes.
    151 
      if (!only_create_branch) {
    152 
        await wmill_sync_pull(
    153 
          items,
    154 
          workspace_id,
    155 
          skip_secret,
    156 
          repo_url_resource_path,
    157 
          use_individual_branch,
    158 
          repo_resource.branch
    159 
        );
    160 
      }
    161 
      try {
    162 
        await git_push(items, repo_resource, only_create_branch);
    163 
      } catch (e) {
    164 
        throw e;
    165 
      } finally {
    166 
        await delete_pgp_keys();
    167 
        // Cleanup: remove safe.directory config
    168 
        if (safeDirectoryPath) {
    169 
          try {
    170 
            await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
    171 
          } catch (e) {
    172 
            console.log(`Warning: Could not unset safe.directory config: ${e}`);
    173 
          }
    174 
        }
    175 
      }
    176 
      console.log("Finished syncing");
    177 
      process.chdir(`${cwd}`);
    178 
    }
    179
    180
    181 
    function get_fork_branch_name(w_id: string, originalBranch: string): string {
    182 
      if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    183 
        return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    184 
      }
    185 
      return w_id;
    186 
    }
    187
    188 
    async function git_clone(
    189 
      cwd: string,
    190 
      repo_resource: any,
    191 
      no_single_branch: boolean,
    192 
    ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    193 
      // TODO: handle private SSH keys as well
    194 
      let repo_url = repo_resource.url;
    195 
      const subfolder = repo_resource.folder ?? "";
    196 
      const branch = repo_resource.branch ?? "";
    197 
      const repo_name = basename(repo_url, ".git");
    198 
      const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    199 
      if (azureMatch) {
    200 
        console.log(
    201 
          "Requires Azure DevOps service account access token, requesting..."
    202 
        );
    203 
        const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    204 
        const response = await fetch(
    205 
          `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
    206 
          {
    207 
            method: "POST",
    208 
            body: new URLSearchParams({
    209 
              client_id: azureResource.azureClientId,
    210 
              client_secret: azureResource.azureClientSecret,
    211 
              grant_type: "client_credentials",
    212 
              resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
    213 
            }),
    214 
          }
    215 
        );
    216 
        const { access_token } = await response.json();
    217 
        repo_url = repo_url.replace(azureMatch[0], access_token);
    218 
      }
    219 
      const args = ["clone", "--quiet", "--depth", "1"];
    220 
      if (no_single_branch) {
    221 
        args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    222 
      }
    223 
      if (subfolder !== "") {
    224 
        args.push("--sparse");
    225 
      }
    226 
      if (branch !== "") {
    227 
        args.push("--branch");
    228 
        args.push(branch);
    229 
      }
    230 
      args.push(repo_url);
    231 
      args.push(repo_name);
    232 
      await sh_run(-1, "git", ...args);
    233 
      try {
    234 
        process.chdir(`${cwd}/${repo_name}`);
    235 
        const safeDirectoryPath = process.cwd();
    236 
        // Add safe.directory to handle dubious ownership in cloned repo
    237 
        try {
    238 
          await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
    239 
        } catch (e) {
    240 
          console.log(`Warning: Could not add safe.directory config: ${e}`);
    241 
        }
    242
    243 
        if (subfolder !== "") {
    244 
          await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    245 
          try {
    246 
            process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    247 
          } catch (err) {
    248 
            console.log(
    249 
              `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
    250 
            );
    251 
            throw err;
    252 
          }
    253 
        }
    254 
        const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
    255 
        return { repo_name, safeDirectoryPath, clonedBranchName };
    256
    257 
      } catch (err) {
    258 
        console.log(
    259 
          `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    260 
        );
    261 
        throw err;
    262 
      }
    263 
    }
    264 
    async function git_checkout_branch(
    265 
      items: SyncObject[],
    266 
      workspace_id: string,
    267 
      use_individual_branch: boolean,
    268 
      group_by_folder: boolean,
    269 
      originalBranchName: string
    270 
    ) {
    271 
      let branchName;
    272 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    273 
        branchName = get_fork_branch_name(workspace_id, originalBranchName);
    274 
      } else {
    275
    276 
        if (!use_individual_branch
    277 
          // If individual branch is true, we can assume items is of length 1, as debouncing is disabled for jobs with this flag
    278 
          || items[0].path_type === "user" || items[0].path_type === "group") {
    279 
          return;
    280 
        }
    281
    282 
        // as mentioned above, it is safe to assume that items.len is 1
    283 
        const [path, parent_path] = [items[0].path, items[0].parent_path];
    284 
        branchName = group_by_folder
    285 
          ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
    286 
            ?.split("/")
    287 
            .slice(0, 2)
    288 
            .join("__")}`
    289 
          : `wm_deploy/${workspace_id}/${items[0].path_type}/${(
    290 
            path ?? parent_path
    291 
          )?.replaceAll("/", "__")}`;
    292 
      }
    293
    294 
      try {
    295 
        await sh_run(undefined, "git", "checkout", branchName);
    296 
      } catch (err) {
    297 
        console.log(
    298 
          `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    299 
        );
    300 
        try {
    301 
          await sh_run(undefined, "git", "checkout", "-b", branchName);
    302 
          await sh_run(
    303 
            undefined,
    304 
            "git",
    305 
            "config",
    306 
            "--add",
    307 
            "--bool",
    308 
            "push.autoSetupRemote",
    309 
            "true"
    310 
          );
    311 
        } catch (err) {
    312 
          console.log(
    313 
            `Error checking out branch '${branchName}'. Error was:\n${err}`
    314 
          );
    315 
          throw err;
    316 
        }
    317 
      }
    318 
      console.log(`Successfully switched to branch ${branchName}`);
    319 
    }
    320
    321 
    async function git_push(
    322 
      items: SyncObject[],
    323 
      repo_resource: any,
    324 
      only_create_branch: boolean,
    325 
    ) {
    326 
      let user_email = process.env["WM_EMAIL"] ?? "";
    327 
      let user_name = process.env["WM_USERNAME"] ?? "";
    328
    329 
      if (repo_resource.gpg_key) {
    330 
        await set_gpg_signing_secret(repo_resource.gpg_key);
    331 
        // Configure git with GPG key email for signing
    332 
        await sh_run(
    333 
          undefined,
    334 
          "git",
    335 
          "config",
    336 
          "user.email",
    337 
          repo_resource.gpg_key.email
    338 
        );
    339 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    340 
      } else {
    341 
        await sh_run(undefined, "git", "config", "user.email", user_email);
    342 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    343 
      }
    344 
      if (only_create_branch) {
    345 
        await sh_run(undefined, "git", "push", "--porcelain");
    346 
      }
    347
    348 
      let commit_description: string[] = [];
    349 
      for (const { path, parent_path, commit_msg } of items) {
    350 
        if (path !== undefined && path !== null && path !== "") {
    351 
          try {
    352 
            await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
    353 
          } catch (e) {
    354 
            console.log(`Unable to stage files matching ${path}**, ${e}`);
    355 
          }
    356 
        }
    357 
        if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    358 
          try {
    359 
            await sh_run(
    360 
              undefined,
    361 
              "git",
    362 
              "add",
    363 
              "wmill-lock.yaml",
    364 
              `${parent_path}**`
    365 
            );
    366 
          } catch (e) {
    367 
            console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    368 
          }
    369 
        }
    370
    371 
        commit_description.push(commit_msg);
    372 
      }
    373
    374 
      try {
    375 
        await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    376 
      } catch {
    377 
        // git diff returns exit-code = 1 when there's at least one staged changes
    378 
        const commitArgs = ["git", "commit"];
    379
    380 
        // Always use --author to set consistent authorship
    381 
        commitArgs.push("--author", `"${user_name} <${user_email}>"`);
    382
    383 
        const [header, description] = (commit_description.length == 1) ? [commit_description[0], ""] : [`[WM]: Deployed ${commit_description.length} objects`, commit_description.join("\n")];
    384
    385 
        commitArgs.push(
    386 
          "-m",
    387 
          `"${header == undefined || header == "" ? "no commit msg" : header}"`,
    388 
          "-m",
    389 
          `"${description}"`
    390 
        );
    391
    392 
        await sh_run(undefined, ...commitArgs);
    393 
        try {
    394 
          await sh_run(undefined, "git", "push", "--porcelain");
    395 
        } catch (e) {
    396 
          console.log(`Could not push, trying to rebase first: ${e}`);
    397 
          await sh_run(undefined, "git", "pull", "--rebase");
    398 
          await sh_run(undefined, "git", "push", "--porcelain");
    399 
        }
    400 
        return;
    401 
      }
    402
    403 
      console.log("No changes detected, nothing to commit. Returning...");
    404 
    }
    405 
    async function sh_run(
    406 
      secret_position: number | undefined,
    407 
      cmd: string,
    408 
      ...args: string[]
    409 
    ) {
    410 
      const nargs = secret_position != undefined ? args.slice() : args;
    411 
      if (secret_position && secret_position < 0) {
    412 
        secret_position = nargs.length - 1 + secret_position;
    413 
      }
    414 
      let secret: string | undefined = undefined;
    415 
      if (secret_position != undefined) {
    416 
        nargs[secret_position] = "***";
    417 
        secret = args[secret_position];
    418 
      }
    419
    420 
      console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    421 
      const command = exec(`${cmd} ${args.join(" ")}`);
    422 
      // new Deno.Command(cmd, {
    423 
      //   args: args,
    424 
      // });
    425 
      try {
    426 
        const { stdout, stderr } = await command;
    427 
        if (stdout.length > 0) {
    428 
          console.log(stdout);
    429 
        }
    430 
        if (stderr.length > 0) {
    431 
          console.log(stderr);
    432 
        }
    433 
        console.log("Command successfully executed");
    434 
        return stdout;
    435 
      } catch (error) {
    436 
        let errorString = error.toString();
    437 
        if (secret) {
    438 
          errorString = errorString.replace(secret, "***");
    439 
        }
    440 
        const err = `SH command '${cmd} ${nargs.join(
    441 
          " "
    442 
        )}' returned with error ${errorString}`;
    443 
        throw Error(err);
    444 
      }
    445 
    }
    446
    447 
    function regexFromPath(path_type: PathType, path: string) {
    448 
      if (path_type == "flow") {
    449 
        return `${path}.flow/*`;
    450 
      }
    451 
      if (path_type == "app") {
    452 
        return `${path}.app/*`;
    453 
      } else if (path_type == "folder") {
    454 
        return `${path}/folder.meta.*`;
    455 
      } else if (path_type == "resourcetype") {
    456 
        return `${path}.resource-type.*`;
    457 
      } else if (path_type == "resource") {
    458 
        return `${path}.resource.*`;
    459 
      } else if (path_type == "variable") {
    460 
        return `${path}.variable.*`;
    461 
      } else if (path_type == "schedule") {
    462 
        return `${path}.schedule.*`;
    463 
      } else if (path_type == "user") {
    464 
        return `${path}.user.*`;
    465 
      } else if (path_type == "group") {
    466 
        return `${path}.group.*`;
    467 
      } else if (path_type == "httptrigger") {
    468 
        return `${path}.http_trigger.*`;
    469 
      } else if (path_type == "websockettrigger") {
    470 
        return `${path}.websocket_trigger.*`;
    471 
      } else if (path_type == "kafkatrigger") {
    472 
        return `${path}.kafka_trigger.*`;
    473 
      } else if (path_type == "natstrigger") {
    474 
        return `${path}.nats_trigger.*`;
    475 
      } else if (path_type == "postgrestrigger") {
    476 
        return `${path}.postgres_trigger.*`;
    477 
      } else if (path_type == "mqtttrigger") {
    478 
        return `${path}.mqtt_trigger.*`;
    479 
      } else if (path_type == "sqstrigger") {
    480 
        return `${path}.sqs_trigger.*`;
    481 
      } else if (path_type == "gcptrigger") {
    482 
        return `${path}.gcp_trigger.*`;
    483 
      } else if (path_type == "emailtrigger") {
    484 
        return `${path}.email_trigger.*`;
    485 
      } else {
    486 
        return `${path}.*`;
    487 
      }
    488 
    }
    489
    490 
    async function wmill_sync_pull(
    491 
      items: SyncObject[],
    492 
      workspace_id: string,
    493 
      skip_secret: boolean,
    494 
      repo_url_resource_path: string,
    495 
      use_individual_branch: boolean,
    496 
      original_branch?: string
    497 
    ) {
    498 
      const includes = [];
    499 
      for (const item of items) {
    500 
        const { path_type, path, parent_path } = item;
    501 
        if (path !== undefined && path !== null && path !== "") {
    502 
          includes.push(regexFromPath(path_type, path));
    503 
        }
    504 
        if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    505 
          includes.push(regexFromPath(path_type, parent_path));
    506 
        }
    507 
      }
    508
    509 
      console.log("Pulling workspace into git repo");
    510
    511 
      const args = [
    512 
        "sync",
    513 
        "pull",
    514 
        "--token",
    515 
        process.env["WM_TOKEN"] ?? "",
    516 
        "--workspace",
    517 
        workspace_id,
    518 
        "--base-url",
    519 
        process.env["BASE_URL"] + "/",
    520 
        "--repository",
    521 
        repo_url_resource_path,
    522 
        "--yes",
    523 
        skip_secret ? "--skip-secrets" : "",
    524 
      ];
    525
    526 
      if (items.some(item => item.path_type === "schedule") && !use_individual_branch) {
    527 
        args.push("--include-schedules");
    528 
      }
    529
    530 
      if (items.some(item => item.path_type === "group") && !use_individual_branch) {
    531 
        args.push("--include-groups");
    532 
      }
    533
    534 
      if (items.some(item => item.path_type === "user") && !use_individual_branch) {
    535 
        args.push("--include-users");
    536 
      }
    537
    538 
      if (items.some(item => item.path_type.includes("trigger")) && !use_individual_branch) {
    539 
        args.push("--include-triggers");
    540 
      }
    541 
      // Only include settings when specifically deploying settings
    542 
      if (items.some(item => item.path_type === "settings") && !use_individual_branch) {
    543 
        args.push("--include-settings");
    544 
      }
    545
    546 
      // Only include key when specifically deploying keys
    547 
      if (items.some(item => item.path_type === "key") && !use_individual_branch) {
    548 
        args.push("--include-key");
    549 
      }
    550
    551 
      args.push("--extra-includes", includes.join(","));
    552
    553 
      // If using individual branches, apply promotion settings from original branch
    554 
      if (use_individual_branch && original_branch) {
    555 
        console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
    556 
        args.push("--promotion", original_branch);
    557 
      }
    558
    559 
      await wmill_run(3, ...args);
    560 
    }
    561
    562 
    async function wmill_run(secret_position: number, ...cmd: string[]) {
    563 
      cmd = cmd.filter((elt) => elt !== "");
    564 
      const cmd2 = cmd.slice();
    565 
      cmd2[secret_position] = "***";
    566 
      console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    567 
      await wmill.parse(cmd);
    568 
      console.log("Command successfully executed");
    569 
    }
    570
    571 
    // Function to set up GPG signing
    572 
    async function set_gpg_signing_secret(gpg_key: GpgKey) {
    573 
      try {
    574 
        console.log("Setting GPG private key for git commits");
    575
    576 
        const formattedGpgContent = gpg_key.private_key.replace(
    577 
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
    578 
          (_: string, header: string, body: string, footer: string) =>
    579 
            header +
    580 
            "\n" +
    581 
            "\n" +
    582 
            body.replace(/ ([^\s])/g, "\n$1").trim() +
    583 
            "\n" +
    584 
            footer
    585 
        );
    586
    587 
        const gpg_path = `/tmp/gpg`;
    588 
        await sh_run(undefined, "mkdir", "-p", gpg_path);
    589 
        await sh_run(undefined, "chmod", "700", gpg_path);
    590 
        process.env.GNUPGHOME = gpg_path;
    591 
        // process.env.GIT_TRACE = 1;
    592
    593 
        try {
    594 
          await sh_run(
    595 
            1,
    596 
            "bash",
    597 
            "-c",
    598 
            `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
    599 
          );
    600 
        } catch (e) {
    601 
          // Original error would contain sensitive data
    602 
          throw new Error("Failed to import GPG key!");
    603 
        }
    604
    605 
        const listKeysOutput = await sh_run(
    606 
          undefined,
    607 
          "gpg",
    608 
          "--list-secret-keys",
    609 
          "--with-colons",
    610 
          "--keyid-format=long"
    611 
        );
    612
    613 
        const keyInfoMatch = listKeysOutput.match(
    614 
          /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
    615 
        );
    616
    617 
        if (!keyInfoMatch) {
    618 
          throw new Error("Failed to extract GPG Key ID and Fingerprint");
    619 
        }
    620
    621 
        const keyId = keyInfoMatch[1];
    622 
        gpgFingerprint = keyInfoMatch[2];
    623
    624 
        if (gpg_key.passphrase) {
    625 
          // This is adummy command to unlock the key
    626 
          // with passphrase to load it into agent
    627 
          await sh_run(
    628 
            1,
    629 
            "bash",
    630 
            "-c",
    631 
            `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
    632 
          );
    633 
        }
    634
    635 
        // Configure Git to use the extracted key
    636 
        await sh_run(undefined, "git", "config", "user.signingkey", keyId);
    637 
        await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
    638 
        console.log(`GPG signing configured with key ID: ${keyId} `);
    639 
      } catch (e) {
    640 
        console.error(`Failure while setting GPG key: ${e} `);
    641 
        await delete_pgp_keys();
    642 
      }
    643 
    }
    644
    645 
    async function delete_pgp_keys() {
    646 
      console.log("deleting gpg keys");
    647 
      if (gpgFingerprint) {
    648 
        await sh_run(
    649 
          undefined,
    650 
          "gpg",
    651 
          "--batch",
    652 
          "--yes",
    653 
          "--pinentry-mode",
    654 
          "loopback",
    655 
          "--delete-secret-key",
    656 
          gpgFingerprint
    657 
        );
    658 
        await sh_run(
    659 
          undefined,
    660 
          "gpg",
    661 
          "--batch",
    662 
          "--yes",
    663 
          "--delete-key",
    664 
          "--pinentry-mode",
    665 
          "loopback",
    666 
          gpgFingerprint
    667 
        );
    668 
      }
    669 
    }
    670
    671 
    async function get_gh_app_token() {
    672 
      const workspace = process.env["WM_WORKSPACE"];
    673 
      const jobToken = process.env["WM_TOKEN"];
    674
    675 
      const baseUrl =
    676 
        process.env["BASE_INTERNAL_URL"] ??
    677 
        process.env["BASE_URL"] ??
    678 
        "http://localhost:8000";
    679
    680 
      const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
    681
    682 
      const response = await fetch(url, {
    683 
        method: "POST",
    684 
        headers: {
    685 
          "Content-Type": "application/json",
    686 
          Authorization: `Bearer ${jobToken}`,
    687 
        },
    688 
        body: JSON.stringify({
    689 
          job_token: jobToken,
    690 
        }),
    691 
      });
    692
    693 
      if (!response.ok) {
    694 
        throw new Error(`Error: ${response.statusText}`);
    695 
      }
    696
    697 
      const data = await response.json();
    698
    699 
      return data.token;
    700 
    }
    701
    702 
    function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    703 
      if (!gitHubUrl || !installationToken) {
    704 
        throw new Error("Both GitHub URL and Installation Token are required.");
    705 
      }
    706
    707 
      try {
    708 
        const url = new URL(gitHubUrl);
    709
    710 
        // GitHub repository URL should be in the format: https://github.com/owner/repo.git
    711 
        if (url.hostname !== "github.com") {
    712 
          throw new Error(
    713 
            "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
    714 
          );
    715 
        }
    716
    717 
        // Convert URL to include the installation token
    718 
        return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    719 
      } catch (e) {
    720 
        const error = e as Error;
    721 
        throw new Error(`Invalid URL: ${error.message}`);
    722 
      }
    723 
    }
    724