logo Windmill Hub
Login
All items

Sync script to Git repo

This script will pull the script from the current workspace in a temporary folder, then commit it to the remote Git repository and push it. Only the script-related file will be pushed, nothing else. It takes as input the `git_repository` resource containing the repository URL, the script path, and the commit message to use for the commit. All params are mandatory.

Script windmill Verified

by hugo697 ยท 12/1/2023

Use in Windmill

The script

Submitted by pyranota275 Bun
Verified 2 days ago
All edits
Permalink
1 
import * as wmillclient from "windmill-client";
2
3 
import wmill from "[email protected]";
4 
import { basename } from "node:path";
5 
const util = require("util");
6 
const exec = util.promisify(require("child_process").exec);
7 
import process from "process";
8
9 
type GpgKey = {
10 
  email: string;
11 
  private_key: string;
12 
  passphrase: string;
13 
};
14
15 
const FORKED_WORKSPACE_PREFIX = "wm-fork-";
16
17 
type PathType =
18 
  | "script"
19 
  | "flow"
20 
  | "app"
21 
  | "raw_app"
22 
  | "folder"
23 
  | "resource"
24 
  | "variable"
25 
  | "resourcetype"
26 
  | "schedule"
27 
  | "user"
28 
  | "group"
29 
  | "httptrigger"
30 
  | "websockettrigger"
31 
  | "kafkatrigger"
32 
  | "natstrigger"
33 
  | "postgrestrigger"
34 
  | "mqtttrigger"
35 
  | "sqstrigger"
36 
  | "gcptrigger"
37 
  | "azuretrigger"
38 
  | "emailtrigger";
39
40 
type SyncObject = {
41 
  path_type: PathType;
42 
  path: string | undefined;
43 
  parent_path: string | undefined;
44 
  commit_msg: string;
45 
};
46
47 
let gpgFingerprint: string | undefined = undefined;
48
49 
export async function main(
50 
  items: SyncObject[],
51 
  // Compat, do not use in code, rely on `items` instead
52 
  path_type: PathType | undefined,
53 
  path: string | undefined,
54 
  parent_path: string | undefined,
55 
  commit_msg: string | undefined,
56 
  //
57 
  workspace_id: string,
58 
  repo_url_resource_path: string,
59 
  skip_secret: boolean = true,
60 
  use_individual_branch: boolean = false,
61 
  group_by_folder: boolean = false,
62 
  only_create_branch: boolean = false,
63 
  parent_workspace_id?: string,
64 
) {
65 
  if (path_type !== undefined && commit_msg !== undefined) {
66 
    items = [
67 
      {
68 
        path_type,
69 
        path,
70 
        parent_path,
71 
        commit_msg,
72 
      },
73 
    ];
74 
  }
75 
  await inner(
76 
    items,
77 
    workspace_id,
78 
    repo_url_resource_path,
79 
    skip_secret,
80 
    use_individual_branch,
81 
    group_by_folder,
82 
    only_create_branch,
83 
    parent_workspace_id,
84 
  );
85 
}
86
87 
// Thin git-sync deployment-callback script.
88 
//
89 
// All git orchestration (wm_deploy/fork branch selection, include/promotion
90 
// derivation, commit, push, fork-disable, parent-of-fork rooting) now lives in
91 
// the bundled CLI's hidden `wmill sync git-deploy`. This script only does the
92 
// parts that need Windmill resource/secret access and must wrap the clone:
93 
// resolve the repo resource, GitHub-App / Azure auth, `git clone`, GPG key
94 
// import, then delegate, then clean up.
95 
async function inner(
96 
  items: SyncObject[],
97 
  workspace_id: string,
98 
  repo_url_resource_path: string,
99 
  skip_secret: boolean = true,
100 
  use_individual_branch: boolean = false,
101 
  group_by_folder: boolean = false,
102 
  only_create_branch: boolean = false,
103 
  parent_workspace_id?: string,
104 
) {
105 
  let safeDirectoryPath: string | undefined;
106 
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
107 
  const cwd = process.cwd();
108 
  process.env["HOME"] = ".";
109 
  if (!only_create_branch) {
110 
    for (const item of items) {
111 
      console.log(
112 
        `Syncing ${item.path_type} ${item.path ?? ""} with parent ${item.parent_path ?? ""}`,
113 
      );
114 
    }
115 
  }
116
117 
  if (repo_resource.is_github_app) {
118 
    const token = await get_gh_app_token();
119 
    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
120 
    repo_resource.url = authRepoUrl;
121 
  }
122
123 
  const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath } =
124 
    await git_clone(
125 
      cwd,
126 
      repo_resource,
127 
      use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX),
128 
    );
129 
  safeDirectoryPath = cloneSafeDirectoryPath;
130
131 
  // GPG signing must be configured (global commit.gpgsign / user.signingkey)
132 
  // BEFORE the CLI commits, so the CLI's commit is signed. The committer
133 
  // email/name are passed through to the CLI so authorship stays identical to
134 
  // the old in-script git_push (committer email = gpg key email when signing).
135 
  if (repo_resource.gpg_key) {
136 
    await set_gpg_signing_secret(repo_resource.gpg_key);
137 
  }
138
139 
  const subfolder = repo_resource.folder ?? "";
140 
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
141 
  console.log(
142 
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`,
143 
  );
144
145 
  try {
146 
    // Raw flags are passed through verbatim; the CLI replicates the
147 
    // fork-disable, parent-of-fork rooting and include/promotion derivation.
148 
    const args = [
149 
      "sync",
150 
      "git-deploy",
151 
      "--token",
152 
      process.env["WM_TOKEN"] ?? "",
153 
      "--workspace",
154 
      workspace_id,
155 
      "--base-url",
156 
      process.env["BASE_URL"] + "/",
157 
      "--repository",
158 
      repo_url_resource_path,
159 
      "--git-deploy-items",
160 
      JSON.stringify(items),
161 
    ];
162 
    if (use_individual_branch) args.push("--use-individual-branch");
163 
    if (group_by_folder) args.push("--group-by-folder");
164 
    if (only_create_branch) args.push("--only-create-branch");
165 
    if (parent_workspace_id) {
166 
      args.push("--parent-workspace-id", parent_workspace_id);
167 
    }
168 
    if (skip_secret) args.push("--skip-secrets");
169 
    if (repo_resource.gpg_key) {
170 
      args.push("--git-committer-email", repo_resource.gpg_key.email);
171 
      args.push("--git-committer-name", process.env["WM_USERNAME"] ?? "");
172 
    }
173 
    await wmill_run(3, ...args);
174 
  } catch (e) {
175 
    throw e;
176 
  } finally {
177 
    await delete_pgp_keys();
178 
    // Cleanup: remove safe.directory config
179 
    if (safeDirectoryPath) {
180 
      try {
181 
        await sh_run(
182 
          undefined,
183 
          "git",
184 
          "config",
185 
          "--global",
186 
          "--unset",
187 
          "safe.directory",
188 
          safeDirectoryPath,
189 
        );
190 
      } catch (e) {
191 
        console.log(`Warning: Could not unset safe.directory config: ${e}`);
192 
      }
193 
    }
194 
  }
195 
  console.log("Finished syncing");
196 
  process.chdir(`${cwd}`);
197 
}
198
199 
async function git_clone(
200 
  cwd: string,
201 
  repo_resource: any,
202 
  no_single_branch: boolean,
203 
): Promise<{
204 
  repo_name: string;
205 
  safeDirectoryPath: string;
206 
  clonedBranchName: string;
207 
}> {
208 
  // TODO: handle private SSH keys as well
209 
  let repo_url = repo_resource.url;
210 
  const subfolder = repo_resource.folder ?? "";
211 
  const branch = repo_resource.branch ?? "";
212 
  const repo_name = basename(repo_url, ".git");
213 
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
214 
  if (azureMatch) {
215 
    console.log(
216 
      "Requires Azure DevOps service account access token, requesting...",
217 
    );
218 
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
219 
    const response = await fetch(
220 
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
221 
      {
222 
        method: "POST",
223 
        body: new URLSearchParams({
224 
          client_id: azureResource.azureClientId,
225 
          client_secret: azureResource.azureClientSecret,
226 
          grant_type: "client_credentials",
227 
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
228 
        }),
229 
      },
230 
    );
231 
    const { access_token } = await response.json();
232 
    repo_url = repo_url.replace(azureMatch[0], access_token);
233 
  }
234 
  const args = ["clone", "--quiet", "--depth", "1"];
235 
  if (no_single_branch) {
236 
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
237 
  }
238 
  if (subfolder !== "") {
239 
    args.push("--sparse");
240 
  }
241 
  if (branch !== "") {
242 
    args.push("--branch");
243 
    args.push(branch);
244 
  }
245 
  args.push(repo_url);
246 
  args.push(repo_name);
247 
  await sh_run(-1, "git", ...args);
248 
  try {
249 
    process.chdir(`${cwd}/${repo_name}`);
250 
    const safeDirectoryPath = process.cwd();
251 
    // Add safe.directory to handle dubious ownership in cloned repo
252 
    try {
253 
      await sh_run(
254 
        undefined,
255 
        "git",
256 
        "config",
257 
        "--global",
258 
        "--add",
259 
        "safe.directory",
260 
        process.cwd(),
261 
      );
262 
    } catch (e) {
263 
      console.log(`Warning: Could not add safe.directory config: ${e}`);
264 
    }
265
266 
    if (subfolder !== "") {
267 
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
268 
      try {
269 
        process.chdir(`${cwd}/${repo_name}/${subfolder}`);
270 
      } catch (err) {
271 
        console.log(
272 
          `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`,
273 
        );
274 
        throw err;
275 
      }
276 
    }
277 
    const clonedBranchName = (
278 
      await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")
279 
    ).trim();
280 
    return { repo_name, safeDirectoryPath, clonedBranchName };
281 
  } catch (err) {
282 
    console.log(
283 
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`,
284 
    );
285 
    throw err;
286 
  }
287 
}
288
289 
async function sh_run(
290 
  secret_position: number | undefined,
291 
  cmd: string,
292 
  ...args: string[]
293 
) {
294 
  const nargs = secret_position != undefined ? args.slice() : args;
295 
  if (secret_position && secret_position < 0) {
296 
    secret_position = nargs.length - 1 + secret_position;
297 
  }
298 
  let secret: string | undefined = undefined;
299 
  if (secret_position != undefined) {
300 
    nargs[secret_position] = "***";
301 
    secret = args[secret_position];
302 
  }
303
304 
  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
305 
  const command = exec(`${cmd} ${args.join(" ")}`);
306 
  // new Deno.Command(cmd, {
307 
  //   args: args,
308 
  // });
309 
  try {
310 
    const { stdout, stderr } = await command;
311 
    if (stdout.length > 0) {
312 
      console.log(stdout);
313 
    }
314 
    if (stderr.length > 0) {
315 
      console.log(stderr);
316 
    }
317 
    console.log("Command successfully executed");
318 
    return stdout;
319 
  } catch (error) {
320 
    let errorString = error.toString();
321 
    if (secret) {
322 
      errorString = errorString.replace(secret, "***");
323 
    }
324 
    const err = `SH command '${cmd} ${nargs.join(
325 
      " ",
326 
    )}' returned with error ${errorString}`;
327 
    throw Error(err);
328 
  }
329 
}
330
331 
async function wmill_run(secret_position: number, ...cmd: string[]) {
332 
  cmd = cmd.filter((elt) => elt !== "");
333 
  const cmd2 = cmd.slice();
334 
  cmd2[secret_position] = "***";
335 
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
336 
  await wmill.parse(cmd);
337 
  console.log("Command successfully executed");
338 
}
339
340 
// Function to set up GPG signing
341 
async function set_gpg_signing_secret(gpg_key: GpgKey) {
342 
  try {
343 
    console.log("Setting GPG private key for git commits");
344
345 
    const formattedGpgContent = gpg_key.private_key.replace(
346 
      /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
347 
      (_: string, header: string, body: string, footer: string) =>
348 
        header +
349 
        "\n" +
350 
        "\n" +
351 
        body.replace(/ ([^\s])/g, "\n$1").trim() +
352 
        "\n" +
353 
        footer,
354 
    );
355
356 
    const gpg_path = `/tmp/gpg`;
357 
    await sh_run(undefined, "mkdir", "-p", gpg_path);
358 
    await sh_run(undefined, "chmod", "700", gpg_path);
359 
    process.env.GNUPGHOME = gpg_path;
360 
    // process.env.GIT_TRACE = 1;
361
362 
    try {
363 
      await sh_run(
364 
        1,
365 
        "bash",
366 
        "-c",
367 
        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`,
368 
      );
369 
    } catch (e) {
370 
      // Original error would contain sensitive data
371 
      throw new Error("Failed to import GPG key!");
372 
    }
373
374 
    const listKeysOutput = await sh_run(
375 
      undefined,
376 
      "gpg",
377 
      "--list-secret-keys",
378 
      "--with-colons",
379 
      "--keyid-format=long",
380 
    );
381
382 
    const keyInfoMatch = listKeysOutput.match(
383 
      /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/,
384 
    );
385
386 
    if (!keyInfoMatch) {
387 
      throw new Error("Failed to extract GPG Key ID and Fingerprint");
388 
    }
389
390 
    const keyId = keyInfoMatch[1];
391 
    gpgFingerprint = keyInfoMatch[2];
392
393 
    if (gpg_key.passphrase) {
394 
      // This is adummy command to unlock the key
395 
      // with passphrase to load it into agent
396 
      await sh_run(
397 
        1,
398 
        "bash",
399 
        "-c",
400 
        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`,
401 
      );
402 
    }
403
404 
    // Configure Git to use the extracted key
405 
    await sh_run(undefined, "git", "config", "user.signingkey", keyId);
406 
    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
407 
    console.log(`GPG signing configured with key ID: ${keyId} `);
408 
  } catch (e) {
409 
    console.error(`Failure while setting GPG key: ${e} `);
410 
    await delete_pgp_keys();
411 
  }
412 
}
413
414 
async function delete_pgp_keys() {
415 
  console.log("deleting gpg keys");
416 
  if (gpgFingerprint) {
417 
    await sh_run(
418 
      undefined,
419 
      "gpg",
420 
      "--batch",
421 
      "--yes",
422 
      "--pinentry-mode",
423 
      "loopback",
424 
      "--delete-secret-key",
425 
      gpgFingerprint,
426 
    );
427 
    await sh_run(
428 
      undefined,
429 
      "gpg",
430 
      "--batch",
431 
      "--yes",
432 
      "--delete-key",
433 
      "--pinentry-mode",
434 
      "loopback",
435 
      gpgFingerprint,
436 
    );
437 
  }
438 
}
439
440 
async function get_gh_app_token() {
441 
  const workspace = process.env["WM_WORKSPACE"];
442 
  const jobToken = process.env["WM_TOKEN"];
443
444 
  const baseUrl =
445 
    process.env["BASE_INTERNAL_URL"] ??
446 
    process.env["BASE_URL"] ??
447 
    "http://localhost:8000";
448
449 
  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
450
451 
  const response = await fetch(url, {
452 
    method: "POST",
453 
    headers: {
454 
      "Content-Type": "application/json",
455 
      Authorization: `Bearer ${jobToken}`,
456 
    },
457 
    body: JSON.stringify({
458 
      job_token: jobToken,
459 
    }),
460 
  });
461
462 
  if (!response.ok) {
463 
    const errorBody = await response.text().catch(() => "");
464 
    throw new Error(
465 
      `GitHub App token error (${response.status}): ${errorBody || response.statusText}`,
466 
    );
467 
  }
468
469 
  const data = await response.json();
470
471 
  return data.token;
472 
}
473
474 
function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
475 
  if (!gitHubUrl || !installationToken) {
476 
    throw new Error("Both GitHub URL and Installation Token are required.");
477 
  }
478
479 
  const url = new URL(gitHubUrl);
480 
  return `https://x-access-token:${installationToken}@${url.hostname}${url.pathname}`;
481 
}
482
Other submissions
  • Submitted by hugo697 Deno
    Created 668 days ago
    All edits
    Permalink
    1 
    import * as wmillclient from "npm:[email protected]";
    2 
    import wmill from "https://deno.land/x/[email protected]/main.ts";
    3 
    import { basename } from "https://deno.land/[email protected]/path/mod.ts";
    4
    5 
    export async function main(
    6 
      workspace_id: string,
    7 
      repo_url_resource_path: string,
    8 
      path_type:
    9 
        | "script"
    10 
        | "flow"
    11 
        | "app"
    12 
        | "folder"
    13 
        | "resource"
    14 
        | "variable"
    15 
        | "resourcetype"
    16 
        | "schedule"
    17 
        | "user"
    18 
        | "group",
    19 
      skip_secret = true,
    20 
      path: string | undefined,
    21 
      parent_path: string | undefined,
    22 
      commit_msg: string,
    23 
      use_individual_branch = false,
    24 
      group_by_folder = false
    25 
    ) {
    26 
      const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    27
    28 
      const cwd = Deno.cwd();
    29 
      Deno.env.set("HOME", ".");
    30 
      console.log(
    31 
        `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    32 
      );
    33
    34 
      const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    35 
      await move_to_git_branch(
    36 
        workspace_id,
    37 
        path_type,
    38 
        path,
    39 
        parent_path,
    40 
        use_individual_branch,
    41 
        group_by_folder
    42 
      );
    43
    44 
      const subfolder = repo_resource.folder ?? "";
    45 
      const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    46 
      console.log(
    47 
        `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    48 
      );
    49
    50 
      await wmill_sync_pull(
    51 
        path_type,
    52 
        workspace_id,
    53 
        path,
    54 
        parent_path,
    55 
        skip_secret
    56 
      );
    57
    58 
      await git_push(path, parent_path, commit_msg);
    59
    60 
      console.log("Finished syncing");
    61 
      Deno.chdir(`${cwd}`);
    62 
    }
    63
    64 
    async function git_clone(
    65 
      cwd: string,
    66 
      repo_resource: any,
    67 
      use_individual_branch: boolean
    68 
    ): Promise<string> {
    69 
      // TODO: handle private SSH keys as well
    70 
      let repo_url = repo_resource.url;
    71 
      const subfolder = repo_resource.folder ?? "";
    72 
      const branch = repo_resource.branch ?? "";
    73 
      const repo_name = basename(repo_url, ".git");
    74
    75 
      const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    76
    77 
      if (azureMatch) {
    78 
        console.log(
    79 
          "Requires Azure DevOps service account access token, requesting..."
    80 
        );
    81 
        const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    82
    83 
        const response = await fetch(
    84 
          `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
    85 
          {
    86 
            method: "POST",
    87 
            body: new URLSearchParams({
    88 
              client_id: azureResource.azureClientId,
    89 
              client_secret: azureResource.azureClientSecret,
    90 
              grant_type: "client_credentials",
    91 
              resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
    92 
            }),
    93 
          }
    94 
        );
    95
    96 
        const { access_token } = await response.json();
    97
    98 
        repo_url = repo_url.replace(azureMatch[0], access_token);
    99 
      }
    100
    101 
      const args = ["clone", "--quiet", "--depth", "1"];
    102 
      if (use_individual_branch) {
    103 
        args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    104 
      }
    105 
      if (subfolder !== "") {
    106 
        args.push("--sparse");
    107 
      }
    108 
      if (branch !== "") {
    109 
        args.push("--branch");
    110 
        args.push(branch);
    111 
      }
    112 
      args.push(repo_url);
    113 
      args.push(repo_name);
    114 
      await sh_run(-1, "git", ...args);
    115
    116 
      try {
    117 
        Deno.chdir(`${cwd}/${repo_name}`);
    118 
      } catch (err) {
    119 
        console.log(
    120 
          `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    121 
        );
    122 
        throw err;
    123 
      }
    124 
      Deno.chdir(`${cwd}/${repo_name}`);
    125 
      if (subfolder !== "") {
    126 
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    127 
      }
    128 
      try {
    129 
        Deno.chdir(`${cwd}/${repo_name}/${subfolder}`);
    130 
      } catch (err) {
    131 
        console.log(
    132 
          `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
    133 
        );
    134 
        throw err;
    135 
      }
    136
    137 
      return repo_name;
    138 
    }
    139
    140 
    async function move_to_git_branch(
    141 
      workspace_id: string,
    142 
      path_type:
    143 
        | "script"
    144 
        | "flow"
    145 
        | "app"
    146 
        | "folder"
    147 
        | "resource"
    148 
        | "variable"
    149 
        | "resourcetype"
    150 
        | "schedule"
    151 
        | "user"
    152 
        | "group",
    153 
      path: string | undefined,
    154 
      parent_path: string | undefined,
    155 
      use_individual_branch: boolean,
    156 
      group_by_folder: boolean
    157 
    ) {
    158 
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
    159 
        return;
    160 
      }
    161
    162 
      const branchName = group_by_folder
    163 
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
    164 
            ?.split("/")
    165 
            .slice(0, 2)
    166 
            .join("__")}`
    167 
        : `wm_deploy/${workspace_id}/${path_type}/${(
    168 
            path ?? parent_path
    169 
          )?.replaceAll("/", "__")}`;
    170
    171 
      try {
    172 
        await sh_run(undefined, "git", "checkout", branchName);
    173 
      } catch (err) {
    174 
        console.log(
    175 
          `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    176 
        );
    177 
        try {
    178 
          await sh_run(undefined, "git", "checkout", "-b", branchName);
    179 
          await sh_run(
    180 
            undefined,
    181 
            "git",
    182 
            "config",
    183 
            "--add",
    184 
            "--bool",
    185 
            "push.autoSetupRemote",
    186 
            "true"
    187 
          );
    188 
        } catch (err) {
    189 
          console.log(
    190 
            `Error checking out branch '${branchName}'. Error was:\n${err}`
    191 
          );
    192 
          throw err;
    193 
        }
    194 
      }
    195 
      console.log(`Successfully switched to branch ${branchName}`);
    196 
    }
    197
    198 
    async function git_push(
    199 
      path: string | undefined,
    200 
      parent_path: string | undefined,
    201 
      commit_msg: string
    202 
    ) {
    203 
      await sh_run(
    204 
        undefined,
    205 
        "git",
    206 
        "config",
    207 
        "user.email",
    208 
        Deno.env.get("WM_EMAIL") ?? ""
    209 
      );
    210 
      await sh_run(
    211 
        undefined,
    212 
        "git",
    213 
        "config",
    214 
        "user.name",
    215 
        Deno.env.get("WM_USERNAME") ?? ""
    216 
      );
    217 
      if (path !== undefined && path !== null && path !== "") {
    218 
        try {
    219 
          await sh_run(undefined, "git", "add", `${path}**`);
    220 
        } catch (e) {
    221 
          console.log(`Unable to stage files matching ${path}**, ${e}`);
    222 
        }
    223 
      }
    224 
      if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    225 
        try {
    226 
          await sh_run(undefined, "git", "add", `${parent_path}**`);
    227 
        } catch (e) {
    228 
          console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    229 
        }
    230 
      }
    231 
      try {
    232 
        await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    233 
      } catch {
    234 
        // git diff returns exit-code = 1 when there's at least one staged changes
    235 
        await sh_run(undefined, "git", "commit", "-m", commit_msg);
    236 
        try {
    237 
            await sh_run(undefined, "git", "push", "--porcelain");
    238 
        } catch {
    239 
            console.log("Could not push, trying to rebase first");
    240 
            await sh_run(undefined, "git", "pull", "--rebase");
    241 
            await sh_run(undefined, "git", "push", "--porcelain");
    242 
        }
    243 
        return;
    244 
      }
    245 
      console.log("No changes detected, nothing to commit. Returning...");
    246 
    }
    247
    248 
    async function sh_run(
    249 
      secret_position: number | undefined,
    250 
      cmd: string,
    251 
      ...args: string[]
    252 
    ) {
    253 
      const nargs = secret_position != undefined ? args.slice() : args;
    254 
      if (secret_position < 0) {
    255 
        secret_position = nargs.length - 1 + secret_position;
    256 
      }
    257 
      if (secret_position != undefined) {
    258 
        nargs[secret_position] = "***";
    259 
      }
    260 
      console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    261 
      const command = new Deno.Command(cmd, {
    262 
        args: args,
    263 
      });
    264
    265 
      const { code, stdout, stderr } = await command.output();
    266 
      if (stdout.length > 0) {
    267 
        console.log(new TextDecoder().decode(stdout));
    268 
      }
    269 
      if (stderr.length > 0) {
    270 
        console.log(new TextDecoder().decode(stderr));
    271 
      }
    272 
      if (code !== 0) {
    273 
        const err = `SH command '${cmd} ${nargs.join(
    274 
          " "
    275 
        )}' returned with a non-zero status ${code}.`;
    276 
        throw Error(err);
    277 
      }
    278 
      console.log("Command successfully executed");
    279 
    }
    280
    281 
    function regexFromPath(
    282 
      path_type:
    283 
        | "script"
    284 
        | "flow"
    285 
        | "app"
    286 
        | "folder"
    287 
        | "resource"
    288 
        | "variable"
    289 
        | "resourcetype"
    290 
        | "schedule"
    291 
        | "user"
    292 
        | "group",
    293 
      path: string
    294 
    ) {
    295 
      if (path_type == "flow") {
    296 
        return `${path}.flow/*`;
    297 
      } if (path_type == "app") {
    298 
        return `${path}.app/*`;
    299 
      } else if (path_type == "folder") {
    300 
        return `${path}/folder.meta.*`;
    301 
      } else if (path_type == "resourcetype") {
    302 
        return `${path}.resource-type.*`;
    303 
      } else if (path_type == "resource") {
    304 
        return `${path}.resource.*`;
    305 
      } else if (path_type == "variable") {
    306 
        return `${path}.variable.*`;
    307 
      } else if (path_type == "schedule") {
    308 
        return `${path}.schedule.*`;
    309 
      } else if (path_type == "user") {
    310 
        return `${path}.user.*`;
    311 
      } else if (path_type == "group") {
    312 
        return `${path}.group.*`;
    313 
      } else {
    314 
        return `${path}.*`;
    315 
      }
    316 
    }
    317 
    async function wmill_sync_pull(
    318 
      path_type:
    319 
        | "script"
    320 
        | "flow"
    321 
        | "app"
    322 
        | "folder"
    323 
        | "resource"
    324 
        | "variable"
    325 
        | "resourcetype"
    326 
        | "schedule"
    327 
        | "user"
    328 
        | "group",
    329 
      workspace_id: string,
    330 
      path: string | undefined,
    331 
      parent_path: string | undefined,
    332 
      skip_secret: boolean
    333 
    ) {
    334 
      const includes = [];
    335 
      if (path !== undefined && path !== null && path !== "") {
    336 
        includes.push(regexFromPath(path_type, path));
    337 
      }
    338 
      if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    339 
        includes.push(regexFromPath(path_type, parent_path));
    340 
      }
    341
    342 
      await wmill_run(
    343 
        6,
    344 
        "workspace",
    345 
        "add",
    346 
        workspace_id,
    347 
        workspace_id,
    348 
        Deno.env.get("BASE_INTERNAL_URL") + "/",
    349 
        "--token",
    350 
        Deno.env.get("WM_TOKEN") ?? ""
    351 
      );
    352 
      console.log("Pulling workspace into git repo");
    353 
      await wmill_run(
    354 
        3,
    355 
        "sync",
    356 
        "pull",
    357 
        "--token",
    358 
        Deno.env.get("WM_TOKEN") ?? "",
    359 
        "--workspace",
    360 
        workspace_id,
    361 
        "--yes",
    362 
        "--raw",
    363 
        skip_secret ? "--skip-secrets" : "",
    364 
        "--include-schedules",
    365 
        "--include-users",
    366 
        "--include-groups",
    367 
        "--includes",
    368 
        includes.join(",")
    369 
      );
    370 
    }
    371
    372 
    async function wmill_run(secret_position: number, ...cmd: string[]) {
    373 
      cmd = cmd.filter((elt) => elt !== "");
    374 
      const cmd2 = cmd.slice();
    375 
      cmd2[secret_position] = "***";
    376 
      console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    377 
      await wmill.parse(cmd);
    378 
      console.log("Command successfully executed");
    379 
    }
    380
  • Submitted by rubenfiszel Bun
    Created 162 days ago
    All edits
    Permalink
    1 
    import * as wmillclient from "windmill-client";
    2 
    import wmill from "[email protected]";
    3 
    import { basename } from "node:path";
    4 
    const util = require("util");
    5 
    const exec = util.promisify(require("child_process").exec);
    6 
    import process from "process";
    7
    8 
    type GpgKey = {
    9 
      email: string;
    10 
      private_key: string;
    11 
      passphrase: string;
    12 
    };
    13
    14 
    const FORKED_WORKSPACE_PREFIX = "wm-fork-";
    15 
    const FORKED_BRANCH_PREFIX = "wm-fork";
    16
    17 
    type PathType =
    18 
      | "script"
    19 
      | "flow"
    20 
      | "app"
    21 
      | "folder"
    22 
      | "resource"
    23 
      | "variable"
    24 
      | "resourcetype"
    25 
      | "schedule"
    26 
      | "user"
    27 
      | "group"
    28 
      | "httptrigger"
    29 
      | "websockettrigger"
    30 
      | "kafkatrigger"
    31 
      | "natstrigger"
    32 
      | "postgrestrigger"
    33 
      | "mqtttrigger"
    34 
      | "sqstrigger"
    35 
      | "gcptrigger"
    36 
      | "emailtrigger";
    37
    38 
    let gpgFingerprint: string | undefined = undefined;
    39
    40 
    export async function main(
    41 
      workspace_id: string,
    42 
      repo_url_resource_path: string,
    43 
      path_type: PathType,
    44 
      skip_secret = true,
    45 
      path: string | undefined,
    46 
      parent_path: string | undefined,
    47 
      commit_msg: string,
    48 
      parent_workspace_id?: string,
    49 
      use_individual_branch = false,
    50 
      group_by_folder = false,
    51 
      only_create_branch: boolean = false
    52 
    ) {
    53 
      let safeDirectoryPath: string | undefined;
    54 
      const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    55 
      const cwd = process.cwd();
    56 
      process.env["HOME"] = ".";
    57 
      if (!only_create_branch) {
    58 
        console.log(
    59 
          `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    60 
        );
    61 
      }
    62
    63 
      if (repo_resource.is_github_app) {
    64 
        const token = await get_gh_app_token();
    65 
        const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
    66 
        repo_resource.url = authRepoUrl;
    67 
      }
    68
    69 
      const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    70 
      safeDirectoryPath = cloneSafeDirectoryPath;
    71
    72
    73 
      // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    74 
      // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    75 
      // settings, but we need to infer it from the workspace id
    76
    77 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    78 
        if (use_individual_branch) {
    79 
          console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
    80 
          use_individual_branch = false;
    81 
        }
    82 
        if (group_by_folder) {
    83 
          console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
    84 
          group_by_folder = false;
    85 
        }
    86 
      }
    87
    88 
      if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    89 
        const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
    90 
        console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
    91 
        await move_to_git_branch(
    92 
          parent_workspace_id,
    93 
          path_type,
    94 
          path,
    95 
          parent_path,
    96 
          use_individual_branch,
    97 
          group_by_folder,
    98 
          clonedBranchName
    99 
        );
    100 
      }
    101
    102 
      await move_to_git_branch(
    103 
        workspace_id,
    104 
        path_type,
    105 
        path,
    106 
        parent_path,
    107 
        use_individual_branch,
    108 
        group_by_folder,
    109 
        clonedBranchName
    110 
      );
    111
    112
    113 
      const subfolder = repo_resource.folder ?? "";
    114 
      const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    115 
      console.log(
    116 
        `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    117 
      );
    118
    119 
      // If we want to just create the branch, we can skip pulling the changes.
    120 
      if (!only_create_branch) {
    121 
        await wmill_sync_pull(
    122 
          path_type,
    123 
          workspace_id,
    124 
          path,
    125 
          parent_path,
    126 
          skip_secret,
    127 
          repo_url_resource_path,
    128 
          use_individual_branch,
    129 
          repo_resource.branch
    130 
        );
    131 
      }
    132 
      try {
    133 
        await git_push(path, parent_path, commit_msg, repo_resource, only_create_branch);
    134 
      } catch (e) {
    135 
        throw e;
    136 
      } finally {
    137 
        await delete_pgp_keys();
    138 
        // Cleanup: remove safe.directory config
    139 
        if (safeDirectoryPath) {
    140 
          try {
    141 
            await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
    142 
          } catch (e) {
    143 
            console.log(`Warning: Could not unset safe.directory config: ${e}`);
    144 
          }
    145 
        }
    146 
      }
    147 
      console.log("Finished syncing");
    148 
      process.chdir(`${cwd}`);
    149 
    }
    150
    151 
    function get_fork_branch_name(w_id: string, originalBranch: string): string {
    152 
      if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    153 
        return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    154 
      }
    155 
      return w_id;
    156 
    }
    157
    158 
    async function git_clone(
    159 
      cwd: string,
    160 
      repo_resource: any,
    161 
      no_single_branch: boolean,
    162 
    ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    163 
      // TODO: handle private SSH keys as well
    164 
      let repo_url = repo_resource.url;
    165 
      const subfolder = repo_resource.folder ?? "";
    166 
      const branch = repo_resource.branch ?? "";
    167 
      const repo_name = basename(repo_url, ".git");
    168 
      const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    169 
      if (azureMatch) {
    170 
        console.log(
    171 
          "Requires Azure DevOps service account access token, requesting..."
    172 
        );
    173 
        const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    174 
        const response = await fetch(
    175 
          `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
    176 
          {
    177 
            method: "POST",
    178 
            body: new URLSearchParams({
    179 
              client_id: azureResource.azureClientId,
    180 
              client_secret: azureResource.azureClientSecret,
    181 
              grant_type: "client_credentials",
    182 
              resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
    183 
            }),
    184 
          }
    185 
        );
    186 
        const { access_token } = await response.json();
    187 
        repo_url = repo_url.replace(azureMatch[0], access_token);
    188 
      }
    189 
      const args = ["clone", "--quiet", "--depth", "1"];
    190 
      if (no_single_branch) {
    191 
        args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    192 
      }
    193 
      if (subfolder !== "") {
    194 
        args.push("--sparse");
    195 
      }
    196 
      if (branch !== "") {
    197 
        args.push("--branch");
    198 
        args.push(branch);
    199 
      }
    200 
      args.push(repo_url);
    201 
      args.push(repo_name);
    202 
      await sh_run(-1, "git", ...args);
    203 
      try {
    204 
        process.chdir(`${cwd}/${repo_name}`);
    205 
        const safeDirectoryPath = process.cwd();
    206 
        // Add safe.directory to handle dubious ownership in cloned repo
    207 
        try {
    208 
          await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
    209 
        } catch (e) {
    210 
          console.log(`Warning: Could not add safe.directory config: ${e}`);
    211 
        }
    212
    213 
        if (subfolder !== "") {
    214 
          await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    215 
          try {
    216 
            process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    217 
          } catch (err) {
    218 
            console.log(
    219 
              `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
    220 
            );
    221 
            throw err;
    222 
          }
    223 
        }
    224 
        const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
    225 
        return { repo_name, safeDirectoryPath, clonedBranchName };
    226
    227 
      } catch (err) {
    228 
        console.log(
    229 
          `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    230 
        );
    231 
        throw err;
    232 
      }
    233 
    }
    234 
    async function move_to_git_branch(
    235 
      workspace_id: string,
    236 
      path_type: PathType,
    237 
      path: string | undefined,
    238 
      parent_path: string | undefined,
    239 
      use_individual_branch: boolean,
    240 
      group_by_folder: boolean,
    241 
      originalBranchName: string
    242 
    ) {
    243 
      let branchName;
    244 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    245 
        branchName = get_fork_branch_name(workspace_id, originalBranchName);
    246 
      } else {
    247 
        if (!use_individual_branch || path_type === "user" || path_type === "group") {
    248 
          return;
    249 
        }
    250 
        branchName = group_by_folder
    251 
          ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
    252 
            ?.split("/")
    253 
            .slice(0, 2)
    254 
            .join("__")}`
    255 
          : `wm_deploy/${workspace_id}/${path_type}/${(
    256 
            path ?? parent_path
    257 
          )?.replaceAll("/", "__")}`;
    258 
      }
    259
    260 
      try {
    261 
        await sh_run(undefined, "git", "checkout", branchName);
    262 
      } catch (err) {
    263 
        console.log(
    264 
          `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    265 
        );
    266 
        try {
    267 
          await sh_run(undefined, "git", "checkout", "-b", branchName);
    268 
          await sh_run(
    269 
            undefined,
    270 
            "git",
    271 
            "config",
    272 
            "--add",
    273 
            "--bool",
    274 
            "push.autoSetupRemote",
    275 
            "true"
    276 
          );
    277 
        } catch (err) {
    278 
          console.log(
    279 
            `Error checking out branch '${branchName}'. Error was:\n${err}`
    280 
          );
    281 
          throw err;
    282 
        }
    283 
      }
    284 
      console.log(`Successfully switched to branch ${branchName}`);
    285 
    }
    286 
    async function git_push(
    287 
      path: string | undefined,
    288 
      parent_path: string | undefined,
    289 
      commit_msg: string,
    290 
      repo_resource: any,
    291 
      only_create_branch: boolean,
    292 
    ) {
    293 
      let user_email = process.env["WM_EMAIL"] ?? "";
    294 
      let user_name = process.env["WM_USERNAME"] ?? "";
    295
    296 
      if (repo_resource.gpg_key) {
    297 
        await set_gpg_signing_secret(repo_resource.gpg_key);
    298 
        // Configure git with GPG key email for signing
    299 
        await sh_run(
    300 
          undefined,
    301 
          "git",
    302 
          "config",
    303 
          "user.email",
    304 
          repo_resource.gpg_key.email
    305 
        );
    306 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    307 
      } else {
    308 
        await sh_run(undefined, "git", "config", "user.email", user_email);
    309 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    310 
      }
    311 
      if (only_create_branch) {
    312 
        await sh_run(undefined, "git", "push", "--porcelain");
    313 
      }
    314
    315 
      if (path !== undefined && path !== null && path !== "") {
    316 
        try {
    317 
          await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
    318 
        } catch (e) {
    319 
          console.log(`Unable to stage files matching ${path}**, ${e}`);
    320 
        }
    321 
      }
    322 
      if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    323 
        try {
    324 
          await sh_run(
    325 
            undefined,
    326 
            "git",
    327 
            "add",
    328 
            "wmill-lock.yaml",
    329 
            `${parent_path}**`
    330 
          );
    331 
        } catch (e) {
    332 
          console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    333 
        }
    334 
      }
    335 
      try {
    336 
        await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    337 
      } catch {
    338 
        // git diff returns exit-code = 1 when there's at least one staged changes
    339 
        const commitArgs = ["git", "commit"];
    340
    341 
        // Always use --author to set consistent authorship
    342 
        commitArgs.push("--author", `"${user_name} <${user_email}>"`);
    343 
        commitArgs.push(
    344 
          "-m",
    345 
          `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
    346 
        );
    347
    348 
        await sh_run(undefined, ...commitArgs);
    349 
        try {
    350 
          await sh_run(undefined, "git", "push", "--porcelain");
    351 
        } catch (e) {
    352 
          console.log(`Could not push, trying to rebase first: ${e}`);
    353 
          await sh_run(undefined, "git", "pull", "--rebase");
    354 
          await sh_run(undefined, "git", "push", "--porcelain");
    355 
        }
    356 
        return;
    357 
      }
    358 
      console.log("No changes detected, nothing to commit. Returning...");
    359 
    }
    360 
    async function sh_run(
    361 
      secret_position: number | undefined,
    362 
      cmd: string,
    363 
      ...args: string[]
    364 
    ) {
    365 
      const nargs = secret_position != undefined ? args.slice() : args;
    366 
      if (secret_position && secret_position < 0) {
    367 
        secret_position = nargs.length - 1 + secret_position;
    368 
      }
    369 
      let secret: string | undefined = undefined;
    370 
      if (secret_position != undefined) {
    371 
        nargs[secret_position] = "***";
    372 
        secret = args[secret_position];
    373 
      }
    374
    375 
      console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    376 
      const command = exec(`${cmd} ${args.join(" ")}`);
    377 
      // new Deno.Command(cmd, {
    378 
      //   args: args,
    379 
      // });
    380 
      try {
    381 
        const { stdout, stderr } = await command;
    382 
        if (stdout.length > 0) {
    383 
          console.log(stdout);
    384 
        }
    385 
        if (stderr.length > 0) {
    386 
          console.log(stderr);
    387 
        }
    388 
        console.log("Command successfully executed");
    389 
        return stdout;
    390 
      } catch (error) {
    391 
        let errorString = error.toString();
    392 
        if (secret) {
    393 
          errorString = errorString.replace(secret, "***");
    394 
        }
    395 
        const err = `SH command '${cmd} ${nargs.join(
    396 
          " "
    397 
        )}' returned with error ${errorString}`;
    398 
        throw Error(err);
    399 
      }
    400 
    }
    401
    402 
    function regexFromPath(path_type: PathType, path: string) {
    403 
      if (path_type == "flow") {
    404 
        return `${path}.flow/*`;
    405 
      }
    406 
      if (path_type == "app") {
    407 
        return `${path}.app/*`;
    408 
      } else if (path_type == "folder") {
    409 
        return `${path}/folder.meta.*`;
    410 
      } else if (path_type == "resourcetype") {
    411 
        return `${path}.resource-type.*`;
    412 
      } else if (path_type == "resource") {
    413 
        return `${path}.resource.*`;
    414 
      } else if (path_type == "variable") {
    415 
        return `${path}.variable.*`;
    416 
      } else if (path_type == "schedule") {
    417 
        return `${path}.schedule.*`;
    418 
      } else if (path_type == "user") {
    419 
        return `${path}.user.*`;
    420 
      } else if (path_type == "group") {
    421 
        return `${path}.group.*`;
    422 
      } else if (path_type == "httptrigger") {
    423 
        return `${path}.http_trigger.*`;
    424 
      } else if (path_type == "websockettrigger") {
    425 
        return `${path}.websocket_trigger.*`;
    426 
      } else if (path_type == "kafkatrigger") {
    427 
        return `${path}.kafka_trigger.*`;
    428 
      } else if (path_type == "natstrigger") {
    429 
        return `${path}.nats_trigger.*`;
    430 
      } else if (path_type == "postgrestrigger") {
    431 
        return `${path}.postgres_trigger.*`;
    432 
      } else if (path_type == "mqtttrigger") {
    433 
        return `${path}.mqtt_trigger.*`;
    434 
      } else if (path_type == "sqstrigger") {
    435 
        return `${path}.sqs_trigger.*`;
    436 
      } else if (path_type == "gcptrigger") {
    437 
        return `${path}.gcp_trigger.*`;
    438 
      } else if (path_type == "emailtrigger") {
    439 
        return `${path}.email_trigger.*`;
    440 
      } else {
    441 
        return `${path}.*`;
    442 
      }
    443 
    }
    444
    445 
    async function wmill_sync_pull(
    446 
      path_type: PathType,
    447 
      workspace_id: string,
    448 
      path: string | undefined,
    449 
      parent_path: string | undefined,
    450 
      skip_secret: boolean,
    451 
      repo_url_resource_path: string,
    452 
      use_individual_branch: boolean,
    453 
      original_branch?: string
    454 
    ) {
    455 
      const includes = [];
    456 
      if (path !== undefined && path !== null && path !== "") {
    457 
        includes.push(regexFromPath(path_type, path));
    458 
      }
    459 
      if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    460 
        includes.push(regexFromPath(path_type, parent_path));
    461 
      }
    462
    463 
      console.log("Pulling workspace into git repo");
    464
    465 
      const args = [
    466 
        "sync",
    467 
        "pull",
    468 
        "--token",
    469 
        process.env["WM_TOKEN"] ?? "",
    470 
        "--workspace",
    471 
        workspace_id,
    472 
        "--base-url",
    473 
        process.env["BASE_URL"] + "/",
    474 
        "--repository",
    475 
        repo_url_resource_path,
    476 
        "--yes",
    477 
        skip_secret ? "--skip-secrets" : "",
    478 
      ];
    479
    480 
      if (path_type === "schedule" && !use_individual_branch) {
    481 
        args.push("--include-schedules");
    482 
      }
    483
    484 
      if (path_type === "group" && !use_individual_branch) {
    485 
        args.push("--include-groups");
    486 
      }
    487
    488 
      if (path_type === "user" && !use_individual_branch) {
    489 
        args.push("--include-users");
    490 
      }
    491
    492 
      if (path_type.includes("trigger") && !use_individual_branch) {
    493 
        args.push("--include-triggers");
    494 
      }
    495 
      // Only include settings when specifically deploying settings
    496 
      if (path_type === "settings" && !use_individual_branch) {
    497 
        args.push("--include-settings");
    498 
      }
    499
    500 
      // Only include key when specifically deploying keys
    501 
      if (path_type === "key" && !use_individual_branch) {
    502 
        args.push("--include-key");
    503 
      }
    504
    505 
      args.push("--extra-includes", includes.join(","));
    506
    507 
      // If using individual branches, apply promotion settings from original branch
    508 
      if (use_individual_branch && original_branch) {
    509 
        console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
    510 
        args.push("--promotion", original_branch);
    511 
      }
    512
    513 
      await wmill_run(3, ...args);
    514 
    }
    515
    516 
    async function wmill_run(secret_position: number, ...cmd: string[]) {
    517 
      cmd = cmd.filter((elt) => elt !== "");
    518 
      const cmd2 = cmd.slice();
    519 
      cmd2[secret_position] = "***";
    520 
      console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    521 
      await wmill.parse(cmd);
    522 
      console.log("Command successfully executed");
    523 
    }
    524
    525 
    // Function to set up GPG signing
    526 
    async function set_gpg_signing_secret(gpg_key: GpgKey) {
    527 
      try {
    528 
        console.log("Setting GPG private key for git commits");
    529
    530 
        const formattedGpgContent = gpg_key.private_key.replace(
    531 
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
    532 
          (_: string, header: string, body: string, footer: string) =>
    533 
            header +
    534 
            "\n" +
    535 
            "\n" +
    536 
            body.replace(/ ([^\s])/g, "\n$1").trim() +
    537 
            "\n" +
    538 
            footer
    539 
        );
    540
    541 
        const gpg_path = `/tmp/gpg`;
    542 
        await sh_run(undefined, "mkdir", "-p", gpg_path);
    543 
        await sh_run(undefined, "chmod", "700", gpg_path);
    544 
        process.env.GNUPGHOME = gpg_path;
    545 
        // process.env.GIT_TRACE = 1;
    546
    547 
        try {
    548 
          await sh_run(
    549 
            1,
    550 
            "bash",
    551 
            "-c",
    552 
            `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
    553 
          );
    554 
        } catch (e) {
    555 
          // Original error would contain sensitive data
    556 
          throw new Error("Failed to import GPG key!");
    557 
        }
    558
    559 
        const listKeysOutput = await sh_run(
    560 
          undefined,
    561 
          "gpg",
    562 
          "--list-secret-keys",
    563 
          "--with-colons",
    564 
          "--keyid-format=long"
    565 
        );
    566
    567 
        const keyInfoMatch = listKeysOutput.match(
    568 
          /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
    569 
        );
    570
    571 
        if (!keyInfoMatch) {
    572 
          throw new Error("Failed to extract GPG Key ID and Fingerprint");
    573 
        }
    574
    575 
        const keyId = keyInfoMatch[1];
    576 
        gpgFingerprint = keyInfoMatch[2];
    577
    578 
        if (gpg_key.passphrase) {
    579 
          // This is adummy command to unlock the key
    580 
          // with passphrase to load it into agent
    581 
          await sh_run(
    582 
            1,
    583 
            "bash",
    584 
            "-c",
    585 
            `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
    586 
          );
    587 
        }
    588
    589 
        // Configure Git to use the extracted key
    590 
        await sh_run(undefined, "git", "config", "user.signingkey", keyId);
    591 
        await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
    592 
        console.log(`GPG signing configured with key ID: ${keyId} `);
    593 
      } catch (e) {
    594 
        console.error(`Failure while setting GPG key: ${e} `);
    595 
        await delete_pgp_keys();
    596 
      }
    597 
    }
    598
    599 
    async function delete_pgp_keys() {
    600 
      console.log("deleting gpg keys");
    601 
      if (gpgFingerprint) {
    602 
        await sh_run(
    603 
          undefined,
    604 
          "gpg",
    605 
          "--batch",
    606 
          "--yes",
    607 
          "--pinentry-mode",
    608 
          "loopback",
    609 
          "--delete-secret-key",
    610 
          gpgFingerprint
    611 
        );
    612 
        await sh_run(
    613 
          undefined,
    614 
          "gpg",
    615 
          "--batch",
    616 
          "--yes",
    617 
          "--delete-key",
    618 
          "--pinentry-mode",
    619 
          "loopback",
    620 
          gpgFingerprint
    621 
        );
    622 
      }
    623 
    }
    624
    625 
    async function get_gh_app_token() {
    626 
      const workspace = process.env["WM_WORKSPACE"];
    627 
      const jobToken = process.env["WM_TOKEN"];
    628
    629 
      const baseUrl =
    630 
        process.env["BASE_INTERNAL_URL"] ??
    631 
        process.env["BASE_URL"] ??
    632 
        "http://localhost:8000";
    633
    634 
      const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
    635
    636 
      const response = await fetch(url, {
    637 
        method: "POST",
    638 
        headers: {
    639 
          "Content-Type": "application/json",
    640 
          Authorization: `Bearer ${jobToken}`,
    641 
        },
    642 
        body: JSON.stringify({
    643 
          job_token: jobToken,
    644 
        }),
    645 
      });
    646
    647 
      if (!response.ok) {
    648 
        throw new Error(`Error: ${response.statusText}`);
    649 
      }
    650
    651 
      const data = await response.json();
    652
    653 
      return data.token;
    654 
    }
    655
    656 
    function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    657 
      if (!gitHubUrl || !installationToken) {
    658 
        throw new Error("Both GitHub URL and Installation Token are required.");
    659 
      }
    660
    661 
      try {
    662 
        const url = new URL(gitHubUrl);
    663
    664 
        // GitHub repository URL should be in the format: https://github.com/owner/repo.git
    665 
        if (url.hostname !== "github.com") {
    666 
          throw new Error(
    667 
            "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
    668 
          );
    669 
        }
    670
    671 
        // Convert URL to include the installation token
    672 
        return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    673 
      } catch (e) {
    674 
        const error = e as Error;
    675 
        throw new Error(`Invalid URL: ${error.message}`);
    676 
      }
    677 
    }
    678
  • Submitted by pyranota275 Bun
    Created 135 days ago
    All edits
    Permalink
    1 
    import * as wmillclient from "windmill-client";
    2 
    import wmill from "[email protected]";
    3 
    import { basename } from "node:path";
    4 
    const util = require("util");
    5 
    const exec = util.promisify(require("child_process").exec);
    6 
    import process from "process";
    7
    8 
    type GpgKey = {
    9 
      email: string;
    10 
      private_key: string;
    11 
      passphrase: string;
    12 
    };
    13
    14 
    const FORKED_WORKSPACE_PREFIX = "wm-fork-";
    15 
    const FORKED_BRANCH_PREFIX = "wm-fork";
    16
    17 
    type PathType =
    18 
      | "script"
    19 
      | "flow"
    20 
      | "app"
    21 
      | "folder"
    22 
      | "resource"
    23 
      | "variable"
    24 
      | "resourcetype"
    25 
      | "schedule"
    26 
      | "user"
    27 
      | "group"
    28 
      | "httptrigger"
    29 
      | "websockettrigger"
    30 
      | "kafkatrigger"
    31 
      | "natstrigger"
    32 
      | "postgrestrigger"
    33 
      | "mqtttrigger"
    34 
      | "sqstrigger"
    35 
      | "gcptrigger"
    36 
      | "emailtrigger";
    37
    38 
    type SyncObject = {
    39 
      path_type: PathType;
    40 
      path: string | undefined;
    41 
      parent_path: string | undefined;
    42 
      commit_msg: string;
    43 
    };
    44
    45 
    let gpgFingerprint: string | undefined = undefined;
    46
    47 
    export async function main(
    48 
      items: SyncObject[],
    49 
      // Compat, do not use in code, rely on `items` instead
    50 
      path_type: PathType | undefined,
    51 
      path: string | undefined,
    52 
      parent_path: string | undefined,
    53 
      commit_msg: string | undefined,
    54 
      //
    55 
      workspace_id: string,
    56 
      repo_url_resource_path: string,
    57 
      skip_secret: boolean = true,
    58 
      use_individual_branch: boolean = false,
    59 
      group_by_folder: boolean = false,
    60 
      only_create_branch: boolean = false,
    61 
      parent_workspace_id?: string,
    62 
    ) {
    63
    64 
      if (path_type !== undefined && commit_msg !== undefined) {
    65 
        items = [{
    66 
          path_type,
    67 
          path,
    68 
          parent_path,
    69 
          commit_msg,
    70 
        }];
    71 
      }
    72 
      await inner(items, workspace_id, repo_url_resource_path, skip_secret, use_individual_branch, group_by_folder, only_create_branch, parent_workspace_id);
    73 
    }
    74
    75 
    async function inner(
    76 
      items: SyncObject[],
    77 
      workspace_id: string,
    78 
      repo_url_resource_path: string,
    79 
      skip_secret: boolean = true,
    80 
      use_individual_branch: boolean = false,
    81 
      group_by_folder: boolean = false,
    82 
      only_create_branch: boolean = false,
    83 
      parent_workspace_id?: string,
    84 
    ) {
    85
    86 
      let safeDirectoryPath: string | undefined;
    87 
      const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    88 
      const cwd = process.cwd();
    89 
      process.env["HOME"] = ".";
    90 
      if (!only_create_branch) {
    91 
        for (const item of items) {
    92 
          console.log(
    93 
            `Syncing ${item.path_type} ${item.path ?? ""} with parent ${item.parent_path ?? ""}`
    94 
          );
    95 
        }
    96 
      }
    97
    98 
      if (repo_resource.is_github_app) {
    99 
        const token = await get_gh_app_token();
    100 
        const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
    101 
        repo_resource.url = authRepoUrl;
    102 
      }
    103
    104 
      const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    105 
      safeDirectoryPath = cloneSafeDirectoryPath;
    106
    107
    108 
      // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    109 
      // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    110 
      // settings, but we need to infer it from the workspace id
    111
    112 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    113 
        if (use_individual_branch) {
    114 
          console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
    115 
          use_individual_branch = false;
    116 
        }
    117 
        if (group_by_folder) {
    118 
          console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
    119 
          group_by_folder = false;
    120 
        }
    121 
      }
    122
    123 
      if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    124 
        const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
    125 
        console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
    126 
        await git_checkout_branch(
    127 
          items,
    128 
          parent_workspace_id,
    129 
          use_individual_branch,
    130 
          group_by_folder,
    131 
          clonedBranchName
    132 
        );
    133 
      }
    134
    135 
      await git_checkout_branch(
    136 
        items,
    137 
        workspace_id,
    138 
        use_individual_branch,
    139 
        group_by_folder,
    140 
        clonedBranchName
    141 
      );
    142
    143
    144 
      const subfolder = repo_resource.folder ?? "";
    145 
      const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    146 
      console.log(
    147 
        `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    148 
      );
    149
    150 
      // If we want to just create the branch, we can skip pulling the changes.
    151 
      if (!only_create_branch) {
    152 
        await wmill_sync_pull(
    153 
          items,
    154 
          workspace_id,
    155 
          skip_secret,
    156 
          repo_url_resource_path,
    157 
          use_individual_branch,
    158 
          repo_resource.branch
    159 
        );
    160 
      }
    161 
      try {
    162 
        await git_push(items, repo_resource, only_create_branch);
    163 
      } catch (e) {
    164 
        throw e;
    165 
      } finally {
    166 
        await delete_pgp_keys();
    167 
        // Cleanup: remove safe.directory config
    168 
        if (safeDirectoryPath) {
    169 
          try {
    170 
            await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
    171 
          } catch (e) {
    172 
            console.log(`Warning: Could not unset safe.directory config: ${e}`);
    173 
          }
    174 
        }
    175 
      }
    176 
      console.log("Finished syncing");
    177 
      process.chdir(`${cwd}`);
    178 
    }
    179
    180
    181 
    function get_fork_branch_name(w_id: string, originalBranch: string): string {
    182 
      if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    183 
        return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    184 
      }
    185 
      return w_id;
    186 
    }
    187
    188 
    async function git_clone(
    189 
      cwd: string,
    190 
      repo_resource: any,
    191 
      no_single_branch: boolean,
    192 
    ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    193 
      // TODO: handle private SSH keys as well
    194 
      let repo_url = repo_resource.url;
    195 
      const subfolder = repo_resource.folder ?? "";
    196 
      const branch = repo_resource.branch ?? "";
    197 
      const repo_name = basename(repo_url, ".git");
    198 
      const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    199 
      if (azureMatch) {
    200 
        console.log(
    201 
          "Requires Azure DevOps service account access token, requesting..."
    202 
        );
    203 
        const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    204 
        const response = await fetch(
    205 
          `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
    206 
          {
    207 
            method: "POST",
    208 
            body: new URLSearchParams({
    209 
              client_id: azureResource.azureClientId,
    210 
              client_secret: azureResource.azureClientSecret,
    211 
              grant_type: "client_credentials",
    212 
              resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
    213 
            }),
    214 
          }
    215 
        );
    216 
        const { access_token } = await response.json();
    217 
        repo_url = repo_url.replace(azureMatch[0], access_token);
    218 
      }
    219 
      const args = ["clone", "--quiet", "--depth", "1"];
    220 
      if (no_single_branch) {
    221 
        args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    222 
      }
    223 
      if (subfolder !== "") {
    224 
        args.push("--sparse");
    225 
      }
    226 
      if (branch !== "") {
    227 
        args.push("--branch");
    228 
        args.push(branch);
    229 
      }
    230 
      args.push(repo_url);
    231 
      args.push(repo_name);
    232 
      await sh_run(-1, "git", ...args);
    233 
      try {
    234 
        process.chdir(`${cwd}/${repo_name}`);
    235 
        const safeDirectoryPath = process.cwd();
    236 
        // Add safe.directory to handle dubious ownership in cloned repo
    237 
        try {
    238 
          await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
    239 
        } catch (e) {
    240 
          console.log(`Warning: Could not add safe.directory config: ${e}`);
    241 
        }
    242
    243 
        if (subfolder !== "") {
    244 
          await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    245 
          try {
    246 
            process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    247 
          } catch (err) {
    248 
            console.log(
    249 
              `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
    250 
            );
    251 
            throw err;
    252 
          }
    253 
        }
    254 
        const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
    255 
        return { repo_name, safeDirectoryPath, clonedBranchName };
    256
    257 
      } catch (err) {
    258 
        console.log(
    259 
          `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    260 
        );
    261 
        throw err;
    262 
      }
    263 
    }
    264 
    async function git_checkout_branch(
    265 
      items: SyncObject[],
    266 
      workspace_id: string,
    267 
      use_individual_branch: boolean,
    268 
      group_by_folder: boolean,
    269 
      originalBranchName: string
    270 
    ) {
    271 
      let branchName;
    272 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    273 
        branchName = get_fork_branch_name(workspace_id, originalBranchName);
    274 
      } else {
    275
    276 
        if (!use_individual_branch
    277 
          // If individual branch is true, we can assume items is of length 1, as debouncing is disabled for jobs with this flag
    278 
          || items[0].path_type === "user" || items[0].path_type === "group") {
    279 
          return;
    280 
        }
    281
    282 
        // as mentioned above, it is safe to assume that items.len is 1
    283 
        const [path, parent_path] = [items[0].path, items[0].parent_path];
    284 
        branchName = group_by_folder
    285 
          ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
    286 
            ?.split("/")
    287 
            .slice(0, 2)
    288 
            .join("__")}`
    289 
          : `wm_deploy/${workspace_id}/${items[0].path_type}/${(
    290 
            path ?? parent_path
    291 
          )?.replaceAll("/", "__")}`;
    292 
      }
    293
    294 
      try {
    295 
        await sh_run(undefined, "git", "checkout", branchName);
    296 
      } catch (err) {
    297 
        console.log(
    298 
          `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    299 
        );
    300 
        try {
    301 
          await sh_run(undefined, "git", "checkout", "-b", branchName);
    302 
          await sh_run(
    303 
            undefined,
    304 
            "git",
    305 
            "config",
    306 
            "--add",
    307 
            "--bool",
    308 
            "push.autoSetupRemote",
    309 
            "true"
    310 
          );
    311 
        } catch (err) {
    312 
          console.log(
    313 
            `Error checking out branch '${branchName}'. Error was:\n${err}`
    314 
          );
    315 
          throw err;
    316 
        }
    317 
      }
    318 
      console.log(`Successfully switched to branch ${branchName}`);
    319 
    }
    320
    321 
    function composeCommitHeader(items: SyncObject[]): string {
    322 
      // Count occurrences of each path_type
    323 
      const typeCounts = new Map<PathType, number>();
    324 
      for (const item of items) {
    325 
        typeCounts.set(item.path_type, (typeCounts.get(item.path_type) ?? 0) + 1);
    326 
      }
    327
    328 
      // Sort by count descending to get the top 2
    329 
      const sortedTypes = Array.from(typeCounts.entries()).sort((a, b) => b[1] - a[1]);
    330
    331 
      const parts: string[] = [];
    332 
      let othersCount = 0;
    333
    334 
      for (let i = 0; i < sortedTypes.length; i++) {
    335 
        const [pathType, count] = sortedTypes[i];
    336 
        if (i < 3) {
    337 
          // Pluralize the path type if count > 1
    338 
          const label = count > 1 ? `${pathType}s` : pathType;
    339
    340 
          if (i == 2 && sortedTypes.length == 3) {
    341 
            parts.push(`and ${count} ${label}`);
    342 
          } else {
    343 
            parts.push(`${count} ${label}`);
    344 
          }
    345 
        } else {
    346 
          othersCount += count;
    347 
        }
    348 
      }
    349
    350 
      let header = `[WM] Deployed ${parts.join(", ")}`;
    351 
      if (othersCount > 0) {
    352 
        header += ` and ${othersCount} other object${othersCount > 1 ? "s" : ""}`;
    353 
      }
    354
    355 
      return header;
    356 
    }
    357
    358 
    async function git_push(
    359 
      items: SyncObject[],
    360 
      repo_resource: any,
    361 
      only_create_branch: boolean,
    362 
    ) {
    363 
      let user_email = process.env["WM_EMAIL"] ?? "";
    364 
      let user_name = process.env["WM_USERNAME"] ?? "";
    365
    366 
      if (repo_resource.gpg_key) {
    367 
        await set_gpg_signing_secret(repo_resource.gpg_key);
    368 
        // Configure git with GPG key email for signing
    369 
        await sh_run(
    370 
          undefined,
    371 
          "git",
    372 
          "config",
    373 
          "user.email",
    374 
          repo_resource.gpg_key.email
    375 
        );
    376 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    377 
      } else {
    378 
        await sh_run(undefined, "git", "config", "user.email", user_email);
    379 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    380 
      }
    381 
      if (only_create_branch) {
    382 
        await sh_run(undefined, "git", "push", "--porcelain");
    383 
      }
    384
    385 
      let commit_description: string[] = [];
    386 
      for (const { path, parent_path, commit_msg } of items) {
    387 
        if (path !== undefined && path !== null && path !== "") {
    388 
          try {
    389 
            await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
    390 
          } catch (e) {
    391 
            console.log(`Unable to stage files matching ${path}**, ${e}`);
    392 
          }
    393 
        }
    394 
        if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    395 
          try {
    396 
            await sh_run(
    397 
              undefined,
    398 
              "git",
    399 
              "add",
    400 
              "wmill-lock.yaml",
    401 
              `${parent_path}**`
    402 
            );
    403 
          } catch (e) {
    404 
            console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    405 
          }
    406 
        }
    407
    408 
        commit_description.push(commit_msg);
    409 
      }
    410
    411 
      try {
    412 
        await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    413 
      } catch {
    414 
        // git diff returns exit-code = 1 when there's at least one staged changes
    415 
        const commitArgs = ["git", "commit"];
    416
    417 
        // Always use --author to set consistent authorship
    418 
        commitArgs.push("--author", `"${user_name} <${user_email}>"`);
    419
    420 
        const [header, description] = (commit_description.length == 1)
    421 
          ? [commit_description[0], ""]
    422 
          : [composeCommitHeader(items), commit_description.join("\n")];
    423
    424 
        commitArgs.push(
    425 
          "-m",
    426 
          `"${header == undefined || header == "" ? "no commit msg" : header}"`,
    427 
          "-m",
    428 
          `"${description}"`
    429 
        );
    430
    431 
        await sh_run(undefined, ...commitArgs);
    432 
        try {
    433 
          await sh_run(undefined, "git", "push", "--porcelain");
    434 
        } catch (e) {
    435 
          console.log(`Could not push, trying to rebase first: ${e}`);
    436 
          await sh_run(undefined, "git", "pull", "--rebase");
    437 
          await sh_run(undefined, "git", "push", "--porcelain");
    438 
        }
    439 
        return;
    440 
      }
    441
    442 
      console.log("No changes detected, nothing to commit. Returning...");
    443 
    }
    444 
    async function sh_run(
    445 
      secret_position: number | undefined,
    446 
      cmd: string,
    447 
      ...args: string[]
    448 
    ) {
    449 
      const nargs = secret_position != undefined ? args.slice() : args;
    450 
      if (secret_position && secret_position < 0) {
    451 
        secret_position = nargs.length - 1 + secret_position;
    452 
      }
    453 
      let secret: string | undefined = undefined;
    454 
      if (secret_position != undefined) {
    455 
        nargs[secret_position] = "***";
    456 
        secret = args[secret_position];
    457 
      }
    458
    459 
      console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    460 
      const command = exec(`${cmd} ${args.join(" ")}`);
    461 
      // new Deno.Command(cmd, {
    462 
      //   args: args,
    463 
      // });
    464 
      try {
    465 
        const { stdout, stderr } = await command;
    466 
        if (stdout.length > 0) {
    467 
          console.log(stdout);
    468 
        }
    469 
        if (stderr.length > 0) {
    470 
          console.log(stderr);
    471 
        }
    472 
        console.log("Command successfully executed");
    473 
        return stdout;
    474 
      } catch (error) {
    475 
        let errorString = error.toString();
    476 
        if (secret) {
    477 
          errorString = errorString.replace(secret, "***");
    478 
        }
    479 
        const err = `SH command '${cmd} ${nargs.join(
    480 
          " "
    481 
        )}' returned with error ${errorString}`;
    482 
        throw Error(err);
    483 
      }
    484 
    }
    485
    486 
    function regexFromPath(path_type: PathType, path: string) {
    487 
      if (path_type == "flow") {
    488 
        return `${path}.flow/*`;
    489 
      }
    490 
      if (path_type == "app") {
    491 
        return `${path}.app/*`;
    492 
      } else if (path_type == "folder") {
    493 
        return `${path}/folder.meta.*`;
    494 
      } else if (path_type == "resourcetype") {
    495 
        return `${path}.resource-type.*`;
    496 
      } else if (path_type == "resource") {
    497 
        return `${path}.resource.*`;
    498 
      } else if (path_type == "variable") {
    499 
        return `${path}.variable.*`;
    500 
      } else if (path_type == "schedule") {
    501 
        return `${path}.schedule.*`;
    502 
      } else if (path_type == "user") {
    503 
        return `${path}.user.*`;
    504 
      } else if (path_type == "group") {
    505 
        return `${path}.group.*`;
    506 
      } else if (path_type == "httptrigger") {
    507 
        return `${path}.http_trigger.*`;
    508 
      } else if (path_type == "websockettrigger") {
    509 
        return `${path}.websocket_trigger.*`;
    510 
      } else if (path_type == "kafkatrigger") {
    511 
        return `${path}.kafka_trigger.*`;
    512 
      } else if (path_type == "natstrigger") {
    513 
        return `${path}.nats_trigger.*`;
    514 
      } else if (path_type == "postgrestrigger") {
    515 
        return `${path}.postgres_trigger.*`;
    516 
      } else if (path_type == "mqtttrigger") {
    517 
        return `${path}.mqtt_trigger.*`;
    518 
      } else if (path_type == "sqstrigger") {
    519 
        return `${path}.sqs_trigger.*`;
    520 
      } else if (path_type == "gcptrigger") {
    521 
        return `${path}.gcp_trigger.*`;
    522 
      } else if (path_type == "emailtrigger") {
    523 
        return `${path}.email_trigger.*`;
    524 
      } else {
    525 
        return `${path}.*`;
    526 
      }
    527 
    }
    528
    529 
    async function wmill_sync_pull(
    530 
      items: SyncObject[],
    531 
      workspace_id: string,
    532 
      skip_secret: boolean,
    533 
      repo_url_resource_path: string,
    534 
      use_individual_branch: boolean,
    535 
      original_branch?: string
    536 
    ) {
    537 
      const includes = [];
    538 
      for (const item of items) {
    539 
        const { path_type, path, parent_path } = item;
    540 
        if (path !== undefined && path !== null && path !== "") {
    541 
          includes.push(regexFromPath(path_type, path));
    542 
        }
    543 
        if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    544 
          includes.push(regexFromPath(path_type, parent_path));
    545 
        }
    546 
      }
    547
    548 
      console.log("Pulling workspace into git repo");
    549
    550 
      const args = [
    551 
        "sync",
    552 
        "pull",
    553 
        "--token",
    554 
        process.env["WM_TOKEN"] ?? "",
    555 
        "--workspace",
    556 
        workspace_id,
    557 
        "--base-url",
    558 
        process.env["BASE_URL"] + "/",
    559 
        "--repository",
    560 
        repo_url_resource_path,
    561 
        "--yes",
    562 
        skip_secret ? "--skip-secrets" : "",
    563 
      ];
    564
    565 
      if (items.some(item => item.path_type === "schedule") && !use_individual_branch) {
    566 
        args.push("--include-schedules");
    567 
      }
    568
    569 
      if (items.some(item => item.path_type === "group") && !use_individual_branch) {
    570 
        args.push("--include-groups");
    571 
      }
    572
    573 
      if (items.some(item => item.path_type === "user") && !use_individual_branch) {
    574 
        args.push("--include-users");
    575 
      }
    576
    577 
      if (items.some(item => item.path_type.includes("trigger")) && !use_individual_branch) {
    578 
        args.push("--include-triggers");
    579 
      }
    580 
      // Only include settings when specifically deploying settings
    581 
      if (items.some(item => item.path_type === "settings") && !use_individual_branch) {
    582 
        args.push("--include-settings");
    583 
      }
    584
    585 
      // Only include key when specifically deploying keys
    586 
      if (items.some(item => item.path_type === "key") && !use_individual_branch) {
    587 
        args.push("--include-key");
    588 
      }
    589
    590 
      args.push("--extra-includes", includes.join(","));
    591
    592 
      // If using individual branches, apply promotion settings from original branch
    593 
      if (use_individual_branch && original_branch) {
    594 
        console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
    595 
        args.push("--promotion", original_branch);
    596 
      }
    597
    598 
      await wmill_run(3, ...args);
    599 
    }
    600
    601 
    async function wmill_run(secret_position: number, ...cmd: string[]) {
    602 
      cmd = cmd.filter((elt) => elt !== "");
    603 
      const cmd2 = cmd.slice();
    604 
      cmd2[secret_position] = "***";
    605 
      console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    606 
      await wmill.parse(cmd);
    607 
      console.log("Command successfully executed");
    608 
    }
    609
    610 
    // Function to set up GPG signing
    611 
    async function set_gpg_signing_secret(gpg_key: GpgKey) {
    612 
      try {
    613 
        console.log("Setting GPG private key for git commits");
    614
    615 
        const formattedGpgContent = gpg_key.private_key.replace(
    616 
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
    617 
          (_: string, header: string, body: string, footer: string) =>
    618 
            header +
    619 
            "\n" +
    620 
            "\n" +
    621 
            body.replace(/ ([^\s])/g, "\n$1").trim() +
    622 
            "\n" +
    623 
            footer
    624 
        );
    625
    626 
        const gpg_path = `/tmp/gpg`;
    627 
        await sh_run(undefined, "mkdir", "-p", gpg_path);
    628 
        await sh_run(undefined, "chmod", "700", gpg_path);
    629 
        process.env.GNUPGHOME = gpg_path;
    630 
        // process.env.GIT_TRACE = 1;
    631
    632 
        try {
    633 
          await sh_run(
    634 
            1,
    635 
            "bash",
    636 
            "-c",
    637 
            `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
    638 
          );
    639 
        } catch (e) {
    640 
          // Original error would contain sensitive data
    641 
          throw new Error("Failed to import GPG key!");
    642 
        }
    643
    644 
        const listKeysOutput = await sh_run(
    645 
          undefined,
    646 
          "gpg",
    647 
          "--list-secret-keys",
    648 
          "--with-colons",
    649 
          "--keyid-format=long"
    650 
        );
    651
    652 
        const keyInfoMatch = listKeysOutput.match(
    653 
          /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
    654 
        );
    655
    656 
        if (!keyInfoMatch) {
    657 
          throw new Error("Failed to extract GPG Key ID and Fingerprint");
    658 
        }
    659
    660 
        const keyId = keyInfoMatch[1];
    661 
        gpgFingerprint = keyInfoMatch[2];
    662
    663 
        if (gpg_key.passphrase) {
    664 
          // This is adummy command to unlock the key
    665 
          // with passphrase to load it into agent
    666 
          await sh_run(
    667 
            1,
    668 
            "bash",
    669 
            "-c",
    670 
            `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
    671 
          );
    672 
        }
    673
    674 
        // Configure Git to use the extracted key
    675 
        await sh_run(undefined, "git", "config", "user.signingkey", keyId);
    676 
        await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
    677 
        console.log(`GPG signing configured with key ID: ${keyId} `);
    678 
      } catch (e) {
    679 
        console.error(`Failure while setting GPG key: ${e} `);
    680 
        await delete_pgp_keys();
    681 
      }
    682 
    }
    683
    684 
    async function delete_pgp_keys() {
    685 
      console.log("deleting gpg keys");
    686 
      if (gpgFingerprint) {
    687 
        await sh_run(
    688 
          undefined,
    689 
          "gpg",
    690 
          "--batch",
    691 
          "--yes",
    692 
          "--pinentry-mode",
    693 
          "loopback",
    694 
          "--delete-secret-key",
    695 
          gpgFingerprint
    696 
        );
    697 
        await sh_run(
    698 
          undefined,
    699 
          "gpg",
    700 
          "--batch",
    701 
          "--yes",
    702 
          "--delete-key",
    703 
          "--pinentry-mode",
    704 
          "loopback",
    705 
          gpgFingerprint
    706 
        );
    707 
      }
    708 
    }
    709
    710 
    async function get_gh_app_token() {
    711 
      const workspace = process.env["WM_WORKSPACE"];
    712 
      const jobToken = process.env["WM_TOKEN"];
    713
    714 
      const baseUrl =
    715 
        process.env["BASE_INTERNAL_URL"] ??
    716 
        process.env["BASE_URL"] ??
    717 
        "http://localhost:8000";
    718
    719 
      const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
    720
    721 
      const response = await fetch(url, {
    722 
        method: "POST",
    723 
        headers: {
    724 
          "Content-Type": "application/json",
    725 
          Authorization: `Bearer ${jobToken}`,
    726 
        },
    727 
        body: JSON.stringify({
    728 
          job_token: jobToken,
    729 
        }),
    730 
      });
    731
    732 
      if (!response.ok) {
    733 
        throw new Error(`Error: ${response.statusText}`);
    734 
      }
    735
    736 
      const data = await response.json();
    737
    738 
      return data.token;
    739 
    }
    740
    741 
    function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    742 
      if (!gitHubUrl || !installationToken) {
    743 
        throw new Error("Both GitHub URL and Installation Token are required.");
    744 
      }
    745
    746 
      try {
    747 
        const url = new URL(gitHubUrl);
    748
    749 
        // GitHub repository URL should be in the format: https://github.com/owner/repo.git
    750 
        if (url.hostname !== "github.com") {
    751 
          throw new Error(
    752 
            "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
    753 
          );
    754 
        }
    755
    756 
        // Convert URL to include the installation token
    757 
        return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    758 
      } catch (e) {
    759 
        const error = e as Error;
    760 
        throw new Error(`Invalid URL: ${error.message}`);
    761 
      }
    762 
    }
  • Submitted by pyranota275 Bun
    Created 136 days ago
    All edits
    Permalink
    1 
    import * as wmillclient from "windmill-client";
    2 
    import wmill from "[email protected]";
    3 
    import { basename } from "node:path";
    4 
    const util = require("util");
    5 
    const exec = util.promisify(require("child_process").exec);
    6 
    import process from "process";
    7
    8 
    type GpgKey = {
    9 
      email: string;
    10 
      private_key: string;
    11 
      passphrase: string;
    12 
    };
    13
    14 
    const FORKED_WORKSPACE_PREFIX = "wm-fork-";
    15 
    const FORKED_BRANCH_PREFIX = "wm-fork";
    16
    17 
    type PathType =
    18 
      | "script"
    19 
      | "flow"
    20 
      | "app"
    21 
      | "folder"
    22 
      | "resource"
    23 
      | "variable"
    24 
      | "resourcetype"
    25 
      | "schedule"
    26 
      | "user"
    27 
      | "group"
    28 
      | "httptrigger"
    29 
      | "websockettrigger"
    30 
      | "kafkatrigger"
    31 
      | "natstrigger"
    32 
      | "postgrestrigger"
    33 
      | "mqtttrigger"
    34 
      | "sqstrigger"
    35 
      | "gcptrigger"
    36 
      | "emailtrigger";
    37
    38 
    type SyncObject = {
    39 
      path_type: PathType;
    40 
      path: string | undefined;
    41 
      parent_path: string | undefined;
    42 
      commit_msg: string;
    43 
    };
    44
    45 
    let gpgFingerprint: string | undefined = undefined;
    46
    47 
    export async function main(
    48 
      items: SyncObject[],
    49 
      // Compat, do not use in code, rely on `items` instead
    50 
      path_type: PathType | undefined,
    51 
      path: string | undefined,
    52 
      parent_path: string | undefined,
    53 
      commit_msg: string | undefined,
    54 
      //
    55 
      workspace_id: string,
    56 
      repo_url_resource_path: string,
    57 
      skip_secret: boolean = true,
    58 
      use_individual_branch: boolean = false,
    59 
      group_by_folder: boolean = false,
    60 
      only_create_branch: boolean = false,
    61 
      parent_workspace_id?: string,
    62 
    ) {
    63
    64 
      if (path_type !== undefined && commit_msg !== undefined) {
    65 
        items = [{
    66 
          path_type,
    67 
          path,
    68 
          parent_path,
    69 
          commit_msg,
    70 
        }];
    71 
      }
    72 
      await inner(items, workspace_id, repo_url_resource_path, skip_secret, use_individual_branch, group_by_folder, only_create_branch, parent_workspace_id);
    73 
    }
    74
    75 
    async function inner(
    76 
      items: SyncObject[],
    77 
      workspace_id: string,
    78 
      repo_url_resource_path: string,
    79 
      skip_secret: boolean = true,
    80 
      use_individual_branch: boolean = false,
    81 
      group_by_folder: boolean = false,
    82 
      only_create_branch: boolean = false,
    83 
      parent_workspace_id?: string,
    84 
    ) {
    85
    86 
      let safeDirectoryPath: string | undefined;
    87 
      const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    88 
      const cwd = process.cwd();
    89 
      process.env["HOME"] = ".";
    90 
      if (!only_create_branch) {
    91 
        for (const item of items) {
    92 
          console.log(
    93 
            `Syncing ${item.path_type} ${item.path ?? ""} with parent ${item.parent_path ?? ""}`
    94 
          );
    95 
        }
    96 
      }
    97
    98 
      if (repo_resource.is_github_app) {
    99 
        const token = await get_gh_app_token();
    100 
        const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
    101 
        repo_resource.url = authRepoUrl;
    102 
      }
    103
    104 
      const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    105 
      safeDirectoryPath = cloneSafeDirectoryPath;
    106
    107
    108 
      // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    109 
      // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    110 
      // settings, but we need to infer it from the workspace id
    111
    112 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    113 
        if (use_individual_branch) {
    114 
          console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
    115 
          use_individual_branch = false;
    116 
        }
    117 
        if (group_by_folder) {
    118 
          console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
    119 
          group_by_folder = false;
    120 
        }
    121 
      }
    122
    123 
      if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    124 
        const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
    125 
        console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
    126 
        await git_checkout_branch(
    127 
          items,
    128 
          parent_workspace_id,
    129 
          use_individual_branch,
    130 
          group_by_folder,
    131 
          clonedBranchName
    132 
        );
    133 
      }
    134
    135 
      await git_checkout_branch(
    136 
        items,
    137 
        workspace_id,
    138 
        use_individual_branch,
    139 
        group_by_folder,
    140 
        clonedBranchName
    141 
      );
    142
    143
    144 
      const subfolder = repo_resource.folder ?? "";
    145 
      const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    146 
      console.log(
    147 
        `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    148 
      );
    149
    150 
      // If we want to just create the branch, we can skip pulling the changes.
    151 
      if (!only_create_branch) {
    152 
        await wmill_sync_pull(
    153 
          items,
    154 
          workspace_id,
    155 
          skip_secret,
    156 
          repo_url_resource_path,
    157 
          use_individual_branch,
    158 
          repo_resource.branch
    159 
        );
    160 
      }
    161 
      try {
    162 
        await git_push(items, repo_resource, only_create_branch);
    163 
      } catch (e) {
    164 
        throw e;
    165 
      } finally {
    166 
        await delete_pgp_keys();
    167 
        // Cleanup: remove safe.directory config
    168 
        if (safeDirectoryPath) {
    169 
          try {
    170 
            await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
    171 
          } catch (e) {
    172 
            console.log(`Warning: Could not unset safe.directory config: ${e}`);
    173 
          }
    174 
        }
    175 
      }
    176 
      console.log("Finished syncing");
    177 
      process.chdir(`${cwd}`);
    178 
    }
    179
    180
    181 
    function get_fork_branch_name(w_id: string, originalBranch: string): string {
    182 
      if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    183 
        return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    184 
      }
    185 
      return w_id;
    186 
    }
    187
    188 
    async function git_clone(
    189 
      cwd: string,
    190 
      repo_resource: any,
    191 
      no_single_branch: boolean,
    192 
    ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    193 
      // TODO: handle private SSH keys as well
    194 
      let repo_url = repo_resource.url;
    195 
      const subfolder = repo_resource.folder ?? "";
    196 
      const branch = repo_resource.branch ?? "";
    197 
      const repo_name = basename(repo_url, ".git");
    198 
      const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    199 
      if (azureMatch) {
    200 
        console.log(
    201 
          "Requires Azure DevOps service account access token, requesting..."
    202 
        );
    203 
        const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    204 
        const response = await fetch(
    205 
          `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
    206 
          {
    207 
            method: "POST",
    208 
            body: new URLSearchParams({
    209 
              client_id: azureResource.azureClientId,
    210 
              client_secret: azureResource.azureClientSecret,
    211 
              grant_type: "client_credentials",
    212 
              resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
    213 
            }),
    214 
          }
    215 
        );
    216 
        const { access_token } = await response.json();
    217 
        repo_url = repo_url.replace(azureMatch[0], access_token);
    218 
      }
    219 
      const args = ["clone", "--quiet", "--depth", "1"];
    220 
      if (no_single_branch) {
    221 
        args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    222 
      }
    223 
      if (subfolder !== "") {
    224 
        args.push("--sparse");
    225 
      }
    226 
      if (branch !== "") {
    227 
        args.push("--branch");
    228 
        args.push(branch);
    229 
      }
    230 
      args.push(repo_url);
    231 
      args.push(repo_name);
    232 
      await sh_run(-1, "git", ...args);
    233 
      try {
    234 
        process.chdir(`${cwd}/${repo_name}`);
    235 
        const safeDirectoryPath = process.cwd();
    236 
        // Add safe.directory to handle dubious ownership in cloned repo
    237 
        try {
    238 
          await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
    239 
        } catch (e) {
    240 
          console.log(`Warning: Could not add safe.directory config: ${e}`);
    241 
        }
    242
    243 
        if (subfolder !== "") {
    244 
          await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    245 
          try {
    246 
            process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    247 
          } catch (err) {
    248 
            console.log(
    249 
              `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
    250 
            );
    251 
            throw err;
    252 
          }
    253 
        }
    254 
        const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
    255 
        return { repo_name, safeDirectoryPath, clonedBranchName };
    256
    257 
      } catch (err) {
    258 
        console.log(
    259 
          `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    260 
        );
    261 
        throw err;
    262 
      }
    263 
    }
    264 
    async function git_checkout_branch(
    265 
      items: SyncObject[],
    266 
      workspace_id: string,
    267 
      use_individual_branch: boolean,
    268 
      group_by_folder: boolean,
    269 
      originalBranchName: string
    270 
    ) {
    271 
      let branchName;
    272 
      if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    273 
        branchName = get_fork_branch_name(workspace_id, originalBranchName);
    274 
      } else {
    275
    276 
        if (!use_individual_branch
    277 
          // If individual branch is true, we can assume items is of length 1, as debouncing is disabled for jobs with this flag
    278 
          || items[0].path_type === "user" || items[0].path_type === "group") {
    279 
          return;
    280 
        }
    281
    282 
        // as mentioned above, it is safe to assume that items.len is 1
    283 
        const [path, parent_path] = [items[0].path, items[0].parent_path];
    284 
        branchName = group_by_folder
    285 
          ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
    286 
            ?.split("/")
    287 
            .slice(0, 2)
    288 
            .join("__")}`
    289 
          : `wm_deploy/${workspace_id}/${items[0].path_type}/${(
    290 
            path ?? parent_path
    291 
          )?.replaceAll("/", "__")}`;
    292 
      }
    293
    294 
      try {
    295 
        await sh_run(undefined, "git", "checkout", branchName);
    296 
      } catch (err) {
    297 
        console.log(
    298 
          `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    299 
        );
    300 
        try {
    301 
          await sh_run(undefined, "git", "checkout", "-b", branchName);
    302 
          await sh_run(
    303 
            undefined,
    304 
            "git",
    305 
            "config",
    306 
            "--add",
    307 
            "--bool",
    308 
            "push.autoSetupRemote",
    309 
            "true"
    310 
          );
    311 
        } catch (err) {
    312 
          console.log(
    313 
            `Error checking out branch '${branchName}'. Error was:\n${err}`
    314 
          );
    315 
          throw err;
    316 
        }
    317 
      }
    318 
      console.log(`Successfully switched to branch ${branchName}`);
    319 
    }
    320
    321 
    async function git_push(
    322 
      items: SyncObject[],
    323 
      repo_resource: any,
    324 
      only_create_branch: boolean,
    325 
    ) {
    326 
      let user_email = process.env["WM_EMAIL"] ?? "";
    327 
      let user_name = process.env["WM_USERNAME"] ?? "";
    328
    329 
      if (repo_resource.gpg_key) {
    330 
        await set_gpg_signing_secret(repo_resource.gpg_key);
    331 
        // Configure git with GPG key email for signing
    332 
        await sh_run(
    333 
          undefined,
    334 
          "git",
    335 
          "config",
    336 
          "user.email",
    337 
          repo_resource.gpg_key.email
    338 
        );
    339 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    340 
      } else {
    341 
        await sh_run(undefined, "git", "config", "user.email", user_email);
    342 
        await sh_run(undefined, "git", "config", "user.name", user_name);
    343 
      }
    344 
      if (only_create_branch) {
    345 
        await sh_run(undefined, "git", "push", "--porcelain");
    346 
      }
    347
    348 
      let commit_description: string[] = [];
    349 
      for (const { path, parent_path, commit_msg } of items) {
    350 
        if (path !== undefined && path !== null && path !== "") {
    351 
          try {
    352 
            await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
    353 
          } catch (e) {
    354 
            console.log(`Unable to stage files matching ${path}**, ${e}`);
    355 
          }
    356 
        }
    357 
        if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    358 
          try {
    359 
            await sh_run(
    360 
              undefined,
    361 
              "git",
    362 
              "add",
    363 
              "wmill-lock.yaml",
    364 
              `${parent_path}**`
    365 
            );
    366 
          } catch (e) {
    367 
            console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    368 
          }
    369 
        }
    370
    371 
        commit_description.push(commit_msg);
    372 
      }
    373
    374 
      try {
    375 
        await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    376 
      } catch {
    377 
        // git diff returns exit-code = 1 when there's at least one staged changes
    378 
        const commitArgs = ["git", "commit"];
    379
    380 
        // Always use --author to set consistent authorship
    381 
        commitArgs.push("--author", `"${user_name} <${user_email}>"`);
    382
    383 
        const [header, description] = (commit_description.length == 1) ? [commit_description[0], ""] : [`[WM]: Deployed ${commit_description.length} objects`, commit_description.join("\n")];
    384
    385 
        commitArgs.push(
    386 
          "-m",
    387 
          `"${header == undefined || header == "" ? "no commit msg" : header}"`,
    388 
          "-m",
    389 
          `"${description}"`
    390 
        );
    391
    392 
        await sh_run(undefined, ...commitArgs);
    393 
        try {
    394 
          await sh_run(undefined, "git", "push", "--porcelain");
    395 
        } catch (e) {
    396 
          console.log(`Could not push, trying to rebase first: ${e}`);
    397 
          await sh_run(undefined, "git", "pull", "--rebase");
    398 
          await sh_run(undefined, "git", "push", "--porcelain");
    399 
        }
    400 
        return;
    401 
      }
    402
    403 
      console.log("No changes detected, nothing to commit. Returning...");
    404 
    }
    405 
    async function sh_run(
    406 
      secret_position: number | undefined,
    407 
      cmd: string,
    408 
      ...args: string[]
    409 
    ) {
    410 
      const nargs = secret_position != undefined ? args.slice() : args;
    411 
      if (secret_position && secret_position < 0) {
    412 
        secret_position = nargs.length - 1 + secret_position;
    413 
      }
    414 
      let secret: string | undefined = undefined;
    415 
      if (secret_position != undefined) {
    416 
        nargs[secret_position] = "***";
    417 
        secret = args[secret_position];
    418 
      }
    419
    420 
      console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    421 
      const command = exec(`${cmd} ${args.join(" ")}`);
    422 
      // new Deno.Command(cmd, {
    423 
      //   args: args,
    424 
      // });
    425 
      try {
    426 
        const { stdout, stderr } = await command;
    427 
        if (stdout.length > 0) {
    428 
          console.log(stdout);
    429 
        }
    430 
        if (stderr.length > 0) {
    431 
          console.log(stderr);
    432 
        }
    433 
        console.log("Command successfully executed");
    434 
        return stdout;
    435 
      } catch (error) {
    436 
        let errorString = error.toString();
    437 
        if (secret) {
    438 
          errorString = errorString.replace(secret, "***");
    439 
        }
    440 
        const err = `SH command '${cmd} ${nargs.join(
    441 
          " "
    442 
        )}' returned with error ${errorString}`;
    443 
        throw Error(err);
    444 
      }
    445 
    }
    446
    447 
    function regexFromPath(path_type: PathType, path: string) {
    448 
      if (path_type == "flow") {
    449 
        return `${path}.flow/*`;
    450 
      }
    451 
      if (path_type == "app") {
    452 
        return `${path}.app/*`;
    453 
      } else if (path_type == "folder") {
    454 
        return `${path}/folder.meta.*`;
    455 
      } else if (path_type == "resourcetype") {
    456 
        return `${path}.resource-type.*`;
    457 
      } else if (path_type == "resource") {
    458 
        return `${path}.resource.*`;
    459 
      } else if (path_type == "variable") {
    460 
        return `${path}.variable.*`;
    461 
      } else if (path_type == "schedule") {
    462 
        return `${path}.schedule.*`;
    463 
      } else if (path_type == "user") {
    464 
        return `${path}.user.*`;
    465 
      } else if (path_type == "group") {
    466 
        return `${path}.group.*`;
    467 
      } else if (path_type == "httptrigger") {
    468 
        return `${path}.http_trigger.*`;
    469 
      } else if (path_type == "websockettrigger") {
    470 
        return `${path}.websocket_trigger.*`;
    471 
      } else if (path_type == "kafkatrigger") {
    472 
        return `${path}.kafka_trigger.*`;
    473 
      } else if (path_type == "natstrigger") {
    474 
        return `${path}.nats_trigger.*`;
    475 
      } else if (path_type == "postgrestrigger") {
    476 
        return `${path}.postgres_trigger.*`;
    477 
      } else if (path_type == "mqtttrigger") {
    478 
        return `${path}.mqtt_trigger.*`;
    479 
      } else if (path_type == "sqstrigger") {
    480 
        return `${path}.sqs_trigger.*`;
    481 
      } else if (path_type == "gcptrigger") {
    482 
        return `${path}.gcp_trigger.*`;
    483 
      } else if (path_type == "emailtrigger") {
    484 
        return `${path}.email_trigger.*`;
    485 
      } else {
    486 
        return `${path}.*`;
    487 
      }
    488 
    }
    489
    490 
    async function wmill_sync_pull(
    491 
      items: SyncObject[],
    492 
      workspace_id: string,
    493 
      skip_secret: boolean,
    494 
      repo_url_resource_path: string,
    495 
      use_individual_branch: boolean,
    496 
      original_branch?: string
    497 
    ) {
    498 
      const includes = [];
    499 
      for (const item of items) {
    500 
        const { path_type, path, parent_path } = item;
    501 
        if (path !== undefined && path !== null && path !== "") {
    502 
          includes.push(regexFromPath(path_type, path));
    503 
        }
    504 
        if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    505 
          includes.push(regexFromPath(path_type, parent_path));
    506 
        }
    507 
      }
    508
    509 
      console.log("Pulling workspace into git repo");
    510
    511 
      const args = [
    512 
        "sync",
    513 
        "pull",
    514 
        "--token",
    515 
        process.env["WM_TOKEN"] ?? "",
    516 
        "--workspace",
    517 
        workspace_id,
    518 
        "--base-url",
    519 
        process.env["BASE_URL"] + "/",
    520 
        "--repository",
    521 
        repo_url_resource_path,
    522 
        "--yes",
    523 
        skip_secret ? "--skip-secrets" : "",
    524 
      ];
    525
    526 
      if (items.some(item => item.path_type === "schedule") && !use_individual_branch) {
    527 
        args.push("--include-schedules");
    528 
      }
    529
    530 
      if (items.some(item => item.path_type === "group") && !use_individual_branch) {
    531 
        args.push("--include-groups");
    532 
      }
    533
    534 
      if (items.some(item => item.path_type === "user") && !use_individual_branch) {
    535 
        args.push("--include-users");
    536 
      }
    537
    538 
      if (items.some(item => item.path_type.includes("trigger")) && !use_individual_branch) {
    539 
        args.push("--include-triggers");
    540 
      }
    541 
      // Only include settings when specifically deploying settings
    542 
      if (items.some(item => item.path_type === "settings") && !use_individual_branch) {
    543 
        args.push("--include-settings");
    544 
      }
    545
    546 
      // Only include key when specifically deploying keys
    547 
      if (items.some(item => item.path_type === "key") && !use_individual_branch) {
    548 
        args.push("--include-key");
    549 
      }
    550
    551 
      args.push("--extra-includes", includes.join(","));
    552
    553 
      // If using individual branches, apply promotion settings from original branch
    554 
      if (use_individual_branch && original_branch) {
    555 
        console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
    556 
        args.push("--promotion", original_branch);
    557 
      }
    558
    559 
      await wmill_run(3, ...args);
    560 
    }
    561
    562 
    async function wmill_run(secret_position: number, ...cmd: string[]) {
    563 
      cmd = cmd.filter((elt) => elt !== "");
    564 
      const cmd2 = cmd.slice();
    565 
      cmd2[secret_position] = "***";
    566 
      console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    567 
      await wmill.parse(cmd);
    568 
      console.log("Command successfully executed");
    569 
    }
    570
    571 
    // Function to set up GPG signing
    572 
    async function set_gpg_signing_secret(gpg_key: GpgKey) {
    573 
      try {
    574 
        console.log("Setting GPG private key for git commits");
    575
    576 
        const formattedGpgContent = gpg_key.private_key.replace(
    577 
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
    578 
          (_: string, header: string, body: string, footer: string) =>
    579 
            header +
    580 
            "\n" +
    581 
            "\n" +
    582 
            body.replace(/ ([^\s])/g, "\n$1").trim() +
    583 
            "\n" +
    584 
            footer
    585 
        );
    586
    587 
        const gpg_path = `/tmp/gpg`;
    588 
        await sh_run(undefined, "mkdir", "-p", gpg_path);
    589 
        await sh_run(undefined, "chmod", "700", gpg_path);
    590 
        process.env.GNUPGHOME = gpg_path;
    591 
        // process.env.GIT_TRACE = 1;
    592
    593 
        try {
    594 
          await sh_run(
    595 
            1,
    596 
            "bash",
    597 
            "-c",
    598 
            `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
    599 
          );
    600 
        } catch (e) {
    601 
          // Original error would contain sensitive data
    602 
          throw new Error("Failed to import GPG key!");
    603 
        }
    604
    605 
        const listKeysOutput = await sh_run(
    606 
          undefined,
    607 
          "gpg",
    608 
          "--list-secret-keys",
    609 
          "--with-colons",
    610 
          "--keyid-format=long"
    611 
        );
    612
    613 
        const keyInfoMatch = listKeysOutput.match(
    614 
          /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
    615 
        );
    616
    617 
        if (!keyInfoMatch) {
    618 
          throw new Error("Failed to extract GPG Key ID and Fingerprint");
    619 
        }
    620
    621 
        const keyId = keyInfoMatch[1];
    622 
        gpgFingerprint = keyInfoMatch[2];
    623
    624 
        if (gpg_key.passphrase) {
    625 
          // This is adummy command to unlock the key
    626 
          // with passphrase to load it into agent
    627 
          await sh_run(
    628 
            1,
    629 
            "bash",
    630 
            "-c",
    631 
            `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
    632 
          );
    633 
        }
    634
    635 
        // Configure Git to use the extracted key
    636 
        await sh_run(undefined, "git", "config", "user.signingkey", keyId);
    637 
        await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
    638 
        console.log(`GPG signing configured with key ID: ${keyId} `);
    639 
      } catch (e) {
    640 
        console.error(`Failure while setting GPG key: ${e} `);
    641 
        await delete_pgp_keys();
    642 
      }
    643 
    }
    644
    645 
    async function delete_pgp_keys() {
    646 
      console.log("deleting gpg keys");
    647 
      if (gpgFingerprint) {
    648 
        await sh_run(
    649 
          undefined,
    650 
          "gpg",
    651 
          "--batch",
    652 
          "--yes",
    653 
          "--pinentry-mode",
    654 
          "loopback",
    655 
          "--delete-secret-key",
    656 
          gpgFingerprint
    657 
        );
    658 
        await sh_run(
    659 
          undefined,
    660 
          "gpg",
    661 
          "--batch",
    662 
          "--yes",
    663 
          "--delete-key",
    664 
          "--pinentry-mode",
    665 
          "loopback",
    666 
          gpgFingerprint
    667 
        );
    668 
      }
    669 
    }
    670
    671 
    async function get_gh_app_token() {
    672 
      const workspace = process.env["WM_WORKSPACE"];
    673 
      const jobToken = process.env["WM_TOKEN"];
    674
    675 
      const baseUrl =
    676 
        process.env["BASE_INTERNAL_URL"] ??
    677 
        process.env["BASE_URL"] ??
    678 
        "http://localhost:8000";
    679
    680 
      const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
    681
    682 
      const response = await fetch(url, {
    683 
        method: "POST",
    684 
        headers: {
    685 
          "Content-Type": "application/json",
    686 
          Authorization: `Bearer ${jobToken}`,
    687 
        },
    688 
        body: JSON.stringify({
    689 
          job_token: jobToken,
    690 
        }),
    691 
      });
    692
    693 
      if (!response.ok) {
    694 
        throw new Error(`Error: ${response.statusText}`);
    695 
      }
    696
    697 
      const data = await response.json();
    698
    699 
      return data.token;
    700 
    }
    701
    702 
    function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    703 
      if (!gitHubUrl || !installationToken) {
    704 
        throw new Error("Both GitHub URL and Installation Token are required.");
    705 
      }
    706
    707 
      try {
    708 
        const url = new URL(gitHubUrl);
    709
    710 
        // GitHub repository URL should be in the format: https://github.com/owner/repo.git
    711 
        if (url.hostname !== "github.com") {
    712 
          throw new Error(
    713 
            "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
    714 
          );
    715 
        }
    716
    717 
        // Convert URL to include the installation token
    718 
        return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    719 
      } catch (e) {
    720 
        const error = e as Error;
    721 
        throw new Error(`Invalid URL: ${error.message}`);
    722 
      }
    723 
    }
    724