1
Sync script to Git repo
One script reply has been approved by the moderators Verified

This script will pull the script from the current workspace in a temporary folder, then commit it to the remote Git repository and push it. Only the script-related file will be pushed, nothing else. It takes as input the git_repository resource containing the repository URL, the script path, and the commit message to use for the commit. All params are mandatory.

Created by hugo697 650 days ago Viewed 111253 times
0
Submitted by rubenfiszel Bun
Verified 367 days ago
1
import * as wmillclient from "windmill-client";
2
import wmill from "windmill-cli";
3
import { basename } from "node:path";
4
const util = require("util");
5
const exec = util.promisify(require("child_process").exec);
6
import process from "process";
7

8
type GpgKey = {
9
  email: string;
10
  private_key: string;
11
  passphrase: string;
12
};
13

14
const FORKED_WORKSPACE_PREFIX = "wm-fork-";
15
const FORKED_BRANCH_PREFIX = "wm-fork";
16

17
type PathType =
18
  | "script"
19
  | "flow"
20
  | "app"
21
  | "folder"
22
  | "resource"
23
  | "variable"
24
  | "resourcetype"
25
  | "schedule"
26
  | "user"
27
  | "group"
28
  | "httptrigger"
29
  | "websockettrigger"
30
  | "kafkatrigger"
31
  | "natstrigger"
32
  | "postgrestrigger"
33
  | "mqtttrigger"
34
  | "sqstrigger"
35
  | "gcptrigger"
36
  | "emailtrigger";
37

38
let gpgFingerprint: string | undefined = undefined;
39

40
export async function main(
41
  workspace_id: string,
42
  repo_url_resource_path: string,
43
  path_type: PathType,
44
  skip_secret = true,
45
  path: string | undefined,
46
  parent_path: string | undefined,
47
  commit_msg: string,
48
  parent_workspace_id?: string,
49
  use_individual_branch = false,
50
  group_by_folder = false
51
) {
52
  let safeDirectoryPath: string | undefined;
53
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
54
  const cwd = process.cwd();
55
  process.env["HOME"] = ".";
56
  console.log(
57
    `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
58
  );
59

60
  if (repo_resource.is_github_app) {
61
    const token = await get_gh_app_token();
62
    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
63
    repo_resource.url = authRepoUrl;
64
  }
65

66
  const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
67
  safeDirectoryPath = cloneSafeDirectoryPath;
68

69

70
  // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
71
  // a fork of a fork workspace. In that case, the original branch is not stored in the resource
72
  // settings, but we need to infer it from the workspace id
73

74
  if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
75
    if (use_individual_branch) {
76
      console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
77
      use_individual_branch = false;
78
    }
79
    if (group_by_folder) {
80
      console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
81
      group_by_folder = false;
82
    }
83
  }
84

85
  if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
86
    const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
87
    console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
88
    await move_to_git_branch(
89
      parent_workspace_id,
90
      path_type,
91
      path,
92
      parent_path,
93
      use_individual_branch,
94
      group_by_folder,
95
      clonedBranchName
96
    );
97
  }
98

99
  await move_to_git_branch(
100
    workspace_id,
101
    path_type,
102
    path,
103
    parent_path,
104
    use_individual_branch,
105
    group_by_folder,
106
    clonedBranchName
107
  );
108
  const subfolder = repo_resource.folder ?? "";
109
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
110
  console.log(
111
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
112
  );
113
  await wmill_sync_pull(
114
    path_type,
115
    workspace_id,
116
    path,
117
    parent_path,
118
    skip_secret,
119
    repo_url_resource_path,
120
    use_individual_branch,
121
    repo_resource.branch
122
  );
123
  try {
124
    await git_push(path, parent_path, commit_msg, repo_resource);
125
  } catch (e) {
126
    throw e;
127
  } finally {
128
    await delete_pgp_keys();
129
    // Cleanup: remove safe.directory config
130
    if (safeDirectoryPath) {
131
      try {
132
        await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
133
      } catch (e) {
134
        console.log(`Warning: Could not unset safe.directory config: ${e}`);
135
      }
136
    }
137
  }
138
  console.log("Finished syncing");
139
  process.chdir(`${cwd}`);
140
}
141

142
function get_fork_branch_name(w_id: string, originalBranch: string): string {
143
  if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
144
    return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
145
  }
146
  return w_id;
147
}
148

149
async function git_clone(
150
  cwd: string,
151
  repo_resource: any,
152
  no_single_branch: boolean,
153
): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
154
  // TODO: handle private SSH keys as well
155
  let repo_url = repo_resource.url;
156
  const subfolder = repo_resource.folder ?? "";
157
  const branch = repo_resource.branch ?? "";
158
  const repo_name = basename(repo_url, ".git");
159
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
160
  if (azureMatch) {
161
    console.log(
162
      "Requires Azure DevOps service account access token, requesting..."
163
    );
164
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
165
    const response = await fetch(
166
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
167
      {
168
        method: "POST",
169
        body: new URLSearchParams({
170
          client_id: azureResource.azureClientId,
171
          client_secret: azureResource.azureClientSecret,
172
          grant_type: "client_credentials",
173
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
174
        }),
175
      }
176
    );
177
    const { access_token } = await response.json();
178
    repo_url = repo_url.replace(azureMatch[0], access_token);
179
  }
180
  const args = ["clone", "--quiet", "--depth", "1"];
181
  if (no_single_branch) {
182
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
183
  }
184
  if (subfolder !== "") {
185
    args.push("--sparse");
186
  }
187
  if (branch !== "") {
188
    args.push("--branch");
189
    args.push(branch);
190
  }
191
  args.push(repo_url);
192
  args.push(repo_name);
193
  await sh_run(-1, "git", ...args);
194
  try {
195
    process.chdir(`${cwd}/${repo_name}`);
196
    const safeDirectoryPath = process.cwd();
197
    // Add safe.directory to handle dubious ownership in cloned repo
198
    try {
199
      await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
200
    } catch (e) {
201
      console.log(`Warning: Could not add safe.directory config: ${e}`);
202
    }
203

204
    if (subfolder !== "") {
205
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
206
      try {
207
        process.chdir(`${cwd}/${repo_name}/${subfolder}`);
208
      } catch (err) {
209
        console.log(
210
          `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
211
        );
212
        throw err;
213
      }
214
    }
215
    const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
216
    return { repo_name, safeDirectoryPath, clonedBranchName };
217

218
  } catch (err) {
219
    console.log(
220
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
221
    );
222
    throw err;
223
  }
224
}
225
async function move_to_git_branch(
226
  workspace_id: string,
227
  path_type: PathType,
228
  path: string | undefined,
229
  parent_path: string | undefined,
230
  use_individual_branch: boolean,
231
  group_by_folder: boolean,
232
  originalBranchName: string
233
) {
234
  let branchName;
235
  if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
236
    branchName = get_fork_branch_name(workspace_id, originalBranchName);
237
  } else {
238
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
239
      return;
240
    }
241
    branchName = group_by_folder
242
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
243
          ?.split("/")
244
          .slice(0, 2)
245
          .join("__")}`
246
      : `wm_deploy/${workspace_id}/${path_type}/${(
247
          path ?? parent_path
248
        )?.replaceAll("/", "__")}`;
249
  }
250

251
  try {
252
    await sh_run(undefined, "git", "checkout", branchName);
253
  } catch (err) {
254
    console.log(
255
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
256
    );
257
    try {
258
      await sh_run(undefined, "git", "checkout", "-b", branchName);
259
      await sh_run(
260
        undefined,
261
        "git",
262
        "config",
263
        "--add",
264
        "--bool",
265
        "push.autoSetupRemote",
266
        "true"
267
      );
268
    } catch (err) {
269
      console.log(
270
        `Error checking out branch '${branchName}'. Error was:\n${err}`
271
      );
272
      throw err;
273
    }
274
  }
275
  console.log(`Successfully switched to branch ${branchName}`);
276
}
277
async function git_push(
278
  path: string | undefined,
279
  parent_path: string | undefined,
280
  commit_msg: string,
281
  repo_resource: any
282
) {
283
  let user_email = process.env["WM_EMAIL"] ?? "";
284
  let user_name = process.env["WM_USERNAME"] ?? "";
285

286
  if (repo_resource.gpg_key) {
287
    await set_gpg_signing_secret(repo_resource.gpg_key);
288
    // Configure git with GPG key email for signing
289
    await sh_run(
290
      undefined,
291
      "git",
292
      "config",
293
      "user.email",
294
      repo_resource.gpg_key.email
295
    );
296
    await sh_run(undefined, "git", "config", "user.name", user_name);
297
  } else {
298
    await sh_run(undefined, "git", "config", "user.email", user_email);
299
    await sh_run(undefined, "git", "config", "user.name", user_name);
300
  }
301

302
  if (path !== undefined && path !== null && path !== "") {
303
    try {
304
      await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
305
    } catch (e) {
306
      console.log(`Unable to stage files matching ${path}**, ${e}`);
307
    }
308
  }
309
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
310
    try {
311
      await sh_run(
312
        undefined,
313
        "git",
314
        "add",
315
        "wmill-lock.yaml",
316
        `${parent_path}**`
317
      );
318
    } catch (e) {
319
      console.log(`Unable to stage files matching ${parent_path}, ${e}`);
320
    }
321
  }
322
  try {
323
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
324
  } catch {
325
    // git diff returns exit-code = 1 when there's at least one staged changes
326
    const commitArgs = ["git", "commit"];
327

328
    // Always use --author to set consistent authorship
329
    commitArgs.push("--author", `"${user_name} <${user_email}>"`);
330
    commitArgs.push(
331
      "-m",
332
      `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
333
    );
334

335
    await sh_run(undefined, ...commitArgs);
336
    try {
337
      await sh_run(undefined, "git", "push", "--porcelain");
338
    } catch (e) {
339
      console.log(`Could not push, trying to rebase first: ${e}`);
340
      await sh_run(undefined, "git", "pull", "--rebase");
341
      await sh_run(undefined, "git", "push", "--porcelain");
342
    }
343
    return;
344
  }
345
  console.log("No changes detected, nothing to commit. Returning...");
346
}
347
async function sh_run(
348
  secret_position: number | undefined,
349
  cmd: string,
350
  ...args: string[]
351
) {
352
  const nargs = secret_position != undefined ? args.slice() : args;
353
  if (secret_position && secret_position < 0) {
354
    secret_position = nargs.length - 1 + secret_position;
355
  }
356
  let secret: string | undefined = undefined;
357
  if (secret_position != undefined) {
358
    nargs[secret_position] = "***";
359
    secret = args[secret_position];
360
  }
361

362
  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
363
  const command = exec(`${cmd} ${args.join(" ")}`);
364
  // new Deno.Command(cmd, {
365
  //   args: args,
366
  // });
367
  try {
368
    const { stdout, stderr } = await command;
369
    if (stdout.length > 0) {
370
      console.log(stdout);
371
    }
372
    if (stderr.length > 0) {
373
      console.log(stderr);
374
    }
375
    console.log("Command successfully executed");
376
    return stdout;
377
  } catch (error) {
378
    let errorString = error.toString();
379
    if (secret) {
380
      errorString = errorString.replace(secret, "***");
381
    }
382
    const err = `SH command '${cmd} ${nargs.join(
383
      " "
384
    )}' returned with error ${errorString}`;
385
    throw Error(err);
386
  }
387
}
388

389
function regexFromPath(path_type: PathType, path: string) {
390
  if (path_type == "flow") {
391
    return `${path}.flow/*`;
392
  }
393
  if (path_type == "app") {
394
    return `${path}.app/*`;
395
  } else if (path_type == "folder") {
396
    return `${path}/folder.meta.*`;
397
  } else if (path_type == "resourcetype") {
398
    return `${path}.resource-type.*`;
399
  } else if (path_type == "resource") {
400
    return `${path}.resource.*`;
401
  } else if (path_type == "variable") {
402
    return `${path}.variable.*`;
403
  } else if (path_type == "schedule") {
404
    return `${path}.schedule.*`;
405
  } else if (path_type == "user") {
406
    return `${path}.user.*`;
407
  } else if (path_type == "group") {
408
    return `${path}.group.*`;
409
  } else if (path_type == "httptrigger") {
410
    return `${path}.http_trigger.*`;
411
  } else if (path_type == "websockettrigger") {
412
    return `${path}.websocket_trigger.*`;
413
  } else if (path_type == "kafkatrigger") {
414
    return `${path}.kafka_trigger.*`;
415
  } else if (path_type == "natstrigger") {
416
    return `${path}.nats_trigger.*`;
417
  } else if (path_type == "postgrestrigger") {
418
    return `${path}.postgres_trigger.*`;
419
  } else if (path_type == "mqtttrigger") {
420
    return `${path}.mqtt_trigger.*`;
421
  } else if (path_type == "sqstrigger") {
422
    return `${path}.sqs_trigger.*`;
423
  } else if (path_type == "gcptrigger") {
424
    return `${path}.gcp_trigger.*`;
425
  } else if (path_type == "emailtrigger") {
426
    return `${path}.email_trigger.*`;
427
  } else {
428
    return `${path}.*`;
429
  }
430
}
431

432
async function wmill_sync_pull(
433
  path_type: PathType,
434
  workspace_id: string,
435
  path: string | undefined,
436
  parent_path: string | undefined,
437
  skip_secret: boolean,
438
  repo_url_resource_path: string,
439
  use_individual_branch: boolean,
440
  original_branch?: string
441
) {
442
  const includes = [];
443
  if (path !== undefined && path !== null && path !== "") {
444
    includes.push(regexFromPath(path_type, path));
445
  }
446
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
447
    includes.push(regexFromPath(path_type, parent_path));
448
  }
449
  await wmill_run(
450
    6,
451
    "workspace",
452
    "add",
453
    workspace_id,
454
    workspace_id,
455
    process.env["BASE_URL"] + "/",
456
    "--token",
457
    process.env["WM_TOKEN"] ?? ""
458
  );
459
  console.log("Pulling workspace into git repo");
460
  const args = [
461
    "sync",
462
    "pull",
463
    "--token",
464
    process.env["WM_TOKEN"] ?? "",
465
    "--workspace",
466
    workspace_id,
467
    "--repository",
468
    repo_url_resource_path,
469
    "--yes",
470
    skip_secret ? "--skip-secrets" : "",
471
    "--include-schedules",
472
    "--include-users",
473
    "--include-groups",
474
    "--include-triggers",
475
  ];
476

477
  // Only include settings when specifically deploying settings
478
  if (path_type === "settings" && !use_individual_branch) {
479
    args.push("--include-settings");
480
  }
481

482
  // Only include key when specifically deploying keys
483
  if (path_type === "key" && !use_individual_branch) {
484
    args.push("--include-key");
485
  }
486

487
  args.push("--extra-includes", includes.join(","));
488

489
  // If using individual branches, apply promotion settings from original branch
490
  if (use_individual_branch && original_branch) {
491
    console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
492
    args.push("--promotion", original_branch);
493
  }
494

495
  await wmill_run(3, ...args);
496
}
497

498
async function wmill_run(secret_position: number, ...cmd: string[]) {
499
  cmd = cmd.filter((elt) => elt !== "");
500
  const cmd2 = cmd.slice();
501
  cmd2[secret_position] = "***";
502
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
503
  await wmill.parse(cmd);
504
  console.log("Command successfully executed");
505
}
506

507
// Function to set up GPG signing
508
async function set_gpg_signing_secret(gpg_key: GpgKey) {
509
  try {
510
    console.log("Setting GPG private key for git commits");
511

512
    const formattedGpgContent = gpg_key.private_key.replace(
513
      /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
514
      (_: string, header: string, body: string, footer: string) =>
515
        header +
516
        "\n" +
517
        "\n" +
518
        body.replace(/ ([^\s])/g, "\n$1").trim() +
519
        "\n" +
520
        footer
521
    );
522

523
    const gpg_path = `/tmp/gpg`;
524
    await sh_run(undefined, "mkdir", "-p", gpg_path);
525
    await sh_run(undefined, "chmod", "700", gpg_path);
526
    process.env.GNUPGHOME = gpg_path;
527
    // process.env.GIT_TRACE = 1;
528

529
    try {
530
      await sh_run(
531
        1,
532
        "bash",
533
        "-c",
534
        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
535
      );
536
    } catch (e) {
537
      // Original error would contain sensitive data
538
      throw new Error("Failed to import GPG key!");
539
    }
540

541
    const listKeysOutput = await sh_run(
542
      undefined,
543
      "gpg",
544
      "--list-secret-keys",
545
      "--with-colons",
546
      "--keyid-format=long"
547
    );
548

549
    const keyInfoMatch = listKeysOutput.match(
550
      /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
551
    );
552

553
    if (!keyInfoMatch) {
554
      throw new Error("Failed to extract GPG Key ID and Fingerprint");
555
    }
556

557
    const keyId = keyInfoMatch[1];
558
    gpgFingerprint = keyInfoMatch[2];
559

560
    if (gpg_key.passphrase) {
561
      // This is adummy command to unlock the key
562
      // with passphrase to load it into agent
563
      await sh_run(
564
        1,
565
        "bash",
566
        "-c",
567
        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
568
      );
569
    }
570

571
    // Configure Git to use the extracted key
572
    await sh_run(undefined, "git", "config", "user.signingkey", keyId);
573
    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
574
    console.log(`GPG signing configured with key ID: ${keyId} `);
575
  } catch (e) {
576
    console.error(`Failure while setting GPG key: ${e} `);
577
    await delete_pgp_keys();
578
  }
579
}
580

581
async function delete_pgp_keys() {
582
  console.log("deleting gpg keys");
583
  if (gpgFingerprint) {
584
    await sh_run(
585
      undefined,
586
      "gpg",
587
      "--batch",
588
      "--yes",
589
      "--pinentry-mode",
590
      "loopback",
591
      "--delete-secret-key",
592
      gpgFingerprint
593
    );
594
    await sh_run(
595
      undefined,
596
      "gpg",
597
      "--batch",
598
      "--yes",
599
      "--delete-key",
600
      "--pinentry-mode",
601
      "loopback",
602
      gpgFingerprint
603
    );
604
  }
605
}
606

607
async function get_gh_app_token() {
608
  const workspace = process.env["WM_WORKSPACE"];
609
  const jobToken = process.env["WM_TOKEN"];
610

611
  const baseUrl =
612
    process.env["BASE_INTERNAL_URL"] ??
613
    process.env["BASE_URL"] ??
614
    "http://localhost:8000";
615

616
  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
617

618
  const response = await fetch(url, {
619
    method: "POST",
620
    headers: {
621
      "Content-Type": "application/json",
622
      Authorization: `Bearer ${jobToken}`,
623
    },
624
    body: JSON.stringify({
625
      job_token: jobToken,
626
    }),
627
  });
628

629
  if (!response.ok) {
630
    throw new Error(`Error: ${response.statusText}`);
631
  }
632

633
  const data = await response.json();
634

635
  return data.token;
636
}
637

638
function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
639
  if (!gitHubUrl || !installationToken) {
640
    throw new Error("Both GitHub URL and Installation Token are required.");
641
  }
642

643
  try {
644
    const url = new URL(gitHubUrl);
645

646
    // GitHub repository URL should be in the format: https://github.com/owner/repo.git
647
    if (url.hostname !== "github.com") {
648
      throw new Error(
649
        "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
650
      );
651
    }
652

653
    // Convert URL to include the installation token
654
    return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
655
  } catch (e) {
656
    const error = e as Error;
657
    throw new Error(`Invalid URL: ${error.message}`);
658
  }
659
}
660

Other submissions