1
Sync script to Git repo
One script reply has been approved by the moderators Verified

This script will pull the script from the current workspace in a temporary folder, then commit it to the remote Git repository and push it. Only the script-related file will be pushed, nothing else. It takes as input the git_repository resource containing the repository URL, the script path, and the commit message to use for the commit. All params are mandatory.

Created by hugo697 593 days ago Viewed 84622 times
0
Submitted by rubenfiszel Bun
Verified 310 days ago
1
import * as wmillclient from "windmill-client";
2
import wmill from "windmill-cli";
3
import { basename } from "node:path";
4
const util = require("util");
5
const exec = util.promisify(require("child_process").exec);
6
import process from "process";
7

8
type GpgKey = {
9
  email: string;
10
  private_key: string;
11
  passphrase: string;
12
};
13

14
type PathType =
15
  | "script"
16
  | "flow"
17
  | "app"
18
  | "folder"
19
  | "resource"
20
  | "variable"
21
  | "resourcetype"
22
  | "schedule"
23
  | "user"
24
  | "group"
25
  | "httptrigger"
26
  | "websockettrigger"
27
  | "kafkatrigger"
28
  | "natstrigger"
29
  | "postgrestrigger"
30
  | "mqtttrigger"
31
  | "sqstrigger"
32
  | "gcptrigger";
33

34
let gpgFingerprint: string | undefined = undefined;
35

36
export async function main(
37
  workspace_id: string,
38
  repo_url_resource_path: string,
39
  path_type: PathType,
40
  skip_secret = true,
41
  path: string | undefined,
42
  parent_path: string | undefined,
43
  commit_msg: string,
44
  use_individual_branch = false,
45
  group_by_folder = false
46
) {
47
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
48
  const cwd = process.cwd();
49
  process.env["HOME"] = ".";
50
  console.log(
51
    `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
52
  );
53

54
  if (repo_resource.is_github_app) {
55
    const token = await get_gh_app_token();
56
    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
57
    repo_resource.url = authRepoUrl;
58
  }
59

60
  const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
61
  await move_to_git_branch(
62
    workspace_id,
63
    path_type,
64
    path,
65
    parent_path,
66
    use_individual_branch,
67
    group_by_folder
68
  );
69
  const subfolder = repo_resource.folder ?? "";
70
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
71
  console.log(
72
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
73
  );
74
  await wmill_sync_pull(
75
    path_type,
76
    workspace_id,
77
    path,
78
    parent_path,
79
    skip_secret
80
  );
81
  try {
82
    await git_push(path, parent_path, commit_msg, repo_resource);
83
  } catch (e) {
84
    throw e;
85
  } finally {
86
    await delete_pgp_keys();
87
  }
88
  console.log("Finished syncing");
89
  process.chdir(`${cwd}`);
90
}
91
async function git_clone(
92
  cwd: string,
93
  repo_resource: any,
94
  use_individual_branch: boolean
95
): Promise<string> {
96
  // TODO: handle private SSH keys as well
97
  let repo_url = repo_resource.url;
98
  const subfolder = repo_resource.folder ?? "";
99
  const branch = repo_resource.branch ?? "";
100
  const repo_name = basename(repo_url, ".git");
101
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
102
  if (azureMatch) {
103
    console.log(
104
      "Requires Azure DevOps service account access token, requesting..."
105
    );
106
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
107
    const response = await fetch(
108
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
109
      {
110
        method: "POST",
111
        body: new URLSearchParams({
112
          client_id: azureResource.azureClientId,
113
          client_secret: azureResource.azureClientSecret,
114
          grant_type: "client_credentials",
115
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
116
        }),
117
      }
118
    );
119
    const { access_token } = await response.json();
120
    repo_url = repo_url.replace(azureMatch[0], access_token);
121
  }
122
  const args = ["clone", "--quiet", "--depth", "1"];
123
  if (use_individual_branch) {
124
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
125
  }
126
  if (subfolder !== "") {
127
    args.push("--sparse");
128
  }
129
  if (branch !== "") {
130
    args.push("--branch");
131
    args.push(branch);
132
  }
133
  args.push(repo_url);
134
  args.push(repo_name);
135
  await sh_run(-1, "git", ...args);
136
  try {
137
    process.chdir(`${cwd}/${repo_name}`);
138
  } catch (err) {
139
    console.log(
140
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
141
    );
142
    throw err;
143
  }
144
  process.chdir(`${cwd}/${repo_name}`);
145
  if (subfolder !== "") {
146
    await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
147
  }
148
  try {
149
    process.chdir(`${cwd}/${repo_name}/${subfolder}`);
150
  } catch (err) {
151
    console.log(
152
      `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
153
    );
154
    throw err;
155
  }
156
  return repo_name;
157
}
158
async function move_to_git_branch(
159
  workspace_id: string,
160
  path_type: PathType,
161
  path: string | undefined,
162
  parent_path: string | undefined,
163
  use_individual_branch: boolean,
164
  group_by_folder: boolean
165
) {
166
  if (!use_individual_branch || path_type === "user" || path_type === "group") {
167
    return;
168
  }
169
  const branchName = group_by_folder
170
    ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
171
        ?.split("/")
172
        .slice(0, 2)
173
        .join("__")}`
174
    : `wm_deploy/${workspace_id}/${path_type}/${(
175
        path ?? parent_path
176
      )?.replaceAll("/", "__")}`;
177
  try {
178
    await sh_run(undefined, "git", "checkout", branchName);
179
  } catch (err) {
180
    console.log(
181
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
182
    );
183
    try {
184
      await sh_run(undefined, "git", "checkout", "-b", branchName);
185
      await sh_run(
186
        undefined,
187
        "git",
188
        "config",
189
        "--add",
190
        "--bool",
191
        "push.autoSetupRemote",
192
        "true"
193
      );
194
    } catch (err) {
195
      console.log(
196
        `Error checking out branch '${branchName}'. Error was:\n${err}`
197
      );
198
      throw err;
199
    }
200
  }
201
  console.log(`Successfully switched to branch ${branchName}`);
202
}
203
async function git_push(
204
  path: string | undefined,
205
  parent_path: string | undefined,
206
  commit_msg: string,
207
  repo_resource: any
208
) {
209
  let user_email = process.env["WM_EMAIL"] ?? "";
210
  let user_name = process.env["WM_USERNAME"] ?? "";
211

212
  if (repo_resource.gpg_key) {
213
    await set_gpg_signing_secret(repo_resource.gpg_key);
214
    // Configure git with GPG key email for signing
215
    await sh_run(
216
      undefined,
217
      "git",
218
      "config",
219
      "user.email",
220
      repo_resource.gpg_key.email
221
    );
222
    await sh_run(undefined, "git", "config", "user.name", user_name);
223
  } else {
224
    await sh_run(undefined, "git", "config", "user.email", user_email);
225
    await sh_run(undefined, "git", "config", "user.name", user_name);
226
  }
227

228
  if (path !== undefined && path !== null && path !== "") {
229
    try {
230
      await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
231
    } catch (e) {
232
      console.log(`Unable to stage files matching ${path}**, ${e}`);
233
    }
234
  }
235
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
236
    try {
237
      await sh_run(
238
        undefined,
239
        "git",
240
        "add",
241
        "wmill-lock.yaml",
242
        `${parent_path}**`
243
      );
244
    } catch (e) {
245
      console.log(`Unable to stage files matching ${parent_path}, ${e}`);
246
    }
247
  }
248
  try {
249
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
250
  } catch {
251
    // git diff returns exit-code = 1 when there's at least one staged changes
252
    const commitArgs = ["git", "commit"];
253

254
    // Always use --author to set consistent authorship
255
    commitArgs.push("--author", `"${user_name} <${user_email}>"`);
256
    commitArgs.push(
257
      "-m",
258
      `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
259
    );
260

261
    await sh_run(undefined, ...commitArgs);
262
    try {
263
      await sh_run(undefined, "git", "push", "--porcelain");
264
    } catch (e) {
265
      console.log(`Could not push, trying to rebase first: ${e}`);
266
      await sh_run(undefined, "git", "pull", "--rebase");
267
      await sh_run(undefined, "git", "push", "--porcelain");
268
    }
269
    return;
270
  }
271
  console.log("No changes detected, nothing to commit. Returning...");
272
}
273
async function sh_run(
274
  secret_position: number | undefined,
275
  cmd: string,
276
  ...args: string[]
277
) {
278
  const nargs = secret_position != undefined ? args.slice() : args;
279
  if (secret_position && secret_position < 0) {
280
    secret_position = nargs.length - 1 + secret_position;
281
  }
282
  let secret: string | undefined = undefined;
283
  if (secret_position != undefined) {
284
    nargs[secret_position] = "***";
285
    secret = args[secret_position];
286
  }
287

288
  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
289
  const command = exec(`${cmd} ${args.join(" ")}`);
290
  // new Deno.Command(cmd, {
291
  //   args: args,
292
  // });
293
  try {
294
    const { stdout, stderr } = await command;
295
    if (stdout.length > 0) {
296
      console.log(stdout);
297
    }
298
    if (stderr.length > 0) {
299
      console.log(stderr);
300
    }
301
    console.log("Command successfully executed");
302
    return stdout;
303
  } catch (error) {
304
    let errorString = error.toString();
305
    if (secret) {
306
      errorString = errorString.replace(secret, "***");
307
    }
308
    const err = `SH command '${cmd} ${nargs.join(
309
      " "
310
    )}' returned with error ${errorString}`;
311
    throw Error(err);
312
  }
313
}
314

315
function regexFromPath(path_type: PathType, path: string) {
316
  if (path_type == "flow") {
317
    return `${path}.flow/*`;
318
  }
319
  if (path_type == "app") {
320
    return `${path}.app/*`;
321
  } else if (path_type == "folder") {
322
    return `${path}/folder.meta.*`;
323
  } else if (path_type == "resourcetype") {
324
    return `${path}.resource-type.*`;
325
  } else if (path_type == "resource") {
326
    return `${path}.resource.*`;
327
  } else if (path_type == "variable") {
328
    return `${path}.variable.*`;
329
  } else if (path_type == "schedule") {
330
    return `${path}.schedule.*`;
331
  } else if (path_type == "user") {
332
    return `${path}.user.*`;
333
  } else if (path_type == "group") {
334
    return `${path}.group.*`;
335
  } else if (path_type == "httptrigger") {
336
    return `${path}.http_trigger.*`;
337
  } else if (path_type == "websockettrigger") {
338
    return `${path}.websocket_trigger.*`;
339
  } else if (path_type == "kafkatrigger") {
340
    return `${path}.kafka_trigger.*`;
341
  } else if (path_type == "natstrigger") {
342
    return `${path}.nats_trigger.*`;
343
  } else if (path_type == "postgrestrigger") {
344
    return `${path}.postgres_trigger.*`;
345
  } else if (path_type == "mqtttrigger") {
346
    return `${path}.mqtt_trigger.*`;
347
  } else if (path_type == "sqstrigger") {
348
    return `${path}.sqs_trigger.*`;
349
  } else if (path_type == "gcptrigger") {
350
    return `${path}.gcp_trigger.*`;
351
  } else {
352
    return `${path}.*`;
353
  }
354
}
355

356
async function wmill_sync_pull(
357
  path_type: PathType,
358
  workspace_id: string,
359
  path: string | undefined,
360
  parent_path: string | undefined,
361
  skip_secret: boolean
362
) {
363
  const includes = [];
364
  if (path !== undefined && path !== null && path !== "") {
365
    includes.push(regexFromPath(path_type, path));
366
  }
367
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
368
    includes.push(regexFromPath(path_type, parent_path));
369
  }
370
  await wmill_run(
371
    6,
372
    "workspace",
373
    "add",
374
    workspace_id,
375
    workspace_id,
376
    process.env["BASE_URL"] + "/",
377
    "--token",
378
    process.env["WM_TOKEN"] ?? ""
379
  );
380
  console.log("Pulling workspace into git repo");
381
  await wmill_run(
382
    3,
383
    "sync",
384
    "pull",
385
    "--token",
386
    process.env["WM_TOKEN"] ?? "",
387
    "--workspace",
388
    workspace_id,
389
    "--yes",
390
    skip_secret ? "--skip-secrets" : "",
391
    "--include-schedules",
392
    "--include-users",
393
    "--include-groups",
394
    "--include-triggers",
395
    "--include-settings",
396
    "--include-key",
397
    "--extra-includes",
398
    includes.join(",")
399
  );
400
}
401

402
async function wmill_run(secret_position: number, ...cmd: string[]) {
403
  cmd = cmd.filter((elt) => elt !== "");
404
  const cmd2 = cmd.slice();
405
  cmd2[secret_position] = "***";
406
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
407
  await wmill.parse(cmd);
408
  console.log("Command successfully executed");
409
}
410

411
// Function to set up GPG signing
412
async function set_gpg_signing_secret(gpg_key: GpgKey) {
413
  try {
414
    console.log("Setting GPG private key for git commits");
415

416
    const formattedGpgContent = gpg_key.private_key.replace(
417
      /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
418
      (_: string, header: string, body: string, footer: string) =>
419
        header +
420
        "\n" +
421
        "\n" +
422
        body.replace(/ ([^\s])/g, "\n$1").trim() +
423
        "\n" +
424
        footer
425
    );
426

427
    const gpg_path = `/tmp/gpg`;
428
    await sh_run(undefined, "mkdir", "-p", gpg_path);
429
    await sh_run(undefined, "chmod", "700", gpg_path);
430
    process.env.GNUPGHOME = gpg_path;
431
    // process.env.GIT_TRACE = 1;
432

433
    try {
434
      await sh_run(
435
        1,
436
        "bash",
437
        "-c",
438
        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
439
      );
440
    } catch (e) {
441
      // Original error would contain sensitive data
442
      throw new Error("Failed to import GPG key!");
443
    }
444

445
    const listKeysOutput = await sh_run(
446
      undefined,
447
      "gpg",
448
      "--list-secret-keys",
449
      "--with-colons",
450
      "--keyid-format=long"
451
    );
452

453
    const keyInfoMatch = listKeysOutput.match(
454
      /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
455
    );
456

457
    if (!keyInfoMatch) {
458
      throw new Error("Failed to extract GPG Key ID and Fingerprint");
459
    }
460

461
    const keyId = keyInfoMatch[1];
462
    gpgFingerprint = keyInfoMatch[2];
463

464
    if (gpg_key.passphrase) {
465
      // This is adummy command to unlock the key
466
      // with passphrase to load it into agent
467
      await sh_run(
468
        1,
469
        "bash",
470
        "-c",
471
        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
472
      );
473
    }
474

475
    // Configure Git to use the extracted key
476
    await sh_run(undefined, "git", "config", "user.signingkey", keyId);
477
    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
478
    console.log(`GPG signing configured with key ID: ${keyId} `);
479
  } catch (e) {
480
    console.error(`Failure while setting GPG key: ${e} `);
481
    await delete_pgp_keys();
482
  }
483
}
484

485
async function delete_pgp_keys() {
486
  console.log("deleting gpg keys");
487
  if (gpgFingerprint) {
488
    await sh_run(
489
      undefined,
490
      "gpg",
491
      "--batch",
492
      "--yes",
493
      "--pinentry-mode",
494
      "loopback",
495
      "--delete-secret-key",
496
      gpgFingerprint
497
    );
498
    await sh_run(
499
      undefined,
500
      "gpg",
501
      "--batch",
502
      "--yes",
503
      "--delete-key",
504
      "--pinentry-mode",
505
      "loopback",
506
      gpgFingerprint
507
    );
508
  }
509
}
510

511
async function get_gh_app_token() {
512
  const workspace = process.env["WM_WORKSPACE"];
513
  const jobToken = process.env["WM_TOKEN"];
514

515
  const baseUrl =
516
    process.env["BASE_INTERNAL_URL"] ??
517
    process.env["BASE_URL"] ??
518
    "http://localhost:8000";
519

520
  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
521

522
  const response = await fetch(url, {
523
    method: "POST",
524
    headers: {
525
      "Content-Type": "application/json",
526
      Authorization: `Bearer ${jobToken}`,
527
    },
528
    body: JSON.stringify({
529
      job_token: jobToken,
530
    }),
531
  });
532

533
  if (!response.ok) {
534
    throw new Error(`Error: ${response.statusText}`);
535
  }
536

537
  const data = await response.json();
538

539
  return data.token;
540
}
541

542
function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
543
  if (!gitHubUrl || !installationToken) {
544
    throw new Error("Both GitHub URL and Installation Token are required.");
545
  }
546

547
  try {
548
    const url = new URL(gitHubUrl);
549

550
    // GitHub repository URL should be in the format: https://github.com/owner/repo.git
551
    if (url.hostname !== "github.com") {
552
      throw new Error(
553
        "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
554
      );
555
    }
556

557
    // Convert URL to include the installation token
558
    return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
559
  } catch (e) {
560
    const error = e as Error;
561
    throw new Error(`Invalid URL: ${error.message}`);
562
  }
563
}
564

Other submissions