1
Sync script to Git repo
One script reply has been approved by the moderators Verified

This script will pull the script from the current workspace in a temporary folder, then commit it to the remote Git repository and push it. Only the script-related file will be pushed, nothing else. It takes as input the git_repository resource containing the repository URL, the script path, and the commit message to use for the commit. All params are mandatory.

Created by hugo697 576 days ago Viewed 79800 times
0
Submitted by rubenfiszel Bun
Verified 293 days ago
1
import * as wmillclient from "windmill-client";
2
import wmill from "windmill-cli";
3
import { basename } from "node:path";
4
const util = require("util");
5
const exec = util.promisify(require("child_process").exec);
6
import process from "process";
7

8
type GpgKey = {
9
  email: string;
10
  private_key: string;
11
  passphrase: string;
12
};
13

14
type PathType =
15
  | "script"
16
  | "flow"
17
  | "app"
18
  | "folder"
19
  | "resource"
20
  | "variable"
21
  | "resourcetype"
22
  | "schedule"
23
  | "user"
24
  | "group"
25
  | "httptrigger"
26
  | "websockettrigger"
27
  | "kafkatrigger"
28
  | "natstrigger"
29
  | "postgrestrigger"
30
  | "mqtttrigger"
31
  | "sqstrigger"
32
  | "gcptrigger";
33

34
let gpgFingerprint: string | undefined = undefined;
35

36
export async function main(
37
  workspace_id: string,
38
  repo_url_resource_path: string,
39
  path_type: PathType,
40
  skip_secret = true,
41
  path: string | undefined,
42
  parent_path: string | undefined,
43
  commit_msg: string,
44
  use_individual_branch = false,
45
  group_by_folder = false
46
) {
47
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
48
  const cwd = process.cwd();
49
  process.env["HOME"] = ".";
50
  console.log(
51
    `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
52
  );
53

54
  if (repo_resource.is_github_app) {
55
    const token = await get_gh_app_token();
56
    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
57
    repo_resource.url = authRepoUrl;
58
  }
59

60
  const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
61
  await move_to_git_branch(
62
    workspace_id,
63
    path_type,
64
    path,
65
    parent_path,
66
    use_individual_branch,
67
    group_by_folder
68
  );
69
  const subfolder = repo_resource.folder ?? "";
70
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
71
  console.log(
72
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
73
  );
74
  await wmill_sync_pull(
75
    path_type,
76
    workspace_id,
77
    path,
78
    parent_path,
79
    skip_secret
80
  );
81
  try {
82
    await git_push(path, parent_path, commit_msg, repo_resource);
83
  } catch (e) {
84
    throw e;
85
  } finally {
86
    await delete_pgp_keys();
87
  }
88
  console.log("Finished syncing");
89
  process.chdir(`${cwd}`);
90
}
91
async function git_clone(
92
  cwd: string,
93
  repo_resource: any,
94
  use_individual_branch: boolean
95
): Promise<string> {
96
  // TODO: handle private SSH keys as well
97
  let repo_url = repo_resource.url;
98
  const subfolder = repo_resource.folder ?? "";
99
  const branch = repo_resource.branch ?? "";
100
  const repo_name = basename(repo_url, ".git");
101
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
102
  if (azureMatch) {
103
    console.log(
104
      "Requires Azure DevOps service account access token, requesting..."
105
    );
106
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
107
    const response = await fetch(
108
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
109
      {
110
        method: "POST",
111
        body: new URLSearchParams({
112
          client_id: azureResource.azureClientId,
113
          client_secret: azureResource.azureClientSecret,
114
          grant_type: "client_credentials",
115
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
116
        }),
117
      }
118
    );
119
    const { access_token } = await response.json();
120
    repo_url = repo_url.replace(azureMatch[0], access_token);
121
  }
122
  const args = ["clone", "--quiet", "--depth", "1"];
123
  if (use_individual_branch) {
124
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
125
  }
126
  if (subfolder !== "") {
127
    args.push("--sparse");
128
  }
129
  if (branch !== "") {
130
    args.push("--branch");
131
    args.push(branch);
132
  }
133
  args.push(repo_url);
134
  args.push(repo_name);
135
  await sh_run(-1, "git", ...args);
136
  try {
137
    process.chdir(`${cwd}/${repo_name}`);
138
  } catch (err) {
139
    console.log(
140
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
141
    );
142
    throw err;
143
  }
144
  process.chdir(`${cwd}/${repo_name}`);
145
  if (subfolder !== "") {
146
    await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
147
  }
148
  try {
149
    process.chdir(`${cwd}/${repo_name}/${subfolder}`);
150
  } catch (err) {
151
    console.log(
152
      `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
153
    );
154
    throw err;
155
  }
156
  return repo_name;
157
}
158
async function move_to_git_branch(
159
  workspace_id: string,
160
  path_type: PathType,
161
  path: string | undefined,
162
  parent_path: string | undefined,
163
  use_individual_branch: boolean,
164
  group_by_folder: boolean
165
) {
166
  if (!use_individual_branch || path_type === "user" || path_type === "group") {
167
    return;
168
  }
169
  const branchName = group_by_folder
170
    ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
171
        ?.split("/")
172
        .slice(0, 2)
173
        .join("__")}`
174
    : `wm_deploy/${workspace_id}/${path_type}/${(
175
        path ?? parent_path
176
      )?.replaceAll("/", "__")}`;
177
  try {
178
    await sh_run(undefined, "git", "checkout", branchName);
179
  } catch (err) {
180
    console.log(
181
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
182
    );
183
    try {
184
      await sh_run(undefined, "git", "checkout", "-b", branchName);
185
      await sh_run(
186
        undefined,
187
        "git",
188
        "config",
189
        "--add",
190
        "--bool",
191
        "push.autoSetupRemote",
192
        "true"
193
      );
194
    } catch (err) {
195
      console.log(
196
        `Error checking out branch '${branchName}'. Error was:\n${err}`
197
      );
198
      throw err;
199
    }
200
  }
201
  console.log(`Successfully switched to branch ${branchName}`);
202
}
203
async function git_push(
204
  path: string | undefined,
205
  parent_path: string | undefined,
206
  commit_msg: string,
207
  repo_resource: any
208
) {
209
  let user_email = process.env["WM_EMAIL"] ?? "";
210
  let user_name = process.env["WM_USERNAME"] ?? "";
211

212
  if (repo_resource.gpg_key) {
213
    await set_gpg_signing_secret(repo_resource.gpg_key);
214
    // Configure git with GPG key email for signing
215
    await sh_run(
216
      undefined,
217
      "git",
218
      "config",
219
      "user.email",
220
      repo_resource.gpg_key.email
221
    );
222
    await sh_run(undefined, "git", "config", "user.name", user_name);
223
  } else {
224
    await sh_run(undefined, "git", "config", "user.email", user_email);
225
    await sh_run(undefined, "git", "config", "user.name", user_name);
226
  }
227

228
  if (path !== undefined && path !== null && path !== "") {
229
    try {
230
      await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
231
    } catch (e) {
232
      console.log(`Unable to stage files matching ${path}**, ${e}`);
233
    }
234
  }
235
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
236
    try {
237
      await sh_run(
238
        undefined,
239
        "git",
240
        "add",
241
        "wmill-lock.yaml",
242
        `${parent_path}**`
243
      );
244
    } catch (e) {
245
      console.log(`Unable to stage files matching ${parent_path}, ${e}`);
246
    }
247
  }
248
  try {
249
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
250
  } catch {
251
    // git diff returns exit-code = 1 when there's at least one staged changes
252
    const commitArgs = ["git", "commit"];
253

254
    // Always use --author to set consistent authorship
255
    commitArgs.push("--author", `"${user_name} <${user_email}>"`);
256
    commitArgs.push(
257
      "-m",
258
      `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
259
    );
260

261
    await sh_run(undefined, ...commitArgs);
262
    try {
263
      await sh_run(undefined, "git", "push", "--porcelain");
264
    } catch (e) {
265
      console.log(`Could not push, trying to rebase first: ${e}`);
266
      await sh_run(undefined, "git", "pull", "--rebase");
267
      await sh_run(undefined, "git", "push", "--porcelain");
268
    }
269
    return;
270
  }
271
  console.log("No changes detected, nothing to commit. Returning...");
272
}
273
async function sh_run(
274
  secret_position: number | undefined,
275
  cmd: string,
276
  ...args: string[]
277
) {
278
  const nargs = secret_position != undefined ? args.slice() : args;
279
  if (secret_position && secret_position < 0) {
280
    secret_position = nargs.length - 1 + secret_position;
281
  }
282
  let secret: string | undefined = undefined;
283
  if (secret_position != undefined) {
284
    nargs[secret_position] = "***";
285
    secret = args[secret_position];
286
  }
287

288
  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
289
  const command = exec(`${cmd} ${args.join(" ")}`);
290
  // new Deno.Command(cmd, {
291
  //   args: args,
292
  // });
293
  try {
294
    const { stdout, stderr } = await command;
295
    if (stdout.length > 0) {
296
      console.log(stdout);
297
    }
298
    if (stderr.length > 0) {
299
      console.log(stderr);
300
    }
301
    console.log("Command successfully executed");
302
    return stdout;
303
  } catch (error) {
304
    let errorString = error.toString();
305
    if (secret) {
306
      errorString = errorString.replace(secret, "***");
307
    }
308
    const err = `SH command '${cmd} ${nargs.join(
309
      " "
310
    )}' returned with error ${errorString}`;
311
    throw Error(err);
312
  }
313
}
314

315
function regexFromPath(path_type: PathType, path: string) {
316
  if (path_type == "flow") {
317
    return `${path}.flow/*`;
318
  }
319
  if (path_type == "app") {
320
    return `${path}.app/*`;
321
  } else if (path_type == "folder") {
322
    return `${path}/folder.meta.*`;
323
  } else if (path_type == "resourcetype") {
324
    return `${path}.resource-type.*`;
325
  } else if (path_type == "resource") {
326
    return `${path}.resource.*`;
327
  } else if (path_type == "variable") {
328
    return `${path}.variable.*`;
329
  } else if (path_type == "schedule") {
330
    return `${path}.schedule.*`;
331
  } else if (path_type == "user") {
332
    return `${path}.user.*`;
333
  } else if (path_type == "group") {
334
    return `${path}.group.*`;
335
  } else if (path_type == "httptrigger") {
336
    return `${path}.http_trigger.*`;
337
  } else if (path_type == "websockettrigger") {
338
    return `${path}.websocket_trigger.*`;
339
  } else if (path_type == "kafkatrigger") {
340
    return `${path}.kafka_trigger.*`;
341
  } else if (path_type == "natstrigger") {
342
    return `${path}.nats_trigger.*`;
343
  } else if (path_type == "postgrestrigger") {
344
    return `${path}.postgres_trigger.*`;
345
  } else if (path_type == "mqtttrigger") {
346
    return `${path}.mqtt_trigger.*`;
347
  } else if (path_type == "sqstrigger") {
348
    return `${path}.sqs_trigger.*`;
349
  } else if (path_type == "gcptrigger") {
350
    return `${path}.gcp_trigger.*`;
351
  } else {
352
    return `${path}.*`;
353
  }
354
}
355

356
async function wmill_sync_pull(
357
  path_type: PathType,
358
  workspace_id: string,
359
  path: string | undefined,
360
  parent_path: string | undefined,
361
  skip_secret: boolean
362
) {
363
  const includes = [];
364
  if (path !== undefined && path !== null && path !== "") {
365
    includes.push(regexFromPath(path_type, path));
366
  }
367
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
368
    includes.push(regexFromPath(path_type, parent_path));
369
  }
370
  await wmill_run(
371
    6,
372
    "workspace",
373
    "add",
374
    workspace_id,
375
    workspace_id,
376
    process.env["BASE_URL"] + "/",
377
    "--token",
378
    process.env["WM_TOKEN"] ?? ""
379
  );
380
  console.log("Pulling workspace into git repo");
381
  await wmill_run(
382
    3,
383
    "sync",
384
    "pull",
385
    "--token",
386
    process.env["WM_TOKEN"] ?? "",
387
    "--workspace",
388
    workspace_id,
389
    "--yes",
390
    skip_secret ? "--skip-secrets" : "",
391
    "--include-schedules",
392
    "--include-users",
393
    "--include-groups",
394
    "--include-triggers",
395
    "--extra-includes",
396
    includes.join(",")
397
  );
398
}
399

400
async function wmill_run(secret_position: number, ...cmd: string[]) {
401
  cmd = cmd.filter((elt) => elt !== "");
402
  const cmd2 = cmd.slice();
403
  cmd2[secret_position] = "***";
404
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
405
  await wmill.parse(cmd);
406
  console.log("Command successfully executed");
407
}
408

409
// Function to set up GPG signing
410
async function set_gpg_signing_secret(gpg_key: GpgKey) {
411
  try {
412
    console.log("Setting GPG private key for git commits");
413

414
    const formattedGpgContent = gpg_key.private_key.replace(
415
      /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
416
      (_: string, header: string, body: string, footer: string) =>
417
        header +
418
        "\n" +
419
        "\n" +
420
        body.replace(/ ([^\s])/g, "\n$1").trim() +
421
        "\n" +
422
        footer
423
    );
424

425
    const gpg_path = `/tmp/gpg`;
426
    await sh_run(undefined, "mkdir", "-p", gpg_path);
427
    await sh_run(undefined, "chmod", "700", gpg_path);
428
    process.env.GNUPGHOME = gpg_path;
429
    // process.env.GIT_TRACE = 1;
430

431
    try {
432
      await sh_run(
433
        1,
434
        "bash",
435
        "-c",
436
        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
437
      );
438
    } catch (e) {
439
      // Original error would contain sensitive data
440
      throw new Error("Failed to import GPG key!");
441
    }
442

443
    const listKeysOutput = await sh_run(
444
      undefined,
445
      "gpg",
446
      "--list-secret-keys",
447
      "--with-colons",
448
      "--keyid-format=long"
449
    );
450

451
    const keyInfoMatch = listKeysOutput.match(
452
      /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
453
    );
454

455
    if (!keyInfoMatch) {
456
      throw new Error("Failed to extract GPG Key ID and Fingerprint");
457
    }
458

459
    const keyId = keyInfoMatch[1];
460
    gpgFingerprint = keyInfoMatch[2];
461

462
    if (gpg_key.passphrase) {
463
      // This is adummy command to unlock the key
464
      // with passphrase to load it into agent
465
      await sh_run(
466
        1,
467
        "bash",
468
        "-c",
469
        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
470
      );
471
    }
472

473
    // Configure Git to use the extracted key
474
    await sh_run(undefined, "git", "config", "user.signingkey", keyId);
475
    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
476
    console.log(`GPG signing configured with key ID: ${keyId} `);
477
  } catch (e) {
478
    console.error(`Failure while setting GPG key: ${e} `);
479
    await delete_pgp_keys();
480
  }
481
}
482

483
async function delete_pgp_keys() {
484
  console.log("deleting gpg keys");
485
  if (gpgFingerprint) {
486
    await sh_run(
487
      undefined,
488
      "gpg",
489
      "--batch",
490
      "--yes",
491
      "--pinentry-mode",
492
      "loopback",
493
      "--delete-secret-key",
494
      gpgFingerprint
495
    );
496
    await sh_run(
497
      undefined,
498
      "gpg",
499
      "--batch",
500
      "--yes",
501
      "--delete-key",
502
      "--pinentry-mode",
503
      "loopback",
504
      gpgFingerprint
505
    );
506
  }
507
}
508

509
async function get_gh_app_token() {
510
  const workspace = process.env["WM_WORKSPACE"];
511
  const jobToken = process.env["WM_TOKEN"];
512

513
  const baseUrl =
514
    process.env["BASE_INTERNAL_URL"] ??
515
    process.env["BASE_URL"] ??
516
    "http://localhost:8000";
517

518
  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
519

520
  const response = await fetch(url, {
521
    method: "POST",
522
    headers: {
523
      "Content-Type": "application/json",
524
      Authorization: `Bearer ${jobToken}`,
525
    },
526
    body: JSON.stringify({
527
      job_token: jobToken,
528
    }),
529
  });
530

531
  if (!response.ok) {
532
    throw new Error(`Error: ${response.statusText}`);
533
  }
534

535
  const data = await response.json();
536

537
  return data.token;
538
}
539

540
function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
541
  if (!gitHubUrl || !installationToken) {
542
    throw new Error("Both GitHub URL and Installation Token are required.");
543
  }
544

545
  try {
546
    const url = new URL(gitHubUrl);
547

548
    // GitHub repository URL should be in the format: https://github.com/owner/repo.git
549
    if (url.hostname !== "github.com") {
550
      throw new Error(
551
        "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
552
      );
553
    }
554

555
    // Convert URL to include the installation token
556
    return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
557
  } catch (e) {
558
    const error = e as Error;
559
    throw new Error(`Invalid URL: ${error.message}`);
560
  }
561
}
Other submissions