1
Sync script to Git repo
One script reply has been approved by the moderators Verified

This script will pull the script from the current workspace in a temporary folder, then commit it to the remote Git repository and push it. Only the script-related file will be pushed, nothing else. It takes as input the git_repository resource containing the repository URL, the script path, and the commit message to use for the commit. All params are mandatory.

Created by hugo697 556 days ago Viewed 73589 times
0
Submitted by rubenfiszel Bun
Verified 273 days ago
1
import * as wmillclient from "windmill-client";
2
import wmill from "windmill-cli";
3
import { basename } from "node:path"
4
const util = require('util');
5
const exec = util.promisify(require('child_process').exec);
6
import process from "process";
7

8
type GpgKey = {
9
  email: string,
10
  private_key: string,
11
  passphrase: string
12
}
13

14
type PathType =
15
  | "script"
16
  | "flow"
17
  | "app"
18
  | "folder"
19
  | "resource"
20
  | "variable"
21
  | "resourcetype"
22
  | "schedule"
23
  | "user"
24
  | "group"
25
  | "httptrigger"
26
  | "websockettrigger"
27
  | "kafkatrigger"
28
  | "natstrigger"
29
  | "postgrestrigger"
30
  | "mqtttrigger"
31
  | "sqstrigger"
32
  | "gcptrigger";
33

34
let gpgFingerprint: string | undefined = undefined;
35

36
export async function main(
37
  workspace_id: string,
38
  repo_url_resource_path: string,
39
  path_type: PathType,
40
  skip_secret = true,
41
  path: string | undefined,
42
  parent_path: string | undefined,
43
  commit_msg: string,
44
  use_individual_branch = false,
45
  group_by_folder = false
46
) {
47
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
48
  const cwd = process.cwd();
49
  process.env["HOME"] = "."
50
  console.log(
51
    `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
52
  );
53

54
  if (repo_resource.is_github_app) {
55
    const token = await get_gh_app_token()
56
    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
57
    repo_resource.url = authRepoUrl;
58
  }
59

60
  const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
61
  await move_to_git_branch(
62
    workspace_id,
63
    path_type,
64
    path,
65
    parent_path,
66
    use_individual_branch,
67
    group_by_folder
68
  );
69
  const subfolder = repo_resource.folder ?? "";
70
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
71
  console.log(
72
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
73
  );
74
  await wmill_sync_pull(
75
    path_type,
76
    workspace_id,
77
    path,
78
    parent_path,
79
    skip_secret
80
  );
81
  try {
82
    await git_push(path, parent_path, commit_msg, repo_resource);
83
  } catch (e) {
84
    throw (e)
85
  } finally {
86
    await delete_pgp_keys()
87
  }
88
  console.log("Finished syncing");
89
  process.chdir(`${cwd}`);
90
}
91
async function git_clone(
92
  cwd: string,
93
  repo_resource: any,
94
  use_individual_branch: boolean
95
): Promise<string> {
96
  // TODO: handle private SSH keys as well
97
  let repo_url = repo_resource.url;
98
  const subfolder = repo_resource.folder ?? "";
99
  const branch = repo_resource.branch ?? "";
100
  const repo_name = basename(repo_url, ".git");
101
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
102
  if (azureMatch) {
103
    console.log(
104
      "Requires Azure DevOps service account access token, requesting..."
105
    );
106
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
107
    const response = await fetch(
108
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
109
      {
110
        method: "POST",
111
        body: new URLSearchParams({
112
          client_id: azureResource.azureClientId,
113
          client_secret: azureResource.azureClientSecret,
114
          grant_type: "client_credentials",
115
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
116
        }),
117
      }
118
    );
119
    const { access_token } = await response.json();
120
    repo_url = repo_url.replace(azureMatch[0], access_token);
121
  }
122
  const args = ["clone", "--quiet", "--depth", "1"];
123
  if (use_individual_branch) {
124
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
125
  }
126
  if (subfolder !== "") {
127
    args.push("--sparse");
128
  }
129
  if (branch !== "") {
130
    args.push("--branch");
131
    args.push(branch);
132
  }
133
  args.push(repo_url);
134
  args.push(repo_name);
135
  await sh_run(-1, "git", ...args);
136
  try {
137
    process.chdir(`${cwd}/${repo_name}`);
138
  } catch (err) {
139
    console.log(
140
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
141
    );
142
    throw err;
143
  }
144
  process.chdir(`${cwd}/${repo_name}`);
145
  if (subfolder !== "") {
146
    await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
147
  }
148
  try {
149
    process.chdir(`${cwd}/${repo_name}/${subfolder}`);
150
  } catch (err) {
151
    console.log(
152
      `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
153
    );
154
    throw err;
155
  }
156
  return repo_name;
157
}
158
async function move_to_git_branch(
159
  workspace_id: string,
160
  path_type: PathType,
161
  path: string | undefined,
162
  parent_path: string | undefined,
163
  use_individual_branch: boolean,
164
  group_by_folder: boolean
165
) {
166
  if (!use_individual_branch || path_type === "user" || path_type === "group") {
167
    return;
168
  }
169
  const branchName = group_by_folder
170
    ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
171
      ?.split("/")
172
      .slice(0, 2)
173
      .join("__")}`
174
    : `wm_deploy/${workspace_id}/${path_type}/${(
175
      path ?? parent_path
176
    )?.replaceAll("/", "__")}`;
177
  try {
178
    await sh_run(undefined, "git", "checkout", branchName);
179
  } catch (err) {
180
    console.log(
181
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
182
    );
183
    try {
184
      await sh_run(undefined, "git", "checkout", "-b", branchName);
185
      await sh_run(
186
        undefined,
187
        "git",
188
        "config",
189
        "--add",
190
        "--bool",
191
        "push.autoSetupRemote",
192
        "true"
193
      );
194
    } catch (err) {
195
      console.log(
196
        `Error checking out branch '${branchName}'. Error was:\n${err}`
197
      );
198
      throw err;
199
    }
200
  }
201
  console.log(`Successfully switched to branch ${branchName}`);
202
}
203
async function git_push(
204
  path: string | undefined,
205
  parent_path: string | undefined,
206
  commit_msg: string,
207
  repo_resource: any
208
) {
209
  let user_email = process.env["WM_EMAIL"] ?? ""
210
  let user_name = process.env["WM_USERNAME"] ?? ""
211

212
  if (repo_resource.gpg_key) {
213
    await set_gpg_signing_secret(repo_resource.gpg_key);
214
    user_email = repo_resource.gpg_key.email
215
  }
216

217
  await sh_run(
218
    undefined,
219
    "git",
220
    "config",
221
    "user.email",
222
    user_email
223
  );
224
  await sh_run(
225
    undefined,
226
    "git",
227
    "config",
228
    "user.name",
229
    user_name
230
  );
231
  if (path !== undefined && path !== null && path !== "") {
232
    try {
233
      await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
234
    } catch (e) {
235
      console.log(`Unable to stage files matching ${path}**, ${e}`);
236
    }
237
  }
238
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
239
    try {
240
      await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`);
241
    } catch (e) {
242
      console.log(`Unable to stage files matching ${parent_path}, ${e}`);
243
    }
244
  }
245
  try {
246
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
247
  } catch {
248
    // git diff returns exit-code = 1 when there's at least one staged changes
249
    await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
250
    try {
251
      await sh_run(undefined, "git", "push", "--porcelain");
252
    } catch (e) {
253
      console.log(`Could not push, trying to rebase first: ${e}`);
254
      await sh_run(undefined, "git", "pull", "--rebase");
255
      await sh_run(undefined, "git", "push", "--porcelain");
256
    }
257
    return;
258
  }
259
  console.log("No changes detected, nothing to commit. Returning...");
260
}
261
async function sh_run(
262
  secret_position: number | undefined,
263
  cmd: string,
264
  ...args: string[]
265
) {
266
  const nargs = secret_position != undefined ? args.slice() : args;
267
  if (secret_position && secret_position < 0) {
268
    secret_position = nargs.length - 1 + secret_position;
269
  }
270
  let secret: string | undefined = undefined
271
  if (secret_position != undefined) {
272
    nargs[secret_position] = "***";
273
    secret = args[secret_position]
274
  }
275

276
  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
277
  const command = exec(`${cmd} ${args.join(" ")}`)
278
  // new Deno.Command(cmd, {
279
  //   args: args,
280
  // });
281
  try {
282
    const { stdout, stderr } = await command
283
    if (stdout.length > 0) {
284
      console.log(stdout);
285
    }
286
    if (stderr.length > 0) {
287
      console.log(stderr);
288
    }
289
    console.log("Command successfully executed");
290
    return stdout;
291

292
  } catch (error) {
293
    let errorString = error.toString();
294
    if (secret) {
295
      errorString = errorString.replace(secret, "***");
296
    }
297
    const err = `SH command '${cmd} ${nargs.join(
298
      " "
299
    )}' returned with error ${errorString}`;
300
    throw Error(err);
301
  }
302
}
303

304
function regexFromPath(
305
  path_type: PathType,
306
  path: string
307
) {
308
  if (path_type == "flow") {
309
    return `${path}.flow/*`;
310
  } if (path_type == "app") {
311
    return `${path}.app/*`;
312
  } else if (path_type == "folder") {
313
    return `${path}/folder.meta.*`;
314
  } else if (path_type == "resourcetype") {
315
    return `${path}.resource-type.*`;
316
  } else if (path_type == "resource") {
317
    return `${path}.resource.*`;
318
  } else if (path_type == "variable") {
319
    return `${path}.variable.*`;
320
  } else if (path_type == "schedule") {
321
    return `${path}.schedule.*`;
322
  } else if (path_type == "user") {
323
    return `${path}.user.*`;
324
  } else if (path_type == "group") {
325
    return `${path}.group.*`;
326
  } else if (path_type == "httptrigger") {
327
    return `${path}.http_trigger.*`
328
  } else if (path_type == "websockettrigger") {
329
    return `${path}.websocket_trigger.*`
330
  } else if (path_type == "kafkatrigger") {
331
    return `${path}.kafka_trigger.*`
332
  } else if (path_type == "natstrigger") {
333
    return `${path}.nats_trigger.*`
334
  } else if (path_type == "postgrestrigger") {
335
    return `${path}.postgres_trigger.*`
336
  } else if (path_type == "mqtttrigger") {
337
    return `${path}.mqtt_trigger.*`
338
  } else if (path_type == "sqstrigger") {
339
    return `${path}.sqs_trigger.*`
340
  } else if (path_type == "gcptrigger") {
341
    return `${path}.gcp_trigger.*`
342
  } else {
343
    return `${path}.*`;
344
  }
345
}
346

347
async function wmill_sync_pull(
348
  path_type: PathType,
349
  workspace_id: string,
350
  path: string | undefined,
351
  parent_path: string | undefined,
352
  skip_secret: boolean
353
) {
354
  const includes = [];
355
  if (path !== undefined && path !== null && path !== "") {
356
    includes.push(regexFromPath(path_type, path));
357
  }
358
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
359
    includes.push(regexFromPath(path_type, parent_path));
360
  }
361
  await wmill_run(
362
    6,
363
    "workspace",
364
    "add",
365
    workspace_id,
366
    workspace_id,
367
    process.env["BASE_URL"] + "/",
368
    "--token",
369
    process.env["WM_TOKEN"] ?? ""
370
  );
371
  console.log("Pulling workspace into git repo");
372
  await wmill_run(
373
    3,
374
    "sync",
375
    "pull",
376
    "--token",
377
    process.env["WM_TOKEN"] ?? "",
378
    "--workspace",
379
    workspace_id,
380
    "--yes",
381
    skip_secret ? "--skip-secrets" : "",
382
    "--include-schedules",
383
    "--include-users",
384
    "--include-groups",
385
    "--include-triggers",
386
    "--extra-includes",
387
    includes.join(",")
388
  );
389
}
390

391
async function wmill_run(secret_position: number, ...cmd: string[]) {
392
  cmd = cmd.filter((elt) => elt !== "");
393
  const cmd2 = cmd.slice();
394
  cmd2[secret_position] = "***";
395
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
396
  await wmill.parse(cmd);
397
  console.log("Command successfully executed");
398
}
399

400
// Function to set up GPG signing
401
async function set_gpg_signing_secret(gpg_key: GpgKey) {
402
  try {
403
    console.log("Setting GPG private key for git commits");
404

405
    const formattedGpgContent = gpg_key.private_key
406
      .replace(
407
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
408
        (_: string, header: string, body: string, footer: string) =>
409
          header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
410
      );
411

412
    const gpg_path = `/tmp/gpg`;
413
    await sh_run(undefined, "mkdir", "-p", gpg_path);
414
    await sh_run(undefined, "chmod", "700", gpg_path)
415
    process.env.GNUPGHOME = gpg_path;
416
    // process.env.GIT_TRACE = 1;
417

418

419
    try {
420
      await sh_run(
421
        1,
422
        "bash",
423
        "-c",
424
        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
425
      );
426
    } catch (e) {
427
      // Original error would contain sensitive data
428
      throw new Error('Failed to import GPG key!')
429
    }
430

431
    const listKeysOutput = await sh_run(
432
      undefined,
433
      "gpg",
434
      "--list-secret-keys",
435
      "--with-colons",
436
      "--keyid-format=long"
437
    );
438

439
    const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
440

441
    if (!keyInfoMatch) {
442
      throw new Error("Failed to extract GPG Key ID and Fingerprint");
443
    }
444

445
    const keyId = keyInfoMatch[1];
446
    gpgFingerprint = keyInfoMatch[2];
447

448
    if (gpg_key.passphrase) {
449
      // This is adummy command to unlock the key
450
      // with passphrase to load it into agent
451
      await sh_run(
452
        1,
453
        "bash",
454
        "-c",
455
        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
456
      );
457
    }
458

459
    // Configure Git to use the extracted key
460
    await sh_run(undefined, "git", "config", "user.signingkey", keyId);
461
    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
462
    console.log(`GPG signing configured with key ID: ${keyId} `);
463

464
  } catch (e) {
465
    console.error(`Failure while setting GPG key: ${e} `);
466
    await delete_pgp_keys();
467
  }
468
}
469

470
async function delete_pgp_keys() {
471
  console.log("deleting gpg keys")
472
  if (gpgFingerprint) {
473
    await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
474
    await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
475
  }
476
}
477

478
async function get_gh_app_token() {
479
  const workspace = process.env["WM_WORKSPACE"];
480
  const jobToken = process.env["WM_TOKEN"];
481

482
  const baseUrl =
483
    process.env["BASE_INTERNAL_URL"] ??
484
    process.env["BASE_URL"] ??
485
    "http://localhost:8000";
486

487
  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
488

489
  const response = await fetch(url, {
490
    method: 'POST',
491
    headers: {
492
      'Content-Type': 'application/json',
493
      'Authorization': `Bearer ${jobToken}`,
494
    },
495
    body: JSON.stringify({
496
      job_token: jobToken,
497
    }),
498
  });
499

500
  if (!response.ok) {
501
    throw new Error(`Error: ${response.statusText}`);
502
  }
503

504
  const data = await response.json();
505

506
  return data.token;
507
}
508

509
function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
510
  if (!gitHubUrl || !installationToken) {
511
    throw new Error("Both GitHub URL and Installation Token are required.");
512
  }
513

514
  try {
515
    const url = new URL(gitHubUrl);
516

517
    // GitHub repository URL should be in the format: https://github.com/owner/repo.git
518
    if (url.hostname !== "github.com") {
519
      throw new Error("Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'.");
520
    }
521

522
    // Convert URL to include the installation token
523
    return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
524
  } catch (e) {
525
    const error = e as Error
526
    throw new Error(`Invalid URL: ${error.message}`)
527
  }
528
}
Other submissions