1
Sync script to Git repo
One script reply has been approved by the moderators Verified

This script will pull the script from the current workspace in a temporary folder, then commit it to the remote Git repository and push it. Only the script-related file will be pushed, nothing else. It takes as input the git_repository resource containing the repository URL, the script path, and the commit message to use for the commit. All params are mandatory.

Created by hugo697 377 days ago Viewed 27753 times
0
Submitted by rubenfiszel Bun
Verified 94 days ago
1
import * as wmillclient from "windmill-client";
2
import wmill from "windmill-cli";
3
import { basename } from "node:path"
4
const util = require('util');
5
const exec = util.promisify(require('child_process').exec);
6
import process from "process";
7

8
export async function main(
9
  workspace_id: string,
10
  repo_url_resource_path: string,
11
  path_type:
12
    | "script"
13
    | "flow"
14
    | "app"
15
    | "folder"
16
    | "resource"
17
    | "variable"
18
    | "resourcetype"
19
    | "schedule"
20
    | "user"
21
    | "group",
22
  skip_secret = true,
23
  path: string | undefined,
24
  parent_path: string | undefined,
25
  commit_msg: string,
26
  use_individual_branch = false,
27
  group_by_folder = false
28
) {
29
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
30
  const cwd = process.cwd();
31
  process.env["HOME"] = "."
32
  console.log(
33
    `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
34
  );
35
  const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
36
  await move_to_git_branch(
37
    workspace_id,
38
    path_type,
39
    path,
40
    parent_path,
41
    use_individual_branch,
42
    group_by_folder
43
  );
44
  const subfolder = repo_resource.folder ?? "";
45
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
46
  console.log(
47
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
48
  );
49
  await wmill_sync_pull(
50
    path_type,
51
    workspace_id,
52
    path,
53
    parent_path,
54
    skip_secret
55
  );
56
  await git_push(path, parent_path, commit_msg);
57
  console.log("Finished syncing");
58
  process.chdir(`${cwd}`);
59
}
60
async function git_clone(
61
  cwd: string,
62
  repo_resource: any,
63
  use_individual_branch: boolean
64
): Promise<string> {
65
  // TODO: handle private SSH keys as well
66
  let repo_url = repo_resource.url;
67
  const subfolder = repo_resource.folder ?? "";
68
  const branch = repo_resource.branch ?? "";
69
  const repo_name = basename(repo_url, ".git");
70
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
71
  if (azureMatch) {
72
    console.log(
73
      "Requires Azure DevOps service account access token, requesting..."
74
    );
75
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
76
    const response = await fetch(
77
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
78
      {
79
        method: "POST",
80
        body: new URLSearchParams({
81
          client_id: azureResource.azureClientId,
82
          client_secret: azureResource.azureClientSecret,
83
          grant_type: "client_credentials",
84
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
85
        }),
86
      }
87
    );
88
    const { access_token } = await response.json();
89
    repo_url = repo_url.replace(azureMatch[0], access_token);
90
  }
91
  const args = ["clone", "--quiet", "--depth", "1"];
92
  if (use_individual_branch) {
93
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
94
  }
95
  if (subfolder !== "") {
96
    args.push("--sparse");
97
  }
98
  if (branch !== "") {
99
    args.push("--branch");
100
    args.push(branch);
101
  }
102
  args.push(repo_url);
103
  args.push(repo_name);
104
  await sh_run(-1, "git", ...args);
105
  try {
106
    process.chdir(`${cwd}/${repo_name}`);
107
  } catch (err) {
108
    console.log(
109
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
110
    );
111
    throw err;
112
  }
113
  process.chdir(`${cwd}/${repo_name}`);
114
  if (subfolder !== "") {
115
    await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
116
  }
117
  try {
118
    process.chdir(`${cwd}/${repo_name}/${subfolder}`);
119
  } catch (err) {
120
    console.log(
121
      `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
122
    );
123
    throw err;
124
  }
125
  return repo_name;
126
}
127
async function move_to_git_branch(
128
  workspace_id: string,
129
  path_type:
130
    | "script"
131
    | "flow"
132
    | "app"
133
    | "folder"
134
    | "resource"
135
    | "variable"
136
    | "resourcetype"
137
    | "schedule"
138
    | "user"
139
    | "group",
140
  path: string | undefined,
141
  parent_path: string | undefined,
142
  use_individual_branch: boolean,
143
  group_by_folder: boolean
144
) {
145
  if (!use_individual_branch || path_type === "user" || path_type === "group") {
146
    return;
147
  }
148
  const branchName = group_by_folder
149
    ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
150
      ?.split("/")
151
      .slice(0, 2)
152
      .join("__")}`
153
    : `wm_deploy/${workspace_id}/${path_type}/${(
154
      path ?? parent_path
155
    )?.replaceAll("/", "__")}`;
156
  try {
157
    await sh_run(undefined, "git", "checkout", branchName);
158
  } catch (err) {
159
    console.log(
160
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
161
    );
162
    try {
163
      await sh_run(undefined, "git", "checkout", "-b", branchName);
164
      await sh_run(
165
        undefined,
166
        "git",
167
        "config",
168
        "--add",
169
        "--bool",
170
        "push.autoSetupRemote",
171
        "true"
172
      );
173
    } catch (err) {
174
      console.log(
175
        `Error checking out branch '${branchName}'. Error was:\n${err}`
176
      );
177
      throw err;
178
    }
179
  }
180
  console.log(`Successfully switched to branch ${branchName}`);
181
}
182
async function git_push(
183
  path: string | undefined,
184
  parent_path: string | undefined,
185
  commit_msg: string
186
) {
187
  await sh_run(
188
    undefined,
189
    "git",
190
    "config",
191
    "user.email",
192
    process.env["WM_EMAIL"] ?? ""
193
  );
194
  await sh_run(
195
    undefined,
196
    "git",
197
    "config",
198
    "user.name",
199
    process.env["WM_USERNAME"] ?? ""
200
  );
201
  if (path !== undefined && path !== null && path !== "") {
202
    try {
203
      await sh_run(undefined, "git", "add", `${path}**`);
204
    } catch (e) {
205
      console.log(`Unable to stage files matching ${path}**, ${e}`);
206
    }
207
  }
208
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
209
    try {
210
      await sh_run(undefined, "git", "add", `${parent_path}**`);
211
    } catch (e) {
212
      console.log(`Unable to stage files matching ${parent_path}, ${e}`);
213
    }
214
  }
215
  try {
216
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
217
  } catch {
218
    // git diff returns exit-code = 1 when there's at least one staged changes
219
    await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
220
    try {
221
      await sh_run(undefined, "git", "push", "--porcelain");
222
    } catch {
223
      console.log("Could not push, trying to rebase first");
224
      await sh_run(undefined, "git", "pull", "--rebase");
225
      await sh_run(undefined, "git", "push", "--porcelain");
226
    }
227
    return;
228
  }
229
  console.log("No changes detected, nothing to commit. Returning...");
230
}
231
async function sh_run(
232
  secret_position: number | undefined,
233
  cmd: string,
234
  ...args: string[]
235
) {
236
  const nargs = secret_position != undefined ? args.slice() : args;
237
  if (secret_position && secret_position < 0) {
238
    secret_position = nargs.length - 1 + secret_position;
239
  }
240
  if (secret_position != undefined) {
241
    nargs[secret_position] = "***";
242
  }
243
  
244
  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
245
  const command = exec(`${cmd} ${args.join(" ")}`)
246
  // new Deno.Command(cmd, {
247
  //   args: args,
248
  // });
249
  const { error, stdout, stderr } = await command
250
  if (stdout.length > 0) {
251
    console.log(stdout);
252
  }
253
  if (stderr.length > 0) {
254
    console.log(stderr);
255
  }
256
  if (error) {
257
    const err = `SH command '${cmd} ${nargs.join(
258
      " "
259
    )}' returned with error ${error}`;
260
    throw Error(err);
261
  }
262
  console.log("Command successfully executed");
263
}
264

265
function regexFromPath(
266
  path_type:
267
    | "script"
268
    | "flow"
269
    | "app"
270
    | "folder"
271
    | "resource"
272
    | "variable"
273
    | "resourcetype"
274
    | "schedule"
275
    | "user"
276
    | "group",
277
  path: string
278
) {
279
  if (path_type == "flow") {
280
    return `${path}.flow/*`;
281
  } if (path_type == "app") {
282
    return `${path}.app/*`;
283
  } else if (path_type == "folder") {
284
    return `${path}/folder.meta.*`;
285
  } else if (path_type == "resourcetype") {
286
    return `${path}.resource-type.*`;
287
  } else if (path_type == "resource") {
288
    return `${path}.resource.*`;
289
  } else if (path_type == "variable") {
290
    return `${path}.variable.*`;
291
  } else if (path_type == "schedule") {
292
    return `${path}.schedule.*`;
293
  } else if (path_type == "user") {
294
    return `${path}.user.*`;
295
  } else if (path_type == "group") {
296
    return `${path}.group.*`;
297
  } else {
298
    return `${path}.*`;
299
  }
300
}
301
async function wmill_sync_pull(
302
  path_type:
303
    | "script"
304
    | "flow"
305
    | "app"
306
    | "folder"
307
    | "resource"
308
    | "variable"
309
    | "resourcetype"
310
    | "schedule"
311
    | "user"
312
    | "group",
313
  workspace_id: string,
314
  path: string | undefined,
315
  parent_path: string | undefined,
316
  skip_secret: boolean
317
) {
318
  const includes = [];
319
  if (path !== undefined && path !== null && path !== "") {
320
    includes.push(regexFromPath(path_type, path));
321
  }
322
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
323
    includes.push(regexFromPath(path_type, parent_path));
324
  }
325
  await wmill_run(
326
    6,
327
    "workspace",
328
    "add",
329
    workspace_id,
330
    workspace_id,
331
    process.env["BASE_URL"] + "/",
332
    "--token",
333
    process.env["WM_TOKEN"] ?? ""
334
  );
335
  console.log("Pulling workspace into git repo");
336
  await wmill_run(
337
    3,
338
    "sync",
339
    "pull",
340
    "--token",
341
    process.env["WM_TOKEN"] ?? "",
342
    "--workspace",
343
    workspace_id,
344
    "--yes",
345
    "--raw",
346
    skip_secret ? "--skip-secrets" : "",
347
    "--include-schedules",
348
    "--include-users",
349
    "--include-groups",
350
    "--extra-includes",
351
    includes.join(",")
352
  );
353
}
354

355
async function wmill_run(secret_position: number, ...cmd: string[]) {
356
  cmd = cmd.filter((elt) => elt !== "");
357
  const cmd2 = cmd.slice();
358
  cmd2[secret_position] = "***";
359
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
360
  await wmill.parse(cmd);
361
  console.log("Command successfully executed");
362
}
Other submissions