1
Sync script to Git repo
One script reply has been approved by the moderators Verified

This script will pull the script from the current workspace in a temporary folder, then commit it to the remote Git repository and push it. Only the script-related file will be pushed, nothing else. It takes as input the git_repository resource containing the repository URL, the script path, and the commit message to use for the commit. All params are mandatory.

Created by hugo697 239 days ago Viewed 18060 times
1
Submitted by hugo697 Deno
Verified 239 days ago
1
import * as wmillclient from "npm:windmill-client@1.333.4";
2
import wmill from "https://deno.land/x/wmill@v1.333.4/main.ts";
3
import { basename } from "https://deno.land/std@0.208.0/path/mod.ts";
4

5
export async function main(
6
  workspace_id: string,
7
  repo_url_resource_path: string,
8
  path_type:
9
    | "script"
10
    | "flow"
11
    | "app"
12
    | "folder"
13
    | "resource"
14
    | "variable"
15
    | "resourcetype"
16
    | "schedule"
17
    | "user"
18
    | "group",
19
  skip_secret = true,
20
  path: string | undefined,
21
  parent_path: string | undefined,
22
  commit_msg: string,
23
  use_individual_branch = false,
24
  group_by_folder = false
25
) {
26
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
27

28
  const cwd = Deno.cwd();
29
  Deno.env.set("HOME", ".");
30
  console.log(
31
    `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
32
  );
33

34
  const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
35
  await move_to_git_branch(
36
    workspace_id,
37
    path_type,
38
    path,
39
    parent_path,
40
    use_individual_branch,
41
    group_by_folder
42
  );
43

44
  const subfolder = repo_resource.folder ?? "";
45
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
46
  console.log(
47
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
48
  );
49

50
  await wmill_sync_pull(
51
    path_type,
52
    workspace_id,
53
    path,
54
    parent_path,
55
    skip_secret
56
  );
57

58
  await git_push(path, parent_path, commit_msg);
59

60
  console.log("Finished syncing");
61
  Deno.chdir(`${cwd}`);
62
}
63

64
async function git_clone(
65
  cwd: string,
66
  repo_resource: any,
67
  use_individual_branch: boolean
68
): Promise<string> {
69
  // TODO: handle private SSH keys as well
70
  let repo_url = repo_resource.url;
71
  const subfolder = repo_resource.folder ?? "";
72
  const branch = repo_resource.branch ?? "";
73
  const repo_name = basename(repo_url, ".git");
74

75
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
76

77
  if (azureMatch) {
78
    console.log(
79
      "Requires Azure DevOps service account access token, requesting..."
80
    );
81
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
82

83
    const response = await fetch(
84
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
85
      {
86
        method: "POST",
87
        body: new URLSearchParams({
88
          client_id: azureResource.azureClientId,
89
          client_secret: azureResource.azureClientSecret,
90
          grant_type: "client_credentials",
91
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
92
        }),
93
      }
94
    );
95

96
    const { access_token } = await response.json();
97

98
    repo_url = repo_url.replace(azureMatch[0], access_token);
99
  }
100

101
  const args = ["clone", "--quiet", "--depth", "1"];
102
  if (use_individual_branch) {
103
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
104
  }
105
  if (subfolder !== "") {
106
    args.push("--sparse");
107
  }
108
  if (branch !== "") {
109
    args.push("--branch");
110
    args.push(branch);
111
  }
112
  args.push(repo_url);
113
  args.push(repo_name);
114
  await sh_run(-1, "git", ...args);
115

116
  try {
117
    Deno.chdir(`${cwd}/${repo_name}`);
118
  } catch (err) {
119
    console.log(
120
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
121
    );
122
    throw err;
123
  }
124
  Deno.chdir(`${cwd}/${repo_name}`);
125
  if (subfolder !== "") {
126
    await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
127
  }
128
  try {
129
    Deno.chdir(`${cwd}/${repo_name}/${subfolder}`);
130
  } catch (err) {
131
    console.log(
132
      `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
133
    );
134
    throw err;
135
  }
136

137
  return repo_name;
138
}
139

140
async function move_to_git_branch(
141
  workspace_id: string,
142
  path_type:
143
    | "script"
144
    | "flow"
145
    | "app"
146
    | "folder"
147
    | "resource"
148
    | "variable"
149
    | "resourcetype"
150
    | "schedule"
151
    | "user"
152
    | "group",
153
  path: string | undefined,
154
  parent_path: string | undefined,
155
  use_individual_branch: boolean,
156
  group_by_folder: boolean
157
) {
158
  if (!use_individual_branch || path_type === "user" || path_type === "group") {
159
    return;
160
  }
161

162
  const branchName = group_by_folder
163
    ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
164
        ?.split("/")
165
        .slice(0, 2)
166
        .join("__")}`
167
    : `wm_deploy/${workspace_id}/${path_type}/${(
168
        path ?? parent_path
169
      )?.replaceAll("/", "__")}`;
170

171
  try {
172
    await sh_run(undefined, "git", "checkout", branchName);
173
  } catch (err) {
174
    console.log(
175
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
176
    );
177
    try {
178
      await sh_run(undefined, "git", "checkout", "-b", branchName);
179
      await sh_run(
180
        undefined,
181
        "git",
182
        "config",
183
        "--add",
184
        "--bool",
185
        "push.autoSetupRemote",
186
        "true"
187
      );
188
    } catch (err) {
189
      console.log(
190
        `Error checking out branch '${branchName}'. Error was:\n${err}`
191
      );
192
      throw err;
193
    }
194
  }
195
  console.log(`Successfully switched to branch ${branchName}`);
196
}
197

198
async function git_push(
199
  path: string | undefined,
200
  parent_path: string | undefined,
201
  commit_msg: string
202
) {
203
  await sh_run(
204
    undefined,
205
    "git",
206
    "config",
207
    "user.email",
208
    Deno.env.get("WM_EMAIL") ?? ""
209
  );
210
  await sh_run(
211
    undefined,
212
    "git",
213
    "config",
214
    "user.name",
215
    Deno.env.get("WM_USERNAME") ?? ""
216
  );
217
  if (path !== undefined && path !== null && path !== "") {
218
    try {
219
      await sh_run(undefined, "git", "add", `${path}**`);
220
    } catch (e) {
221
      console.log(`Unable to stage files matching ${path}**, ${e}`);
222
    }
223
  }
224
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
225
    try {
226
      await sh_run(undefined, "git", "add", `${parent_path}**`);
227
    } catch (e) {
228
      console.log(`Unable to stage files matching ${parent_path}, ${e}`);
229
    }
230
  }
231
  try {
232
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
233
  } catch {
234
    // git diff returns exit-code = 1 when there's at least one staged changes
235
    await sh_run(undefined, "git", "commit", "-m", commit_msg);
236
    try {
237
        await sh_run(undefined, "git", "push", "--porcelain");
238
    } catch {
239
        console.log("Could not push, trying to rebase first");
240
        await sh_run(undefined, "git", "pull", "--rebase");
241
        await sh_run(undefined, "git", "push", "--porcelain");
242
    }
243
    return;
244
  }
245
  console.log("No changes detected, nothing to commit. Returning...");
246
}
247

248
async function sh_run(
249
  secret_position: number | undefined,
250
  cmd: string,
251
  ...args: string[]
252
) {
253
  const nargs = secret_position != undefined ? args.slice() : args;
254
  if (secret_position < 0) {
255
    secret_position = nargs.length - 1 + secret_position;
256
  }
257
  if (secret_position != undefined) {
258
    nargs[secret_position] = "***";
259
  }
260
  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
261
  const command = new Deno.Command(cmd, {
262
    args: args,
263
  });
264

265
  const { code, stdout, stderr } = await command.output();
266
  if (stdout.length > 0) {
267
    console.log(new TextDecoder().decode(stdout));
268
  }
269
  if (stderr.length > 0) {
270
    console.log(new TextDecoder().decode(stderr));
271
  }
272
  if (code !== 0) {
273
    const err = `SH command '${cmd} ${nargs.join(
274
      " "
275
    )}' returned with a non-zero status ${code}.`;
276
    throw Error(err);
277
  }
278
  console.log("Command successfully executed");
279
}
280

281
function regexFromPath(
282
  path_type:
283
    | "script"
284
    | "flow"
285
    | "app"
286
    | "folder"
287
    | "resource"
288
    | "variable"
289
    | "resourcetype"
290
    | "schedule"
291
    | "user"
292
    | "group",
293
  path: string
294
) {
295
  if (path_type == "flow") {
296
    return `${path}.flow/*`;
297
  } if (path_type == "app") {
298
    return `${path}.app/*`;
299
  } else if (path_type == "folder") {
300
    return `${path}/folder.meta.*`;
301
  } else if (path_type == "resourcetype") {
302
    return `${path}.resource-type.*`;
303
  } else if (path_type == "resource") {
304
    return `${path}.resource.*`;
305
  } else if (path_type == "variable") {
306
    return `${path}.variable.*`;
307
  } else if (path_type == "schedule") {
308
    return `${path}.schedule.*`;
309
  } else if (path_type == "user") {
310
    return `${path}.user.*`;
311
  } else if (path_type == "group") {
312
    return `${path}.group.*`;
313
  } else {
314
    return `${path}.*`;
315
  }
316
}
317
async function wmill_sync_pull(
318
  path_type:
319
    | "script"
320
    | "flow"
321
    | "app"
322
    | "folder"
323
    | "resource"
324
    | "variable"
325
    | "resourcetype"
326
    | "schedule"
327
    | "user"
328
    | "group",
329
  workspace_id: string,
330
  path: string | undefined,
331
  parent_path: string | undefined,
332
  skip_secret: boolean
333
) {
334
  const includes = [];
335
  if (path !== undefined && path !== null && path !== "") {
336
    includes.push(regexFromPath(path_type, path));
337
  }
338
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
339
    includes.push(regexFromPath(path_type, parent_path));
340
  }
341

342
  await wmill_run(
343
    6,
344
    "workspace",
345
    "add",
346
    workspace_id,
347
    workspace_id,
348
    Deno.env.get("BASE_INTERNAL_URL") + "/",
349
    "--token",
350
    Deno.env.get("WM_TOKEN") ?? ""
351
  );
352
  console.log("Pulling workspace into git repo");
353
  await wmill_run(
354
    3,
355
    "sync",
356
    "pull",
357
    "--token",
358
    Deno.env.get("WM_TOKEN") ?? "",
359
    "--workspace",
360
    workspace_id,
361
    "--yes",
362
    "--raw",
363
    skip_secret ? "--skip-secrets" : "",
364
    "--include-schedules",
365
    "--include-users",
366
    "--include-groups",
367
    "--includes",
368
    includes.join(",")
369
  );
370
}
371

372
async function wmill_run(secret_position: number, ...cmd: string[]) {
373
  cmd = cmd.filter((elt) => elt !== "");
374
  const cmd2 = cmd.slice();
375
  cmd2[secret_position] = "***";
376
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
377
  await wmill.parse(cmd);
378
  console.log("Command successfully executed");
379
}
380