Script 'Sync script to Git repo' for windmill

Sync script to Git repo

Verified

This script will pull the script from the current workspace in a temporary folder, then commit it to the remote Git repository and push it. Only the script-related file will be pushed, nothing else. It takes as input the git_repository resource containing the repository URL, the script path, and the commit message to use for the commit. All params are mandatory.

Created by hugo697 154 days ago Viewed 10419 times

Submitted by hugo697 Deno

Verified 154 days ago

All edits

Permalink

import * as wmillclient from "npm:windmill-client@1.321.2";
import wmill from "https://deno.land/x/wmill@v1.321.2/main.ts";
import { basename } from "https://deno.land/std@0.208.0/path/mod.ts";

export async function main(
  workspace_id: string,
  repo_url_resource_path: string,
  path_type:
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group",
  skip_secret = true,
  path: string | undefined,
  parent_path: string | undefined,
  commit_msg: string,
  use_individual_branch = false,
  group_by_folder = false
) {
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);

  const cwd = Deno.cwd();
  Deno.env.set("HOME", ".");
  console.log(
    `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
  );

  const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
  await move_to_git_branch(
    workspace_id,
    path_type,
    path,
    parent_path,
    use_individual_branch,
    group_by_folder
  );

  const subfolder = repo_resource.folder ?? "";
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
  console.log(
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
  );

  await wmill_sync_pull(
    path_type,
    workspace_id,
    path,
    parent_path,
    skip_secret
  );

  await git_push(path, parent_path, commit_msg);

  console.log("Finished syncing");
  Deno.chdir(`${cwd}`);
}

async function git_clone(
  cwd: string,
  repo_resource: any,
  use_individual_branch: boolean
): Promise<string> {
  // TODO: handle private SSH keys as well
  let repo_url = repo_resource.url;
  const subfolder = repo_resource.folder ?? "";
  const branch = repo_resource.branch ?? "";
  const repo_name = basename(repo_url, ".git");

  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);

  if (azureMatch) {
    console.log(
      "Requires Azure DevOps service account access token, requesting..."
    );
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);

    const response = await fetch(
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
      {
        method: "POST",
        body: new URLSearchParams({
          client_id: azureResource.azureClientId,
          client_secret: azureResource.azureClientSecret,
          grant_type: "client_credentials",
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
        }),
      }
    );

    const { access_token } = await response.json();

    repo_url = repo_url.replace(azureMatch[0], access_token);
  }

  const args = ["clone", "--quiet", "--depth", "1"];
  if (use_individual_branch) {
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
  }
  if (subfolder !== "") {
    args.push("--sparse");
  }
  if (branch !== "") {
    args.push("--branch");
    args.push(branch);
  }
  args.push(repo_url);
  args.push(repo_name);
  await sh_run(-1, "git", ...args);

  try {
    Deno.chdir(`${cwd}/${repo_name}`);
  } catch (err) {
    console.log(
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    );
    throw err;
  }
  Deno.chdir(`${cwd}/${repo_name}`);
  if (subfolder !== "") {
    await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
  }
  try {
    Deno.chdir(`${cwd}/${repo_name}/${subfolder}`);
  } catch (err) {
    console.log(
      `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
    );
    throw err;
  }

  return repo_name;
}

async function move_to_git_branch(
  workspace_id: string,
  path_type:
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group",
  path: string | undefined,
  parent_path: string | undefined,
  use_individual_branch: boolean,
  group_by_folder: boolean
) {
  if (!use_individual_branch || path_type === "user" || path_type === "group") {
    return;
  }

  const branchName = group_by_folder
    ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
    : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;

  try {
    await sh_run(undefined, "git", "checkout", branchName);
  } catch (err) {
    console.log(
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    );
    try {
      await sh_run(undefined, "git", "checkout", "-b", branchName);
      await sh_run(
        undefined,
        "git",
        "config",
        "--add",
        "--bool",
        "push.autoSetupRemote",
        "true"
      );
    } catch (err) {
      console.log(
        `Error checking out branch '${branchName}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  console.log(`Successfully switched to branch ${branchName}`);
}

async function git_push(
  path: string | undefined,
  parent_path: string | undefined,
  commit_msg: string
) {
  await sh_run(
    undefined,
    "git",
    "config",
    "user.email",
    Deno.env.get("WM_EMAIL") ?? ""
  );
  await sh_run(
    undefined,
    "git",
    "config",
    "user.name",
    Deno.env.get("WM_USERNAME") ?? ""
  );
  if (path !== undefined && path !== null && path !== "") {
    try {
      await sh_run(undefined, "git", "add", `${path}**`);
    } catch (e) {
      console.log(`Unable to stage files matching ${path}**, ${e}`);
    }
  }
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    try {
      await sh_run(undefined, "git", "add", `${parent_path}**`);
    } catch (e) {
      console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    }
  }
  try {
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
  } catch {
    // git diff returns exit-code = 1 when there's at least on staged changes
    await sh_run(undefined, "git", "commit", "-m", commit_msg);
    await sh_run(undefined, "git", "push", "--porcelain");
    return;
  }
  console.log("No changes detected, nothing to commit. Returning...");
}

async function sh_run(
  secret_position: number | undefined,
  cmd: string,
  ...args: string[]
) {
  const nargs = secret_position != undefined ? args.slice() : args;
  if (secret_position < 0) {
    secret_position = nargs.length - 1 + secret_position;
  }
  if (secret_position != undefined) {
    nargs[secret_position] = "***";
  }
  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
  const command = new Deno.Command(cmd, {
    args: args,
  });

  const { code, stdout, stderr } = await command.output();
  if (stdout.length > 0) {
    console.log(new TextDecoder().decode(stdout));
  }
  if (stderr.length > 0) {
    console.log(new TextDecoder().decode(stderr));
  }
  if (code !== 0) {
    const err = `SH command '${cmd} ${args.join(
      " "
    )}' returned with a non-zero status ${code}.`;
    throw err;
  }
  console.log("Command successfully executed");
}

function regexFromPath(
  path_type:
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group",
  path: string
) {
  if (path_type == "flow") {
    return `${path}.flow/*`;
  } if (path_type == "app") {
    return `${path}.app/*`;
  } else if (path_type == "folder") {
    return `${path}/folder.meta.*`;
  } else if (path_type == "resourcetype") {
    return `${path}.resource-type.*`;
  } else if (path_type == "resource") {
    return `${path}.resource.*`;
  } else if (path_type == "variable") {
    return `${path}.variable.*`;
  } else if (path_type == "schedule") {
    return `${path}.schedule.*`;
  } else if (path_type == "user") {
    return `${path}.user.*`;
  } else if (path_type == "group") {
    return `${path}.group.*`;
  } else {
    return `${path}.*`;
  }
}
async function wmill_sync_pull(
  path_type:
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group",
  workspace_id: string,
  path: string | undefined,
  parent_path: string | undefined,
  skip_secret: boolean
) {
  const includes = [];
  if (path !== undefined && path !== null && path !== "") {
    includes.push(regexFromPath(path_type, path));
  }
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    includes.push(regexFromPath(path_type, parent_path));
  }

  await wmill_run(
    6,
    "workspace",
    "add",
    workspace_id,
    workspace_id,
    Deno.env.get("BASE_INTERNAL_URL") + "/",
    "--token",
    Deno.env.get("WM_TOKEN") ?? ""
  );
  console.log("Pulling workspace into git repo");
  await wmill_run(
    3,
    "sync",
    "pull",
    "--token",
    Deno.env.get("WM_TOKEN") ?? "",
    "--workspace",
    workspace_id,
    "--yes",
    "--raw",
    skip_secret ? "--skip-secrets" : "",
    "--include-schedules",
    "--include-users",
    "--include-groups",
    "--includes",
    includes.join(",")
  );
}

async function wmill_run(secret_position: number, ...cmd: string[]) {
  cmd = cmd.filter((elt) => elt !== "");
  const cmd2 = cmd.slice();
  cmd2[secret_position] = "***";
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
  await wmill.parse(cmd);
  console.log("Command successfully executed");
}


`1`	`import * as wmillclient from "npm:windmill-client@1.321.2";`
`2`	`import wmill from "https://deno.land/x/wmill@v1.321.2/main.ts";`
`3`	`import { basename } from "https://deno.land/std@0.208.0/path/mod.ts";`
`4`
`5`	`export async function main(`
`6`	`workspace_id: string,`
`7`	`repo_url_resource_path: string,`
`8`	`path_type:`
`9`	`\| "script"`
`10`	`\| "flow"`
`11`	`\| "app"`
`12`	`\| "folder"`
`13`	`\| "resource"`
`14`	`\| "variable"`
`15`	`\| "resourcetype"`
`16`	`\| "schedule"`
`17`	`\| "user"`
`18`	`\| "group",`
`19`	`skip_secret = true,`
`20`	`path: string \| undefined,`
`21`	`parent_path: string \| undefined,`
`22`	`commit_msg: string,`
`23`	`use_individual_branch = false,`
`24`	`group_by_folder = false`
`25`	`) {`
`26`	`const repo_resource = await wmillclient.getResource(repo_url_resource_path);`
`27`
`28`	`const cwd = Deno.cwd();`
`29`	`Deno.env.set("HOME", ".");`
`30`	`console.log(`
`31`	`Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
`32`	`);`
`33`
`34`	`const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);`
`35`	`await move_to_git_branch(`
`36`	`workspace_id,`
`37`	`path_type,`
`38`	`path,`
`39`	`parent_path,`
`40`	`use_individual_branch,`
`41`	`group_by_folder`
`42`	`);`
`43`
`44`	`const subfolder = repo_resource.folder ?? "";`
`45`	`const branch_or_default = repo_resource.branch ?? "<DEFAULT>";`
`46`	`console.log(`
`47`	`Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
`48`	`);`
`49`
`50`	`await wmill_sync_pull(`
`51`	`path_type,`
`52`	`workspace_id,`
`53`	`path,`
`54`	`parent_path,`
`55`	`skip_secret`
`56`	`);`
`57`
`58`	`await git_push(path, parent_path, commit_msg);`
`59`
`60`	`console.log("Finished syncing");`
`61`	Deno.chdir(`${cwd}`);
`62`	`}`
`63`
`64`	`async function git_clone(`
`65`	`cwd: string,`
`66`	`repo_resource: any,`
`67`	`use_individual_branch: boolean`
`68`	`): Promise<string> {`
`69`	`// TODO: handle private SSH keys as well`
`70`	`let repo_url = repo_resource.url;`
`71`	`const subfolder = repo_resource.folder ?? "";`
`72`	`const branch = repo_resource.branch ?? "";`
`73`	`const repo_name = basename(repo_url, ".git");`
`74`
`75`	`const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);`
`76`
`77`	`if (azureMatch) {`
`78`	`console.log(`
`79`	`"Requires Azure DevOps service account access token, requesting..."`
`80`	`);`
`81`	`const azureResource = await wmillclient.getResource(azureMatch.groups.url);`
`82`
`83`	`const response = await fetch(`
`84`	`https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
`85`	`{`
`86`	`method: "POST",`
`87`	`body: new URLSearchParams({`
`88`	`client_id: azureResource.azureClientId,`
`89`	`client_secret: azureResource.azureClientSecret,`
`90`	`grant_type: "client_credentials",`
`91`	`resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",`
`92`	`}),`
`93`	`}`
`94`	`);`
`95`
`96`	`const { access_token } = await response.json();`
`97`
`98`	`repo_url = repo_url.replace(azureMatch[0], access_token);`
`99`	`}`
`100`
`101`	`const args = ["clone", "--quiet", "--depth", "1"];`
`102`	`if (use_individual_branch) {`
`103`	`args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo`
`104`	`}`
`105`	`if (subfolder !== "") {`
`106`	`args.push("--sparse");`
`107`	`}`
`108`	`if (branch !== "") {`
`109`	`args.push("--branch");`
`110`	`args.push(branch);`
`111`	`}`
`112`	`args.push(repo_url);`
`113`	`args.push(repo_name);`
`114`	`await sh_run(-1, "git", ...args);`
`115`
`116`	`try {`
`117`	Deno.chdir(`${cwd}/${repo_name}`);
`118`	`} catch (err) {`
`119`	`console.log(`
`120`	`Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
`121`	`);`
`122`	`throw err;`
`123`	`}`
`124`	Deno.chdir(`${cwd}/${repo_name}`);
`125`	`if (subfolder !== "") {`
`126`	`await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);`
`127`	`}`
`128`	`try {`
`129`	Deno.chdir(`${cwd}/${repo_name}/${subfolder}`);
`130`	`} catch (err) {`
`131`	`console.log(`
`132`	`Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
`133`	`);`
`134`	`throw err;`
`135`	`}`
`136`
`137`	`return repo_name;`
`138`	`}`
`139`
`140`	`async function move_to_git_branch(`
`141`	`workspace_id: string,`
`142`	`path_type:`
`143`	`\| "script"`
`144`	`\| "flow"`
`145`	`\| "app"`
`146`	`\| "folder"`
`147`	`\| "resource"`
`148`	`\| "variable"`
`149`	`\| "resourcetype"`
`150`	`\| "schedule"`
`151`	`\| "user"`
`152`	`\| "group",`
`153`	`path: string \| undefined,`
`154`	`parent_path: string \| undefined,`
`155`	`use_individual_branch: boolean,`
`156`	`group_by_folder: boolean`
`157`	`) {`
`158`	`if (!use_individual_branch \|\| path_type === "user" \|\| path_type === "group") {`
`159`	`return;`
`160`	`}`
`161`
`162`	`const branchName = group_by_folder`
`163`	? `wm_deploy/${workspace_id}/${(path ?? parent_path)
`164`	`?.split("/")`
`165`	`.slice(0, 2)`
`166`	.join("__")}`
`167`	: `wm_deploy/${workspace_id}/${path_type}/${(
`168`	`path ?? parent_path`
`169`	)?.replaceAll("/", "__")}`;
`170`
`171`	`try {`
`172`	`await sh_run(undefined, "git", "checkout", branchName);`
`173`	`} catch (err) {`
`174`	`console.log(`
`175`	`Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
`176`	`);`
`177`	`try {`
`178`	`await sh_run(undefined, "git", "checkout", "-b", branchName);`
`179`	`await sh_run(`
`180`	`undefined,`
`181`	`"git",`
`182`	`"config",`
`183`	`"--add",`
`184`	`"--bool",`
`185`	`"push.autoSetupRemote",`
`186`	`"true"`
`187`	`);`
`188`	`} catch (err) {`
`189`	`console.log(`
`190`	`Error checking out branch '${branchName}'. Error was:\n${err}`
`191`	`);`
`192`	`throw err;`
`193`	`}`
`194`	`}`
`195`	console.log(`Successfully switched to branch ${branchName}`);
`196`	`}`
`197`
`198`	`async function git_push(`
`199`	`path: string \| undefined,`
`200`	`parent_path: string \| undefined,`
`201`	`commit_msg: string`
`202`	`) {`
`203`	`await sh_run(`
`204`	`undefined,`
`205`	`"git",`
`206`	`"config",`
`207`	`"user.email",`
`208`	`Deno.env.get("WM_EMAIL") ?? ""`
`209`	`);`
`210`	`await sh_run(`
`211`	`undefined,`
`212`	`"git",`
`213`	`"config",`
`214`	`"user.name",`
`215`	`Deno.env.get("WM_USERNAME") ?? ""`
`216`	`);`
`217`	`if (path !== undefined && path !== null && path !== "") {`
`218`	`try {`
`219`	await sh_run(undefined, "git", "add", `${path}**`);
`220`	`} catch (e) {`
`221`	console.log(`Unable to stage files matching ${path}**, ${e}`);
`222`	`}`
`223`	`}`
`224`	`if (parent_path !== undefined && parent_path !== null && parent_path !== "") {`
`225`	`try {`
`226`	await sh_run(undefined, "git", "add", `${parent_path}**`);
`227`	`} catch (e) {`
`228`	console.log(`Unable to stage files matching ${parent_path}, ${e}`);
`229`	`}`
`230`	`}`
`231`	`try {`
`232`	`await sh_run(undefined, "git", "diff", "--cached", "--quiet");`
`233`	`} catch {`
`234`	`// git diff returns exit-code = 1 when there's at least on staged changes`
`235`	`await sh_run(undefined, "git", "commit", "-m", commit_msg);`
`236`	`await sh_run(undefined, "git", "push", "--porcelain");`
`237`	`return;`
`238`	`}`
`239`	`console.log("No changes detected, nothing to commit. Returning...");`
`240`	`}`
`241`
`242`	`async function sh_run(`
`243`	`secret_position: number \| undefined,`
`244`	`cmd: string,`
`245`	`...args: string[]`
`246`	`) {`
`247`	`const nargs = secret_position != undefined ? args.slice() : args;`
`248`	`if (secret_position < 0) {`
`249`	`secret_position = nargs.length - 1 + secret_position;`
`250`	`}`
`251`	`if (secret_position != undefined) {`
`252`	`nargs[secret_position] = "***";`
`253`	`}`
`254`	console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
`255`	`const command = new Deno.Command(cmd, {`
`256`	`args: args,`
`257`	`});`
`258`
`259`	`const { code, stdout, stderr } = await command.output();`
`260`	`if (stdout.length > 0) {`
`261`	`console.log(new TextDecoder().decode(stdout));`
`262`	`}`
`263`	`if (stderr.length > 0) {`
`264`	`console.log(new TextDecoder().decode(stderr));`
`265`	`}`
`266`	`if (code !== 0) {`
`267`	const err = `SH command '${cmd} ${args.join(
`268`	`" "`
`269`	)}' returned with a non-zero status ${code}.`;
`270`	`throw err;`
`271`	`}`
`272`	`console.log("Command successfully executed");`
`273`	`}`
`274`
`275`	`function regexFromPath(`
`276`	`path_type:`
`277`	`\| "script"`
`278`	`\| "flow"`
`279`	`\| "app"`
`280`	`\| "folder"`
`281`	`\| "resource"`
`282`	`\| "variable"`
`283`	`\| "resourcetype"`
`284`	`\| "schedule"`
`285`	`\| "user"`
`286`	`\| "group",`
`287`	`path: string`
`288`	`) {`
`289`	`if (path_type == "flow") {`
`290`	return `${path}.flow/*`;
`291`	`} if (path_type == "app") {`
`292`	return `${path}.app/*`;
`293`	`} else if (path_type == "folder") {`
`294`	return `${path}/folder.meta.*`;
`295`	`} else if (path_type == "resourcetype") {`
`296`	return `${path}.resource-type.*`;
`297`	`} else if (path_type == "resource") {`
`298`	return `${path}.resource.*`;
`299`	`} else if (path_type == "variable") {`
`300`	return `${path}.variable.*`;
`301`	`} else if (path_type == "schedule") {`
`302`	return `${path}.schedule.*`;
`303`	`} else if (path_type == "user") {`
`304`	return `${path}.user.*`;
`305`	`} else if (path_type == "group") {`
`306`	return `${path}.group.*`;
`307`	`} else {`
`308`	return `${path}.*`;
`309`	`}`
`310`	`}`
`311`	`async function wmill_sync_pull(`
`312`	`path_type:`
`313`	`\| "script"`
`314`	`\| "flow"`
`315`	`\| "app"`
`316`	`\| "folder"`
`317`	`\| "resource"`
`318`	`\| "variable"`
`319`	`\| "resourcetype"`
`320`	`\| "schedule"`
`321`	`\| "user"`
`322`	`\| "group",`
`323`	`workspace_id: string,`
`324`	`path: string \| undefined,`
`325`	`parent_path: string \| undefined,`
`326`	`skip_secret: boolean`
`327`	`) {`
`328`	`const includes = [];`
`329`	`if (path !== undefined && path !== null && path !== "") {`
`330`	`includes.push(regexFromPath(path_type, path));`
`331`	`}`
`332`	`if (parent_path !== undefined && parent_path !== null && parent_path !== "") {`
`333`	`includes.push(regexFromPath(path_type, parent_path));`
`334`	`}`
`335`
`336`	`await wmill_run(`
`337`	`6,`
`338`	`"workspace",`
`339`	`"add",`
`340`	`workspace_id,`
`341`	`workspace_id,`
`342`	`Deno.env.get("BASE_INTERNAL_URL") + "/",`
`343`	`"--token",`
`344`	`Deno.env.get("WM_TOKEN") ?? ""`
`345`	`);`
`346`	`console.log("Pulling workspace into git repo");`
`347`	`await wmill_run(`
`348`	`3,`
`349`	`"sync",`
`350`	`"pull",`
`351`	`"--token",`
`352`	`Deno.env.get("WM_TOKEN") ?? "",`
`353`	`"--workspace",`
`354`	`workspace_id,`
`355`	`"--yes",`
`356`	`"--raw",`
`357`	`skip_secret ? "--skip-secrets" : "",`
`358`	`"--include-schedules",`
`359`	`"--include-users",`
`360`	`"--include-groups",`
`361`	`"--includes",`
`362`	`includes.join(",")`
`363`	`);`
`364`	`}`
`365`
`366`	`async function wmill_run(secret_position: number, ...cmd: string[]) {`
`367`	`cmd = cmd.filter((elt) => elt !== "");`
`368`	`const cmd2 = cmd.slice();`
`369`	`cmd2[secret_position] = "***";`
`370`	console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
`371`	`await wmill.parse(cmd);`
`372`	`console.log("Command successfully executed");`
`373`	`}`
`374`