Script 'Sync script to Git repo' for windmill

Sync script to Git repo

Verified

This script will pull the script from the current workspace in a temporary folder, then commit it to the remote Git repository and push it. Only the script-related file will be pushed, nothing else. It takes as input the git_repository resource containing the repository URL, the script path, and the commit message to use for the commit. All params are mandatory.

Created by hugo697 825 days ago Picked 90 times

Submitted by pyranota275 Bun

Verified 8 days ago

All edits

Permalink

import * as wmillclient from "windmill-client";
import wmill from "[email protected]";
import { basename } from "node:path";
const util = require("util");
const exec = util.promisify(require("child_process").exec);
import process from "process";

type GpgKey = {
  email: string;
  private_key: string;
  passphrase: string;
};

const FORKED_WORKSPACE_PREFIX = "wm-fork-";
const FORKED_BRANCH_PREFIX = "wm-fork";

type PathType =
  | "script"
  | "flow"
  | "app"
  | "raw_app"
  | "folder"
  | "resource"
  | "variable"
  | "resourcetype"
  | "schedule"
  | "user"
  | "group"
  | "httptrigger"
  | "websockettrigger"
  | "kafkatrigger"
  | "natstrigger"
  | "postgrestrigger"
  | "mqtttrigger"
  | "sqstrigger"
  | "gcptrigger"
  | "emailtrigger";

type SyncObject = {
  path_type: PathType;
  path: string | undefined;
  parent_path: string | undefined;
  commit_msg: string;
};

let gpgFingerprint: string | undefined = undefined;

export async function main(
  items: SyncObject[],
  // Compat, do not use in code, rely on `items` instead
  path_type: PathType | undefined,
  path: string | undefined,
  parent_path: string | undefined,
  commit_msg: string | undefined,
  //
  workspace_id: string,
  repo_url_resource_path: string,
  skip_secret: boolean = true,
  use_individual_branch: boolean = false,
  group_by_folder: boolean = false,
  only_create_branch: boolean = false,
  parent_workspace_id?: string,
  force_branch?: string,
) {

  if (path_type !== undefined && commit_msg !== undefined) {
    items = [{
      path_type,
      path,
      parent_path,
      commit_msg,
    }];
  }
  await inner(items, workspace_id, repo_url_resource_path, skip_secret, use_individual_branch, group_by_folder, only_create_branch, parent_workspace_id, force_branch);
}

async function inner(
  items: SyncObject[],
  workspace_id: string,
  repo_url_resource_path: string,
  skip_secret: boolean = true,
  use_individual_branch: boolean = false,
  group_by_folder: boolean = false,
  only_create_branch: boolean = false,
  parent_workspace_id?: string,
  force_branch?: string,
) {

  let safeDirectoryPath: string | undefined;
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
  const cwd = process.cwd();
  process.env["HOME"] = ".";
  if (!only_create_branch) {
    for (const item of items) {
      console.log(
        `Syncing ${item.path_type} ${item.path ?? ""} with parent ${item.parent_path ?? ""}`
      );
    }
  }

  if (repo_resource.is_github_app) {
    const token = await get_gh_app_token();
    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
    repo_resource.url = authRepoUrl;
  }

  const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
  safeDirectoryPath = cloneSafeDirectoryPath;


  // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
  // a fork of a fork workspace. In that case, the original branch is not stored in the resource
  // settings, but we need to infer it from the workspace id

  if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    if (use_individual_branch) {
      console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
      use_individual_branch = false;
    }
    if (group_by_folder) {
      console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
      group_by_folder = false;
    }
  }

  if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
    console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
    await git_checkout_branch(
      items,
      parent_workspace_id,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
  }

  await git_checkout_branch(
    items,
    workspace_id,
    use_individual_branch,
    group_by_folder,
    clonedBranchName
  );


  const subfolder = repo_resource.folder ?? "";
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
  console.log(
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
  );

  // If we want to just create the branch, we can skip pulling the changes.
  if (!only_create_branch) {
    await wmill_sync_pull(
      items,
      workspace_id,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch,
      force_branch
    );
  }
  try {
    await git_push(items, repo_resource, only_create_branch);
  } catch (e) {
    throw e;
  } finally {
    await delete_pgp_keys();
    // Cleanup: remove safe.directory config
    if (safeDirectoryPath) {
      try {
        await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
      } catch (e) {
        console.log(`Warning: Could not unset safe.directory config: ${e}`);
      }
    }
  }
  console.log("Finished syncing");
  process.chdir(`${cwd}`);
}


function get_fork_branch_name(w_id: string, originalBranch: string): string {
  if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
  }
  return w_id;
}

async function git_clone(
  cwd: string,
  repo_resource: any,
  no_single_branch: boolean,
): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
  // TODO: handle private SSH keys as well
  let repo_url = repo_resource.url;
  const subfolder = repo_resource.folder ?? "";
  const branch = repo_resource.branch ?? "";
  const repo_name = basename(repo_url, ".git");
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
  if (azureMatch) {
    console.log(
      "Requires Azure DevOps service account access token, requesting..."
    );
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    const response = await fetch(
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
      {
        method: "POST",
        body: new URLSearchParams({
          client_id: azureResource.azureClientId,
          client_secret: azureResource.azureClientSecret,
          grant_type: "client_credentials",
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
        }),
      }
    );
    const { access_token } = await response.json();
    repo_url = repo_url.replace(azureMatch[0], access_token);
  }
  const args = ["clone", "--quiet", "--depth", "1"];
  if (no_single_branch) {
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
  }
  if (subfolder !== "") {
    args.push("--sparse");
  }
  if (branch !== "") {
    args.push("--branch");
    args.push(branch);
  }
  args.push(repo_url);
  args.push(repo_name);
  await sh_run(-1, "git", ...args);
  try {
    process.chdir(`${cwd}/${repo_name}`);
    const safeDirectoryPath = process.cwd();
    // Add safe.directory to handle dubious ownership in cloned repo
    try {
      await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
    } catch (e) {
      console.log(`Warning: Could not add safe.directory config: ${e}`);
    }

    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
      try {
        process.chdir(`${cwd}/${repo_name}/${subfolder}`);
      } catch (err) {
        console.log(
          `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
    return { repo_name, safeDirectoryPath, clonedBranchName };

  } catch (err) {
    console.log(
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    );
    throw err;
  }
}
async function git_checkout_branch(
  items: SyncObject[],
  workspace_id: string,
  use_individual_branch: boolean,
  group_by_folder: boolean,
  originalBranchName: string
) {
  let branchName;
  if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    branchName = get_fork_branch_name(workspace_id, originalBranchName);
  } else {

    if (!use_individual_branch
      // If individual branch is true, we can assume items is of length 1, as debouncing is disabled for jobs with this flag
      || items[0].path_type === "user" || items[0].path_type === "group") {
      return;
    }

    // as mentioned above, it is safe to assume that items.len is 1
    const [path, parent_path] = [items[0].path, items[0].parent_path];
    branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${items[0].path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
  }

  try {
    await sh_run(undefined, "git", "checkout", branchName);
  } catch (err) {
    console.log(
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    );
    try {
      await sh_run(undefined, "git", "checkout", "-b", branchName);
      await sh_run(
        undefined,
        "git",
        "config",
        "--add",
        "--bool",
        "push.autoSetupRemote",
        "true"
      );
    } catch (err) {
      console.log(
        `Error checking out branch '${branchName}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  console.log(`Successfully switched to branch ${branchName}`);
}

function composeCommitHeader(items: SyncObject[]): string {
  // Count occurrences of each path_type
  const typeCounts = new Map<PathType, number>();
  for (const item of items) {
    typeCounts.set(item.path_type, (typeCounts.get(item.path_type) ?? 0) + 1);
  }

  // Sort by count descending to get the top 2
  const sortedTypes = Array.from(typeCounts.entries()).sort((a, b) => b[1] - a[1]);

  const parts: string[] = [];
  let othersCount = 0;

  for (let i = 0; i < sortedTypes.length; i++) {
    const [pathType, count] = sortedTypes[i];
    if (i < 3) {
      // Pluralize the path type if count > 1
      const label = count > 1 ? `${pathType}s` : pathType;

      if (i == 2 && sortedTypes.length == 3) {
        parts.push(`and ${count} ${label}`);
      } else {
        parts.push(`${count} ${label}`);
      }
    } else {
      othersCount += count;
    }
  }

  let header = `[WM]: Deployed ${parts.join(", ")}`;
  if (othersCount > 0) {
    header += ` and ${othersCount} other object${othersCount > 1 ? "s" : ""}`;
  }

  return header;
}

async function git_push(
  items: SyncObject[],
  repo_resource: any,
  only_create_branch: boolean,
) {
  let user_email = process.env["WM_EMAIL"] ?? "";
  let user_name = process.env["WM_USERNAME"] ?? "";

  if (repo_resource.gpg_key) {
    await set_gpg_signing_secret(repo_resource.gpg_key);
    // Configure git with GPG key email for signing
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      repo_resource.gpg_key.email
    );
    await sh_run(undefined, "git", "config", "user.name", user_name);
  } else {
    await sh_run(undefined, "git", "config", "user.email", user_email);
    await sh_run(undefined, "git", "config", "user.name", user_name);
  }
  if (only_create_branch) {
    await sh_run(undefined, "git", "push", "--porcelain");
  }

  let commit_description: string[] = [];
  for (const { path, parent_path, commit_msg } of items) {
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `'${path}**'`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `'${parent_path}**'`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }

    commit_description.push(commit_msg);
  }

  try {
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
  } catch {
    // git diff returns exit-code = 1 when there's at least one staged changes
    const commitArgs = ["git", "commit"];

    // Always use --author to set consistent authorship
    commitArgs.push("--author", `"${user_name} <${user_email}>"`);

    const [header, description] = (commit_description.length == 1)
      ? [commit_description[0], ""]
      : [composeCommitHeader(items), commit_description.join("\n")];

    commitArgs.push(
      "-m",
      `"${header == undefined || header == "" ? "no commit msg" : header}"`,
      "-m",
      `"${description}"`
    );

    await sh_run(undefined, ...commitArgs);
    try {
      await sh_run(undefined, "git", "push", "--porcelain");
    } catch (e) {
      console.log(`Could not push, trying to rebase first: ${e}`);
      await sh_run(undefined, "git", "pull", "--rebase");
      await sh_run(undefined, "git", "push", "--porcelain");
    }
    return;
  }

  console.log("No changes detected, nothing to commit. Returning...");
}
async function sh_run(
  secret_position: number | undefined,
  cmd: string,
  ...args: string[]
) {
  const nargs = secret_position != undefined ? args.slice() : args;
  if (secret_position && secret_position < 0) {
    secret_position = nargs.length - 1 + secret_position;
  }
  let secret: string | undefined = undefined;
  if (secret_position != undefined) {
    nargs[secret_position] = "***";
    secret = args[secret_position];
  }

  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
  const command = exec(`${cmd} ${args.join(" ")}`);
  // new Deno.Command(cmd, {
  //   args: args,
  // });
  try {
    const { stdout, stderr } = await command;
    if (stdout.length > 0) {
      console.log(stdout);
    }
    if (stderr.length > 0) {
      console.log(stderr);
    }
    console.log("Command successfully executed");
    return stdout;
  } catch (error) {
    let errorString = error.toString();
    if (secret) {
      errorString = errorString.replace(secret, "***");
    }
    const err = `SH command '${cmd} ${nargs.join(
      " "
    )}' returned with error ${errorString}`;
    throw Error(err);
  }
}

function regexFromPath(path_type: PathType, path: string) {
  if (path_type == "flow") {
    return `${path}.flow/*,${path}__flow/*`;
  } else if (path_type == "app") {
    return `${path}.app/*,${path}__app/*`;
  } else if (path_type == "raw_app") {
    return `${path}.raw_app/**,${path}__raw_app/**`;
  } else if (path_type == "folder") {
    return `${path}/folder.meta.*`;
  } else if (path_type == "resourcetype") {
    return `${path}.resource-type.*`;
  } else if (path_type == "resource") {
    return `${path}.resource.*`;
  } else if (path_type == "variable") {
    return `${path}.variable.*`;
  } else if (path_type == "schedule") {
    return `${path}.schedule.*`;
  } else if (path_type == "user") {
    return `${path}.user.*`;
  } else if (path_type == "group") {
    return `${path}.group.*`;
  } else if (path_type == "httptrigger") {
    return `${path}.http_trigger.*`;
  } else if (path_type == "websockettrigger") {
    return `${path}.websocket_trigger.*`;
  } else if (path_type == "kafkatrigger") {
    return `${path}.kafka_trigger.*`;
  } else if (path_type == "natstrigger") {
    return `${path}.nats_trigger.*`;
  } else if (path_type == "postgrestrigger") {
    return `${path}.postgres_trigger.*`;
  } else if (path_type == "mqtttrigger") {
    return `${path}.mqtt_trigger.*`;
  } else if (path_type == "sqstrigger") {
    return `${path}.sqs_trigger.*`;
  } else if (path_type == "gcptrigger") {
    return `${path}.gcp_trigger.*`;
  } else if (path_type == "emailtrigger") {
    return `${path}.email_trigger.*`;
  } else {
    return `${path}.*`;
  }
}

async function wmill_sync_pull(
  items: SyncObject[],
  workspace_id: string,
  skip_secret: boolean,
  repo_url_resource_path: string,
  use_individual_branch: boolean,
  original_branch?: string,
  force_branch?: string,
) {
  const includes = [];
  for (const item of items) {
    const { path_type, path, parent_path } = item;
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
  }

  console.log("Pulling workspace into git repo");

  const args = [
    "sync",
    "pull",
    "--token",
    process.env["WM_TOKEN"] ?? "",
    "--workspace",
    workspace_id,
    "--base-url",
    process.env["BASE_URL"] + "/",
    "--repository",
    repo_url_resource_path,
    "--yes",
    skip_secret ? "--skip-secrets" : "",
  ];

  if (items.some(item => item.path_type === "schedule") && !use_individual_branch) {
    args.push("--include-schedules");
  }

  if (items.some(item => item.path_type === "group") && !use_individual_branch) {
    args.push("--include-groups");
  }

  if (items.some(item => item.path_type === "user") && !use_individual_branch) {
    args.push("--include-users");
  }

  if (items.some(item => item.path_type.includes("trigger")) && !use_individual_branch) {
    args.push("--include-triggers");
  }
  // Only include settings when specifically deploying settings
  if (items.some(item => item.path_type === "settings") && !use_individual_branch) {
    args.push("--include-settings");
  }

  // Only include key when specifically deploying keys
  if (items.some(item => item.path_type === "key") && !use_individual_branch) {
    args.push("--include-key");
  }

  if (force_branch) {
    args.push("--branch", force_branch);
  }

  args.push("--extra-includes", includes.join(","));

  // If using individual branches, apply promotion settings from original branch
  if (use_individual_branch && original_branch) {
    console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
    args.push("--promotion", original_branch);
  }

  await wmill_run(3, ...args);
}

async function wmill_run(secret_position: number, ...cmd: string[]) {
  cmd = cmd.filter((elt) => elt !== "");
  const cmd2 = cmd.slice();
  cmd2[secret_position] = "***";
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
  await wmill.parse(cmd);
  console.log("Command successfully executed");
}

// Function to set up GPG signing
async function set_gpg_signing_secret(gpg_key: GpgKey) {
  try {
    console.log("Setting GPG private key for git commits");

    const formattedGpgContent = gpg_key.private_key.replace(
      /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
      (_: string, header: string, body: string, footer: string) =>
        header +
        "\n" +
        "\n" +
        body.replace(/ ([^\s])/g, "\n$1").trim() +
        "\n" +
        footer
    );

    const gpg_path = `/tmp/gpg`;
    await sh_run(undefined, "mkdir", "-p", gpg_path);
    await sh_run(undefined, "chmod", "700", gpg_path);
    process.env.GNUPGHOME = gpg_path;
    // process.env.GIT_TRACE = 1;

    try {
      await sh_run(
        1,
        "bash",
        "-c",
        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
      );
    } catch (e) {
      // Original error would contain sensitive data
      throw new Error("Failed to import GPG key!");
    }

    const listKeysOutput = await sh_run(
      undefined,
      "gpg",
      "--list-secret-keys",
      "--with-colons",
      "--keyid-format=long"
    );

    const keyInfoMatch = listKeysOutput.match(
      /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
    );

    if (!keyInfoMatch) {
      throw new Error("Failed to extract GPG Key ID and Fingerprint");
    }

    const keyId = keyInfoMatch[1];
    gpgFingerprint = keyInfoMatch[2];

    if (gpg_key.passphrase) {
      // This is adummy command to unlock the key
      // with passphrase to load it into agent
      await sh_run(
        1,
        "bash",
        "-c",
        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
      );
    }

    // Configure Git to use the extracted key
    await sh_run(undefined, "git", "config", "user.signingkey", keyId);
    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
    console.log(`GPG signing configured with key ID: ${keyId} `);
  } catch (e) {
    console.error(`Failure while setting GPG key: ${e} `);
    await delete_pgp_keys();
  }
}

async function delete_pgp_keys() {
  console.log("deleting gpg keys");
  if (gpgFingerprint) {
    await sh_run(
      undefined,
      "gpg",
      "--batch",
      "--yes",
      "--pinentry-mode",
      "loopback",
      "--delete-secret-key",
      gpgFingerprint
    );
    await sh_run(
      undefined,
      "gpg",
      "--batch",
      "--yes",
      "--delete-key",
      "--pinentry-mode",
      "loopback",
      gpgFingerprint
    );
  }
}

async function get_gh_app_token() {
  const workspace = process.env["WM_WORKSPACE"];
  const jobToken = process.env["WM_TOKEN"];

  const baseUrl =
    process.env["BASE_INTERNAL_URL"] ??
    process.env["BASE_URL"] ??
    "http://localhost:8000";

  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;

  const response = await fetch(url, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      Authorization: `Bearer ${jobToken}`,
    },
    body: JSON.stringify({
      job_token: jobToken,
    }),
  });

  if (!response.ok) {
    throw new Error(`Error: ${response.statusText}`);
  }

  const data = await response.json();

  return data.token;
}

function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
  if (!gitHubUrl || !installationToken) {
    throw new Error("Both GitHub URL and Installation Token are required.");
  }

  try {
    const url = new URL(gitHubUrl);

    // GitHub repository URL should be in the format: https://github.com/owner/repo.git
    if (url.hostname !== "github.com") {
      throw new Error(
        "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
      );
    }

    // Convert URL to include the installation token
    return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
  } catch (e) {
    const error = e as Error;
    throw new Error(`Invalid URL: ${error.message}`);
  }
}

Other submissions

Submitted by hugo697 Deno

Created 591 days ago

All edits

Permalink

import * as wmillclient from "npm:[email protected]";
import wmill from "https://deno.land/x/[email protected]/main.ts";
import { basename } from "https://deno.land/[email protected]/path/mod.ts";

export async function main(
  workspace_id: string,
  repo_url_resource_path: string,
  path_type:
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group",
  skip_secret = true,
  path: string | undefined,
  parent_path: string | undefined,
  commit_msg: string,
  use_individual_branch = false,
  group_by_folder = false
) {
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);

  const cwd = Deno.cwd();
  Deno.env.set("HOME", ".");
  console.log(
    `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
  );

  const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
  await move_to_git_branch(
    workspace_id,
    path_type,
    path,
    parent_path,
    use_individual_branch,
    group_by_folder
  );

  const subfolder = repo_resource.folder ?? "";
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
  console.log(
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
  );

  await wmill_sync_pull(
    path_type,
    workspace_id,
    path,
    parent_path,
    skip_secret
  );

  await git_push(path, parent_path, commit_msg);

  console.log("Finished syncing");
  Deno.chdir(`${cwd}`);
}

async function git_clone(
  cwd: string,
  repo_resource: any,
  use_individual_branch: boolean
): Promise<string> {
  // TODO: handle private SSH keys as well
  let repo_url = repo_resource.url;
  const subfolder = repo_resource.folder ?? "";
  const branch = repo_resource.branch ?? "";
  const repo_name = basename(repo_url, ".git");

  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);

  if (azureMatch) {
    console.log(
      "Requires Azure DevOps service account access token, requesting..."
    );
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);

    const response = await fetch(
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
      {
        method: "POST",
        body: new URLSearchParams({
          client_id: azureResource.azureClientId,
          client_secret: azureResource.azureClientSecret,
          grant_type: "client_credentials",
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
        }),
      }
    );

    const { access_token } = await response.json();

    repo_url = repo_url.replace(azureMatch[0], access_token);
  }

  const args = ["clone", "--quiet", "--depth", "1"];
  if (use_individual_branch) {
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
  }
  if (subfolder !== "") {
    args.push("--sparse");
  }
  if (branch !== "") {
    args.push("--branch");
    args.push(branch);
  }
  args.push(repo_url);
  args.push(repo_name);
  await sh_run(-1, "git", ...args);

  try {
    Deno.chdir(`${cwd}/${repo_name}`);
  } catch (err) {
    console.log(
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    );
    throw err;
  }
  Deno.chdir(`${cwd}/${repo_name}`);
  if (subfolder !== "") {
    await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
  }
  try {
    Deno.chdir(`${cwd}/${repo_name}/${subfolder}`);
  } catch (err) {
    console.log(
      `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
    );
    throw err;
  }

  return repo_name;
}

async function move_to_git_branch(
  workspace_id: string,
  path_type:
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group",
  path: string | undefined,
  parent_path: string | undefined,
  use_individual_branch: boolean,
  group_by_folder: boolean
) {
  if (!use_individual_branch || path_type === "user" || path_type === "group") {
    return;
  }

  const branchName = group_by_folder
    ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
    : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;

  try {
    await sh_run(undefined, "git", "checkout", branchName);
  } catch (err) {
    console.log(
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    );
    try {
      await sh_run(undefined, "git", "checkout", "-b", branchName);
      await sh_run(
        undefined,
        "git",
        "config",
        "--add",
        "--bool",
        "push.autoSetupRemote",
        "true"
      );
    } catch (err) {
      console.log(
        `Error checking out branch '${branchName}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  console.log(`Successfully switched to branch ${branchName}`);
}

async function git_push(
  path: string | undefined,
  parent_path: string | undefined,
  commit_msg: string
) {
  await sh_run(
    undefined,
    "git",
    "config",
    "user.email",
    Deno.env.get("WM_EMAIL") ?? ""
  );
  await sh_run(
    undefined,
    "git",
    "config",
    "user.name",
    Deno.env.get("WM_USERNAME") ?? ""
  );
  if (path !== undefined && path !== null && path !== "") {
    try {
      await sh_run(undefined, "git", "add", `${path}**`);
    } catch (e) {
      console.log(`Unable to stage files matching ${path}**, ${e}`);
    }
  }
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    try {
      await sh_run(undefined, "git", "add", `${parent_path}**`);
    } catch (e) {
      console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    }
  }
  try {
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
  } catch {
    // git diff returns exit-code = 1 when there's at least one staged changes
    await sh_run(undefined, "git", "commit", "-m", commit_msg);
    try {
        await sh_run(undefined, "git", "push", "--porcelain");
    } catch {
        console.log("Could not push, trying to rebase first");
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
    }
    return;
  }
  console.log("No changes detected, nothing to commit. Returning...");
}

async function sh_run(
  secret_position: number | undefined,
  cmd: string,
  ...args: string[]
) {
  const nargs = secret_position != undefined ? args.slice() : args;
  if (secret_position < 0) {
    secret_position = nargs.length - 1 + secret_position;
  }
  if (secret_position != undefined) {
    nargs[secret_position] = "***";
  }
  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
  const command = new Deno.Command(cmd, {
    args: args,
  });

  const { code, stdout, stderr } = await command.output();
  if (stdout.length > 0) {
    console.log(new TextDecoder().decode(stdout));
  }
  if (stderr.length > 0) {
    console.log(new TextDecoder().decode(stderr));
  }
  if (code !== 0) {
    const err = `SH command '${cmd} ${nargs.join(
      " "
    )}' returned with a non-zero status ${code}.`;
    throw Error(err);
  }
  console.log("Command successfully executed");
}

function regexFromPath(
  path_type:
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group",
  path: string
) {
  if (path_type == "flow") {
    return `${path}.flow/*`;
  } if (path_type == "app") {
    return `${path}.app/*`;
  } else if (path_type == "folder") {
    return `${path}/folder.meta.*`;
  } else if (path_type == "resourcetype") {
    return `${path}.resource-type.*`;
  } else if (path_type == "resource") {
    return `${path}.resource.*`;
  } else if (path_type == "variable") {
    return `${path}.variable.*`;
  } else if (path_type == "schedule") {
    return `${path}.schedule.*`;
  } else if (path_type == "user") {
    return `${path}.user.*`;
  } else if (path_type == "group") {
    return `${path}.group.*`;
  } else {
    return `${path}.*`;
  }
}
async function wmill_sync_pull(
  path_type:
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group",
  workspace_id: string,
  path: string | undefined,
  parent_path: string | undefined,
  skip_secret: boolean
) {
  const includes = [];
  if (path !== undefined && path !== null && path !== "") {
    includes.push(regexFromPath(path_type, path));
  }
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    includes.push(regexFromPath(path_type, parent_path));
  }

  await wmill_run(
    6,
    "workspace",
    "add",
    workspace_id,
    workspace_id,
    Deno.env.get("BASE_INTERNAL_URL") + "/",
    "--token",
    Deno.env.get("WM_TOKEN") ?? ""
  );
  console.log("Pulling workspace into git repo");
  await wmill_run(
    3,
    "sync",
    "pull",
    "--token",
    Deno.env.get("WM_TOKEN") ?? "",
    "--workspace",
    workspace_id,
    "--yes",
    "--raw",
    skip_secret ? "--skip-secrets" : "",
    "--include-schedules",
    "--include-users",
    "--include-groups",
    "--includes",
    includes.join(",")
  );
}

async function wmill_run(secret_position: number, ...cmd: string[]) {
  cmd = cmd.filter((elt) => elt !== "");
  const cmd2 = cmd.slice();
  cmd2[secret_position] = "***";
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
  await wmill.parse(cmd);
  console.log("Command successfully executed");
}


Submitted by rubenfiszel Bun

Created 84 days ago

All edits

Permalink

import * as wmillclient from "windmill-client";
import wmill from "[email protected]";
import { basename } from "node:path";
const util = require("util");
const exec = util.promisify(require("child_process").exec);
import process from "process";

type GpgKey = {
  email: string;
  private_key: string;
  passphrase: string;
};

const FORKED_WORKSPACE_PREFIX = "wm-fork-";
const FORKED_BRANCH_PREFIX = "wm-fork";

type PathType =
  | "script"
  | "flow"
  | "app"
  | "folder"
  | "resource"
  | "variable"
  | "resourcetype"
  | "schedule"
  | "user"
  | "group"
  | "httptrigger"
  | "websockettrigger"
  | "kafkatrigger"
  | "natstrigger"
  | "postgrestrigger"
  | "mqtttrigger"
  | "sqstrigger"
  | "gcptrigger"
  | "emailtrigger";

let gpgFingerprint: string | undefined = undefined;

export async function main(
  workspace_id: string,
  repo_url_resource_path: string,
  path_type: PathType,
  skip_secret = true,
  path: string | undefined,
  parent_path: string | undefined,
  commit_msg: string,
  parent_workspace_id?: string,
  use_individual_branch = false,
  group_by_folder = false,
  only_create_branch: boolean = false
) {
  let safeDirectoryPath: string | undefined;
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
  const cwd = process.cwd();
  process.env["HOME"] = ".";
  if (!only_create_branch) {
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  }

  if (repo_resource.is_github_app) {
    const token = await get_gh_app_token();
    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
    repo_resource.url = authRepoUrl;
  }

  const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
  safeDirectoryPath = cloneSafeDirectoryPath;


  // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
  // a fork of a fork workspace. In that case, the original branch is not stored in the resource
  // settings, but we need to infer it from the workspace id

  if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    if (use_individual_branch) {
      console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
      use_individual_branch = false;
    }
    if (group_by_folder) {
      console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
      group_by_folder = false;
    }
  }

  if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
    console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
    await move_to_git_branch(
      parent_workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
  }

  await move_to_git_branch(
    workspace_id,
    path_type,
    path,
    parent_path,
    use_individual_branch,
    group_by_folder,
    clonedBranchName
  );


  const subfolder = repo_resource.folder ?? "";
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
  console.log(
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
  );

  // If we want to just create the branch, we can skip pulling the changes.
  if (!only_create_branch) {
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
  }
  try {
    await git_push(path, parent_path, commit_msg, repo_resource, only_create_branch);
  } catch (e) {
    throw e;
  } finally {
    await delete_pgp_keys();
    // Cleanup: remove safe.directory config
    if (safeDirectoryPath) {
      try {
        await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
      } catch (e) {
        console.log(`Warning: Could not unset safe.directory config: ${e}`);
      }
    }
  }
  console.log("Finished syncing");
  process.chdir(`${cwd}`);
}

function get_fork_branch_name(w_id: string, originalBranch: string): string {
  if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
  }
  return w_id;
}

async function git_clone(
  cwd: string,
  repo_resource: any,
  no_single_branch: boolean,
): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
  // TODO: handle private SSH keys as well
  let repo_url = repo_resource.url;
  const subfolder = repo_resource.folder ?? "";
  const branch = repo_resource.branch ?? "";
  const repo_name = basename(repo_url, ".git");
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
  if (azureMatch) {
    console.log(
      "Requires Azure DevOps service account access token, requesting..."
    );
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    const response = await fetch(
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
      {
        method: "POST",
        body: new URLSearchParams({
          client_id: azureResource.azureClientId,
          client_secret: azureResource.azureClientSecret,
          grant_type: "client_credentials",
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
        }),
      }
    );
    const { access_token } = await response.json();
    repo_url = repo_url.replace(azureMatch[0], access_token);
  }
  const args = ["clone", "--quiet", "--depth", "1"];
  if (no_single_branch) {
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
  }
  if (subfolder !== "") {
    args.push("--sparse");
  }
  if (branch !== "") {
    args.push("--branch");
    args.push(branch);
  }
  args.push(repo_url);
  args.push(repo_name);
  await sh_run(-1, "git", ...args);
  try {
    process.chdir(`${cwd}/${repo_name}`);
    const safeDirectoryPath = process.cwd();
    // Add safe.directory to handle dubious ownership in cloned repo
    try {
      await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
    } catch (e) {
      console.log(`Warning: Could not add safe.directory config: ${e}`);
    }

    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
      try {
        process.chdir(`${cwd}/${repo_name}/${subfolder}`);
      } catch (err) {
        console.log(
          `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
    return { repo_name, safeDirectoryPath, clonedBranchName };

  } catch (err) {
    console.log(
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    );
    throw err;
  }
}
async function move_to_git_branch(
  workspace_id: string,
  path_type: PathType,
  path: string | undefined,
  parent_path: string | undefined,
  use_individual_branch: boolean,
  group_by_folder: boolean,
  originalBranchName: string
) {
  let branchName;
  if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    branchName = get_fork_branch_name(workspace_id, originalBranchName);
  } else {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
  }

  try {
    await sh_run(undefined, "git", "checkout", branchName);
  } catch (err) {
    console.log(
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    );
    try {
      await sh_run(undefined, "git", "checkout", "-b", branchName);
      await sh_run(
        undefined,
        "git",
        "config",
        "--add",
        "--bool",
        "push.autoSetupRemote",
        "true"
      );
    } catch (err) {
      console.log(
        `Error checking out branch '${branchName}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  console.log(`Successfully switched to branch ${branchName}`);
}
async function git_push(
  path: string | undefined,
  parent_path: string | undefined,
  commit_msg: string,
  repo_resource: any,
  only_create_branch: boolean,
) {
  let user_email = process.env["WM_EMAIL"] ?? "";
  let user_name = process.env["WM_USERNAME"] ?? "";

  if (repo_resource.gpg_key) {
    await set_gpg_signing_secret(repo_resource.gpg_key);
    // Configure git with GPG key email for signing
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      repo_resource.gpg_key.email
    );
    await sh_run(undefined, "git", "config", "user.name", user_name);
  } else {
    await sh_run(undefined, "git", "config", "user.email", user_email);
    await sh_run(undefined, "git", "config", "user.name", user_name);
  }
  if (only_create_branch) {
    await sh_run(undefined, "git", "push", "--porcelain");
  }

  if (path !== undefined && path !== null && path !== "") {
    try {
      await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
    } catch (e) {
      console.log(`Unable to stage files matching ${path}**, ${e}`);
    }
  }
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    try {
      await sh_run(
        undefined,
        "git",
        "add",
        "wmill-lock.yaml",
        `${parent_path}**`
      );
    } catch (e) {
      console.log(`Unable to stage files matching ${parent_path}, ${e}`);
    }
  }
  try {
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
  } catch {
    // git diff returns exit-code = 1 when there's at least one staged changes
    const commitArgs = ["git", "commit"];

    // Always use --author to set consistent authorship
    commitArgs.push("--author", `"${user_name} <${user_email}>"`);
    commitArgs.push(
      "-m",
      `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
    );

    await sh_run(undefined, ...commitArgs);
    try {
      await sh_run(undefined, "git", "push", "--porcelain");
    } catch (e) {
      console.log(`Could not push, trying to rebase first: ${e}`);
      await sh_run(undefined, "git", "pull", "--rebase");
      await sh_run(undefined, "git", "push", "--porcelain");
    }
    return;
  }
  console.log("No changes detected, nothing to commit. Returning...");
}
async function sh_run(
  secret_position: number | undefined,
  cmd: string,
  ...args: string[]
) {
  const nargs = secret_position != undefined ? args.slice() : args;
  if (secret_position && secret_position < 0) {
    secret_position = nargs.length - 1 + secret_position;
  }
  let secret: string | undefined = undefined;
  if (secret_position != undefined) {
    nargs[secret_position] = "***";
    secret = args[secret_position];
  }

  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
  const command = exec(`${cmd} ${args.join(" ")}`);
  // new Deno.Command(cmd, {
  //   args: args,
  // });
  try {
    const { stdout, stderr } = await command;
    if (stdout.length > 0) {
      console.log(stdout);
    }
    if (stderr.length > 0) {
      console.log(stderr);
    }
    console.log("Command successfully executed");
    return stdout;
  } catch (error) {
    let errorString = error.toString();
    if (secret) {
      errorString = errorString.replace(secret, "***");
    }
    const err = `SH command '${cmd} ${nargs.join(
      " "
    )}' returned with error ${errorString}`;
    throw Error(err);
  }
}

function regexFromPath(path_type: PathType, path: string) {
  if (path_type == "flow") {
    return `${path}.flow/*`;
  }
  if (path_type == "app") {
    return `${path}.app/*`;
  } else if (path_type == "folder") {
    return `${path}/folder.meta.*`;
  } else if (path_type == "resourcetype") {
    return `${path}.resource-type.*`;
  } else if (path_type == "resource") {
    return `${path}.resource.*`;
  } else if (path_type == "variable") {
    return `${path}.variable.*`;
  } else if (path_type == "schedule") {
    return `${path}.schedule.*`;
  } else if (path_type == "user") {
    return `${path}.user.*`;
  } else if (path_type == "group") {
    return `${path}.group.*`;
  } else if (path_type == "httptrigger") {
    return `${path}.http_trigger.*`;
  } else if (path_type == "websockettrigger") {
    return `${path}.websocket_trigger.*`;
  } else if (path_type == "kafkatrigger") {
    return `${path}.kafka_trigger.*`;
  } else if (path_type == "natstrigger") {
    return `${path}.nats_trigger.*`;
  } else if (path_type == "postgrestrigger") {
    return `${path}.postgres_trigger.*`;
  } else if (path_type == "mqtttrigger") {
    return `${path}.mqtt_trigger.*`;
  } else if (path_type == "sqstrigger") {
    return `${path}.sqs_trigger.*`;
  } else if (path_type == "gcptrigger") {
    return `${path}.gcp_trigger.*`;
  } else if (path_type == "emailtrigger") {
    return `${path}.email_trigger.*`;
  } else {
    return `${path}.*`;
  }
}

async function wmill_sync_pull(
  path_type: PathType,
  workspace_id: string,
  path: string | undefined,
  parent_path: string | undefined,
  skip_secret: boolean,
  repo_url_resource_path: string,
  use_individual_branch: boolean,
  original_branch?: string
) {
  const includes = [];
  if (path !== undefined && path !== null && path !== "") {
    includes.push(regexFromPath(path_type, path));
  }
  if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
    includes.push(regexFromPath(path_type, parent_path));
  }

  console.log("Pulling workspace into git repo");

  const args = [
    "sync",
    "pull",
    "--token",
    process.env["WM_TOKEN"] ?? "",
    "--workspace",
    workspace_id,
    "--base-url",
    process.env["BASE_URL"] + "/",
    "--repository",
    repo_url_resource_path,
    "--yes",
    skip_secret ? "--skip-secrets" : "",
  ];

  if (path_type === "schedule" && !use_individual_branch) {
    args.push("--include-schedules");
  }

  if (path_type === "group" && !use_individual_branch) {
    args.push("--include-groups");
  }

  if (path_type === "user" && !use_individual_branch) {
    args.push("--include-users");
  }

  if (path_type.includes("trigger") && !use_individual_branch) {
    args.push("--include-triggers");
  }
  // Only include settings when specifically deploying settings
  if (path_type === "settings" && !use_individual_branch) {
    args.push("--include-settings");
  }

  // Only include key when specifically deploying keys
  if (path_type === "key" && !use_individual_branch) {
    args.push("--include-key");
  }

  args.push("--extra-includes", includes.join(","));

  // If using individual branches, apply promotion settings from original branch
  if (use_individual_branch && original_branch) {
    console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
    args.push("--promotion", original_branch);
  }

  await wmill_run(3, ...args);
}

async function wmill_run(secret_position: number, ...cmd: string[]) {
  cmd = cmd.filter((elt) => elt !== "");
  const cmd2 = cmd.slice();
  cmd2[secret_position] = "***";
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
  await wmill.parse(cmd);
  console.log("Command successfully executed");
}

// Function to set up GPG signing
async function set_gpg_signing_secret(gpg_key: GpgKey) {
  try {
    console.log("Setting GPG private key for git commits");

    const formattedGpgContent = gpg_key.private_key.replace(
      /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
      (_: string, header: string, body: string, footer: string) =>
        header +
        "\n" +
        "\n" +
        body.replace(/ ([^\s])/g, "\n$1").trim() +
        "\n" +
        footer
    );

    const gpg_path = `/tmp/gpg`;
    await sh_run(undefined, "mkdir", "-p", gpg_path);
    await sh_run(undefined, "chmod", "700", gpg_path);
    process.env.GNUPGHOME = gpg_path;
    // process.env.GIT_TRACE = 1;

    try {
      await sh_run(
        1,
        "bash",
        "-c",
        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
      );
    } catch (e) {
      // Original error would contain sensitive data
      throw new Error("Failed to import GPG key!");
    }

    const listKeysOutput = await sh_run(
      undefined,
      "gpg",
      "--list-secret-keys",
      "--with-colons",
      "--keyid-format=long"
    );

    const keyInfoMatch = listKeysOutput.match(
      /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
    );

    if (!keyInfoMatch) {
      throw new Error("Failed to extract GPG Key ID and Fingerprint");
    }

    const keyId = keyInfoMatch[1];
    gpgFingerprint = keyInfoMatch[2];

    if (gpg_key.passphrase) {
      // This is adummy command to unlock the key
      // with passphrase to load it into agent
      await sh_run(
        1,
        "bash",
        "-c",
        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
      );
    }

    // Configure Git to use the extracted key
    await sh_run(undefined, "git", "config", "user.signingkey", keyId);
    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
    console.log(`GPG signing configured with key ID: ${keyId} `);
  } catch (e) {
    console.error(`Failure while setting GPG key: ${e} `);
    await delete_pgp_keys();
  }
}

async function delete_pgp_keys() {
  console.log("deleting gpg keys");
  if (gpgFingerprint) {
    await sh_run(
      undefined,
      "gpg",
      "--batch",
      "--yes",
      "--pinentry-mode",
      "loopback",
      "--delete-secret-key",
      gpgFingerprint
    );
    await sh_run(
      undefined,
      "gpg",
      "--batch",
      "--yes",
      "--delete-key",
      "--pinentry-mode",
      "loopback",
      gpgFingerprint
    );
  }
}

async function get_gh_app_token() {
  const workspace = process.env["WM_WORKSPACE"];
  const jobToken = process.env["WM_TOKEN"];

  const baseUrl =
    process.env["BASE_INTERNAL_URL"] ??
    process.env["BASE_URL"] ??
    "http://localhost:8000";

  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;

  const response = await fetch(url, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      Authorization: `Bearer ${jobToken}`,
    },
    body: JSON.stringify({
      job_token: jobToken,
    }),
  });

  if (!response.ok) {
    throw new Error(`Error: ${response.statusText}`);
  }

  const data = await response.json();

  return data.token;
}

function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
  if (!gitHubUrl || !installationToken) {
    throw new Error("Both GitHub URL and Installation Token are required.");
  }

  try {
    const url = new URL(gitHubUrl);

    // GitHub repository URL should be in the format: https://github.com/owner/repo.git
    if (url.hostname !== "github.com") {
      throw new Error(
        "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
      );
    }

    // Convert URL to include the installation token
    return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
  } catch (e) {
    const error = e as Error;
    throw new Error(`Invalid URL: ${error.message}`);
  }
}


Submitted by pyranota275 Bun

Created 58 days ago

All edits

Permalink

import * as wmillclient from "windmill-client";
import wmill from "[email protected]";
import { basename } from "node:path";
const util = require("util");
const exec = util.promisify(require("child_process").exec);
import process from "process";

type GpgKey = {
  email: string;
  private_key: string;
  passphrase: string;
};

const FORKED_WORKSPACE_PREFIX = "wm-fork-";
const FORKED_BRANCH_PREFIX = "wm-fork";

type PathType =
  | "script"
  | "flow"
  | "app"
  | "folder"
  | "resource"
  | "variable"
  | "resourcetype"
  | "schedule"
  | "user"
  | "group"
  | "httptrigger"
  | "websockettrigger"
  | "kafkatrigger"
  | "natstrigger"
  | "postgrestrigger"
  | "mqtttrigger"
  | "sqstrigger"
  | "gcptrigger"
  | "emailtrigger";

type SyncObject = {
  path_type: PathType;
  path: string | undefined;
  parent_path: string | undefined;
  commit_msg: string;
};

let gpgFingerprint: string | undefined = undefined;

export async function main(
  items: SyncObject[],
  // Compat, do not use in code, rely on `items` instead
  path_type: PathType | undefined,
  path: string | undefined,
  parent_path: string | undefined,
  commit_msg: string | undefined,
  //
  workspace_id: string,
  repo_url_resource_path: string,
  skip_secret: boolean = true,
  use_individual_branch: boolean = false,
  group_by_folder: boolean = false,
  only_create_branch: boolean = false,
  parent_workspace_id?: string,
) {

  if (path_type !== undefined && commit_msg !== undefined) {
    items = [{
      path_type,
      path,
      parent_path,
      commit_msg,
    }];
  }
  await inner(items, workspace_id, repo_url_resource_path, skip_secret, use_individual_branch, group_by_folder, only_create_branch, parent_workspace_id);
}

async function inner(
  items: SyncObject[],
  workspace_id: string,
  repo_url_resource_path: string,
  skip_secret: boolean = true,
  use_individual_branch: boolean = false,
  group_by_folder: boolean = false,
  only_create_branch: boolean = false,
  parent_workspace_id?: string,
) {

  let safeDirectoryPath: string | undefined;
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
  const cwd = process.cwd();
  process.env["HOME"] = ".";
  if (!only_create_branch) {
    for (const item of items) {
      console.log(
        `Syncing ${item.path_type} ${item.path ?? ""} with parent ${item.parent_path ?? ""}`
      );
    }
  }

  if (repo_resource.is_github_app) {
    const token = await get_gh_app_token();
    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
    repo_resource.url = authRepoUrl;
  }

  const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
  safeDirectoryPath = cloneSafeDirectoryPath;


  // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
  // a fork of a fork workspace. In that case, the original branch is not stored in the resource
  // settings, but we need to infer it from the workspace id

  if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    if (use_individual_branch) {
      console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
      use_individual_branch = false;
    }
    if (group_by_folder) {
      console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
      group_by_folder = false;
    }
  }

  if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
    console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
    await git_checkout_branch(
      items,
      parent_workspace_id,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
  }

  await git_checkout_branch(
    items,
    workspace_id,
    use_individual_branch,
    group_by_folder,
    clonedBranchName
  );


  const subfolder = repo_resource.folder ?? "";
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
  console.log(
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
  );

  // If we want to just create the branch, we can skip pulling the changes.
  if (!only_create_branch) {
    await wmill_sync_pull(
      items,
      workspace_id,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
  }
  try {
    await git_push(items, repo_resource, only_create_branch);
  } catch (e) {
    throw e;
  } finally {
    await delete_pgp_keys();
    // Cleanup: remove safe.directory config
    if (safeDirectoryPath) {
      try {
        await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
      } catch (e) {
        console.log(`Warning: Could not unset safe.directory config: ${e}`);
      }
    }
  }
  console.log("Finished syncing");
  process.chdir(`${cwd}`);
}


function get_fork_branch_name(w_id: string, originalBranch: string): string {
  if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
  }
  return w_id;
}

async function git_clone(
  cwd: string,
  repo_resource: any,
  no_single_branch: boolean,
): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
  // TODO: handle private SSH keys as well
  let repo_url = repo_resource.url;
  const subfolder = repo_resource.folder ?? "";
  const branch = repo_resource.branch ?? "";
  const repo_name = basename(repo_url, ".git");
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
  if (azureMatch) {
    console.log(
      "Requires Azure DevOps service account access token, requesting..."
    );
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    const response = await fetch(
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
      {
        method: "POST",
        body: new URLSearchParams({
          client_id: azureResource.azureClientId,
          client_secret: azureResource.azureClientSecret,
          grant_type: "client_credentials",
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
        }),
      }
    );
    const { access_token } = await response.json();
    repo_url = repo_url.replace(azureMatch[0], access_token);
  }
  const args = ["clone", "--quiet", "--depth", "1"];
  if (no_single_branch) {
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
  }
  if (subfolder !== "") {
    args.push("--sparse");
  }
  if (branch !== "") {
    args.push("--branch");
    args.push(branch);
  }
  args.push(repo_url);
  args.push(repo_name);
  await sh_run(-1, "git", ...args);
  try {
    process.chdir(`${cwd}/${repo_name}`);
    const safeDirectoryPath = process.cwd();
    // Add safe.directory to handle dubious ownership in cloned repo
    try {
      await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
    } catch (e) {
      console.log(`Warning: Could not add safe.directory config: ${e}`);
    }

    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
      try {
        process.chdir(`${cwd}/${repo_name}/${subfolder}`);
      } catch (err) {
        console.log(
          `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
    return { repo_name, safeDirectoryPath, clonedBranchName };

  } catch (err) {
    console.log(
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    );
    throw err;
  }
}
async function git_checkout_branch(
  items: SyncObject[],
  workspace_id: string,
  use_individual_branch: boolean,
  group_by_folder: boolean,
  originalBranchName: string
) {
  let branchName;
  if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    branchName = get_fork_branch_name(workspace_id, originalBranchName);
  } else {

    if (!use_individual_branch
      // If individual branch is true, we can assume items is of length 1, as debouncing is disabled for jobs with this flag
      || items[0].path_type === "user" || items[0].path_type === "group") {
      return;
    }

    // as mentioned above, it is safe to assume that items.len is 1
    const [path, parent_path] = [items[0].path, items[0].parent_path];
    branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${items[0].path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
  }

  try {
    await sh_run(undefined, "git", "checkout", branchName);
  } catch (err) {
    console.log(
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    );
    try {
      await sh_run(undefined, "git", "checkout", "-b", branchName);
      await sh_run(
        undefined,
        "git",
        "config",
        "--add",
        "--bool",
        "push.autoSetupRemote",
        "true"
      );
    } catch (err) {
      console.log(
        `Error checking out branch '${branchName}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  console.log(`Successfully switched to branch ${branchName}`);
}

function composeCommitHeader(items: SyncObject[]): string {
  // Count occurrences of each path_type
  const typeCounts = new Map<PathType, number>();
  for (const item of items) {
    typeCounts.set(item.path_type, (typeCounts.get(item.path_type) ?? 0) + 1);
  }

  // Sort by count descending to get the top 2
  const sortedTypes = Array.from(typeCounts.entries()).sort((a, b) => b[1] - a[1]);

  const parts: string[] = [];
  let othersCount = 0;

  for (let i = 0; i < sortedTypes.length; i++) {
    const [pathType, count] = sortedTypes[i];
    if (i < 3) {
      // Pluralize the path type if count > 1
      const label = count > 1 ? `${pathType}s` : pathType;

      if (i == 2 && sortedTypes.length == 3) {
        parts.push(`and ${count} ${label}`);
      } else {
        parts.push(`${count} ${label}`);
      }
    } else {
      othersCount += count;
    }
  }

  let header = `[WM] Deployed ${parts.join(", ")}`;
  if (othersCount > 0) {
    header += ` and ${othersCount} other object${othersCount > 1 ? "s" : ""}`;
  }

  return header;
}

async function git_push(
  items: SyncObject[],
  repo_resource: any,
  only_create_branch: boolean,
) {
  let user_email = process.env["WM_EMAIL"] ?? "";
  let user_name = process.env["WM_USERNAME"] ?? "";

  if (repo_resource.gpg_key) {
    await set_gpg_signing_secret(repo_resource.gpg_key);
    // Configure git with GPG key email for signing
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      repo_resource.gpg_key.email
    );
    await sh_run(undefined, "git", "config", "user.name", user_name);
  } else {
    await sh_run(undefined, "git", "config", "user.email", user_email);
    await sh_run(undefined, "git", "config", "user.name", user_name);
  }
  if (only_create_branch) {
    await sh_run(undefined, "git", "push", "--porcelain");
  }

  let commit_description: string[] = [];
  for (const { path, parent_path, commit_msg } of items) {
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }

    commit_description.push(commit_msg);
  }

  try {
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
  } catch {
    // git diff returns exit-code = 1 when there's at least one staged changes
    const commitArgs = ["git", "commit"];

    // Always use --author to set consistent authorship
    commitArgs.push("--author", `"${user_name} <${user_email}>"`);

    const [header, description] = (commit_description.length == 1)
      ? [commit_description[0], ""]
      : [composeCommitHeader(items), commit_description.join("\n")];

    commitArgs.push(
      "-m",
      `"${header == undefined || header == "" ? "no commit msg" : header}"`,
      "-m",
      `"${description}"`
    );

    await sh_run(undefined, ...commitArgs);
    try {
      await sh_run(undefined, "git", "push", "--porcelain");
    } catch (e) {
      console.log(`Could not push, trying to rebase first: ${e}`);
      await sh_run(undefined, "git", "pull", "--rebase");
      await sh_run(undefined, "git", "push", "--porcelain");
    }
    return;
  }

  console.log("No changes detected, nothing to commit. Returning...");
}
async function sh_run(
  secret_position: number | undefined,
  cmd: string,
  ...args: string[]
) {
  const nargs = secret_position != undefined ? args.slice() : args;
  if (secret_position && secret_position < 0) {
    secret_position = nargs.length - 1 + secret_position;
  }
  let secret: string | undefined = undefined;
  if (secret_position != undefined) {
    nargs[secret_position] = "***";
    secret = args[secret_position];
  }

  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
  const command = exec(`${cmd} ${args.join(" ")}`);
  // new Deno.Command(cmd, {
  //   args: args,
  // });
  try {
    const { stdout, stderr } = await command;
    if (stdout.length > 0) {
      console.log(stdout);
    }
    if (stderr.length > 0) {
      console.log(stderr);
    }
    console.log("Command successfully executed");
    return stdout;
  } catch (error) {
    let errorString = error.toString();
    if (secret) {
      errorString = errorString.replace(secret, "***");
    }
    const err = `SH command '${cmd} ${nargs.join(
      " "
    )}' returned with error ${errorString}`;
    throw Error(err);
  }
}

function regexFromPath(path_type: PathType, path: string) {
  if (path_type == "flow") {
    return `${path}.flow/*`;
  }
  if (path_type == "app") {
    return `${path}.app/*`;
  } else if (path_type == "folder") {
    return `${path}/folder.meta.*`;
  } else if (path_type == "resourcetype") {
    return `${path}.resource-type.*`;
  } else if (path_type == "resource") {
    return `${path}.resource.*`;
  } else if (path_type == "variable") {
    return `${path}.variable.*`;
  } else if (path_type == "schedule") {
    return `${path}.schedule.*`;
  } else if (path_type == "user") {
    return `${path}.user.*`;
  } else if (path_type == "group") {
    return `${path}.group.*`;
  } else if (path_type == "httptrigger") {
    return `${path}.http_trigger.*`;
  } else if (path_type == "websockettrigger") {
    return `${path}.websocket_trigger.*`;
  } else if (path_type == "kafkatrigger") {
    return `${path}.kafka_trigger.*`;
  } else if (path_type == "natstrigger") {
    return `${path}.nats_trigger.*`;
  } else if (path_type == "postgrestrigger") {
    return `${path}.postgres_trigger.*`;
  } else if (path_type == "mqtttrigger") {
    return `${path}.mqtt_trigger.*`;
  } else if (path_type == "sqstrigger") {
    return `${path}.sqs_trigger.*`;
  } else if (path_type == "gcptrigger") {
    return `${path}.gcp_trigger.*`;
  } else if (path_type == "emailtrigger") {
    return `${path}.email_trigger.*`;
  } else {
    return `${path}.*`;
  }
}

async function wmill_sync_pull(
  items: SyncObject[],
  workspace_id: string,
  skip_secret: boolean,
  repo_url_resource_path: string,
  use_individual_branch: boolean,
  original_branch?: string
) {
  const includes = [];
  for (const item of items) {
    const { path_type, path, parent_path } = item;
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
  }

  console.log("Pulling workspace into git repo");

  const args = [
    "sync",
    "pull",
    "--token",
    process.env["WM_TOKEN"] ?? "",
    "--workspace",
    workspace_id,
    "--base-url",
    process.env["BASE_URL"] + "/",
    "--repository",
    repo_url_resource_path,
    "--yes",
    skip_secret ? "--skip-secrets" : "",
  ];

  if (items.some(item => item.path_type === "schedule") && !use_individual_branch) {
    args.push("--include-schedules");
  }

  if (items.some(item => item.path_type === "group") && !use_individual_branch) {
    args.push("--include-groups");
  }

  if (items.some(item => item.path_type === "user") && !use_individual_branch) {
    args.push("--include-users");
  }

  if (items.some(item => item.path_type.includes("trigger")) && !use_individual_branch) {
    args.push("--include-triggers");
  }
  // Only include settings when specifically deploying settings
  if (items.some(item => item.path_type === "settings") && !use_individual_branch) {
    args.push("--include-settings");
  }

  // Only include key when specifically deploying keys
  if (items.some(item => item.path_type === "key") && !use_individual_branch) {
    args.push("--include-key");
  }

  args.push("--extra-includes", includes.join(","));

  // If using individual branches, apply promotion settings from original branch
  if (use_individual_branch && original_branch) {
    console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
    args.push("--promotion", original_branch);
  }

  await wmill_run(3, ...args);
}

async function wmill_run(secret_position: number, ...cmd: string[]) {
  cmd = cmd.filter((elt) => elt !== "");
  const cmd2 = cmd.slice();
  cmd2[secret_position] = "***";
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
  await wmill.parse(cmd);
  console.log("Command successfully executed");
}

// Function to set up GPG signing
async function set_gpg_signing_secret(gpg_key: GpgKey) {
  try {
    console.log("Setting GPG private key for git commits");

    const formattedGpgContent = gpg_key.private_key.replace(
      /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
      (_: string, header: string, body: string, footer: string) =>
        header +
        "\n" +
        "\n" +
        body.replace(/ ([^\s])/g, "\n$1").trim() +
        "\n" +
        footer
    );

    const gpg_path = `/tmp/gpg`;
    await sh_run(undefined, "mkdir", "-p", gpg_path);
    await sh_run(undefined, "chmod", "700", gpg_path);
    process.env.GNUPGHOME = gpg_path;
    // process.env.GIT_TRACE = 1;

    try {
      await sh_run(
        1,
        "bash",
        "-c",
        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
      );
    } catch (e) {
      // Original error would contain sensitive data
      throw new Error("Failed to import GPG key!");
    }

    const listKeysOutput = await sh_run(
      undefined,
      "gpg",
      "--list-secret-keys",
      "--with-colons",
      "--keyid-format=long"
    );

    const keyInfoMatch = listKeysOutput.match(
      /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
    );

    if (!keyInfoMatch) {
      throw new Error("Failed to extract GPG Key ID and Fingerprint");
    }

    const keyId = keyInfoMatch[1];
    gpgFingerprint = keyInfoMatch[2];

    if (gpg_key.passphrase) {
      // This is adummy command to unlock the key
      // with passphrase to load it into agent
      await sh_run(
        1,
        "bash",
        "-c",
        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
      );
    }

    // Configure Git to use the extracted key
    await sh_run(undefined, "git", "config", "user.signingkey", keyId);
    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
    console.log(`GPG signing configured with key ID: ${keyId} `);
  } catch (e) {
    console.error(`Failure while setting GPG key: ${e} `);
    await delete_pgp_keys();
  }
}

async function delete_pgp_keys() {
  console.log("deleting gpg keys");
  if (gpgFingerprint) {
    await sh_run(
      undefined,
      "gpg",
      "--batch",
      "--yes",
      "--pinentry-mode",
      "loopback",
      "--delete-secret-key",
      gpgFingerprint
    );
    await sh_run(
      undefined,
      "gpg",
      "--batch",
      "--yes",
      "--delete-key",
      "--pinentry-mode",
      "loopback",
      gpgFingerprint
    );
  }
}

async function get_gh_app_token() {
  const workspace = process.env["WM_WORKSPACE"];
  const jobToken = process.env["WM_TOKEN"];

  const baseUrl =
    process.env["BASE_INTERNAL_URL"] ??
    process.env["BASE_URL"] ??
    "http://localhost:8000";

  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;

  const response = await fetch(url, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      Authorization: `Bearer ${jobToken}`,
    },
    body: JSON.stringify({
      job_token: jobToken,
    }),
  });

  if (!response.ok) {
    throw new Error(`Error: ${response.statusText}`);
  }

  const data = await response.json();

  return data.token;
}

function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
  if (!gitHubUrl || !installationToken) {
    throw new Error("Both GitHub URL and Installation Token are required.");
  }

  try {
    const url = new URL(gitHubUrl);

    // GitHub repository URL should be in the format: https://github.com/owner/repo.git
    if (url.hostname !== "github.com") {
      throw new Error(
        "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
      );
    }

    // Convert URL to include the installation token
    return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
  } catch (e) {
    const error = e as Error;
    throw new Error(`Invalid URL: ${error.message}`);
  }
}

Submitted by pyranota275 Bun

Created 59 days ago

All edits

Permalink

import * as wmillclient from "windmill-client";
import wmill from "[email protected]";
import { basename } from "node:path";
const util = require("util");
const exec = util.promisify(require("child_process").exec);
import process from "process";

type GpgKey = {
  email: string;
  private_key: string;
  passphrase: string;
};

const FORKED_WORKSPACE_PREFIX = "wm-fork-";
const FORKED_BRANCH_PREFIX = "wm-fork";

type PathType =
  | "script"
  | "flow"
  | "app"
  | "folder"
  | "resource"
  | "variable"
  | "resourcetype"
  | "schedule"
  | "user"
  | "group"
  | "httptrigger"
  | "websockettrigger"
  | "kafkatrigger"
  | "natstrigger"
  | "postgrestrigger"
  | "mqtttrigger"
  | "sqstrigger"
  | "gcptrigger"
  | "emailtrigger";

type SyncObject = {
  path_type: PathType;
  path: string | undefined;
  parent_path: string | undefined;
  commit_msg: string;
};

let gpgFingerprint: string | undefined = undefined;

export async function main(
  items: SyncObject[],
  // Compat, do not use in code, rely on `items` instead
  path_type: PathType | undefined,
  path: string | undefined,
  parent_path: string | undefined,
  commit_msg: string | undefined,
  //
  workspace_id: string,
  repo_url_resource_path: string,
  skip_secret: boolean = true,
  use_individual_branch: boolean = false,
  group_by_folder: boolean = false,
  only_create_branch: boolean = false,
  parent_workspace_id?: string,
) {

  if (path_type !== undefined && commit_msg !== undefined) {
    items = [{
      path_type,
      path,
      parent_path,
      commit_msg,
    }];
  }
  await inner(items, workspace_id, repo_url_resource_path, skip_secret, use_individual_branch, group_by_folder, only_create_branch, parent_workspace_id);
}

async function inner(
  items: SyncObject[],
  workspace_id: string,
  repo_url_resource_path: string,
  skip_secret: boolean = true,
  use_individual_branch: boolean = false,
  group_by_folder: boolean = false,
  only_create_branch: boolean = false,
  parent_workspace_id?: string,
) {

  let safeDirectoryPath: string | undefined;
  const repo_resource = await wmillclient.getResource(repo_url_resource_path);
  const cwd = process.cwd();
  process.env["HOME"] = ".";
  if (!only_create_branch) {
    for (const item of items) {
      console.log(
        `Syncing ${item.path_type} ${item.path ?? ""} with parent ${item.parent_path ?? ""}`
      );
    }
  }

  if (repo_resource.is_github_app) {
    const token = await get_gh_app_token();
    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
    repo_resource.url = authRepoUrl;
  }

  const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
  safeDirectoryPath = cloneSafeDirectoryPath;


  // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
  // a fork of a fork workspace. In that case, the original branch is not stored in the resource
  // settings, but we need to infer it from the workspace id

  if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    if (use_individual_branch) {
      console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
      use_individual_branch = false;
    }
    if (group_by_folder) {
      console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
      group_by_folder = false;
    }
  }

  if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
    console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
    await git_checkout_branch(
      items,
      parent_workspace_id,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
  }

  await git_checkout_branch(
    items,
    workspace_id,
    use_individual_branch,
    group_by_folder,
    clonedBranchName
  );


  const subfolder = repo_resource.folder ?? "";
  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
  console.log(
    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
  );

  // If we want to just create the branch, we can skip pulling the changes.
  if (!only_create_branch) {
    await wmill_sync_pull(
      items,
      workspace_id,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
  }
  try {
    await git_push(items, repo_resource, only_create_branch);
  } catch (e) {
    throw e;
  } finally {
    await delete_pgp_keys();
    // Cleanup: remove safe.directory config
    if (safeDirectoryPath) {
      try {
        await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
      } catch (e) {
        console.log(`Warning: Could not unset safe.directory config: ${e}`);
      }
    }
  }
  console.log("Finished syncing");
  process.chdir(`${cwd}`);
}


function get_fork_branch_name(w_id: string, originalBranch: string): string {
  if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
  }
  return w_id;
}

async function git_clone(
  cwd: string,
  repo_resource: any,
  no_single_branch: boolean,
): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
  // TODO: handle private SSH keys as well
  let repo_url = repo_resource.url;
  const subfolder = repo_resource.folder ?? "";
  const branch = repo_resource.branch ?? "";
  const repo_name = basename(repo_url, ".git");
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
  if (azureMatch) {
    console.log(
      "Requires Azure DevOps service account access token, requesting..."
    );
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    const response = await fetch(
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
      {
        method: "POST",
        body: new URLSearchParams({
          client_id: azureResource.azureClientId,
          client_secret: azureResource.azureClientSecret,
          grant_type: "client_credentials",
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
        }),
      }
    );
    const { access_token } = await response.json();
    repo_url = repo_url.replace(azureMatch[0], access_token);
  }
  const args = ["clone", "--quiet", "--depth", "1"];
  if (no_single_branch) {
    args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
  }
  if (subfolder !== "") {
    args.push("--sparse");
  }
  if (branch !== "") {
    args.push("--branch");
    args.push(branch);
  }
  args.push(repo_url);
  args.push(repo_name);
  await sh_run(-1, "git", ...args);
  try {
    process.chdir(`${cwd}/${repo_name}`);
    const safeDirectoryPath = process.cwd();
    // Add safe.directory to handle dubious ownership in cloned repo
    try {
      await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
    } catch (e) {
      console.log(`Warning: Could not add safe.directory config: ${e}`);
    }

    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
      try {
        process.chdir(`${cwd}/${repo_name}/${subfolder}`);
      } catch (err) {
        console.log(
          `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
    return { repo_name, safeDirectoryPath, clonedBranchName };

  } catch (err) {
    console.log(
      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
    );
    throw err;
  }
}
async function git_checkout_branch(
  items: SyncObject[],
  workspace_id: string,
  use_individual_branch: boolean,
  group_by_folder: boolean,
  originalBranchName: string
) {
  let branchName;
  if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
    branchName = get_fork_branch_name(workspace_id, originalBranchName);
  } else {

    if (!use_individual_branch
      // If individual branch is true, we can assume items is of length 1, as debouncing is disabled for jobs with this flag
      || items[0].path_type === "user" || items[0].path_type === "group") {
      return;
    }

    // as mentioned above, it is safe to assume that items.len is 1
    const [path, parent_path] = [items[0].path, items[0].parent_path];
    branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${items[0].path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
  }

  try {
    await sh_run(undefined, "git", "checkout", branchName);
  } catch (err) {
    console.log(
      `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
    );
    try {
      await sh_run(undefined, "git", "checkout", "-b", branchName);
      await sh_run(
        undefined,
        "git",
        "config",
        "--add",
        "--bool",
        "push.autoSetupRemote",
        "true"
      );
    } catch (err) {
      console.log(
        `Error checking out branch '${branchName}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  console.log(`Successfully switched to branch ${branchName}`);
}

async function git_push(
  items: SyncObject[],
  repo_resource: any,
  only_create_branch: boolean,
) {
  let user_email = process.env["WM_EMAIL"] ?? "";
  let user_name = process.env["WM_USERNAME"] ?? "";

  if (repo_resource.gpg_key) {
    await set_gpg_signing_secret(repo_resource.gpg_key);
    // Configure git with GPG key email for signing
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      repo_resource.gpg_key.email
    );
    await sh_run(undefined, "git", "config", "user.name", user_name);
  } else {
    await sh_run(undefined, "git", "config", "user.email", user_email);
    await sh_run(undefined, "git", "config", "user.name", user_name);
  }
  if (only_create_branch) {
    await sh_run(undefined, "git", "push", "--porcelain");
  }

  let commit_description: string[] = [];
  for (const { path, parent_path, commit_msg } of items) {
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }

    commit_description.push(commit_msg);
  }

  try {
    await sh_run(undefined, "git", "diff", "--cached", "--quiet");
  } catch {
    // git diff returns exit-code = 1 when there's at least one staged changes
    const commitArgs = ["git", "commit"];

    // Always use --author to set consistent authorship
    commitArgs.push("--author", `"${user_name} <${user_email}>"`);

    const [header, description] = (commit_description.length == 1) ? [commit_description[0], ""] : [`[WM]: Deployed ${commit_description.length} objects`, commit_description.join("\n")];

    commitArgs.push(
      "-m",
      `"${header == undefined || header == "" ? "no commit msg" : header}"`,
      "-m",
      `"${description}"`
    );

    await sh_run(undefined, ...commitArgs);
    try {
      await sh_run(undefined, "git", "push", "--porcelain");
    } catch (e) {
      console.log(`Could not push, trying to rebase first: ${e}`);
      await sh_run(undefined, "git", "pull", "--rebase");
      await sh_run(undefined, "git", "push", "--porcelain");
    }
    return;
  }

  console.log("No changes detected, nothing to commit. Returning...");
}
async function sh_run(
  secret_position: number | undefined,
  cmd: string,
  ...args: string[]
) {
  const nargs = secret_position != undefined ? args.slice() : args;
  if (secret_position && secret_position < 0) {
    secret_position = nargs.length - 1 + secret_position;
  }
  let secret: string | undefined = undefined;
  if (secret_position != undefined) {
    nargs[secret_position] = "***";
    secret = args[secret_position];
  }

  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
  const command = exec(`${cmd} ${args.join(" ")}`);
  // new Deno.Command(cmd, {
  //   args: args,
  // });
  try {
    const { stdout, stderr } = await command;
    if (stdout.length > 0) {
      console.log(stdout);
    }
    if (stderr.length > 0) {
      console.log(stderr);
    }
    console.log("Command successfully executed");
    return stdout;
  } catch (error) {
    let errorString = error.toString();
    if (secret) {
      errorString = errorString.replace(secret, "***");
    }
    const err = `SH command '${cmd} ${nargs.join(
      " "
    )}' returned with error ${errorString}`;
    throw Error(err);
  }
}

function regexFromPath(path_type: PathType, path: string) {
  if (path_type == "flow") {
    return `${path}.flow/*`;
  }
  if (path_type == "app") {
    return `${path}.app/*`;
  } else if (path_type == "folder") {
    return `${path}/folder.meta.*`;
  } else if (path_type == "resourcetype") {
    return `${path}.resource-type.*`;
  } else if (path_type == "resource") {
    return `${path}.resource.*`;
  } else if (path_type == "variable") {
    return `${path}.variable.*`;
  } else if (path_type == "schedule") {
    return `${path}.schedule.*`;
  } else if (path_type == "user") {
    return `${path}.user.*`;
  } else if (path_type == "group") {
    return `${path}.group.*`;
  } else if (path_type == "httptrigger") {
    return `${path}.http_trigger.*`;
  } else if (path_type == "websockettrigger") {
    return `${path}.websocket_trigger.*`;
  } else if (path_type == "kafkatrigger") {
    return `${path}.kafka_trigger.*`;
  } else if (path_type == "natstrigger") {
    return `${path}.nats_trigger.*`;
  } else if (path_type == "postgrestrigger") {
    return `${path}.postgres_trigger.*`;
  } else if (path_type == "mqtttrigger") {
    return `${path}.mqtt_trigger.*`;
  } else if (path_type == "sqstrigger") {
    return `${path}.sqs_trigger.*`;
  } else if (path_type == "gcptrigger") {
    return `${path}.gcp_trigger.*`;
  } else if (path_type == "emailtrigger") {
    return `${path}.email_trigger.*`;
  } else {
    return `${path}.*`;
  }
}

async function wmill_sync_pull(
  items: SyncObject[],
  workspace_id: string,
  skip_secret: boolean,
  repo_url_resource_path: string,
  use_individual_branch: boolean,
  original_branch?: string
) {
  const includes = [];
  for (const item of items) {
    const { path_type, path, parent_path } = item;
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
  }

  console.log("Pulling workspace into git repo");

  const args = [
    "sync",
    "pull",
    "--token",
    process.env["WM_TOKEN"] ?? "",
    "--workspace",
    workspace_id,
    "--base-url",
    process.env["BASE_URL"] + "/",
    "--repository",
    repo_url_resource_path,
    "--yes",
    skip_secret ? "--skip-secrets" : "",
  ];

  if (items.some(item => item.path_type === "schedule") && !use_individual_branch) {
    args.push("--include-schedules");
  }

  if (items.some(item => item.path_type === "group") && !use_individual_branch) {
    args.push("--include-groups");
  }

  if (items.some(item => item.path_type === "user") && !use_individual_branch) {
    args.push("--include-users");
  }

  if (items.some(item => item.path_type.includes("trigger")) && !use_individual_branch) {
    args.push("--include-triggers");
  }
  // Only include settings when specifically deploying settings
  if (items.some(item => item.path_type === "settings") && !use_individual_branch) {
    args.push("--include-settings");
  }

  // Only include key when specifically deploying keys
  if (items.some(item => item.path_type === "key") && !use_individual_branch) {
    args.push("--include-key");
  }

  args.push("--extra-includes", includes.join(","));

  // If using individual branches, apply promotion settings from original branch
  if (use_individual_branch && original_branch) {
    console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
    args.push("--promotion", original_branch);
  }

  await wmill_run(3, ...args);
}

async function wmill_run(secret_position: number, ...cmd: string[]) {
  cmd = cmd.filter((elt) => elt !== "");
  const cmd2 = cmd.slice();
  cmd2[secret_position] = "***";
  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
  await wmill.parse(cmd);
  console.log("Command successfully executed");
}

// Function to set up GPG signing
async function set_gpg_signing_secret(gpg_key: GpgKey) {
  try {
    console.log("Setting GPG private key for git commits");

    const formattedGpgContent = gpg_key.private_key.replace(
      /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
      (_: string, header: string, body: string, footer: string) =>
        header +
        "\n" +
        "\n" +
        body.replace(/ ([^\s])/g, "\n$1").trim() +
        "\n" +
        footer
    );

    const gpg_path = `/tmp/gpg`;
    await sh_run(undefined, "mkdir", "-p", gpg_path);
    await sh_run(undefined, "chmod", "700", gpg_path);
    process.env.GNUPGHOME = gpg_path;
    // process.env.GIT_TRACE = 1;

    try {
      await sh_run(
        1,
        "bash",
        "-c",
        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
      );
    } catch (e) {
      // Original error would contain sensitive data
      throw new Error("Failed to import GPG key!");
    }

    const listKeysOutput = await sh_run(
      undefined,
      "gpg",
      "--list-secret-keys",
      "--with-colons",
      "--keyid-format=long"
    );

    const keyInfoMatch = listKeysOutput.match(
      /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
    );

    if (!keyInfoMatch) {
      throw new Error("Failed to extract GPG Key ID and Fingerprint");
    }

    const keyId = keyInfoMatch[1];
    gpgFingerprint = keyInfoMatch[2];

    if (gpg_key.passphrase) {
      // This is adummy command to unlock the key
      // with passphrase to load it into agent
      await sh_run(
        1,
        "bash",
        "-c",
        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
      );
    }

    // Configure Git to use the extracted key
    await sh_run(undefined, "git", "config", "user.signingkey", keyId);
    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
    console.log(`GPG signing configured with key ID: ${keyId} `);
  } catch (e) {
    console.error(`Failure while setting GPG key: ${e} `);
    await delete_pgp_keys();
  }
}

async function delete_pgp_keys() {
  console.log("deleting gpg keys");
  if (gpgFingerprint) {
    await sh_run(
      undefined,
      "gpg",
      "--batch",
      "--yes",
      "--pinentry-mode",
      "loopback",
      "--delete-secret-key",
      gpgFingerprint
    );
    await sh_run(
      undefined,
      "gpg",
      "--batch",
      "--yes",
      "--delete-key",
      "--pinentry-mode",
      "loopback",
      gpgFingerprint
    );
  }
}

async function get_gh_app_token() {
  const workspace = process.env["WM_WORKSPACE"];
  const jobToken = process.env["WM_TOKEN"];

  const baseUrl =
    process.env["BASE_INTERNAL_URL"] ??
    process.env["BASE_URL"] ??
    "http://localhost:8000";

  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;

  const response = await fetch(url, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      Authorization: `Bearer ${jobToken}`,
    },
    body: JSON.stringify({
      job_token: jobToken,
    }),
  });

  if (!response.ok) {
    throw new Error(`Error: ${response.statusText}`);
  }

  const data = await response.json();

  return data.token;
}

function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
  if (!gitHubUrl || !installationToken) {
    throw new Error("Both GitHub URL and Installation Token are required.");
  }

  try {
    const url = new URL(gitHubUrl);

    // GitHub repository URL should be in the format: https://github.com/owner/repo.git
    if (url.hostname !== "github.com") {
      throw new Error(
        "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
      );
    }

    // Convert URL to include the installation token
    return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
  } catch (e) {
    const error = e as Error;
    throw new Error(`Invalid URL: ${error.message}`);
  }
}


`1`	`import * as wmillclient from "npm:[email protected]";`
`2`	`import wmill from "https://deno.land/x/[email protected]/main.ts";`
`3`	`import { basename } from "https://deno.land/[email protected]/path/mod.ts";`
`4`
`5`	`export async function main(`
`6`	`workspace_id: string,`
`7`	`repo_url_resource_path: string,`
`8`	`path_type:`
`9`	`\| "script"`
`10`	`\| "flow"`
`11`	`\| "app"`
`12`	`\| "folder"`
`13`	`\| "resource"`
`14`	`\| "variable"`
`15`	`\| "resourcetype"`
`16`	`\| "schedule"`
`17`	`\| "user"`
`18`	`\| "group",`
`19`	`skip_secret = true,`
`20`	`path: string \| undefined,`
`21`	`parent_path: string \| undefined,`
`22`	`commit_msg: string,`
`23`	`use_individual_branch = false,`
`24`	`group_by_folder = false`
`25`	`) {`
`26`	`const repo_resource = await wmillclient.getResource(repo_url_resource_path);`
`27`
`28`	`const cwd = Deno.cwd();`
`29`	`Deno.env.set("HOME", ".");`
`30`	`console.log(`
`31`	`Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
`32`	`);`
`33`
`34`	`const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);`
`35`	`await move_to_git_branch(`
`36`	`workspace_id,`
`37`	`path_type,`
`38`	`path,`
`39`	`parent_path,`
`40`	`use_individual_branch,`
`41`	`group_by_folder`
`42`	`);`
`43`
`44`	`const subfolder = repo_resource.folder ?? "";`
`45`	`const branch_or_default = repo_resource.branch ?? "<DEFAULT>";`
`46`	`console.log(`
`47`	`Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
`48`	`);`
`49`
`50`	`await wmill_sync_pull(`
`51`	`path_type,`
`52`	`workspace_id,`
`53`	`path,`
`54`	`parent_path,`
`55`	`skip_secret`
`56`	`);`
`57`
`58`	`await git_push(path, parent_path, commit_msg);`
`59`
`60`	`console.log("Finished syncing");`
`61`	Deno.chdir(`${cwd}`);
`62`	`}`
`63`
`64`	`async function git_clone(`
`65`	`cwd: string,`
`66`	`repo_resource: any,`
`67`	`use_individual_branch: boolean`
`68`	`): Promise<string> {`
`69`	`// TODO: handle private SSH keys as well`
`70`	`let repo_url = repo_resource.url;`
`71`	`const subfolder = repo_resource.folder ?? "";`
`72`	`const branch = repo_resource.branch ?? "";`
`73`	`const repo_name = basename(repo_url, ".git");`
`74`
`75`	`const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);`
`76`
`77`	`if (azureMatch) {`
`78`	`console.log(`
`79`	`"Requires Azure DevOps service account access token, requesting..."`
`80`	`);`
`81`	`const azureResource = await wmillclient.getResource(azureMatch.groups.url);`
`82`
`83`	`const response = await fetch(`
`84`	`https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
`85`	`{`
`86`	`method: "POST",`
`87`	`body: new URLSearchParams({`
`88`	`client_id: azureResource.azureClientId,`
`89`	`client_secret: azureResource.azureClientSecret,`
`90`	`grant_type: "client_credentials",`
`91`	`resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",`
`92`	`}),`
`93`	`}`
`94`	`);`
`95`
`96`	`const { access_token } = await response.json();`
`97`
`98`	`repo_url = repo_url.replace(azureMatch[0], access_token);`
`99`	`}`
`100`
`101`	`const args = ["clone", "--quiet", "--depth", "1"];`
`102`	`if (use_individual_branch) {`
`103`	`args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo`
`104`	`}`
`105`	`if (subfolder !== "") {`
`106`	`args.push("--sparse");`
`107`	`}`
`108`	`if (branch !== "") {`
`109`	`args.push("--branch");`
`110`	`args.push(branch);`
`111`	`}`
`112`	`args.push(repo_url);`
`113`	`args.push(repo_name);`
`114`	`await sh_run(-1, "git", ...args);`
`115`
`116`	`try {`
`117`	Deno.chdir(`${cwd}/${repo_name}`);
`118`	`} catch (err) {`
`119`	`console.log(`
`120`	`Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
`121`	`);`
`122`	`throw err;`
`123`	`}`
`124`	Deno.chdir(`${cwd}/${repo_name}`);
`125`	`if (subfolder !== "") {`
`126`	`await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);`
`127`	`}`
`128`	`try {`
`129`	Deno.chdir(`${cwd}/${repo_name}/${subfolder}`);
`130`	`} catch (err) {`
`131`	`console.log(`
`132`	`Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
`133`	`);`
`134`	`throw err;`
`135`	`}`
`136`
`137`	`return repo_name;`
`138`	`}`
`139`
`140`	`async function move_to_git_branch(`
`141`	`workspace_id: string,`
`142`	`path_type:`
`143`	`\| "script"`
`144`	`\| "flow"`
`145`	`\| "app"`
`146`	`\| "folder"`
`147`	`\| "resource"`
`148`	`\| "variable"`
`149`	`\| "resourcetype"`
`150`	`\| "schedule"`
`151`	`\| "user"`
`152`	`\| "group",`
`153`	`path: string \| undefined,`
`154`	`parent_path: string \| undefined,`
`155`	`use_individual_branch: boolean,`
`156`	`group_by_folder: boolean`
`157`	`) {`
`158`	`if (!use_individual_branch \|\| path_type === "user" \|\| path_type === "group") {`
`159`	`return;`
`160`	`}`
`161`
`162`	`const branchName = group_by_folder`
`163`	? `wm_deploy/${workspace_id}/${(path ?? parent_path)
`164`	`?.split("/")`
`165`	`.slice(0, 2)`
`166`	.join("__")}`
`167`	: `wm_deploy/${workspace_id}/${path_type}/${(
`168`	`path ?? parent_path`
`169`	)?.replaceAll("/", "__")}`;
`170`
`171`	`try {`
`172`	`await sh_run(undefined, "git", "checkout", branchName);`
`173`	`} catch (err) {`
`174`	`console.log(`
`175`	`Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
`176`	`);`
`177`	`try {`
`178`	`await sh_run(undefined, "git", "checkout", "-b", branchName);`
`179`	`await sh_run(`
`180`	`undefined,`
`181`	`"git",`
`182`	`"config",`
`183`	`"--add",`
`184`	`"--bool",`
`185`	`"push.autoSetupRemote",`
`186`	`"true"`
`187`	`);`
`188`	`} catch (err) {`
`189`	`console.log(`
`190`	`Error checking out branch '${branchName}'. Error was:\n${err}`
`191`	`);`
`192`	`throw err;`
`193`	`}`
`194`	`}`
`195`	console.log(`Successfully switched to branch ${branchName}`);
`196`	`}`
`197`
`198`	`async function git_push(`
`199`	`path: string \| undefined,`
`200`	`parent_path: string \| undefined,`
`201`	`commit_msg: string`
`202`	`) {`
`203`	`await sh_run(`
`204`	`undefined,`
`205`	`"git",`
`206`	`"config",`
`207`	`"user.email",`
`208`	`Deno.env.get("WM_EMAIL") ?? ""`
`209`	`);`
`210`	`await sh_run(`
`211`	`undefined,`
`212`	`"git",`
`213`	`"config",`
`214`	`"user.name",`
`215`	`Deno.env.get("WM_USERNAME") ?? ""`
`216`	`);`
`217`	`if (path !== undefined && path !== null && path !== "") {`
`218`	`try {`
`219`	await sh_run(undefined, "git", "add", `${path}**`);
`220`	`} catch (e) {`
`221`	console.log(`Unable to stage files matching ${path}**, ${e}`);
`222`	`}`
`223`	`}`
`224`	`if (parent_path !== undefined && parent_path !== null && parent_path !== "") {`
`225`	`try {`
`226`	await sh_run(undefined, "git", "add", `${parent_path}**`);
`227`	`} catch (e) {`
`228`	console.log(`Unable to stage files matching ${parent_path}, ${e}`);
`229`	`}`
`230`	`}`
`231`	`try {`
`232`	`await sh_run(undefined, "git", "diff", "--cached", "--quiet");`
`233`	`} catch {`
`234`	`// git diff returns exit-code = 1 when there's at least one staged changes`
`235`	`await sh_run(undefined, "git", "commit", "-m", commit_msg);`
`236`	`try {`
`237`	`await sh_run(undefined, "git", "push", "--porcelain");`
`238`	`} catch {`
`239`	`console.log("Could not push, trying to rebase first");`
`240`	`await sh_run(undefined, "git", "pull", "--rebase");`
`241`	`await sh_run(undefined, "git", "push", "--porcelain");`
`242`	`}`
`243`	`return;`
`244`	`}`
`245`	`console.log("No changes detected, nothing to commit. Returning...");`
`246`	`}`
`247`
`248`	`async function sh_run(`
`249`	`secret_position: number \| undefined,`
`250`	`cmd: string,`
`251`	`...args: string[]`
`252`	`) {`
`253`	`const nargs = secret_position != undefined ? args.slice() : args;`
`254`	`if (secret_position < 0) {`
`255`	`secret_position = nargs.length - 1 + secret_position;`
`256`	`}`
`257`	`if (secret_position != undefined) {`
`258`	`nargs[secret_position] = "***";`
`259`	`}`
`260`	console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
`261`	`const command = new Deno.Command(cmd, {`
`262`	`args: args,`
`263`	`});`
`264`
`265`	`const { code, stdout, stderr } = await command.output();`
`266`	`if (stdout.length > 0) {`
`267`	`console.log(new TextDecoder().decode(stdout));`
`268`	`}`
`269`	`if (stderr.length > 0) {`
`270`	`console.log(new TextDecoder().decode(stderr));`
`271`	`}`
`272`	`if (code !== 0) {`
`273`	const err = `SH command '${cmd} ${nargs.join(
`274`	`" "`
`275`	)}' returned with a non-zero status ${code}.`;
`276`	`throw Error(err);`
`277`	`}`
`278`	`console.log("Command successfully executed");`
`279`	`}`
`280`
`281`	`function regexFromPath(`
`282`	`path_type:`
`283`	`\| "script"`
`284`	`\| "flow"`
`285`	`\| "app"`
`286`	`\| "folder"`
`287`	`\| "resource"`
`288`	`\| "variable"`
`289`	`\| "resourcetype"`
`290`	`\| "schedule"`
`291`	`\| "user"`
`292`	`\| "group",`
`293`	`path: string`
`294`	`) {`
`295`	`if (path_type == "flow") {`
`296`	return `${path}.flow/*`;
`297`	`} if (path_type == "app") {`
`298`	return `${path}.app/*`;
`299`	`} else if (path_type == "folder") {`
`300`	return `${path}/folder.meta.*`;
`301`	`} else if (path_type == "resourcetype") {`
`302`	return `${path}.resource-type.*`;
`303`	`} else if (path_type == "resource") {`
`304`	return `${path}.resource.*`;
`305`	`} else if (path_type == "variable") {`
`306`	return `${path}.variable.*`;
`307`	`} else if (path_type == "schedule") {`
`308`	return `${path}.schedule.*`;
`309`	`} else if (path_type == "user") {`
`310`	return `${path}.user.*`;
`311`	`} else if (path_type == "group") {`
`312`	return `${path}.group.*`;
`313`	`} else {`
`314`	return `${path}.*`;
`315`	`}`
`316`	`}`
`317`	`async function wmill_sync_pull(`
`318`	`path_type:`
`319`	`\| "script"`
`320`	`\| "flow"`
`321`	`\| "app"`
`322`	`\| "folder"`
`323`	`\| "resource"`
`324`	`\| "variable"`
`325`	`\| "resourcetype"`
`326`	`\| "schedule"`
`327`	`\| "user"`
`328`	`\| "group",`
`329`	`workspace_id: string,`
`330`	`path: string \| undefined,`
`331`	`parent_path: string \| undefined,`
`332`	`skip_secret: boolean`
`333`	`) {`
`334`	`const includes = [];`
`335`	`if (path !== undefined && path !== null && path !== "") {`
`336`	`includes.push(regexFromPath(path_type, path));`
`337`	`}`
`338`	`if (parent_path !== undefined && parent_path !== null && parent_path !== "") {`
`339`	`includes.push(regexFromPath(path_type, parent_path));`
`340`	`}`
`341`
`342`	`await wmill_run(`
`343`	`6,`
`344`	`"workspace",`
`345`	`"add",`
`346`	`workspace_id,`
`347`	`workspace_id,`
`348`	`Deno.env.get("BASE_INTERNAL_URL") + "/",`
`349`	`"--token",`
`350`	`Deno.env.get("WM_TOKEN") ?? ""`
`351`	`);`
`352`	`console.log("Pulling workspace into git repo");`
`353`	`await wmill_run(`
`354`	`3,`
`355`	`"sync",`
`356`	`"pull",`
`357`	`"--token",`
`358`	`Deno.env.get("WM_TOKEN") ?? "",`
`359`	`"--workspace",`
`360`	`workspace_id,`
`361`	`"--yes",`
`362`	`"--raw",`
`363`	`skip_secret ? "--skip-secrets" : "",`
`364`	`"--include-schedules",`
`365`	`"--include-users",`
`366`	`"--include-groups",`
`367`	`"--includes",`
`368`	`includes.join(",")`
`369`	`);`
`370`	`}`
`371`
`372`	`async function wmill_run(secret_position: number, ...cmd: string[]) {`
`373`	`cmd = cmd.filter((elt) => elt !== "");`
`374`	`const cmd2 = cmd.slice();`
`375`	`cmd2[secret_position] = "***";`
`376`	console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
`377`	`await wmill.parse(cmd);`
`378`	`console.log("Command successfully executed");`
`379`	`}`
`380`