Edits history of script submission #9015 for ' Sync script to Git repo (windmill)'

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "[email protected]";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger"
    | "emailtrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false,
    only_create_branch: boolean = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    if (!only_create_branch) {
      console.log(
        `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
      );
    }
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
  
  
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
  
    // If we want to just create the branch, we can skip pulling the changes.
    if (!only_create_branch) {
      await wmill_sync_pull(
        path_type,
        workspace_id,
        path,
        parent_path,
        skip_secret,
        repo_url_resource_path,
        use_individual_branch,
        repo_resource.branch
      );
    }
    try {
      await git_push(path, parent_path, commit_msg, repo_resource, only_create_branch);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    no_single_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (no_single_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any,
    only_create_branch: boolean,
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
    if (only_create_branch) {
      await sh_run(undefined, "git", "push", "--porcelain");
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else if (path_type == "emailtrigger") {
      return `${path}.email_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
  
    console.log("Pulling workspace into git repo");
  
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--base-url",
      process.env["BASE_URL"] + "/",
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
    ];
  
    if (path_type === "schedule" && !use_individual_branch) {
      args.push("--include-schedules");
    }
  
    if (path_type === "group" && !use_individual_branch) {
      args.push("--include-groups");
    }
  
    if (path_type === "user" && !use_individual_branch) {
      args.push("--include-users");
    }
  
    if (path_type.includes("trigger") && !use_individual_branch) {
      args.push("--include-triggers");
    }
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by ruben fiszel 2376 162 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "[email protected]";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger"
    | "emailtrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false,
    only_create_branch: boolean = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    if (!only_create_branch) {
      console.log(
        `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
      );
    }
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
  
  
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
  
    // If we want to just create the branch, we can skip pulling the changes.
    if (!only_create_branch) {
      await wmill_sync_pull(
        path_type,
        workspace_id,
        path,
        parent_path,
        skip_secret,
        repo_url_resource_path,
        use_individual_branch,
        repo_resource.branch
      );
    }
    try {
      await git_push(path, parent_path, commit_msg, repo_resource, only_create_branch);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    no_single_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (no_single_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any,
    only_create_branch: boolean,
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
    if (only_create_branch) {
      await sh_run(undefined, "git", "push", "--porcelain");
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else if (path_type == "emailtrigger") {
      return `${path}.email_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
  
    console.log("Pulling workspace into git repo");
  
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--base-url",
      process.env["BASE_URL"] + "/",
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
    ];
  
    if (path_type === "schedule" && !use_individual_branch) {
      args.push("--include-schedules");
    }
  
    if (path_type === "group" && !use_individual_branch) {
      args.push("--include-groups");
    }
  
    if (path_type === "user" && !use_individual_branch) {
      args.push("--include-users");
    }
  
    if (path_type.includes("trigger") && !use_individual_branch) {
      args.push("--include-triggers");
    }
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by ruben fiszel 2376 172 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "[email protected]";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger"
    | "emailtrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false,
    only_create_branch: boolean = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    if (!only_create_branch) {
      console.log(
        `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
      );
    }
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
  
  
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
  
    // If we want to just create the branch, we can skip pulling the changes.
    if (!only_create_branch) {
      await wmill_sync_pull(
        path_type,
        workspace_id,
        path,
        parent_path,
        skip_secret,
        repo_url_resource_path,
        use_individual_branch,
        repo_resource.branch
      );
    }
    try {
      await git_push(path, parent_path, commit_msg, repo_resource, only_create_branch);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    no_single_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (no_single_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any,
    only_create_branch: boolean,
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
    if (only_create_branch) {
      await sh_run(undefined, "git", "push", "--porcelain");
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else if (path_type == "emailtrigger") {
      return `${path}.email_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
  
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--base-url",
      process.env["BASE_URL"] + "/",
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by ruben fiszel 2376 172 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "[email protected]";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger"
    | "emailtrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false,
    only_create_branch: boolean = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    if (!only_create_branch) {
      console.log(
        `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
      );
    }
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
  
  
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
  
    // If we want to just create the branch, we can skip pulling the changes.
    if (!only_create_branch) {
      await wmill_sync_pull(
        path_type,
        workspace_id,
        path,
        parent_path,
        skip_secret,
        repo_url_resource_path,
        use_individual_branch,
        repo_resource.branch
      );
    }
    try {
      await git_push(path, parent_path, commit_msg, repo_resource, only_create_branch);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    no_single_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (no_single_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any,
    only_create_branch: boolean,
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
    if (only_create_branch) {
      await sh_run(undefined, "git", "push", "--porcelain");
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else if (path_type == "emailtrigger") {
      return `${path}.email_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by ruben fiszel 2376 172 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger"
    | "emailtrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false,
    only_create_branch: boolean = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    if (!only_create_branch) {
      console.log(
        `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
      );
    }
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
  
  
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
  
    // If we want to just create the branch, we can skip pulling the changes.
    if (!only_create_branch) {
      await wmill_sync_pull(
        path_type,
        workspace_id,
        path,
        parent_path,
        skip_secret,
        repo_url_resource_path,
        use_individual_branch,
        repo_resource.branch
      );
    }
    try {
      await git_push(path, parent_path, commit_msg, repo_resource, only_create_branch);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    no_single_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (no_single_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any,
    only_create_branch: boolean,
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
    if (only_create_branch) {
      await sh_run(undefined, "git", "push", "--porcelain");
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else if (path_type == "emailtrigger") {
      return `${path}.email_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo989 176 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger"
    | "emailtrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false,
    only_create_branch: boolean = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    if (!only_create_branch) {
      console.log(
        `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
      );
    }
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
  
  
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
  
    // If we want to just create the branch, we can skip pulling the changes.
    if (!only_create_branch) {
      await wmill_sync_pull(
        path_type,
        workspace_id,
        path,
        parent_path,
        skip_secret,
        repo_url_resource_path,
        use_individual_branch,
        repo_resource.branch
      );
    }
    try {
      await git_push(path, parent_path, commit_msg, repo_resource, only_create_branch);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    no_single_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (no_single_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any,
    only_create_branch: boolean,
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
    if (only_create_branch) {
      await sh_run(undefined, "git", "push", "--porcelain");
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else if (path_type == "emailtrigger") {
      return `${path}.email_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 198 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger"
    | "emailtrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    no_single_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (no_single_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
            ?.split("/")
            .slice(0, 2)
            .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
            path ?? parent_path
          )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else if (path_type == "emailtrigger") {
      return `${path}.email_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 252 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger"
    | "emailtrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    no_single_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (no_single_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
            ?.split("/")
            .slice(0, 2)
            .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
            path ?? parent_path
          )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else if (path_type == "emailtrigger") {
      return `${path}.email_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 254 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch || workspace_id.startsWith(FORKED_WORKSPACE_PREFIX));
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    no_single_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (no_single_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
            ?.split("/")
            .slice(0, 2)
            .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
            path ?? parent_path
          )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 254 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch);
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      const clonedBranchName = (await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")).trim();
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
            ?.split("/")
            .slice(0, 2)
            .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
            path ?? parent_path
          )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 259 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-";
  const FORKED_BRANCH_PREFIX = "wm-fork-";
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch);
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; clonedBranchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      let clonedBranchName = await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD");
      return { repo_name, safeDirectoryPath, clonedBranchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
            ?.split("/")
            .slice(0, 2)
            .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
            path ?? parent_path
          )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 260 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  const FORKED_WORKSPACE_PREFIX = "wm-fork-"
  const FORKED_BRANCH_PREFIX = "wm-fork-"
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath, clonedBranchName } = await git_clone(cwd, repo_resource, use_individual_branch);
    safeDirectoryPath = cloneSafeDirectoryPath;
  
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
  
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      if (use_individual_branch) {
        console.log("Cannot have `use_individual_branch` in a forked workspace, disabling option`");
        use_individual_branch = false;
      }
      if (group_by_folder) {
        console.log("Cannot have `group_by_folder` in a forked workspace, disabling option`");
        group_by_folder = false;
      }
    }
  
    if (parent_workspace_id && parent_workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      const parentBranch = get_fork_branch_name(parent_workspace_id, clonedBranchName);
      console.log(`This workspace's parent is also a fork, moving to branch ${parentBranch} in case a new branch needs to be created with the appropriate root`);
      await move_to_git_branch(
        parent_workspace_id,
        path_type,
        path,
        parent_path,
        use_individual_branch,
        group_by_folder,
        clonedBranchName
      );
    }
  
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder,
      clonedBranchName
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string, originalBranch: string): string {
    if (w_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      return w_id.replace(FORKED_WORKSPACE_PREFIX, `${FORKED_BRANCH_PREFIX}/${originalBranch}/`);
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean,
  ): Promise<{ repo_name: string; safeDirectoryPath: string; branchName: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      let branchName = await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD");
      return { repo_name, safeDirectoryPath, branchName };
  
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean,
    originalBranchName: string
  ) {
    let branchName;
    if (workspace_id.startsWith(FORKED_WORKSPACE_PREFIX)) {
      branchName = get_fork_branch_name(workspace_id, originalBranchName);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
            ?.split("/")
            .slice(0, 2)
            .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
            path ?? parent_path
          )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 260 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    parent_workspace_id?: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    let originalBranch;
  
    // Since we don't modify the resource on the forked workspaces, we have to cosnider the case of
    // a fork of a fork workspace. In that case, the original branch is not stored in the resource
    // settings, but we need to infer it from the workspace id
    if (parent_workspace_id && parent_workspace_id.startsWith("wm-forked-")) {
      use_individual_branch = false;
      group_by_folder = false;
      originalBranch = get_fork_branch_name(parent_workspace_id);
    } else {
      originalBranch = repo_resource.branch
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath } = await git_clone(cwd, repo_resource, use_individual_branch, originalBranch);
    safeDirectoryPath = cloneSafeDirectoryPath;
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  
  function get_fork_branch_name(w_id: string): string {
    if (w_id.startsWith("wm-forked-")) {
      return w_id.replace("wm-forked-", "wm-forked/");
    }
    return w_id;
  }
  
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean,
    branchNameOverride?: string
  ): Promise<{ repo_name: string; safeDirectoryPath: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = branchNameOverride ?? repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      return { repo_name, safeDirectoryPath };
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    let branchName;
    if (workspace_id.startsWith("wm-forked-")) {
      branchName = get_fork_branch_name(workspace_id);
    } else {
      if (!use_individual_branch || path_type === "user" || path_type === "group") {
        return;
      }
      branchName = group_by_folder
        ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
            ?.split("/")
            .slice(0, 2)
            .join("__")}`
        : `wm_deploy/${workspace_id}/${path_type}/${(
            path ?? parent_path
          )?.replaceAll("/", "__")}`;
    }
  
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 261 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath } = await git_clone(cwd, repo_resource, use_individual_branch);
    safeDirectoryPath = cloneSafeDirectoryPath;
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<{ repo_name: string; safeDirectoryPath: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      return { repo_name, safeDirectoryPath };
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by alex308 267 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath } = await git_clone(cwd, repo_resource, use_individual_branch);
    safeDirectoryPath = cloneSafeDirectoryPath;
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<{ repo_name: string; safeDirectoryPath: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      return { repo_name, safeDirectoryPath };
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by alex308 272 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    let safeDirectoryPath: string | undefined;
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath } = await git_clone(cwd, repo_resource, use_individual_branch);
    safeDirectoryPath = cloneSafeDirectoryPath;
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
      // Cleanup: remove safe.directory config
      if (safeDirectoryPath) {
        try {
          await sh_run(undefined, "git", "config", "--global", "--unset", "safe.directory", safeDirectoryPath);
        } catch (e) {
          console.log(`Warning: Could not unset safe.directory config: ${e}`);
        }
      }
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<{ repo_name: string; safeDirectoryPath: string }> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
      const safeDirectoryPath = process.cwd();
      // Add safe.directory to handle dubious ownership in cloned repo
      try {
        await sh_run(undefined, "git", "config", "--global", "--add", "safe.directory", process.cwd());
      } catch (e) {
        console.log(`Warning: Could not add safe.directory config: ${e}`);
      }
  
      if (subfolder !== "") {
        await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
        try {
          process.chdir(`${cwd}/${repo_name}/${subfolder}`);
        } catch (err) {
          console.log(
            `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
          );
          throw err;
        }
      }
      return { repo_name, safeDirectoryPath };
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 279 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by alex308 288 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path,
      use_individual_branch,
      repo_resource.branch
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string,
    use_individual_branch: boolean,
    original_branch?: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings" && !use_individual_branch) {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key" && !use_individual_branch) {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    // If using individual branches, apply promotion settings from original branch
    if (use_individual_branch && original_branch) {
      console.log(`Individual branch deployment detected - using promotion settings from '${original_branch}'`);
      args.push("--promotion", original_branch);
    }
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 288 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    const args = [
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
    ];
  
    // Only include settings when specifically deploying settings
    if (path_type === "settings") {
      args.push("--include-settings");
    }
  
    // Only include key when specifically deploying keys
    if (path_type === "key") {
      args.push("--include-key");
    }
  
    args.push("--extra-includes", includes.join(","));
  
    await wmill_run(3, ...args);
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 296 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
      "--include-settings",
      "--include-key",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by alex308 296 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
      "--include-settings",
      "--include-key",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by alex308 304 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
      "--include-settings",
      "--include-key",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by alex308 309 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
      "--include-settings",
      "--include-key",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by alex308 309 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
      "--include-settings",
      "--include-key",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by alex308 309 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
      "--include-settings",
      "--include-key",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by alex308 309 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret,
      repo_url_resource_path
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean,
    repo_url_resource_path: string
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--repository",
      repo_url_resource_path,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
      "--include-settings",
      "--include-key",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 309 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
      "--include-settings",
      "--include-key",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }
  

  Submitted by hugo697 310 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path";
  const util = require("util");
  const exec = util.promisify(require("child_process").exec);
  import process from "process";
  
  type GpgKey = {
    email: string;
    private_key: string;
    passphrase: string;
  };
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = ".";
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token();
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw e;
    } finally {
      await delete_pgp_keys();
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
          ?.split("/")
          .slice(0, 2)
          .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
          path ?? parent_path
        )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? "";
    let user_name = process.env["WM_USERNAME"] ?? "";
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      // Configure git with GPG key email for signing
      await sh_run(
        undefined,
        "git",
        "config",
        "user.email",
        repo_resource.gpg_key.email
      );
      await sh_run(undefined, "git", "config", "user.name", user_name);
    } else {
      await sh_run(undefined, "git", "config", "user.email", user_email);
      await sh_run(undefined, "git", "config", "user.name", user_name);
    }
  
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(
          undefined,
          "git",
          "add",
          "wmill-lock.yaml",
          `${parent_path}**`
        );
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      const commitArgs = ["git", "commit"];
  
      // Always use --author to set consistent authorship
      commitArgs.push("--author", `"${user_name} <${user_email}>"`);
      commitArgs.push(
        "-m",
        `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`
      );
  
      await sh_run(undefined, ...commitArgs);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`);
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command;
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(path_type: PathType, path: string) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    }
    if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`;
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`;
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`;
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`;
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`;
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`;
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`;
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`;
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key.replace(
        /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
        (_: string, header: string, body: string, footer: string) =>
          header +
          "\n" +
          "\n" +
          body.replace(/ ([^\s])/g, "\n$1").trim() +
          "\n" +
          footer
      );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path);
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error("Failed to import GPG key!");
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(
        /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/
      );
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys");
    if (gpgFingerprint) {
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--pinentry-mode",
        "loopback",
        "--delete-secret-key",
        gpgFingerprint
      );
      await sh_run(
        undefined,
        "gpg",
        "--batch",
        "--yes",
        "--delete-key",
        "--pinentry-mode",
        "loopback",
        gpgFingerprint
      );
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error(
          "Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'."
        );
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error;
      throw new Error(`Invalid URL: ${error.message}`);
    }
  }

  Submitted by hugo697 328 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  type PathType =
    | "script"
    | "flow"
    | "app"
    | "folder"
    | "resource"
    | "variable"
    | "resourcetype"
    | "schedule"
    | "user"
    | "group"
    | "httptrigger"
    | "websockettrigger"
    | "kafkatrigger"
    | "natstrigger"
    | "postgrestrigger"
    | "mqtttrigger"
    | "sqstrigger"
    | "gcptrigger";
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type: PathType,
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token()
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type: PathType,
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
  
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type: PathType,
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else if (path_type == "httptrigger") {
      return `${path}.http_trigger.*`
    } else if (path_type == "websockettrigger") {
      return `${path}.websocket_trigger.*`
    } else if (path_type == "kafkatrigger") {
      return `${path}.kafka_trigger.*`
    } else if (path_type == "natstrigger") {
      return `${path}.nats_trigger.*`
    } else if (path_type == "postgrestrigger") {
      return `${path}.postgres_trigger.*`
    } else if (path_type == "mqtttrigger") {
      return `${path}.mqtt_trigger.*`
    } else if (path_type == "sqstrigger") {
      return `${path}.sqs_trigger.*`
    } else if (path_type == "gcptrigger") {
      return `${path}.gcp_trigger.*`
    } else {
      return `${path}.*`;
    }
  }
  
  async function wmill_sync_pull(
    path_type: PathType,
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--include-triggers",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error("Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'.");
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error
      throw new Error(`Invalid URL: ${error.message}`)
    }
  }

  Submitted by hugo697 367 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token()
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
  
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
  
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error("Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'.");
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error
      throw new Error(`Invalid URL: ${error.message}`)
    }
  }

  Submitted by hugo697 421 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token()
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
  
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
  
    const url = `http://localhost:8000/api/w/${workspace}/github_app/token`;
  
    console.log(url)
    const response = await fetch(url, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${jobToken}`,
      },
      body: JSON.stringify({
        job_token: jobToken,
      }),
    });
  
    if (!response.ok) {
      throw new Error(`Error: ${response.statusText}`);
    }
  
    const data = await response.json();
  
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error("Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'.");
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error
      throw new Error(`Invalid URL: ${error.message}`)
    }
  }

  Submitted by hugo697 421 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app) {
      const token = await get_gh_app_token()
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
  
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
  
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }
  
  async function get_gh_app_token() {
    const res = await wmillclient.WorkspaceService.getGithubAppToken({
      workspace: process.env["WM_WORKSPACE"],
      requestBody: { job_token: process.env["WM_TOKEN"] }
    })
    return res.token
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error("Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'.");
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error
      throw new Error(`Invalid URL: ${error.message}`)
    }
  }

  Submitted by hugo697 422 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
  
    if (repo_resource.is_github_app && repo_resource.github_app_installation_id) {
      const token = await get_gh_app_token()
      const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);
      repo_resource.url = authRepoUrl;
    }
  
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
      
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }
  
  async function get_gh_app_token() {
    const res = await wmillclient.WorkspaceService.getGithubAppToken({
      requestBody: { job_token: process.env["WM_TOKEN"] }
    })
  
    return JSON.parse(res).token
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    if (!gitHubUrl || !installationToken) {
      throw new Error("Both GitHub URL and Installation Token are required.");
    }
  
    try {
      const url = new URL(gitHubUrl);
  
      // GitHub repository URL should be in the format: https://github.com/owner/repo.git
      if (url.hostname !== "github.com") {
        throw new Error("Invalid GitHub URL. Must be in the format 'https://github.com/owner/repo.git'.");
      }
  
      // Convert URL to include the installation token
      return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
    } catch (e) {
      const error = e as Error
      throw new Error(`Invalid URL: ${error.message}`)
    }
  }

  Submitted by hugo697 430 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
      
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }

  Submitted by rubenfiszel 435 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
      
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      "--raw",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }

  Submitted by rubenfiszel 435 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
      
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      "--raw",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }

  Submitted by rubenfiszel 469 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
      
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      "--raw",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }

  Submitted by rubenfiszel 480 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      return stdout;
      
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      "--raw",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }

  Submitted by hugo697 493 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  type GpgKey = {
    email: string,
    private_key: string,
    passphrase: string
  }
  
  let gpgFingerprint: string | undefined = undefined;
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    try {
      await git_push(path, parent_path, commit_msg, repo_resource);
    } catch (e) {
      throw (e)
    } finally {
      await delete_pgp_keys()
    }
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    repo_resource: any
  ) {
    let user_email = process.env["WM_EMAIL"] ?? ""
    let user_name = process.env["WM_USERNAME"] ?? ""
  
    if (repo_resource.gpg_key) {
      await set_gpg_signing_secret(repo_resource.gpg_key);
      user_email = repo_resource.gpg_key.email
    }
  
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      user_email
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      user_name
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      "--raw",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }
  
  // Function to set up GPG signing
  async function set_gpg_signing_secret(gpg_key: GpgKey) {
    try {
      console.log("Setting GPG private key for git commits");
  
      const formattedGpgContent = gpg_key.private_key
        .replace(
          /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,
          (_: string, header: string, body: string, footer: string) =>
            header + "\n" + "\n" + body.replace(/ ([^\s])/g, "\n$1").trim() + "\n" + footer
        );
  
      const gpg_path = `/tmp/gpg`;
      await sh_run(undefined, "mkdir", "-p", gpg_path);
      await sh_run(undefined, "chmod", "700", gpg_path)
      process.env.GNUPGHOME = gpg_path;
      // process.env.GIT_TRACE = 1;
  
  
      try {
        await sh_run(
          1,
          "bash",
          "-c",
          `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`
        );
      } catch (e) {
        // Original error would contain sensitive data
        throw new Error('Failed to import GPG key!')
      }
  
      const listKeysOutput = await sh_run(
        undefined,
        "gpg",
        "--list-secret-keys",
        "--with-colons",
        "--keyid-format=long"
      );
  
      const keyInfoMatch = listKeysOutput.match(/sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/);
  
      if (!keyInfoMatch) {
        throw new Error("Failed to extract GPG Key ID and Fingerprint");
      }
  
      const keyId = keyInfoMatch[1];
      gpgFingerprint = keyInfoMatch[2];
  
      if (gpg_key.passphrase) {
        // This is adummy command to unlock the key
        // with passphrase to load it into agent
        await sh_run(
          1,
          "bash",
          "-c",
          `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`
        );
      }
  
      // Configure Git to use the extracted key
      await sh_run(undefined, "git", "config", "user.signingkey", keyId);
      await sh_run(undefined, "git", "config", "commit.gpgsign", "true");
      console.log(`GPG signing configured with key ID: ${keyId} `);
  
    } catch (e) {
      console.error(`Failure while setting GPG key: ${e} `);
      await delete_pgp_keys();
    }
  }
  
  async function delete_pgp_keys() {
    console.log("deleting gpg keys")
    if (gpgFingerprint) {
      await sh_run(undefined, "gpg", "--batch", "--yes", "--pinentry-mode", "loopback", "--delete-secret-key", gpgFingerprint)
      await sh_run(undefined, "gpg", "--batch", "--yes", "--delete-key", "--pinentry-mode", "loopback", gpgFingerprint)
    }
  }

  Submitted by hugo697 497 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    await git_push(path, parent_path, commit_msg);
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string
  ) {
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      process.env["WM_EMAIL"] ?? ""
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      process.env["WM_USERNAME"] ?? ""
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    let secret: string | undefined = undefined
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position]
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    try {
      const { stdout, stderr } = await command
      if (stdout.length > 0) {
        console.log(stdout);
      }
      if (stderr.length > 0) {
        console.log(stderr);
      }
      console.log("Command successfully executed");
      
    } catch (error) {
      let errorString = error.toString();
      if (secret) {
        errorString = errorString.replace(secret, "***");
      }
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${errorString}`;
      throw Error(err);
    }
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      "--raw",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }

  Submitted by hugo697 503 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    await git_push(path, parent_path, commit_msg);
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string
  ) {
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      process.env["WM_EMAIL"] ?? ""
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      process.env["WM_USERNAME"] ?? ""
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch (e) {
        console.log(`Could not push, trying to rebase first: ${e}`);
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    const { error, stdout, stderr } = await command
    if (stdout.length > 0) {
      console.log(stdout);
    }
    if (stderr.length > 0) {
      console.log(stderr);
    }
    if (error) {
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${error}`;
      throw Error(err);
    }
    console.log("Command successfully executed");
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      "--raw",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }

  Submitted by alex308 525 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  import process from "process";
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    await git_push(path, parent_path, commit_msg);
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string
  ) {
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      process.env["WM_EMAIL"] ?? ""
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      process.env["WM_USERNAME"] ?? ""
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch {
        console.log("Could not push, trying to rebase first");
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    const { error, stdout, stderr } = await command
    if (stdout.length > 0) {
      console.log(stdout);
    }
    if (stderr.length > 0) {
      console.log(stderr);
    }
    if (error) {
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${error}`;
      throw Error(err);
    }
    console.log("Command successfully executed");
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      "--raw",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }

  Submitted by rubenfiszel 608 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import wmill from "windmill-cli";
  import { basename } from "node:path"
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  
  export async function main(
    workspace_id: string,
    repo_url_resource_path: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    skip_secret = true,
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string,
    use_individual_branch = false,
    group_by_folder = false
  ) {
    const repo_resource = await wmillclient.getResource(repo_url_resource_path);
    const cwd = process.cwd();
    process.env["HOME"] = "."
    console.log(
      `Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
    );
    const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
    await move_to_git_branch(
      workspace_id,
      path_type,
      path,
      parent_path,
      use_individual_branch,
      group_by_folder
    );
    const subfolder = repo_resource.folder ?? "";
    const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
    console.log(
      `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
    );
    await wmill_sync_pull(
      path_type,
      workspace_id,
      path,
      parent_path,
      skip_secret
    );
    await git_push(path, parent_path, commit_msg);
    console.log("Finished syncing");
    process.chdir(`${cwd}`);
  }
  async function git_clone(
    cwd: string,
    repo_resource: any,
    use_individual_branch: boolean
  ): Promise<string> {
    // TODO: handle private SSH keys as well
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    const branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log(
        "Requires Azure DevOps service account access token, requesting..."
      );
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
    const args = ["clone", "--quiet", "--depth", "1"];
    if (use_individual_branch) {
      args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
    }
    if (subfolder !== "") {
      args.push("--sparse");
    }
    if (branch !== "") {
      args.push("--branch");
      args.push(branch);
    }
    args.push(repo_url);
    args.push(repo_name);
    await sh_run(-1, "git", ...args);
    try {
      process.chdir(`${cwd}/${repo_name}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
      );
      throw err;
    }
    process.chdir(`${cwd}/${repo_name}`);
    if (subfolder !== "") {
      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
    }
    try {
      process.chdir(`${cwd}/${repo_name}/${subfolder}`);
    } catch (err) {
      console.log(
        `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
      );
      throw err;
    }
    return repo_name;
  }
  async function move_to_git_branch(
    workspace_id: string,
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string | undefined,
    parent_path: string | undefined,
    use_individual_branch: boolean,
    group_by_folder: boolean
  ) {
    if (!use_individual_branch || path_type === "user" || path_type === "group") {
      return;
    }
    const branchName = group_by_folder
      ? `wm_deploy/${workspace_id}/${(path ?? parent_path)
        ?.split("/")
        .slice(0, 2)
        .join("__")}`
      : `wm_deploy/${workspace_id}/${path_type}/${(
        path ?? parent_path
      )?.replaceAll("/", "__")}`;
    try {
      await sh_run(undefined, "git", "checkout", branchName);
    } catch (err) {
      console.log(
        `Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
      );
      try {
        await sh_run(undefined, "git", "checkout", "-b", branchName);
        await sh_run(
          undefined,
          "git",
          "config",
          "--add",
          "--bool",
          "push.autoSetupRemote",
          "true"
        );
      } catch (err) {
        console.log(
          `Error checking out branch '${branchName}'. Error was:\n${err}`
        );
        throw err;
      }
    }
    console.log(`Successfully switched to branch ${branchName}`);
  }
  async function git_push(
    path: string | undefined,
    parent_path: string | undefined,
    commit_msg: string
  ) {
    await sh_run(
      undefined,
      "git",
      "config",
      "user.email",
      process.env["WM_EMAIL"] ?? ""
    );
    await sh_run(
      undefined,
      "git",
      "config",
      "user.name",
      process.env["WM_USERNAME"] ?? ""
    );
    if (path !== undefined && path !== null && path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${path}**, ${e}`);
      }
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      try {
        await sh_run(undefined, "git", "add", `${parent_path}**`);
      } catch (e) {
        console.log(`Unable to stage files matching ${parent_path}, ${e}`);
      }
    }
    try {
      await sh_run(undefined, "git", "diff", "--cached", "--quiet");
    } catch {
      // git diff returns exit-code = 1 when there's at least one staged changes
      await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
      try {
        await sh_run(undefined, "git", "push", "--porcelain");
      } catch {
        console.log("Could not push, trying to rebase first");
        await sh_run(undefined, "git", "pull", "--rebase");
        await sh_run(undefined, "git", "push", "--porcelain");
      }
      return;
    }
    console.log("No changes detected, nothing to commit. Returning...");
  }
  async function sh_run(
    secret_position: number | undefined,
    cmd: string,
    ...args: string[]
  ) {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0) {
      secret_position = nargs.length - 1 + secret_position;
    }
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
    }
    
    console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
    const command = exec(`${cmd} ${args.join(" ")}`)
    // new Deno.Command(cmd, {
    //   args: args,
    // });
    const { error, stdout, stderr } = await command
    if (stdout.length > 0) {
      console.log(stdout);
    }
    if (stderr.length > 0) {
      console.log(stderr);
    }
    if (error) {
      const err = `SH command '${cmd} ${nargs.join(
        " "
      )}' returned with error ${error}`;
      throw Error(err);
    }
    console.log("Command successfully executed");
  }
  
  function regexFromPath(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    path: string
  ) {
    if (path_type == "flow") {
      return `${path}.flow/*`;
    } if (path_type == "app") {
      return `${path}.app/*`;
    } else if (path_type == "folder") {
      return `${path}/folder.meta.*`;
    } else if (path_type == "resourcetype") {
      return `${path}.resource-type.*`;
    } else if (path_type == "resource") {
      return `${path}.resource.*`;
    } else if (path_type == "variable") {
      return `${path}.variable.*`;
    } else if (path_type == "schedule") {
      return `${path}.schedule.*`;
    } else if (path_type == "user") {
      return `${path}.user.*`;
    } else if (path_type == "group") {
      return `${path}.group.*`;
    } else {
      return `${path}.*`;
    }
  }
  async function wmill_sync_pull(
    path_type:
      | "script"
      | "flow"
      | "app"
      | "folder"
      | "resource"
      | "variable"
      | "resourcetype"
      | "schedule"
      | "user"
      | "group",
    workspace_id: string,
    path: string | undefined,
    parent_path: string | undefined,
    skip_secret: boolean
  ) {
    const includes = [];
    if (path !== undefined && path !== null && path !== "") {
      includes.push(regexFromPath(path_type, path));
    }
    if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
      includes.push(regexFromPath(path_type, parent_path));
    }
    await wmill_run(
      6,
      "workspace",
      "add",
      workspace_id,
      workspace_id,
      process.env["BASE_URL"] + "/",
      "--token",
      process.env["WM_TOKEN"] ?? ""
    );
    console.log("Pulling workspace into git repo");
    await wmill_run(
      3,
      "sync",
      "pull",
      "--token",
      process.env["WM_TOKEN"] ?? "",
      "--workspace",
      workspace_id,
      "--yes",
      "--raw",
      skip_secret ? "--skip-secrets" : "",
      "--include-schedules",
      "--include-users",
      "--include-groups",
      "--extra-includes",
      includes.join(",")
    );
  }
  
  async function wmill_run(secret_position: number, ...cmd: string[]) {
    cmd = cmd.filter((elt) => elt !== "");
    const cmd2 = cmd.slice();
    cmd2[secret_position] = "***";
    console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
    await wmill.parse(cmd);
    console.log("Command successfully executed");
  }

  Submitted by rubenfiszel 619 days ago