Sync script to Git repo

import * as wmillclient from "windmill-client";

import wmill from "[email protected]";

import { basename } from "node:path";

import { existsSync } from "node:fs";

const util = require("util");

const exec = util.promisify(require("child_process").exec);

import process from "process";

type GpgKey = {

  email: string;

  private_key: string;

  passphrase: string;

};

const FORKED_WORKSPACE_PREFIX = "wm-fork-";

type PathType =

  | "script"

  | "flow"

  | "app"

  | "raw_app"

  | "folder"

  | "resource"

  | "variable"

  | "resourcetype"

  | "schedule"

  | "user"

  | "group"

  | "httptrigger"

  | "websockettrigger"

  | "kafkatrigger"

  | "natstrigger"

  | "postgrestrigger"

  | "mqtttrigger"

  | "sqstrigger"

  | "gcptrigger"

  | "azuretrigger"

  | "emailtrigger";

type SyncObject = {

  path_type: PathType;

  path: string | undefined;

  parent_path: string | undefined;

  commit_msg: string;

};

let gpgFingerprint: string | undefined = undefined;

export async function main(

  items: SyncObject[],

  // Compat, do not use in code, rely on `items` instead

  path_type: PathType | undefined,

  path: string | undefined,

  parent_path: string | undefined,

  commit_msg: string | undefined,

  //

  workspace_id: string,

  repo_url_resource_path: string,

  skip_secret: boolean = true,

  use_individual_branch: boolean = false,

  group_by_folder: boolean = false,

  only_create_branch: boolean = false,

  parent_workspace_id?: string,

  dev_workspace_label?: string,

  parent_dev_workspace_label?: string,

) {

  if (path_type !== undefined && commit_msg !== undefined) {

    items = [

      {

        path_type,

        path,

        parent_path,

        commit_msg,

      },

    ];

  }

  // The push outcome is the job result: the completion hook reads `pushed` to

  // decide on PR creation and `sha`/`branch` to record the pushed head.

  return await inner(

    items,

    workspace_id,

    repo_url_resource_path,

    skip_secret,

    use_individual_branch,

    group_by_folder,

    only_create_branch,

    parent_workspace_id,

    dev_workspace_label,

    parent_dev_workspace_label,

  );

}

async function inner(

  items: SyncObject[],

  workspace_id: string,

  repo_url_resource_path: string,

  skip_secret: boolean = true,

  use_individual_branch: boolean = false,

  group_by_folder: boolean = false,

  only_create_branch: boolean = false,

  parent_workspace_id?: string,

  dev_workspace_label?: string,

  parent_dev_workspace_label?: string,

) {

  let safeDirectoryPath: string | undefined;

  // Whether a commit was actually pushed, and which commit on which branch:

  // surfaced in the result so the completion hook can skip PR creation for

  // no-op pushes and record the pushed head as the workspace's synced state.

  let push: PushOutcome = { pushed: false };

  const repo_resource = await wmillclient.getResource(repo_url_resource_path);

  const cwd = process.cwd();

  process.env["HOME"] = ".";

  if (!only_create_branch) {

    for (const item of items) {

      console.log(

        `Syncing ${item.path_type} ${item.path ?? ""} with parent ${item.parent_path ?? ""}`,

      );

    }

  }

  if (repo_resource.is_github_app) {

    const token = await get_gh_app_token();

    const authRepoUrl = prependTokenToGitHubUrl(repo_resource.url, token);

    repo_resource.url = authRepoUrl;

  } else if (!urlCarriesCredential(repo_resource.url)) {

    // A repository whose credential Windmill holds gets it from the same

    // endpoint the app path uses; one that carries a token in its URL, or needs

    // none at all, is left exactly as it is.

    const token = await get_stored_git_token();

    if (token) {

      console.log("Using the git credential stored for this repository");

      repo_resource.url = prependTokenToGitUrl(repo_resource.url, token);

    }

  }

  const { repo_name, safeDirectoryPath: cloneSafeDirectoryPath } =

    await git_clone(

      cwd,

      repo_resource,

      // Dev workspaces are forks with a prefix-less id, so also detect them via

      // parent_workspace_id (mirrors the CLI's `parent || wm-fork- prefix`).

      use_individual_branch ||

        workspace_id.startsWith(FORKED_WORKSPACE_PREFIX) ||

        !!parent_workspace_id,

    );

  safeDirectoryPath = cloneSafeDirectoryPath;

  // GPG signing must be set up BEFORE git_push (which is in-script and runs

  // back-to-back with the agent pre-warm — that's the whole point of doing

  // commit/push in the script instead of the CLI).

  if (repo_resource.gpg_key) {

    await set_gpg_signing_secret(repo_resource.gpg_key);

  }

  const subfolder = repo_resource.folder ?? "";

  const branch_or_default = repo_resource.branch ?? "<DEFAULT>";

  console.log(

    `Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`,

  );

  try {

    // CLI owns: branch checkout (wm_deploy/<workspace>/... or wm-fork/...),

    // include-set derivation from items, promotion-overrides resolution,

    // workspace zip pull. With --skip-commit, the CLI returns without

    // committing — the script does that next.

    const args = [

      "sync",

      "git-deploy",

      "--token",

      process.env["WM_TOKEN"] ?? "",

      "--workspace",

      workspace_id,

      "--base-url",

      process.env["BASE_URL"] + "/",

      "--repository",

      repo_url_resource_path,

      "--git-deploy-items",

      JSON.stringify(items),

    ];

    if (use_individual_branch) args.push("--use-individual-branch");

    if (group_by_folder) args.push("--group-by-folder");

    if (only_create_branch) args.push("--only-create-branch");

    if (parent_workspace_id) {

      args.push("--parent-workspace-id", parent_workspace_id);

    }

    if (dev_workspace_label) {

      // Dev workspaces deploy to their environment-label branch (dev/staging).

      args.push("--dev-workspace-label", dev_workspace_label);

    }

    if (parent_dev_workspace_label) {

      // A fork of a dev workspace roots its branch on the dev's label branch.

      args.push("--parent-dev-workspace-label", parent_dev_workspace_label);

    }

    if (skip_secret) args.push("--skip-secrets");

    await wmill_run(3, ...args);

    // In-script commit + push. Stays in the SAME process as the dummy gpg

    // sign in set_gpg_signing_secret — agent cache is warm, signing works.

    if (!only_create_branch) {

      push = await git_push(items, repo_resource, only_create_branch);

    }

  } catch (e) {

    throw e;

  } finally {

    await delete_pgp_keys();

    if (safeDirectoryPath) {

      try {

        await sh_run(

          undefined,

          "git",

          "config",

          "--global",

          "--unset",

          "safe.directory",

          safeDirectoryPath,

        );

      } catch (e) {

        console.log(`Warning: Could not unset safe.directory config: ${e}`);

      }

    }

  }

  console.log("Finished syncing");

  process.chdir(`${cwd}`);

  return push;

}

type PushOutcome = {

  pushed: boolean;

  /** Head commit and branch after a successful push. */

  sha?: string;

  branch?: string;

  /** The push was rejected and retried after rebasing onto the remote, so the

   * pushed commit sits on top of commits this workspace has not pulled. */

  rebased?: boolean;

};

async function git_clone(

  cwd: string,

  repo_resource: any,

  no_single_branch: boolean,

): Promise<{

  repo_name: string;

  safeDirectoryPath: string;

  clonedBranchName: string;

}> {

  let repo_url = repo_resource.url;

  const subfolder = repo_resource.folder ?? "";

  const branch = repo_resource.branch ?? "";

  const repo_name = basename(repo_url, ".git");

  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);

  if (azureMatch) {

    console.log(

      "Requires Azure DevOps service account access token, requesting...",

    );

    const azureResource = await wmillclient.getResource(azureMatch.groups.url);

    const response = await fetch(

      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,

      {

        method: "POST",

        body: new URLSearchParams({

          client_id: azureResource.azureClientId,

          client_secret: azureResource.azureClientSecret,

          grant_type: "client_credentials",

          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",

        }),

      },

    );

    const { access_token } = await response.json();

    repo_url = repo_url.replace(azureMatch[0], access_token);

  }

  const args = ["clone", "--quiet", "--depth", "1"];

  if (no_single_branch) {

    args.push("--no-single-branch");

  }

  if (subfolder !== "") {

    args.push("--sparse");

  }

  if (branch !== "") {

    args.push("--branch");

    args.push(branch);

  }

  args.push(repo_url);

  args.push(repo_name);

  await sh_run(-1, "git", ...args);

  try {

    process.chdir(`${cwd}/${repo_name}`);

    const safeDirectoryPath = process.cwd();

    try {

      await sh_run(

        undefined,

        "git",

        "config",

        "--global",

        "--add",

        "safe.directory",

        process.cwd(),

      );

    } catch (e) {

      console.log(`Warning: Could not add safe.directory config: ${e}`);

    }

    if (subfolder !== "") {

      await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);

      try {

        process.chdir(`${cwd}/${repo_name}/${subfolder}`);

      } catch (err) {

        console.log(

          `Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`,

        );

        throw err;

      }

    }

    const clonedBranchName = (

      await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")

    ).trim();

    return { repo_name, safeDirectoryPath, clonedBranchName };

  } catch (err) {

    console.log(

      `Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`,

    );

    throw err;

  }

}

function composeCommitHeader(items: SyncObject[]): string {

  const typeCounts = new Map<PathType, number>();

  for (const item of items) {

    typeCounts.set(item.path_type, (typeCounts.get(item.path_type) ?? 0) + 1);

  }

  const sortedTypes = Array.from(typeCounts.entries()).sort(

    (a, b) => b[1] - a[1],

  );

  const parts: string[] = [];

  let othersCount = 0;

  for (let i = 0; i < sortedTypes.length; i++) {

    const [pathType, count] = sortedTypes[i];

    if (i < 3) {

      const label = count > 1 ? `${pathType}s` : pathType;

      if (i == 2 && sortedTypes.length == 3) {

        parts.push(`and ${count} ${label}`);

      } else {

        parts.push(`${count} ${label}`);

      }

    } else {

      othersCount += count;

    }

  }

  let header = `[WM]: Deployed ${parts.join(", ")}`;

  if (othersCount > 0) {

    header += ` and ${othersCount} other object${othersCount > 1 ? "s" : ""}`;

  }

  return header;

}

// Stage / commit / push the deploy. Mirrors hub/28217's git_push EXACTLY

// (same staging globs, same commit-message construction quirks, same

// rebase-retry fallback). Keeping this in-script — instead of letting the

// CLI's gitSyncDeployPush handle it — is what restores GPG correctness:

// set_gpg_signing_secret already pre-warmed the agent in THIS process

// milliseconds ago, so signing inherits a warm cache via plain exec.

async function git_push(

  items: SyncObject[],

  repo_resource: any,

  only_create_branch: boolean,

): Promise<PushOutcome> {

  const user_email = process.env["WM_EMAIL"] ?? "";

  const user_name = process.env["WM_USERNAME"] ?? "";

  if (repo_resource.gpg_key) {

    // GPG path: committer identity = the GPG key's email (else git rejects

    // the signed commit's committer/key mismatch).

    await sh_run(

      undefined,

      "git",

      "config",

      "user.email",

      repo_resource.gpg_key.email,

    );

    await sh_run(undefined, "git", "config", "user.name", user_name);

  } else {

    await sh_run(undefined, "git", "config", "user.email", user_email);

    await sh_run(undefined, "git", "config", "user.name", user_name);

  }

  if (only_create_branch) {

    await sh_run(undefined, "git", "push", "--porcelain");

  }

  const commit_description: string[] = [];

  for (const { path, parent_path, commit_msg } of items) {

    if (path !== undefined && path !== null && path !== "") {

      try {

        await sh_run(undefined, "git", "add", "wmill-lock.yaml", `'${path}**'`);

      } catch (e) {

        console.log(`Unable to stage files matching ${path}**, ${e}`);

      }

    }

    if (

      parent_path !== undefined &&

      parent_path !== null &&

      parent_path !== ""

    ) {

      try {

        await sh_run(

          undefined,

          "git",

          "add",

          "wmill-lock.yaml",

          `'${parent_path}**'`,

        );

      } catch (e) {

        console.log(`Unable to stage files matching ${parent_path}, ${e}`);

      }

    }

    commit_description.push(commit_msg);

  }

  // A shared lockfile (`dedupeLockfiles`) lives under `locks/`, outside every

  // item's path glob, and the pull rewrites it when a deployed script's lock

  // changed. `-A` also stages the deletion of a swept one. Only when the

  // directory exists or is tracked: `git add` fails on a pathspec matching

  // nothing, and a repository without shared lockfiles must stay silent.

  const trackedSharedLocks = await sh_run(

    undefined,

    "git",

    "ls-files",

    "--",

    "locks",

  );

  if (existsSync("locks") || trackedSharedLocks.trim() !== "") {

    await sh_run(undefined, "git", "add", "-A", "--", "locks");

  }

  try {

    await sh_run(undefined, "git", "diff", "--cached", "--quiet");

  } catch {

    // git diff --cached --quiet exits 1 iff there is something staged.

    const commitArgs = ["git", "commit"];

    commitArgs.push("--author", `"${user_name} <${user_email}>"`);

    const [header, description] =

      commit_description.length == 1

        ? [commit_description[0], ""]

        : [composeCommitHeader(items), commit_description.join("\n")];

    commitArgs.push(

      "-m",

      `"${header == undefined || header == "" ? "no commit msg" : header}"`,

      "-m",

      `"${description}"`,

    );

    await sh_run(undefined, ...commitArgs);

    let rebased = false;

    try {

      await sh_run(undefined, "git", "push", "--porcelain");

    } catch (e) {

      console.log(`Could not push, trying to rebase first: ${e}`);

      await sh_run(undefined, "git", "pull", "--rebase");

      await sh_run(undefined, "git", "push", "--porcelain");

      rebased = true;

    }

    // Read after the push: a rebase rewrites the commit.

    const sha = (await sh_run(undefined, "git", "rev-parse", "HEAD")).trim();

    const branch = (

      await sh_run(undefined, "git", "rev-parse", "--abbrev-ref", "HEAD")

    ).trim();

    return { pushed: true, sha, branch, rebased };

  }

  console.log("No changes detected, nothing to commit. Returning...");

  return { pushed: false };

}

async function sh_run(

  secret_position: number | undefined,

  cmd: string,

  ...args: string[]

) {

  const nargs = secret_position != undefined ? args.slice() : args;

  if (secret_position && secret_position < 0) {

    secret_position = nargs.length - 1 + secret_position;

  }

  let secret: string | undefined = undefined;

  if (secret_position != undefined) {

    nargs[secret_position] = "***";

    secret = args[secret_position];

  }

  console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);

  const command = exec(`${cmd} ${args.join(" ")}`);

  try {

    const { stdout, stderr } = await command;

    if (stdout.length > 0) {

      console.log(stdout);

    }

    if (stderr.length > 0) {

      console.log(stderr);

    }

    console.log("Command successfully executed");

    return stdout;

  } catch (error) {

    let errorString = error.toString();

    if (secret) {

      errorString = errorString.replace(secret, "***");

    }

    const err = `SH command '${cmd} ${nargs.join(

      " ",

    )}' returned with error ${errorString}`;

    throw Error(err);

  }

}

async function wmill_run(secret_position: number, ...cmd: string[]) {

  cmd = cmd.filter((elt) => elt !== "");

  const cmd2 = cmd.slice();

  cmd2[secret_position] = "***";

  console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);

  await wmill.parse(cmd);

  console.log("Command successfully executed");

}

// Identical to hub/28217's set_gpg_signing_secret. Sets up GNUPGHOME,

// imports the key, pre-warms the agent's passphrase cache with a dummy

// `gpg -bsau`, and configures git's local user.signingkey + commit.gpgsign.

// Because git_push runs immediately after in the SAME process (no CLI

// boundary), the agent cache is still warm at sign time.

async function set_gpg_signing_secret(gpg_key: GpgKey) {

  try {

    console.log("Setting GPG private key for git commits");

    const formattedGpgContent = gpg_key.private_key.replace(

      /(-----BEGIN PGP PRIVATE KEY BLOCK-----)([\s\S]*?)(-----END PGP PRIVATE KEY BLOCK-----)/,

      (_: string, header: string, body: string, footer: string) =>

        header +

        "\n" +

        "\n" +

        body.replace(/ ([^\s])/g, "\n$1").trim() +

        "\n" +

        footer,

    );

    const gpg_path = `/tmp/gpg`;

    await sh_run(undefined, "mkdir", "-p", gpg_path);

    await sh_run(undefined, "chmod", "700", gpg_path);

    process.env.GNUPGHOME = gpg_path;

    try {

      await sh_run(

        1,

        "bash",

        "-c",

        `cat <<EOF | gpg --batch --import \n${formattedGpgContent}\nEOF`,

      );

    } catch (e) {

      throw new Error("Failed to import GPG key!");

    }

    const listKeysOutput = await sh_run(

      undefined,

      "gpg",

      "--list-secret-keys",

      "--with-colons",

      "--keyid-format=long",

    );

    const keyInfoMatch = listKeysOutput.match(

      /sec:[^:]*:[^:]*:[^:]*:([A-F0-9]+):.*\nfpr:::::::::([A-F0-9]{40}):/,

    );

    if (!keyInfoMatch) {

      throw new Error("Failed to extract GPG Key ID and Fingerprint");

    }

    const keyId = keyInfoMatch[1];

    gpgFingerprint = keyInfoMatch[2];

    if (gpg_key.passphrase) {

      // Pre-warm the agent: this dummy sign loads the passphrase into

      // gpg-agent's cache so the actual `git commit` later in this same

      // process doesn't need to prompt.

      await sh_run(

        1,

        "bash",

        "-c",

        `echo "dummy" | gpg --batch --pinentry-mode loopback --passphrase '${gpg_key.passphrase}' --status-fd=2 -bsau ${keyId}`,

      );

    }

    await sh_run(undefined, "git", "config", "user.signingkey", keyId);

    await sh_run(undefined, "git", "config", "commit.gpgsign", "true");

    console.log(`GPG signing configured with key ID: ${keyId} `);

  } catch (e) {

    console.error(`Failure while setting GPG key: ${e} `);

    await delete_pgp_keys();

  }

}

async function delete_pgp_keys() {

  console.log("deleting gpg keys");

  if (gpgFingerprint) {

    await sh_run(

      undefined,

      "gpg",

      "--batch",

      "--yes",

      "--pinentry-mode",

      "loopback",

      "--delete-secret-key",

      gpgFingerprint,

    );

    await sh_run(

      undefined,

      "gpg",

      "--batch",

      "--yes",

      "--delete-key",

      "--pinentry-mode",

      "loopback",

      gpgFingerprint,

    );

  }

}

// True when the remote already authenticates itself, i.e. it has a `user@` or

// `user:password@` userinfo component.

function urlCarriesCredential(url: string | undefined): boolean {

  return /:\/\/[^/@]+@/.test(url ?? "");

}

// The credential Windmill stores for this repository, or undefined when it holds

// none. Absence is normal (a public remote), so it must not fail the sync; the

// reason is logged so a real outage is still diagnosable from the job log.

async function get_stored_git_token() {

  try {

    return await get_gh_app_token();

  } catch (error) {

    console.log(

      "No stored git credential for this repository:",

      error?.message ?? String(error)

    );

    return undefined;

  }

}

async function get_gh_app_token() {

  const workspace = process.env["WM_WORKSPACE"];

  const jobToken = process.env["WM_TOKEN"];

  const baseUrl =

    process.env["BASE_INTERNAL_URL"] ??

    process.env["BASE_URL"] ??

    "http://localhost:8000";

  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;

  const response = await fetch(url, {

    method: "POST",

    headers: {

      "Content-Type": "application/json",

      Authorization: `Bearer ${jobToken}`,

    },

    body: JSON.stringify({

      job_token: jobToken,

    }),

  });

  if (!response.ok) {

    const errorBody = await response.text().catch(() => "");

    throw new Error(

      `GitHub App token error (${response.status}): ${errorBody || response.statusText}`,

    );

  }

  const data = await response.json();

  return data.token;

}

// Rewrite the git remote formats `new URL` can't parse into an https URL:

// scp-like ssh ([user@]host:owner/repo) and scheme-less (host/owner/repo).

// A host:8080/... port form is kept as authority rather than read as scp.

function normalizeGitHubUrl(raw: string): string {

  const cleaned = raw.trim();

  if (cleaned.includes("://")) {

    return cleaned;

  }

  const colonIdx = cleaned.indexOf(":");

  if (colonIdx !== -1) {

    const before = cleaned.slice(0, colonIdx);

    const after = cleaned.slice(colonIdx + 1);

    const firstSeg = after.split("/")[0];

    // `user@host:...` is always scp form, so digits there are an owner

    // (GitHub allows all-numeric ones), not a port.

    const isPort = !before.includes("@") && firstSeg !== "" && /^\d+$/.test(firstSeg);

    // A `@` right after the `:` means `user:token@host/...` — an authority

    // (userinfo) form, not scp.

    const isUserinfo = firstSeg.includes("@");

    if (before !== "" && !before.includes("/") && after !== "" && !isPort && !isUserinfo) {

      const host = before.split("@").pop();

      return `https://${host}/${after.replace(/^\/+/, "")}`;

    }

  }

  return `https://${cleaned}`;

}

function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {

  if (!gitHubUrl || !installationToken) {

    throw new Error("Both GitHub URL and Installation Token are required.");

  }

  // `host` (not `hostname`) so a custom port on a self-hosted git server survives.

  const url = new URL(normalizeGitHubUrl(gitHubUrl));

  return `https://x-access-token:${installationToken}@${url.host}${url.pathname}`;

}

// Authenticate a remote with a stored token. Unlike the app-token form above,

// this keeps the scheme the resource chose, so a self-managed git server reached

// over http is not dialled as https; `oauth2` is the username GitLab documents

// for token auth, and the setters percent-encode the token.

function prependTokenToGitUrl(gitUrl: string, token: string) {

  const url = new URL(normalizeGitHubUrl(gitUrl));

  url.username = "oauth2";

  url.password = token;

  return url.toString();

}