Ctrl K

clone repo and upload to instance storage

Published Oct 14, 2025

Clones a github repo defined in a git_repository resource, then uploads the files to the instance storage for them to be cached

Script windmill Verified

Use in Windmill

The script

Submitted by hugo697 Bun

Verified 2 days ago

All edits

Permalink

import * as wmillclient from "windmill-client";
import { basename, join } from "node:path";
import { existsSync, rmSync } from "fs";
import process from "process";
import { spawn } from 'child_process';
import * as fs_async from 'fs/promises';
import * as fs from 'node:fs';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';

const UPLOAD_CONCURRENCY = 16;
const CLONE_MARKER_FILE = ".windmill_clone_complete";

type GitRepository = {
  url: string;
  branch: string;
  folder: string;
  gpg_key: any;
  is_github_app: boolean;
};

export async function main(
  resource_path: string,
  workspace: string,
  git_ssh_identity?: string[],
  commit?: string
) {
  let clonedRepoPath: string | undefined;

  try {
    console.log("Starting git clone and Blob storage upload process");

    // Get the git repository resource
    const repo_resource: GitRepository = await wmillclient.getResource(resource_path);

    const cwd = process.cwd();

    if (git_ssh_identity) {
      process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity)
    }

    process.env["HOME"] = ".";
    process.env.GIT_TERMINAL_PROMPT = "0";

    // App-backed repos carry no credential in their URL, so git can't
    // authenticate against them. The server serves a snapshot of the commit
    // instead, keeping the installation token on its side.
    const { repo_name, commitHash } = repo_resource.is_github_app
      ? await download_repo_archive(cwd, workspace, resource_path, repo_resource, commit)
      : await git_clone(cwd, repo_resource, commit);
    clonedRepoPath = join(cwd, repo_name);

    // Remove .git directory to avoid uploading git history
    const gitDir = join(clonedRepoPath, ".git");
    if (existsSync(gitDir)) {
      rmSync(gitDir, { recursive: true, force: true });
      console.log("Removed .git directory");
    }

    // Upload to S3
    const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`;
    const fileCount = await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace);

    return {
      success: true,
      message: "Repository cloned and uploaded to S3 successfully",
      s3_path: s3Path,
      commit_hash: commitHash,
      file_count: fileCount,
    };

  } catch (error) {
    console.error("Error in git clone and upload:", error);
    throw error;
  } finally {
    // Clean up cloned repository
    if (clonedRepoPath && existsSync(clonedRepoPath)) {
      rmSync(clonedRepoPath, { recursive: true, force: true });
      console.log("Cleaned up cloned repository");
    }
  }
}

async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise<string> {
  const sshIdFiles = await Promise.all(
    git_ssh_identity.map(async (varPath, i) => {
      const filePath = join(cwd, `./ssh_id_priv_${i}`);

      try {
        // Get variable value using windmill
        let content = await wmillclient.getVariable(varPath);
        content += '\n';

        // Write file with content
        await fs_async.writeFile(filePath, content, { encoding: 'utf8' });

        // Set file permissions to 0o600 (read/write for owner only)
        await fs_async.chmod(filePath, 0o600);

        // Escape single quotes for shell command
        const escapedPath = filePath.replace(/'/g, "'\\''");
        return ` -i '${escapedPath}'`;
      } catch (error) {
        console.error(
          `Variable ${varPath} not found for git ssh identity: ${error}`
        );
        return '';
      }
    })
  );

  const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`;
  return gitSshCmd;
}

async function download_repo_archive(
  cwd: string,
  workspace: string,
  resource_path: string,
  repo_resource: GitRepository,
  commit?: string,
): Promise<{ repo_name: string; commitHash: string }> {
  const repo_name = basename(repo_resource.url, ".git");
  const baseUrl =
    process.env["BASE_INTERNAL_URL"] ??
    process.env["BASE_URL"] ??
    "http://localhost:8000";
  const ref = commit ?? repo_resource.branch ?? "";
  const query = ref !== "" ? `?ref=${encodeURIComponent(ref)}` : "";
  const url = `${baseUrl}/api/w/${workspace}/github_app/repo_archive/${resource_path}${query}`;

  console.log(`Downloading archive of ${resource_path}${ref !== "" ? ` at ${ref}` : ""}`);
  const response = await fetch(url, {
    headers: { Authorization: `Bearer ${process.env["WM_TOKEN"]}` },
  });
  if (!response.ok || response.body == null) {
    const errorBody = await response.text().catch(() => "");
    throw new Error(
      `Repository archive error (${response.status}): ${errorBody || response.statusText}`
    );
  }

  const commitHash = response.headers.get("x-commit-sha");
  if (!commitHash) {
    throw new Error("Repository archive carried no commit sha");
  }

  const archivePath = join(cwd, `${repo_name}.tar.gz`);
  await pipeline(
    Readable.fromWeb(response.body as any),
    fs.createWriteStream(archivePath)
  );

  const repoPath = join(cwd, repo_name);
  await fs_async.mkdir(repoPath, { recursive: true });
  // Every entry sits under a single `{owner}-{repo}-{sha}` directory.
  await runCommand(undefined, "tar", "-xzf", archivePath, "-C", repoPath, "--strip-components=1");
  rmSync(archivePath, { force: true });

  const subfolder = repo_resource.folder ?? "";
  if (subfolder !== "") {
    if (!existsSync(join(repoPath, subfolder))) {
      throw new Error(`Subfolder ${subfolder} does not exist.`);
    }
    // Keep what the sparse checkout of the credentialed path would: the root
    // files, plus the tree the subfolder lives in.
    const keep = subfolder.split("/")[0];
    for (const entry of fs.readdirSync(repoPath, { withFileTypes: true })) {
      if (entry.isDirectory() && entry.name !== keep) {
        rmSync(join(repoPath, entry.name), { recursive: true, force: true });
      }
    }
  }

  return { repo_name, commitHash };
}

async function git_clone(
  cwd: string,
  repo_resource: GitRepository,
  commit?: string,
): Promise<{ repo_name: string; commitHash: string }> {
    if (commit) {
      return git_clone_at_commit(cwd, repo_resource, commit);
    } else {
      return git_clone_at_latest(cwd, repo_resource);
    }
}

async function git_clone_at_commit(
  cwd: string,
  repo_resource: GitRepository,
  commit: string,
): Promise<{ repo_name: string; commitHash: string }> {
  let repo_url = repo_resource.url;
  const subfolder = repo_resource.folder ?? "";
  let branch = repo_resource.branch ?? "";
  const repo_name = basename(repo_url, ".git");

  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
  if (azureMatch) {
    console.log("Fetching Azure DevOps access token...");
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    const response = await fetch(
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
      {
        method: "POST",
        body: new URLSearchParams({
          client_id: azureResource.azureClientId,
          client_secret: azureResource.azureClientSecret,
          grant_type: "client_credentials",
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
        }),
      }
    );
    const { access_token } = await response.json();
    repo_url = repo_url.replace(azureMatch[0], access_token);
  }

  const repoPath = join(cwd, repo_name);
  await fs_async.mkdir(repoPath, { recursive: true });

  process.chdir(repoPath);

  let args = ['init', '--quiet']
  if (branch) {
    args.push(`--initial-branch=${branch}`)
  }
  await runCommand(undefined, 'git', ...args);

  await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url);

  await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit);

  await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD');

  const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();

  // Return to original directory
  process.chdir(cwd);

  return { repo_name, commitHash };
}

async function git_clone_at_latest(
  cwd: string,
  repo_resource: GitRepository
): Promise<{ repo_name: string; commitHash: string }> {
  let repo_url = repo_resource.url;
  const subfolder = repo_resource.folder ?? "";
  let branch = repo_resource.branch ?? "";
  const repo_name = basename(repo_url, ".git");

  // Handle Azure DevOps token if needed
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
  if (azureMatch) {
    console.log("Fetching Azure DevOps access token...");
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    const response = await fetch(
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
      {
        method: "POST",
        body: new URLSearchParams({
          client_id: azureResource.azureClientId,
          client_secret: azureResource.azureClientSecret,
          grant_type: "client_credentials",
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
        }),
      }
    );
    const { access_token } = await response.json();
    repo_url = repo_url.replace(azureMatch[0], access_token);
  }

  const args = ["clone", "--quiet", "--depth", "1"];
  if (subfolder !== "") args.push("--sparse");
  if (branch !== "") args.push("--branch", branch);
  args.push(repo_url, repo_name);

  await runCommand(-1, "git", ...args);

  const fullPath = join(cwd, repo_name);
  process.chdir(fullPath);

  if (subfolder !== "") {
    await runCommand(undefined, "git", "sparse-checkout", "add", subfolder);
    const subfolderPath = join(fullPath, subfolder);

    if (!existsSync(subfolderPath)) {
      throw new Error(`Subfolder ${subfolder} does not exist.`);
    }

    process.chdir(subfolderPath);
  }

  // Get the commit hash
  const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();

  // Return to original directory
  process.chdir(cwd);

  return { repo_name, commitHash };
}

async function uploadDirectoryToS3(
  directoryPath: string,
  s3BasePath: string,
  workspace: string,
): Promise<number> {
  console.log(`Uploading ${directoryPath} -> ${s3BasePath}`);

  // Walk once into a flat task list so we can drive a bounded-concurrency pool.
  const tasks: { localPath: string; s3Key: string }[] = [];
  function walk(dir: string, s3Path: string) {
    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
      const fullPath = join(dir, entry.name);
      const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name;
      if (entry.isDirectory()) {
        walk(fullPath, s3Key);
      } else if (entry.isFile()) {
        tasks.push({ localPath: fullPath, s3Key });
      }
    }
  }
  walk(directoryPath, s3BasePath);
  console.log(`Discovered ${tasks.length} files to upload`);

  let nextIndex = 0;
  let uploaded = 0;
  let lastReport = 0;
  async function worker() {
    while (true) {
      const idx = nextIndex++;
      if (idx >= tasks.length) return;
      const { localPath, s3Key } = tasks[idx];
      const fileContent = fs.readFileSync(localPath);
      const blob = new Blob([fileContent], { type: 'application/octet-stream' });
      await wmillclient.HelpersService.gitRepoViewerFileUpload({
        workspace,
        fileKey: s3Key,
        requestBody: blob,
      });
      uploaded++;
      if (uploaded - lastReport >= 25 || uploaded === tasks.length) {
        lastReport = uploaded;
        console.log(`Uploaded ${uploaded} / ${tasks.length} files`);
      }
    }
  }
  await Promise.all(
    Array.from({ length: Math.min(UPLOAD_CONCURRENCY, tasks.length) }, () => worker())
  );

  // Marker is the LAST write — its presence is what the viewer checks for.
  const markerKey = `${s3BasePath}/${CLONE_MARKER_FILE}`;
  const markerBody = JSON.stringify({
    completed_at: new Date().toISOString(),
    file_count: tasks.length,
  });
  await wmillclient.HelpersService.gitRepoViewerFileUpload({
    workspace,
    fileKey: markerKey,
    requestBody: new Blob([markerBody], { type: 'application/json' }),
  });
  console.log(`Wrote completion marker: ${markerKey}`);

  return tasks.length;
}

function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise<string> {
  const nargs = secret_position != undefined ? args.slice() : args;
  if (secret_position && secret_position < 0)
    secret_position = nargs.length - 1 + secret_position;

  let secret: string | undefined = undefined;
  if (secret_position != undefined) {
    nargs[secret_position] = "***";
    secret = args[secret_position];
  }
  console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);

  return new Promise((resolve, reject) => {
    const process = spawn(cmd, args);

    let stdout = '';
    let stderr = '';

    process.stdout.on('data', (data) => {
      stdout += data.toString();
    });

    process.stderr.on('data', (data) => {
      stderr += data.toString();
    });

    process.on('error', (error) => {
      let errorString = error.toString();
      if (secret) errorString = errorString.replace(secret, "***");
      console.log(`Shell command FAILED: ${cmd}`, errorString);
      const e = new Error(
        `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
      );
      reject(e);
    });

    process.on('close', (code) => {
      if (stdout.length > 0) {
        console.log("Shell stdout:", stdout);
      }
      if (stderr.length > 0) {
        console.log("Shell stderr:", stderr);
      }
      if (code === 0) {
        console.log(`Shell command completed successfully: ${cmd}`);
        resolve(stdout);
      } else {
        reject(new Error(`Command failed with code ${code}: ${stderr}`));
      }
    });
  });
}


`1`	`import * as wmillclient from "windmill-client";`
`2`	`import { basename, join } from "node:path";`
`3`	`import { existsSync, rmSync } from "fs";`
`4`	`import process from "process";`
`5`	`import { spawn } from 'child_process';`
`6`	`import * as fs_async from 'fs/promises';`
`7`	`import * as fs from 'node:fs';`
`8`	`import { Readable } from 'node:stream';`
`9`	`import { pipeline } from 'node:stream/promises';`
`10`
`11`	`const UPLOAD_CONCURRENCY = 16;`
`12`	`const CLONE_MARKER_FILE = ".windmill_clone_complete";`
`13`
`14`	`type GitRepository = {`
`15`	`url: string;`
`16`	`branch: string;`
`17`	`folder: string;`
`18`	`gpg_key: any;`
`19`	`is_github_app: boolean;`
`20`	`};`
`21`
`22`	`export async function main(`
`23`	`resource_path: string,`
`24`	`workspace: string,`
`25`	`git_ssh_identity?: string[],`
`26`	`commit?: string`
`27`	`) {`
`28`	`let clonedRepoPath: string \| undefined;`
`29`
`30`	`try {`
`31`	`console.log("Starting git clone and Blob storage upload process");`
`32`
`33`	`// Get the git repository resource`
`34`	`const repo_resource: GitRepository = await wmillclient.getResource(resource_path);`
`35`
`36`	`const cwd = process.cwd();`
`37`
`38`	`if (git_ssh_identity) {`
`39`	`process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity)`
`40`	`}`
`41`
`42`	`process.env["HOME"] = ".";`
`43`	`process.env.GIT_TERMINAL_PROMPT = "0";`
`44`
`45`	`// App-backed repos carry no credential in their URL, so git can't`
`46`	`// authenticate against them. The server serves a snapshot of the commit`
`47`	`// instead, keeping the installation token on its side.`
`48`	`const { repo_name, commitHash } = repo_resource.is_github_app`
`49`	`? await download_repo_archive(cwd, workspace, resource_path, repo_resource, commit)`
`50`	`: await git_clone(cwd, repo_resource, commit);`
`51`	`clonedRepoPath = join(cwd, repo_name);`
`52`
`53`	`// Remove .git directory to avoid uploading git history`
`54`	`const gitDir = join(clonedRepoPath, ".git");`
`55`	`if (existsSync(gitDir)) {`
`56`	`rmSync(gitDir, { recursive: true, force: true });`
`57`	`console.log("Removed .git directory");`
`58`	`}`
`59`
`60`	`// Upload to S3`
`61`	const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`;
`62`	`const fileCount = await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace);`
`63`
`64`	`return {`
`65`	`success: true,`
`66`	`message: "Repository cloned and uploaded to S3 successfully",`
`67`	`s3_path: s3Path,`
`68`	`commit_hash: commitHash,`
`69`	`file_count: fileCount,`
`70`	`};`
`71`
`72`	`} catch (error) {`
`73`	`console.error("Error in git clone and upload:", error);`
`74`	`throw error;`
`75`	`} finally {`
`76`	`// Clean up cloned repository`
`77`	`if (clonedRepoPath && existsSync(clonedRepoPath)) {`
`78`	`rmSync(clonedRepoPath, { recursive: true, force: true });`
`79`	`console.log("Cleaned up cloned repository");`
`80`	`}`
`81`	`}`
`82`	`}`
`83`
`84`	`async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise<string> {`
`85`	`const sshIdFiles = await Promise.all(`
`86`	`git_ssh_identity.map(async (varPath, i) => {`
`87`	const filePath = join(cwd, `./ssh_id_priv_${i}`);
`88`
`89`	`try {`
`90`	`// Get variable value using windmill`
`91`	`let content = await wmillclient.getVariable(varPath);`
`92`	`content += '\n';`
`93`
`94`	`// Write file with content`
`95`	`await fs_async.writeFile(filePath, content, { encoding: 'utf8' });`
`96`
`97`	`// Set file permissions to 0o600 (read/write for owner only)`
`98`	`await fs_async.chmod(filePath, 0o600);`
`99`
`100`	`// Escape single quotes for shell command`
`101`	`const escapedPath = filePath.replace(/'/g, "'\\''");`
`102`	return ` -i '${escapedPath}'`;
`103`	`} catch (error) {`
`104`	`console.error(`
`105`	`Variable ${varPath} not found for git ssh identity: ${error}`
`106`	`);`
`107`	`return '';`
`108`	`}`
`109`	`})`
`110`	`);`
`111`
`112`	const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`;
`113`	`return gitSshCmd;`
`114`	`}`
`115`
`116`	`async function download_repo_archive(`
`117`	`cwd: string,`
`118`	`workspace: string,`
`119`	`resource_path: string,`
`120`	`repo_resource: GitRepository,`
`121`	`commit?: string,`
`122`	`): Promise<{ repo_name: string; commitHash: string }> {`
`123`	`const repo_name = basename(repo_resource.url, ".git");`
`124`	`const baseUrl =`
`125`	`process.env["BASE_INTERNAL_URL"] ??`
`126`	`process.env["BASE_URL"] ??`
`127`	`"http://localhost:8000";`
`128`	`const ref = commit ?? repo_resource.branch ?? "";`
`129`	const query = ref !== "" ? `?ref=${encodeURIComponent(ref)}` : "";
`130`	const url = `${baseUrl}/api/w/${workspace}/github_app/repo_archive/${resource_path}${query}`;
`131`
`132`	console.log(`Downloading archive of ${resource_path}${ref !== "" ? ` at ${ref}` : ""}`);
`133`	`const response = await fetch(url, {`
`134`	headers: { Authorization: `Bearer ${process.env["WM_TOKEN"]}` },
`135`	`});`
`136`	`if (!response.ok \|\| response.body == null) {`
`137`	`const errorBody = await response.text().catch(() => "");`
`138`	`throw new Error(`
`139`	`Repository archive error (${response.status}): ${errorBody \|\| response.statusText}`
`140`	`);`
`141`	`}`
`142`
`143`	`const commitHash = response.headers.get("x-commit-sha");`
`144`	`if (!commitHash) {`
`145`	`throw new Error("Repository archive carried no commit sha");`
`146`	`}`
`147`
`148`	const archivePath = join(cwd, `${repo_name}.tar.gz`);
`149`	`await pipeline(`
`150`	`Readable.fromWeb(response.body as any),`
`151`	`fs.createWriteStream(archivePath)`
`152`	`);`
`153`
`154`	`const repoPath = join(cwd, repo_name);`
`155`	`await fs_async.mkdir(repoPath, { recursive: true });`
`156`	// Every entry sits under a single `{owner}-{repo}-{sha}` directory.
`157`	`await runCommand(undefined, "tar", "-xzf", archivePath, "-C", repoPath, "--strip-components=1");`
`158`	`rmSync(archivePath, { force: true });`
`159`
`160`	`const subfolder = repo_resource.folder ?? "";`
`161`	`if (subfolder !== "") {`
`162`	`if (!existsSync(join(repoPath, subfolder))) {`
`163`	throw new Error(`Subfolder ${subfolder} does not exist.`);
`164`	`}`
`165`	`// Keep what the sparse checkout of the credentialed path would: the root`
`166`	`// files, plus the tree the subfolder lives in.`
`167`	`const keep = subfolder.split("/")[0];`
`168`	`for (const entry of fs.readdirSync(repoPath, { withFileTypes: true })) {`
`169`	`if (entry.isDirectory() && entry.name !== keep) {`
`170`	`rmSync(join(repoPath, entry.name), { recursive: true, force: true });`
`171`	`}`
`172`	`}`
`173`	`}`
`174`
`175`	`return { repo_name, commitHash };`
`176`	`}`
`177`
`178`	`async function git_clone(`
`179`	`cwd: string,`
`180`	`repo_resource: GitRepository,`
`181`	`commit?: string,`
`182`	`): Promise<{ repo_name: string; commitHash: string }> {`
`183`	`if (commit) {`
`184`	`return git_clone_at_commit(cwd, repo_resource, commit);`
`185`	`} else {`
`186`	`return git_clone_at_latest(cwd, repo_resource);`
`187`	`}`
`188`	`}`
`189`
`190`	`async function git_clone_at_commit(`
`191`	`cwd: string,`
`192`	`repo_resource: GitRepository,`
`193`	`commit: string,`
`194`	`): Promise<{ repo_name: string; commitHash: string }> {`
`195`	`let repo_url = repo_resource.url;`
`196`	`const subfolder = repo_resource.folder ?? "";`
`197`	`let branch = repo_resource.branch ?? "";`
`198`	`const repo_name = basename(repo_url, ".git");`
`199`
`200`	`const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);`
`201`	`if (azureMatch) {`
`202`	`console.log("Fetching Azure DevOps access token...");`
`203`	`const azureResource = await wmillclient.getResource(azureMatch.groups.url);`
`204`	`const response = await fetch(`
`205`	`https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
`206`	`{`
`207`	`method: "POST",`
`208`	`body: new URLSearchParams({`
`209`	`client_id: azureResource.azureClientId,`
`210`	`client_secret: azureResource.azureClientSecret,`
`211`	`grant_type: "client_credentials",`
`212`	`resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",`
`213`	`}),`
`214`	`}`
`215`	`);`
`216`	`const { access_token } = await response.json();`
`217`	`repo_url = repo_url.replace(azureMatch[0], access_token);`
`218`	`}`
`219`
`220`	`const repoPath = join(cwd, repo_name);`
`221`	`await fs_async.mkdir(repoPath, { recursive: true });`
`222`
`223`	`process.chdir(repoPath);`
`224`
`225`	`let args = ['init', '--quiet']`
`226`	`if (branch) {`
`227`	args.push(`--initial-branch=${branch}`)
`228`	`}`
`229`	`await runCommand(undefined, 'git', ...args);`
`230`
`231`	`await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url);`
`232`
`233`	`await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit);`
`234`
`235`	`await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD');`
`236`
`237`	`const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();`
`238`
`239`	`// Return to original directory`
`240`	`process.chdir(cwd);`
`241`
`242`	`return { repo_name, commitHash };`
`243`	`}`
`244`
`245`	`async function git_clone_at_latest(`
`246`	`cwd: string,`
`247`	`repo_resource: GitRepository`
`248`	`): Promise<{ repo_name: string; commitHash: string }> {`
`249`	`let repo_url = repo_resource.url;`
`250`	`const subfolder = repo_resource.folder ?? "";`
`251`	`let branch = repo_resource.branch ?? "";`
`252`	`const repo_name = basename(repo_url, ".git");`
`253`
`254`	`// Handle Azure DevOps token if needed`
`255`	`const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);`
`256`	`if (azureMatch) {`
`257`	`console.log("Fetching Azure DevOps access token...");`
`258`	`const azureResource = await wmillclient.getResource(azureMatch.groups.url);`
`259`	`const response = await fetch(`
`260`	`https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
`261`	`{`
`262`	`method: "POST",`
`263`	`body: new URLSearchParams({`
`264`	`client_id: azureResource.azureClientId,`
`265`	`client_secret: azureResource.azureClientSecret,`
`266`	`grant_type: "client_credentials",`
`267`	`resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",`
`268`	`}),`
`269`	`}`
`270`	`);`
`271`	`const { access_token } = await response.json();`
`272`	`repo_url = repo_url.replace(azureMatch[0], access_token);`
`273`	`}`
`274`
`275`	`const args = ["clone", "--quiet", "--depth", "1"];`
`276`	`if (subfolder !== "") args.push("--sparse");`
`277`	`if (branch !== "") args.push("--branch", branch);`
`278`	`args.push(repo_url, repo_name);`
`279`
`280`	`await runCommand(-1, "git", ...args);`
`281`
`282`	`const fullPath = join(cwd, repo_name);`
`283`	`process.chdir(fullPath);`
`284`
`285`	`if (subfolder !== "") {`
`286`	`await runCommand(undefined, "git", "sparse-checkout", "add", subfolder);`
`287`	`const subfolderPath = join(fullPath, subfolder);`
`288`
`289`	`if (!existsSync(subfolderPath)) {`
`290`	throw new Error(`Subfolder ${subfolder} does not exist.`);
`291`	`}`
`292`
`293`	`process.chdir(subfolderPath);`
`294`	`}`
`295`
`296`	`// Get the commit hash`
`297`	`const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();`
`298`
`299`	`// Return to original directory`
`300`	`process.chdir(cwd);`
`301`
`302`	`return { repo_name, commitHash };`
`303`	`}`
`304`
`305`	`async function uploadDirectoryToS3(`
`306`	`directoryPath: string,`
`307`	`s3BasePath: string,`
`308`	`workspace: string,`
`309`	`): Promise<number> {`
`310`	console.log(`Uploading ${directoryPath} -> ${s3BasePath}`);
`311`
`312`	`// Walk once into a flat task list so we can drive a bounded-concurrency pool.`
`313`	`const tasks: { localPath: string; s3Key: string }[] = [];`
`314`	`function walk(dir: string, s3Path: string) {`
`315`	`for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {`
`316`	`const fullPath = join(dir, entry.name);`
`317`	const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name;
`318`	`if (entry.isDirectory()) {`
`319`	`walk(fullPath, s3Key);`
`320`	`} else if (entry.isFile()) {`
`321`	`tasks.push({ localPath: fullPath, s3Key });`
`322`	`}`
`323`	`}`
`324`	`}`
`325`	`walk(directoryPath, s3BasePath);`
`326`	console.log(`Discovered ${tasks.length} files to upload`);
`327`
`328`	`let nextIndex = 0;`
`329`	`let uploaded = 0;`
`330`	`let lastReport = 0;`
`331`	`async function worker() {`
`332`	`while (true) {`
`333`	`const idx = nextIndex++;`
`334`	`if (idx >= tasks.length) return;`
`335`	`const { localPath, s3Key } = tasks[idx];`
`336`	`const fileContent = fs.readFileSync(localPath);`
`337`	`const blob = new Blob([fileContent], { type: 'application/octet-stream' });`
`338`	`await wmillclient.HelpersService.gitRepoViewerFileUpload({`
`339`	`workspace,`
`340`	`fileKey: s3Key,`
`341`	`requestBody: blob,`
`342`	`});`
`343`	`uploaded++;`
`344`	`if (uploaded - lastReport >= 25 \|\| uploaded === tasks.length) {`
`345`	`lastReport = uploaded;`
`346`	console.log(`Uploaded ${uploaded} / ${tasks.length} files`);
`347`	`}`
`348`	`}`
`349`	`}`
`350`	`await Promise.all(`
`351`	`Array.from({ length: Math.min(UPLOAD_CONCURRENCY, tasks.length) }, () => worker())`
`352`	`);`
`353`
`354`	`// Marker is the LAST write — its presence is what the viewer checks for.`
`355`	const markerKey = `${s3BasePath}/${CLONE_MARKER_FILE}`;
`356`	`const markerBody = JSON.stringify({`
`357`	`completed_at: new Date().toISOString(),`
`358`	`file_count: tasks.length,`
`359`	`});`
`360`	`await wmillclient.HelpersService.gitRepoViewerFileUpload({`
`361`	`workspace,`
`362`	`fileKey: markerKey,`
`363`	`requestBody: new Blob([markerBody], { type: 'application/json' }),`
`364`	`});`
`365`	console.log(`Wrote completion marker: ${markerKey}`);
`366`
`367`	`return tasks.length;`
`368`	`}`
`369`
`370`	`function runCommand(secret_position: number \| undefined, cmd: string, ...args: string[]): Promise<string> {`
`371`	`const nargs = secret_position != undefined ? args.slice() : args;`
`372`	`if (secret_position && secret_position < 0)`
`373`	`secret_position = nargs.length - 1 + secret_position;`
`374`
`375`	`let secret: string \| undefined = undefined;`
`376`	`if (secret_position != undefined) {`
`377`	`nargs[secret_position] = "***";`
`378`	`secret = args[secret_position];`
`379`	`}`
`380`	console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);
`381`
`382`	`return new Promise((resolve, reject) => {`
`383`	`const process = spawn(cmd, args);`
`384`
`385`	`let stdout = '';`
`386`	`let stderr = '';`
`387`
`388`	`process.stdout.on('data', (data) => {`
`389`	`stdout += data.toString();`
`390`	`});`
`391`
`392`	`process.stderr.on('data', (data) => {`
`393`	`stderr += data.toString();`
`394`	`});`
`395`
`396`	`process.on('error', (error) => {`
`397`	`let errorString = error.toString();`
`398`	`if (secret) errorString = errorString.replace(secret, "***");`
`399`	console.log(`Shell command FAILED: ${cmd}`, errorString);
`400`	`const e = new Error(`
`401`	`SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
`402`	`);`
`403`	`reject(e);`
`404`	`});`
`405`
`406`	`process.on('close', (code) => {`
`407`	`if (stdout.length > 0) {`
`408`	`console.log("Shell stdout:", stdout);`
`409`	`}`
`410`	`if (stderr.length > 0) {`
`411`	`console.log("Shell stderr:", stderr);`
`412`	`}`
`413`	`if (code === 0) {`
`414`	console.log(`Shell command completed successfully: ${cmd}`);
`415`	`resolve(stdout);`
`416`	`} else {`
`417`	reject(new Error(`Command failed with code ${code}: ${stderr}`));
`418`	`}`
`419`	`});`
`420`	`});`
`421`	`}`
`422`