Ctrl K

clone repo and upload to instance storage

Published Oct 14, 2025

Clones a github repo defined in a git_repository resource, then uploads the files to the instance storage for them to be cached

Script windmill Verified

Use in Windmill

The script

Submitted by hugo697 Bun

Verified 7 days ago

All edits

Permalink

import * as wmillclient from "windmill-client";
import { basename, join, sep } from "node:path";
import { existsSync, rmSync } from "fs";
import process from "process";
import { spawn } from 'child_process';
import * as fs_async from 'fs/promises';
import * as fs from 'node:fs';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';

const UPLOAD_CONCURRENCY = 16;
const CLONE_MARKER_FILE = ".windmill_clone_complete";
const MAX_SYMLINKED_ENTRIES = 20_000;
const MAX_SYMLINKED_BYTES = 512 * 1024 * 1024;

type GitRepository = {
  url: string;
  branch: string;
  folder: string;
  gpg_key: any;
  is_github_app: boolean;
};

export async function main(
  resource_path: string,
  workspace: string,
  git_ssh_identity?: string[],
  commit?: string
) {
  let clonedRepoPath: string | undefined;

  try {
    console.log("Starting git clone and Blob storage upload process");

    // Get the git repository resource
    const repo_resource: GitRepository = await wmillclient.getResource(resource_path);

    const cwd = process.cwd();

    if (git_ssh_identity) {
      process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity)
    }

    // Absolute: a relative HOME follows the chdir into the checkout, and git
    // then reads the repository's own `.gitconfig` as its global config.
    process.env["HOME"] = cwd;
    process.env.GIT_TERMINAL_PROMPT = "0";

    // App-backed repos carry no credential in their URL, so git can't
    // authenticate against them. The server serves a snapshot of the commit
    // instead, keeping the installation token on its side.
    const { repo_name, commitHash } = repo_resource.is_github_app
      ? await download_repo_archive(cwd, workspace, resource_path, repo_resource, commit)
      : await git_clone(cwd, repo_resource, commit);
    clonedRepoPath = join(cwd, repo_name);

    // Remove .git directory to avoid uploading git history
    const gitDir = join(clonedRepoPath, ".git");
    if (existsSync(gitDir)) {
      rmSync(gitDir, { recursive: true, force: true });
      console.log("Removed .git directory");
    }

    // Upload to S3
    const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`;
    const fileCount = await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace);

    return {
      success: true,
      message: "Repository cloned and uploaded to S3 successfully",
      s3_path: s3Path,
      commit_hash: commitHash,
      file_count: fileCount,
    };

  } catch (error) {
    console.error("Error in git clone and upload:", error);
    throw error;
  } finally {
    // Clean up cloned repository
    if (clonedRepoPath && existsSync(clonedRepoPath)) {
      rmSync(clonedRepoPath, { recursive: true, force: true });
      console.log("Cleaned up cloned repository");
    }
  }
}

async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise<string> {
  const sshIdFiles = await Promise.all(
    git_ssh_identity.map(async (varPath, i) => {
      const filePath = join(cwd, `./ssh_id_priv_${i}`);

      try {
        // Get variable value using windmill
        let content = await wmillclient.getVariable(varPath);
        content += '\n';

        // Write file with content
        await fs_async.writeFile(filePath, content, { encoding: 'utf8' });

        // Set file permissions to 0o600 (read/write for owner only)
        await fs_async.chmod(filePath, 0o600);

        // Escape single quotes for shell command
        const escapedPath = filePath.replace(/'/g, "'\\''");
        return ` -i '${escapedPath}'`;
      } catch (error) {
        console.error(
          `Variable ${varPath} not found for git ssh identity: ${error}`
        );
        return '';
      }
    })
  );

  const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`;
  return gitSshCmd;
}

async function download_repo_archive(
  cwd: string,
  workspace: string,
  resource_path: string,
  repo_resource: GitRepository,
  commit?: string,
): Promise<{ repo_name: string; commitHash: string }> {
  const repo_name = basename(repo_resource.url, ".git");
  const baseUrl =
    process.env["BASE_INTERNAL_URL"] ??
    process.env["BASE_URL"] ??
    "http://localhost:8000";
  const ref = commit ?? repo_resource.branch ?? "";
  const query = ref !== "" ? `?ref=${encodeURIComponent(ref)}` : "";
  const url = `${baseUrl}/api/w/${workspace}/github_app/repo_archive/${resource_path}${query}`;

  console.log(`Downloading archive of ${resource_path}${ref !== "" ? ` at ${ref}` : ""}`);
  const response = await fetch(url, {
    headers: { Authorization: `Bearer ${process.env["WM_TOKEN"]}` },
  });
  if (!response.ok || response.body == null) {
    const errorBody = await response.text().catch(() => "");
    throw new Error(
      `Repository archive error (${response.status}): ${errorBody || response.statusText}`
    );
  }

  const commitHash = response.headers.get("x-commit-sha");
  if (!commitHash) {
    throw new Error("Repository archive carried no commit sha");
  }

  const archivePath = join(cwd, `${repo_name}.tar.gz`);
  await pipeline(
    Readable.fromWeb(response.body as any),
    fs.createWriteStream(archivePath)
  );

  const repoPath = join(cwd, repo_name);
  await fs_async.mkdir(repoPath, { recursive: true });
  // Every entry sits under a single `{owner}-{repo}-{sha}` directory.
  await runCommand(undefined, "tar", "-xzf", archivePath, "-C", repoPath, "--strip-components=1");
  rmSync(archivePath, { force: true });

  const subfolder = repo_resource.folder ?? "";
  if (subfolder !== "") {
    if (!existsSync(join(repoPath, subfolder))) {
      throw new Error(`Subfolder ${subfolder} does not exist.`);
    }
    // Keep what the sparse checkout of the credentialed path would: the root
    // files, plus the tree the subfolder lives in.
    const keep = subfolder.split("/")[0];
    for (const entry of fs.readdirSync(repoPath, { withFileTypes: true })) {
      if (entry.isDirectory() && entry.name !== keep) {
        rmSync(join(repoPath, entry.name), { recursive: true, force: true });
      }
    }
  }

  return { repo_name, commitHash };
}

async function git_clone(
  cwd: string,
  repo_resource: GitRepository,
  commit?: string,
): Promise<{ repo_name: string; commitHash: string }> {
    if (commit) {
      return git_clone_at_commit(cwd, repo_resource, commit);
    } else {
      return git_clone_at_latest(cwd, repo_resource);
    }
}

async function git_clone_at_commit(
  cwd: string,
  repo_resource: GitRepository,
  commit: string,
): Promise<{ repo_name: string; commitHash: string }> {
  let repo_url = repo_resource.url;
  const subfolder = repo_resource.folder ?? "";
  let branch = repo_resource.branch ?? "";
  const repo_name = basename(repo_url, ".git");

  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
  if (azureMatch) {
    console.log("Fetching Azure DevOps access token...");
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    const response = await fetch(
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
      {
        method: "POST",
        body: new URLSearchParams({
          client_id: azureResource.azureClientId,
          client_secret: azureResource.azureClientSecret,
          grant_type: "client_credentials",
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
        }),
      }
    );
    const { access_token } = await response.json();
    repo_url = repo_url.replace(azureMatch[0], access_token);
  }

  const repoPath = join(cwd, repo_name);
  await fs_async.mkdir(repoPath, { recursive: true });

  process.chdir(repoPath);

  let args = ['init', '--quiet']
  if (branch) {
    args.push(`--initial-branch=${branch}`)
  }
  await runCommand(undefined, 'git', ...args);

  await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url);

  await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit);

  await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD');

  const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();

  // Return to original directory
  process.chdir(cwd);

  return { repo_name, commitHash };
}

async function git_clone_at_latest(
  cwd: string,
  repo_resource: GitRepository
): Promise<{ repo_name: string; commitHash: string }> {
  let repo_url = repo_resource.url;
  const subfolder = repo_resource.folder ?? "";
  let branch = repo_resource.branch ?? "";
  const repo_name = basename(repo_url, ".git");

  // Handle Azure DevOps token if needed
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
  if (azureMatch) {
    console.log("Fetching Azure DevOps access token...");
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
    const response = await fetch(
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
      {
        method: "POST",
        body: new URLSearchParams({
          client_id: azureResource.azureClientId,
          client_secret: azureResource.azureClientSecret,
          grant_type: "client_credentials",
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
        }),
      }
    );
    const { access_token } = await response.json();
    repo_url = repo_url.replace(azureMatch[0], access_token);
  }

  const args = ["clone", "--quiet", "--depth", "1"];
  if (subfolder !== "") args.push("--sparse");
  if (branch !== "") args.push("--branch", branch);
  args.push(repo_url, repo_name);

  await runCommand(-1, "git", ...args);

  const fullPath = join(cwd, repo_name);
  process.chdir(fullPath);

  if (subfolder !== "") {
    await runCommand(undefined, "git", "sparse-checkout", "add", subfolder);
    const subfolderPath = join(fullPath, subfolder);

    if (!existsSync(subfolderPath)) {
      throw new Error(`Subfolder ${subfolder} does not exist.`);
    }

    process.chdir(subfolderPath);
  }

  // Get the commit hash
  const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();

  // Return to original directory
  process.chdir(cwd);

  return { repo_name, commitHash };
}

async function uploadDirectoryToS3(
  directoryPath: string,
  s3BasePath: string,
  workspace: string,
): Promise<number> {
  console.log(`Uploading ${directoryPath} -> ${s3BasePath}`);

  // Walk once into a flat task list so we can drive a bounded-concurrency pool.
  const tasks: { localPath: string; s3Key: string }[] = [];
  const root = fs.realpathSync(directoryPath);
  // Real paths of the directories being descended through: a link back up to
  // one of them would otherwise recurse forever.
  const ancestors = new Set<string>();
  // Links can reach one directory along many paths, so a small repository can
  // expand exponentially through them. Every entry reached through a link is
  // charged before it is resolved, skipped ones included, and past either budget
  // the rest are skipped; the checkout's own files are always uploaded.
  let symlinkedEntries = 0;
  let symlinkedBytes = 0;
  let symlinkBudgetSpent = false;
  function chargeSymlinkBudget(relPath: string, entries: number, bytes: number): boolean {
    if (symlinkBudgetSpent) return false;
    symlinkedEntries += entries;
    symlinkedBytes += bytes;
    if (symlinkedEntries <= MAX_SYMLINKED_ENTRIES && symlinkedBytes <= MAX_SYMLINKED_BYTES) {
      return true;
    }
    symlinkBudgetSpent = true;
    console.log(
      `Skipping ${relPath} and every symlinked entry after it: symlinks reach more than ` +
      `${MAX_SYMLINKED_ENTRIES} entries or ${MAX_SYMLINKED_BYTES / 2 ** 20} MiB`
    );
    return false;
  }
  function walk(dir: string, relDir: string, viaLink: boolean) {
    ancestors.add(dir);
    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
      const relPath = relDir ? `${relDir}/${entry.name}` : entry.name;
      const linked = viaLink || entry.isSymbolicLink();
      if (linked && !chargeSymlinkBudget(relPath, 1, 0)) continue;
      let localPath = join(dir, entry.name);
      if (entry.isSymbolicLink()) {
        // The repository chooses the target, so only follow links that stay in
        // the checkout: one to `/proc/self/environ` or to the ssh key written
        // next to it would put that file in front of every reader of the resource.
        const link = fs.readlinkSync(localPath);
        try {
          localPath = fs.realpathSync(localPath);
        } catch (e: any) {
          console.log(`Skipping symlink ${relPath} -> ${link}: cannot resolve target (${e.code})`);
          continue;
        }
        if (localPath !== root && !localPath.startsWith(root + sep)) {
          console.log(`Skipping symlink ${relPath} -> ${link}: target is outside the repository`);
          continue;
        }
      }
      const stat = fs.statSync(localPath);
      if (stat.isDirectory() && ancestors.has(localPath)) {
        console.log(`Skipping ${relPath}: links back to a directory it is inside`);
        continue;
      }
      if (linked && stat.isFile() && !chargeSymlinkBudget(relPath, 0, stat.size)) continue;
      if (stat.isDirectory()) {
        walk(localPath, relPath, linked);
      } else if (stat.isFile()) {
        tasks.push({ localPath, s3Key: `${s3BasePath}/${relPath}` });
      }
    }
    ancestors.delete(dir);
  }
  walk(root, "", false);
  console.log(`Discovered ${tasks.length} files to upload`);

  let nextIndex = 0;
  let uploaded = 0;
  let lastReport = 0;
  async function worker() {
    while (true) {
      const idx = nextIndex++;
      if (idx >= tasks.length) return;
      const { localPath, s3Key } = tasks[idx];
      const fileContent = fs.readFileSync(localPath);
      const blob = new Blob([fileContent], { type: 'application/octet-stream' });
      await wmillclient.HelpersService.gitRepoViewerFileUpload({
        workspace,
        fileKey: s3Key,
        requestBody: blob,
      });
      uploaded++;
      if (uploaded - lastReport >= 25 || uploaded === tasks.length) {
        lastReport = uploaded;
        console.log(`Uploaded ${uploaded} / ${tasks.length} files`);
      }
    }
  }
  await Promise.all(
    Array.from({ length: Math.min(UPLOAD_CONCURRENCY, tasks.length) }, () => worker())
  );

  // Marker is the LAST write — its presence is what the viewer checks for.
  const markerKey = `${s3BasePath}/${CLONE_MARKER_FILE}`;
  const markerBody = JSON.stringify({
    completed_at: new Date().toISOString(),
    file_count: tasks.length,
  });
  await wmillclient.HelpersService.gitRepoViewerFileUpload({
    workspace,
    fileKey: markerKey,
    requestBody: new Blob([markerBody], { type: 'application/json' }),
  });
  console.log(`Wrote completion marker: ${markerKey}`);

  return tasks.length;
}

function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise<string> {
  const nargs = secret_position != undefined ? args.slice() : args;
  if (secret_position && secret_position < 0)
    secret_position = nargs.length - 1 + secret_position;

  let secret: string | undefined = undefined;
  if (secret_position != undefined) {
    nargs[secret_position] = "***";
    secret = args[secret_position];
  }
  console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);

  return new Promise((resolve, reject) => {
    const process = spawn(cmd, args);

    let stdout = '';
    let stderr = '';

    process.stdout.on('data', (data) => {
      stdout += data.toString();
    });

    process.stderr.on('data', (data) => {
      stderr += data.toString();
    });

    process.on('error', (error) => {
      let errorString = error.toString();
      if (secret) errorString = errorString.replace(secret, "***");
      console.log(`Shell command FAILED: ${cmd}`, errorString);
      const e = new Error(
        `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
      );
      reject(e);
    });

    process.on('close', (code) => {
      if (stdout.length > 0) {
        console.log("Shell stdout:", stdout);
      }
      if (stderr.length > 0) {
        console.log("Shell stderr:", stderr);
      }
      if (code === 0) {
        console.log(`Shell command completed successfully: ${cmd}`);
        resolve(stdout);
      } else {
        reject(new Error(`Command failed with code ${code}: ${stderr}`));
      }
    });
  });
}


`1`	`import * as wmillclient from "windmill-client";`
`2`	`import { basename, join, sep } from "node:path";`
`3`	`import { existsSync, rmSync } from "fs";`
`4`	`import process from "process";`
`5`	`import { spawn } from 'child_process';`
`6`	`import * as fs_async from 'fs/promises';`
`7`	`import * as fs from 'node:fs';`
`8`	`import { Readable } from 'node:stream';`
`9`	`import { pipeline } from 'node:stream/promises';`
`10`
`11`	`const UPLOAD_CONCURRENCY = 16;`
`12`	`const CLONE_MARKER_FILE = ".windmill_clone_complete";`
`13`	`const MAX_SYMLINKED_ENTRIES = 20_000;`
`14`	`const MAX_SYMLINKED_BYTES = 512 * 1024 * 1024;`
`15`
`16`	`type GitRepository = {`
`17`	`url: string;`
`18`	`branch: string;`
`19`	`folder: string;`
`20`	`gpg_key: any;`
`21`	`is_github_app: boolean;`
`22`	`};`
`23`
`24`	`export async function main(`
`25`	`resource_path: string,`
`26`	`workspace: string,`
`27`	`git_ssh_identity?: string[],`
`28`	`commit?: string`
`29`	`) {`
`30`	`let clonedRepoPath: string \| undefined;`
`31`
`32`	`try {`
`33`	`console.log("Starting git clone and Blob storage upload process");`
`34`
`35`	`// Get the git repository resource`
`36`	`const repo_resource: GitRepository = await wmillclient.getResource(resource_path);`
`37`
`38`	`const cwd = process.cwd();`
`39`
`40`	`if (git_ssh_identity) {`
`41`	`process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity)`
`42`	`}`
`43`
`44`	`// Absolute: a relative HOME follows the chdir into the checkout, and git`
`45`	// then reads the repository's own `.gitconfig` as its global config.
`46`	`process.env["HOME"] = cwd;`
`47`	`process.env.GIT_TERMINAL_PROMPT = "0";`
`48`
`49`	`// App-backed repos carry no credential in their URL, so git can't`
`50`	`// authenticate against them. The server serves a snapshot of the commit`
`51`	`// instead, keeping the installation token on its side.`
`52`	`const { repo_name, commitHash } = repo_resource.is_github_app`
`53`	`? await download_repo_archive(cwd, workspace, resource_path, repo_resource, commit)`
`54`	`: await git_clone(cwd, repo_resource, commit);`
`55`	`clonedRepoPath = join(cwd, repo_name);`
`56`
`57`	`// Remove .git directory to avoid uploading git history`
`58`	`const gitDir = join(clonedRepoPath, ".git");`
`59`	`if (existsSync(gitDir)) {`
`60`	`rmSync(gitDir, { recursive: true, force: true });`
`61`	`console.log("Removed .git directory");`
`62`	`}`
`63`
`64`	`// Upload to S3`
`65`	const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`;
`66`	`const fileCount = await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace);`
`67`
`68`	`return {`
`69`	`success: true,`
`70`	`message: "Repository cloned and uploaded to S3 successfully",`
`71`	`s3_path: s3Path,`
`72`	`commit_hash: commitHash,`
`73`	`file_count: fileCount,`
`74`	`};`
`75`
`76`	`} catch (error) {`
`77`	`console.error("Error in git clone and upload:", error);`
`78`	`throw error;`
`79`	`} finally {`
`80`	`// Clean up cloned repository`
`81`	`if (clonedRepoPath && existsSync(clonedRepoPath)) {`
`82`	`rmSync(clonedRepoPath, { recursive: true, force: true });`
`83`	`console.log("Cleaned up cloned repository");`
`84`	`}`
`85`	`}`
`86`	`}`
`87`
`88`	`async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise<string> {`
`89`	`const sshIdFiles = await Promise.all(`
`90`	`git_ssh_identity.map(async (varPath, i) => {`
`91`	const filePath = join(cwd, `./ssh_id_priv_${i}`);
`92`
`93`	`try {`
`94`	`// Get variable value using windmill`
`95`	`let content = await wmillclient.getVariable(varPath);`
`96`	`content += '\n';`
`97`
`98`	`// Write file with content`
`99`	`await fs_async.writeFile(filePath, content, { encoding: 'utf8' });`
`100`
`101`	`// Set file permissions to 0o600 (read/write for owner only)`
`102`	`await fs_async.chmod(filePath, 0o600);`
`103`
`104`	`// Escape single quotes for shell command`
`105`	`const escapedPath = filePath.replace(/'/g, "'\\''");`
`106`	return ` -i '${escapedPath}'`;
`107`	`} catch (error) {`
`108`	`console.error(`
`109`	`Variable ${varPath} not found for git ssh identity: ${error}`
`110`	`);`
`111`	`return '';`
`112`	`}`
`113`	`})`
`114`	`);`
`115`
`116`	const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`;
`117`	`return gitSshCmd;`
`118`	`}`
`119`
`120`	`async function download_repo_archive(`
`121`	`cwd: string,`
`122`	`workspace: string,`
`123`	`resource_path: string,`
`124`	`repo_resource: GitRepository,`
`125`	`commit?: string,`
`126`	`): Promise<{ repo_name: string; commitHash: string }> {`
`127`	`const repo_name = basename(repo_resource.url, ".git");`
`128`	`const baseUrl =`
`129`	`process.env["BASE_INTERNAL_URL"] ??`
`130`	`process.env["BASE_URL"] ??`
`131`	`"http://localhost:8000";`
`132`	`const ref = commit ?? repo_resource.branch ?? "";`
`133`	const query = ref !== "" ? `?ref=${encodeURIComponent(ref)}` : "";
`134`	const url = `${baseUrl}/api/w/${workspace}/github_app/repo_archive/${resource_path}${query}`;
`135`
`136`	console.log(`Downloading archive of ${resource_path}${ref !== "" ? ` at ${ref}` : ""}`);
`137`	`const response = await fetch(url, {`
`138`	headers: { Authorization: `Bearer ${process.env["WM_TOKEN"]}` },
`139`	`});`
`140`	`if (!response.ok \|\| response.body == null) {`
`141`	`const errorBody = await response.text().catch(() => "");`
`142`	`throw new Error(`
`143`	`Repository archive error (${response.status}): ${errorBody \|\| response.statusText}`
`144`	`);`
`145`	`}`
`146`
`147`	`const commitHash = response.headers.get("x-commit-sha");`
`148`	`if (!commitHash) {`
`149`	`throw new Error("Repository archive carried no commit sha");`
`150`	`}`
`151`
`152`	const archivePath = join(cwd, `${repo_name}.tar.gz`);
`153`	`await pipeline(`
`154`	`Readable.fromWeb(response.body as any),`
`155`	`fs.createWriteStream(archivePath)`
`156`	`);`
`157`
`158`	`const repoPath = join(cwd, repo_name);`
`159`	`await fs_async.mkdir(repoPath, { recursive: true });`
`160`	// Every entry sits under a single `{owner}-{repo}-{sha}` directory.
`161`	`await runCommand(undefined, "tar", "-xzf", archivePath, "-C", repoPath, "--strip-components=1");`
`162`	`rmSync(archivePath, { force: true });`
`163`
`164`	`const subfolder = repo_resource.folder ?? "";`
`165`	`if (subfolder !== "") {`
`166`	`if (!existsSync(join(repoPath, subfolder))) {`
`167`	throw new Error(`Subfolder ${subfolder} does not exist.`);
`168`	`}`
`169`	`// Keep what the sparse checkout of the credentialed path would: the root`
`170`	`// files, plus the tree the subfolder lives in.`
`171`	`const keep = subfolder.split("/")[0];`
`172`	`for (const entry of fs.readdirSync(repoPath, { withFileTypes: true })) {`
`173`	`if (entry.isDirectory() && entry.name !== keep) {`
`174`	`rmSync(join(repoPath, entry.name), { recursive: true, force: true });`
`175`	`}`
`176`	`}`
`177`	`}`
`178`
`179`	`return { repo_name, commitHash };`
`180`	`}`
`181`
`182`	`async function git_clone(`
`183`	`cwd: string,`
`184`	`repo_resource: GitRepository,`
`185`	`commit?: string,`
`186`	`): Promise<{ repo_name: string; commitHash: string }> {`
`187`	`if (commit) {`
`188`	`return git_clone_at_commit(cwd, repo_resource, commit);`
`189`	`} else {`
`190`	`return git_clone_at_latest(cwd, repo_resource);`
`191`	`}`
`192`	`}`
`193`
`194`	`async function git_clone_at_commit(`
`195`	`cwd: string,`
`196`	`repo_resource: GitRepository,`
`197`	`commit: string,`
`198`	`): Promise<{ repo_name: string; commitHash: string }> {`
`199`	`let repo_url = repo_resource.url;`
`200`	`const subfolder = repo_resource.folder ?? "";`
`201`	`let branch = repo_resource.branch ?? "";`
`202`	`const repo_name = basename(repo_url, ".git");`
`203`
`204`	`const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);`
`205`	`if (azureMatch) {`
`206`	`console.log("Fetching Azure DevOps access token...");`
`207`	`const azureResource = await wmillclient.getResource(azureMatch.groups.url);`
`208`	`const response = await fetch(`
`209`	`https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
`210`	`{`
`211`	`method: "POST",`
`212`	`body: new URLSearchParams({`
`213`	`client_id: azureResource.azureClientId,`
`214`	`client_secret: azureResource.azureClientSecret,`
`215`	`grant_type: "client_credentials",`
`216`	`resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",`
`217`	`}),`
`218`	`}`
`219`	`);`
`220`	`const { access_token } = await response.json();`
`221`	`repo_url = repo_url.replace(azureMatch[0], access_token);`
`222`	`}`
`223`
`224`	`const repoPath = join(cwd, repo_name);`
`225`	`await fs_async.mkdir(repoPath, { recursive: true });`
`226`
`227`	`process.chdir(repoPath);`
`228`
`229`	`let args = ['init', '--quiet']`
`230`	`if (branch) {`
`231`	args.push(`--initial-branch=${branch}`)
`232`	`}`
`233`	`await runCommand(undefined, 'git', ...args);`
`234`
`235`	`await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url);`
`236`
`237`	`await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit);`
`238`
`239`	`await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD');`
`240`
`241`	`const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();`
`242`
`243`	`// Return to original directory`
`244`	`process.chdir(cwd);`
`245`
`246`	`return { repo_name, commitHash };`
`247`	`}`
`248`
`249`	`async function git_clone_at_latest(`
`250`	`cwd: string,`
`251`	`repo_resource: GitRepository`
`252`	`): Promise<{ repo_name: string; commitHash: string }> {`
`253`	`let repo_url = repo_resource.url;`
`254`	`const subfolder = repo_resource.folder ?? "";`
`255`	`let branch = repo_resource.branch ?? "";`
`256`	`const repo_name = basename(repo_url, ".git");`
`257`
`258`	`// Handle Azure DevOps token if needed`
`259`	`const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);`
`260`	`if (azureMatch) {`
`261`	`console.log("Fetching Azure DevOps access token...");`
`262`	`const azureResource = await wmillclient.getResource(azureMatch.groups.url);`
`263`	`const response = await fetch(`
`264`	`https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
`265`	`{`
`266`	`method: "POST",`
`267`	`body: new URLSearchParams({`
`268`	`client_id: azureResource.azureClientId,`
`269`	`client_secret: azureResource.azureClientSecret,`
`270`	`grant_type: "client_credentials",`
`271`	`resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",`
`272`	`}),`
`273`	`}`
`274`	`);`
`275`	`const { access_token } = await response.json();`
`276`	`repo_url = repo_url.replace(azureMatch[0], access_token);`
`277`	`}`
`278`
`279`	`const args = ["clone", "--quiet", "--depth", "1"];`
`280`	`if (subfolder !== "") args.push("--sparse");`
`281`	`if (branch !== "") args.push("--branch", branch);`
`282`	`args.push(repo_url, repo_name);`
`283`
`284`	`await runCommand(-1, "git", ...args);`
`285`
`286`	`const fullPath = join(cwd, repo_name);`
`287`	`process.chdir(fullPath);`
`288`
`289`	`if (subfolder !== "") {`
`290`	`await runCommand(undefined, "git", "sparse-checkout", "add", subfolder);`
`291`	`const subfolderPath = join(fullPath, subfolder);`
`292`
`293`	`if (!existsSync(subfolderPath)) {`
`294`	throw new Error(`Subfolder ${subfolder} does not exist.`);
`295`	`}`
`296`
`297`	`process.chdir(subfolderPath);`
`298`	`}`
`299`
`300`	`// Get the commit hash`
`301`	`const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();`
`302`
`303`	`// Return to original directory`
`304`	`process.chdir(cwd);`
`305`
`306`	`return { repo_name, commitHash };`
`307`	`}`
`308`
`309`	`async function uploadDirectoryToS3(`
`310`	`directoryPath: string,`
`311`	`s3BasePath: string,`
`312`	`workspace: string,`
`313`	`): Promise<number> {`
`314`	console.log(`Uploading ${directoryPath} -> ${s3BasePath}`);
`315`
`316`	`// Walk once into a flat task list so we can drive a bounded-concurrency pool.`
`317`	`const tasks: { localPath: string; s3Key: string }[] = [];`
`318`	`const root = fs.realpathSync(directoryPath);`
`319`	`// Real paths of the directories being descended through: a link back up to`
`320`	`// one of them would otherwise recurse forever.`
`321`	`const ancestors = new Set<string>();`
`322`	`// Links can reach one directory along many paths, so a small repository can`
`323`	`// expand exponentially through them. Every entry reached through a link is`
`324`	`// charged before it is resolved, skipped ones included, and past either budget`
`325`	`// the rest are skipped; the checkout's own files are always uploaded.`
`326`	`let symlinkedEntries = 0;`
`327`	`let symlinkedBytes = 0;`
`328`	`let symlinkBudgetSpent = false;`
`329`	`function chargeSymlinkBudget(relPath: string, entries: number, bytes: number): boolean {`
`330`	`if (symlinkBudgetSpent) return false;`
`331`	`symlinkedEntries += entries;`
`332`	`symlinkedBytes += bytes;`
`333`	`if (symlinkedEntries <= MAX_SYMLINKED_ENTRIES && symlinkedBytes <= MAX_SYMLINKED_BYTES) {`
`334`	`return true;`
`335`	`}`
`336`	`symlinkBudgetSpent = true;`
`337`	`console.log(`
`338`	`Skipping ${relPath} and every symlinked entry after it: symlinks reach more than ` +
`339`	`${MAX_SYMLINKED_ENTRIES} entries or ${MAX_SYMLINKED_BYTES / 2 ** 20} MiB`
`340`	`);`
`341`	`return false;`
`342`	`}`
`343`	`function walk(dir: string, relDir: string, viaLink: boolean) {`
`344`	`ancestors.add(dir);`
`345`	`for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {`
`346`	const relPath = relDir ? `${relDir}/${entry.name}` : entry.name;
`347`	`const linked = viaLink \|\| entry.isSymbolicLink();`
`348`	`if (linked && !chargeSymlinkBudget(relPath, 1, 0)) continue;`
`349`	`let localPath = join(dir, entry.name);`
`350`	`if (entry.isSymbolicLink()) {`
`351`	`// The repository chooses the target, so only follow links that stay in`
`352`	// the checkout: one to `/proc/self/environ` or to the ssh key written
`353`	`// next to it would put that file in front of every reader of the resource.`
`354`	`const link = fs.readlinkSync(localPath);`
`355`	`try {`
`356`	`localPath = fs.realpathSync(localPath);`
`357`	`} catch (e: any) {`
`358`	console.log(`Skipping symlink ${relPath} -> ${link}: cannot resolve target (${e.code})`);
`359`	`continue;`
`360`	`}`
`361`	`if (localPath !== root && !localPath.startsWith(root + sep)) {`
`362`	console.log(`Skipping symlink ${relPath} -> ${link}: target is outside the repository`);
`363`	`continue;`
`364`	`}`
`365`	`}`
`366`	`const stat = fs.statSync(localPath);`
`367`	`if (stat.isDirectory() && ancestors.has(localPath)) {`
`368`	console.log(`Skipping ${relPath}: links back to a directory it is inside`);
`369`	`continue;`
`370`	`}`
`371`	`if (linked && stat.isFile() && !chargeSymlinkBudget(relPath, 0, stat.size)) continue;`
`372`	`if (stat.isDirectory()) {`
`373`	`walk(localPath, relPath, linked);`
`374`	`} else if (stat.isFile()) {`
`375`	tasks.push({ localPath, s3Key: `${s3BasePath}/${relPath}` });
`376`	`}`
`377`	`}`
`378`	`ancestors.delete(dir);`
`379`	`}`
`380`	`walk(root, "", false);`
`381`	console.log(`Discovered ${tasks.length} files to upload`);
`382`
`383`	`let nextIndex = 0;`
`384`	`let uploaded = 0;`
`385`	`let lastReport = 0;`
`386`	`async function worker() {`
`387`	`while (true) {`
`388`	`const idx = nextIndex++;`
`389`	`if (idx >= tasks.length) return;`
`390`	`const { localPath, s3Key } = tasks[idx];`
`391`	`const fileContent = fs.readFileSync(localPath);`
`392`	`const blob = new Blob([fileContent], { type: 'application/octet-stream' });`
`393`	`await wmillclient.HelpersService.gitRepoViewerFileUpload({`
`394`	`workspace,`
`395`	`fileKey: s3Key,`
`396`	`requestBody: blob,`
`397`	`});`
`398`	`uploaded++;`
`399`	`if (uploaded - lastReport >= 25 \|\| uploaded === tasks.length) {`
`400`	`lastReport = uploaded;`
`401`	console.log(`Uploaded ${uploaded} / ${tasks.length} files`);
`402`	`}`
`403`	`}`
`404`	`}`
`405`	`await Promise.all(`
`406`	`Array.from({ length: Math.min(UPLOAD_CONCURRENCY, tasks.length) }, () => worker())`
`407`	`);`
`408`
`409`	`// Marker is the LAST write — its presence is what the viewer checks for.`
`410`	const markerKey = `${s3BasePath}/${CLONE_MARKER_FILE}`;
`411`	`const markerBody = JSON.stringify({`
`412`	`completed_at: new Date().toISOString(),`
`413`	`file_count: tasks.length,`
`414`	`});`
`415`	`await wmillclient.HelpersService.gitRepoViewerFileUpload({`
`416`	`workspace,`
`417`	`fileKey: markerKey,`
`418`	`requestBody: new Blob([markerBody], { type: 'application/json' }),`
`419`	`});`
`420`	console.log(`Wrote completion marker: ${markerKey}`);
`421`
`422`	`return tasks.length;`
`423`	`}`
`424`
`425`	`function runCommand(secret_position: number \| undefined, cmd: string, ...args: string[]): Promise<string> {`
`426`	`const nargs = secret_position != undefined ? args.slice() : args;`
`427`	`if (secret_position && secret_position < 0)`
`428`	`secret_position = nargs.length - 1 + secret_position;`
`429`
`430`	`let secret: string \| undefined = undefined;`
`431`	`if (secret_position != undefined) {`
`432`	`nargs[secret_position] = "***";`
`433`	`secret = args[secret_position];`
`434`	`}`
`435`	console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);
`436`
`437`	`return new Promise((resolve, reject) => {`
`438`	`const process = spawn(cmd, args);`
`439`
`440`	`let stdout = '';`
`441`	`let stderr = '';`
`442`
`443`	`process.stdout.on('data', (data) => {`
`444`	`stdout += data.toString();`
`445`	`});`
`446`
`447`	`process.stderr.on('data', (data) => {`
`448`	`stderr += data.toString();`
`449`	`});`
`450`
`451`	`process.on('error', (error) => {`
`452`	`let errorString = error.toString();`
`453`	`if (secret) errorString = errorString.replace(secret, "***");`
`454`	console.log(`Shell command FAILED: ${cmd}`, errorString);
`455`	`const e = new Error(`
`456`	`SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
`457`	`);`
`458`	`reject(e);`
`459`	`});`
`460`
`461`	`process.on('close', (code) => {`
`462`	`if (stdout.length > 0) {`
`463`	`console.log("Shell stdout:", stdout);`
`464`	`}`
`465`	`if (stderr.length > 0) {`
`466`	`console.log("Shell stderr:", stderr);`
`467`	`}`
`468`	`if (code === 0) {`
`469`	console.log(`Shell command completed successfully: ${cmd}`);
`470`	`resolve(stdout);`
`471`	`} else {`
`472`	reject(new Error(`Command failed with code ${code}: ${stderr}`));
`473`	`}`
`474`	`});`
`475`	`});`
`476`	`}`
`477`