Edits history of script submission #14002 for ' clone repo and upload to instance storage (windmill)'

• bun

  Ap­pro­ved version

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import { basename, join } from "node:path";
  import { existsSync, rmSync } from "fs";
  import process from "process";
  import { spawn } from 'child_process';
  import * as fs_async from 'fs/promises';
  import * as fs from 'node:fs';
  
  type GitRepository = {
    url: string;
    branch: string;
    folder: string;
    gpg_key: any;
    is_github_app: boolean;
  };
  
  export async function main(
    resource_path: string,
    workspace: string,
    git_ssh_identity?: string[],
    commit?: string
  ) {
    let clonedRepoPath: string | undefined;
  
    try {
      console.log("Starting git clone and Blob storage upload process");
  
      // Get the git repository resource
      const repo_resource: GitRepository = await wmillclient.getResource(resource_path);
  
      const cwd = process.cwd();
  
      if (git_ssh_identity) {
        process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity)
      }
  
      // Handle GitHub App authentication if needed
      if (repo_resource.is_github_app) {
        const token = await get_gh_app_token();
        repo_resource.url = prependTokenToGitHubUrl(repo_resource.url, token);
      }
  
      process.env["HOME"] = ".";
      process.env.GIT_TERMINAL_PROMPT = "0";
  
      // Clone the repository
      const { repo_name, commitHash } = await git_clone(cwd, repo_resource, commit);
      clonedRepoPath = join(cwd, repo_name);
  
      // Remove .git directory to avoid uploading git history
      const gitDir = join(clonedRepoPath, ".git");
      if (existsSync(gitDir)) {
        rmSync(gitDir, { recursive: true, force: true });
        console.log("Removed .git directory");
      }
  
      // Upload to S3
      const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`;
      await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace);
  
      return {
        success: true,
        message: "Repository cloned and uploaded to S3 successfully",
        s3_path: s3Path,
        commit_hash: commitHash
      };
  
    } catch (error) {
      console.error("Error in git clone and upload:", error);
      throw error;
    } finally {
      // Clean up cloned repository
      if (clonedRepoPath && existsSync(clonedRepoPath)) {
        rmSync(clonedRepoPath, { recursive: true, force: true });
        console.log("Cleaned up cloned repository");
      }
    }
  }
  
  async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise<string> {
    const sshIdFiles = await Promise.all(
      git_ssh_identity.map(async (varPath, i) => {
        const filePath = join(cwd, `./ssh_id_priv_${i}`);
  
        try {
          // Get variable value using windmill
          let content = await wmillclient.getVariable(varPath);
          content += '\n';
  
          // Write file with content
          await fs_async.writeFile(filePath, content, { encoding: 'utf8' });
  
          // Set file permissions to 0o600 (read/write for owner only)
          await fs_async.chmod(filePath, 0o600);
  
          // Escape single quotes for shell command
          const escapedPath = filePath.replace(/'/g, "'\\''");
          return ` -i '${escapedPath}'`;
        } catch (error) {
          console.error(
            `Variable ${varPath} not found for git ssh identity: ${error}`
          );
          return '';
        }
      })
    );
  
    const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`;
    return gitSshCmd;
  }
  async function git_clone(
    cwd: string,
    repo_resource: GitRepository,
    commit?: string,
  ): Promise<{ repo_name: string; commitHash: string }> {
      if (commit) {
        return git_clone_at_commit(cwd, repo_resource, commit);
      } else {
        return git_clone_at_latest(cwd, repo_resource);
      }
  }
  
  async function git_clone_at_commit(
    cwd: string,
    repo_resource: GitRepository,
    commit: string,
  ): Promise<{ repo_name: string; commitHash: string }> {
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    let branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
  
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log("Fetching Azure DevOps access token...");
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
  
    const repoPath = join(cwd, repo_name);
    await fs_async.mkdir(repoPath, { recursive: true });
  
    process.chdir(repoPath);
  
    let args = ['init', '--quiet']
    if (branch) {
      args.push(`--initial-branch=${branch}`)
    }
    await runCommand(undefined, 'git', ...args);
  
    await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url);
  
    await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit);
  
    await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD');
  
    const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();
  
    // Return to original directory
    process.chdir(cwd);
  
    return { repo_name, commitHash };
  }
  
  async function git_clone_at_latest(
    cwd: string,
    repo_resource: GitRepository
  ): Promise<{ repo_name: string; commitHash: string }> {
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    let branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
  
    // Handle Azure DevOps token if needed
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log("Fetching Azure DevOps access token...");
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
  
    const args = ["clone", "--quiet", "--depth", "1"];
    if (subfolder !== "") args.push("--sparse");
    if (branch !== "") args.push("--branch", branch);
    args.push(repo_url, repo_name);
  
    await runCommand(-1, "git", ...args);
  
    const fullPath = join(cwd, repo_name);
    process.chdir(fullPath);
  
    if (subfolder !== "") {
      await runCommand(undefined, "git", "sparse-checkout", "add", subfolder);
      const subfolderPath = join(fullPath, subfolder);
  
      if (!existsSync(subfolderPath)) {
        throw new Error(`Subfolder ${subfolder} does not exist.`);
      }
  
      process.chdir(subfolderPath);
    }
  
    // Get the commit hash
    const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();
  
    // Return to original directory
    process.chdir(cwd);
  
    return { repo_name, commitHash };
  }
  
  async function uploadDirectoryToS3(
    directoryPath: string,
    s3BasePath: string,
    workspace: string,
  ) {
    console.log(`Uploading directory ${directoryPath} to S3 path ${s3BasePath}`);
  
    async function uploadDirRecursive(currentDir: string, currentS3Path: string) {
      const entries = fs.readdirSync(currentDir, { withFileTypes: true });
  
      for (const entry of entries) {
        const fullPath = join(currentDir, entry.name);
        const s3Key = currentS3Path ? `${currentS3Path}/${entry.name}` : entry.name;
  
        if (entry.isDirectory()) {
          // Recursively upload subdirectory
          await uploadDirRecursive(fullPath, s3Key);
        } else if (entry.isFile()) {
          // Upload file
          const fileContent = fs.readFileSync(fullPath);
          const blob = new Blob([fileContent], { type: 'application/octet-stream' });
          await wmillclient.HelpersService.gitRepoViewerFileUpload({
            workspace,
            fileKey: s3Key,
            requestBody: blob
          });
          console.log(`Uploaded: ${s3Key}`);
        }
      }
    }
  
    await uploadDirRecursive(directoryPath, s3BasePath);
    console.log("Directory upload completed");
  }
  
  function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise<string> {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0)
      secret_position = nargs.length - 1 + secret_position;
  
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
    console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);
  
    return new Promise((resolve, reject) => {
      const process = spawn(cmd, args);
  
      let stdout = '';
      let stderr = '';
  
      process.stdout.on('data', (data) => {
        stdout += data.toString();
      });
  
      process.stderr.on('data', (data) => {
        stderr += data.toString();
      });
  
      process.on('error', (error) => {
        let errorString = error.toString();
        if (secret) errorString = errorString.replace(secret, "***");
        console.log(`Shell command FAILED: ${cmd}`, errorString);
        const e = new Error(
          `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
        );
        reject(e);
      });
  
      process.on('close', (code) => {
        if (stdout.length > 0) {
          console.log("Shell stdout:", stdout);
        }
        if (stderr.length > 0) {
          console.log("Shell stderr:", stderr);
        }
        if (code === 0) {
          console.log(`Shell command completed successfully: ${cmd}`);
          resolve(stdout);
        } else {
          reject(new Error(`Command failed with code ${code}: ${stderr}`));
        }
      });
    });
  }
  
  // async function sh_run(
  //   secret_position: number | undefined,
  //   cmd: string,
  //   ...args: string[]
  // ) {
  //   const nargs = secret_position != undefined ? args.slice() : args;
  //   if (secret_position && secret_position < 0)
  //     secret_position = nargs.length - 1 + secret_position;
  //
  //   let secret: string | undefined = undefined;
  //   if (secret_position != undefined) {
  //     nargs[secret_position] = "***";
  //     secret = args[secret_position];
  //   }
  //
  //   console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);
  //   try {
  //     const { stdout, stderr } = await exec(`${cmd} ${args.join(" ")}`);
  //     if (stdout.length > 0) {
  //       console.log("Shell stdout:", stdout);
  //     }
  //     if (stderr.length > 0) {
  //       console.log("Shell stderr:", stderr);
  //     }
  //     console.log(`Shell command completed successfully: ${cmd}`);
  //     return stdout;
  //   } catch (error: any) {
  //     let errorString = error.toString();
  //     if (secret) errorString = errorString.replace(secret, "***");
  //     console.log(`Shell command FAILED: ${cmd}`, errorString);
  //     throw new Error(
  //       `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
  //     );
  //   }
  // }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({ job_token: jobToken }),
    });
  
    if (!response.ok) {
      const errorBody = await response.text().catch(() => "");
      throw new Error(`GitHub App token error (${response.status}): ${errorBody || response.statusText}`);
    }
    const data = await response.json();
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    const url = new URL(gitHubUrl);
    return `https://x-access-token:${installationToken}@${url.hostname}${url.pathname}`;
  }
  

  Submitted by hugo989 23 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import { basename, join } from "node:path";
  import { existsSync, rmSync } from "fs";
  import process from "process";
  import { spawn } from 'child_process';
  import * as fs_async from 'fs/promises';
  import * as fs from 'node:fs';
  
  type GitRepository = {
    url: string;
    branch: string;
    folder: string;
    gpg_key: any;
    is_github_app: boolean;
  };
  
  export async function main(
    resource_path: string,
    workspace: string,
    git_ssh_identity?: string[],
    commit?: string
  ) {
    let clonedRepoPath: string | undefined;
  
    try {
      console.log("Starting git clone and Blob storage upload process");
  
      // Get the git repository resource
      const repo_resource: GitRepository = await wmillclient.getResource(resource_path);
  
      const cwd = process.cwd();
  
      if (git_ssh_identity) {
        process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity)
      }
  
      // Handle GitHub App authentication if needed
      if (repo_resource.is_github_app) {
        const token = await get_gh_app_token();
        repo_resource.url = prependTokenToGitHubUrl(repo_resource.url, token);
      }
  
      process.env["HOME"] = ".";
      process.env.GIT_TERMINAL_PROMPT = "0";
  
      // Clone the repository
      const { repo_name, commitHash } = await git_clone(cwd, repo_resource, commit);
      clonedRepoPath = join(cwd, repo_name);
  
      // Remove .git directory to avoid uploading git history
      const gitDir = join(clonedRepoPath, ".git");
      if (existsSync(gitDir)) {
        rmSync(gitDir, { recursive: true, force: true });
        console.log("Removed .git directory");
      }
  
      // Upload to S3
      const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`;
      await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace);
  
      return {
        success: true,
        message: "Repository cloned and uploaded to S3 successfully",
        s3_path: s3Path,
        commit_hash: commitHash
      };
  
    } catch (error) {
      console.error("Error in git clone and upload:", error);
      throw error;
    } finally {
      // Clean up cloned repository
      if (clonedRepoPath && existsSync(clonedRepoPath)) {
        rmSync(clonedRepoPath, { recursive: true, force: true });
        console.log("Cleaned up cloned repository");
      }
    }
  }
  
  async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise<string> {
    const sshIdFiles = await Promise.all(
      git_ssh_identity.map(async (varPath, i) => {
        const filePath = join(cwd, `./ssh_id_priv_${i}`);
  
        try {
          // Get variable value using windmill
          let content = await wmillclient.getVariable(varPath);
          content += '\n';
  
          // Write file with content
          await fs_async.writeFile(filePath, content, { encoding: 'utf8' });
  
          // Set file permissions to 0o600 (read/write for owner only)
          await fs_async.chmod(filePath, 0o600);
  
          // Escape single quotes for shell command
          const escapedPath = filePath.replace(/'/g, "'\\''");
          return ` -i '${escapedPath}'`;
        } catch (error) {
          console.error(
            `Variable ${varPath} not found for git ssh identity: ${error}`
          );
          return '';
        }
      })
    );
  
    const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`;
    return gitSshCmd;
  }
  async function git_clone(
    cwd: string,
    repo_resource: GitRepository,
    commit?: string,
  ): Promise<{ repo_name: string; commitHash: string }> {
      if (commit) {
        return git_clone_at_commit(cwd, repo_resource, commit);
      } else {
        return git_clone_at_latest(cwd, repo_resource);
      }
  }
  
  async function git_clone_at_commit(
    cwd: string,
    repo_resource: GitRepository,
    commit: string,
  ): Promise<{ repo_name: string; commitHash: string }> {
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    let branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
  
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log("Fetching Azure DevOps access token...");
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
  
    const repoPath = join(cwd, repo_name);
    await fs_async.mkdir(repoPath, { recursive: true });
  
    process.chdir(repoPath);
  
    let args = ['init', '--quiet']
    if (branch) {
      args.push(`--initial-branch=${branch}`)
    }
    await runCommand(undefined, 'git', ...args);
  
    await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url);
  
    await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit);
  
    await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD');
  
    const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();
  
    // Return to original directory
    process.chdir(cwd);
  
    return { repo_name, commitHash };
  }
  
  async function git_clone_at_latest(
    cwd: string,
    repo_resource: GitRepository
  ): Promise<{ repo_name: string; commitHash: string }> {
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    let branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
  
    // Handle Azure DevOps token if needed
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log("Fetching Azure DevOps access token...");
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
  
    const args = ["clone", "--quiet", "--depth", "1"];
    if (subfolder !== "") args.push("--sparse");
    if (branch !== "") args.push("--branch", branch);
    args.push(repo_url, repo_name);
  
    await runCommand(-1, "git", ...args);
  
    const fullPath = join(cwd, repo_name);
    process.chdir(fullPath);
  
    if (subfolder !== "") {
      await runCommand(undefined, "git", "sparse-checkout", "add", subfolder);
      const subfolderPath = join(fullPath, subfolder);
  
      if (!existsSync(subfolderPath)) {
        throw new Error(`Subfolder ${subfolder} does not exist.`);
      }
  
      process.chdir(subfolderPath);
    }
  
    // Get the commit hash
    const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();
  
    // Return to original directory
    process.chdir(cwd);
  
    return { repo_name, commitHash };
  }
  
  async function uploadDirectoryToS3(
    directoryPath: string,
    s3BasePath: string,
    workspace: string,
  ) {
    console.log(`Uploading directory ${directoryPath} to S3 path ${s3BasePath}`);
  
    async function uploadDirRecursive(currentDir: string, currentS3Path: string) {
      const entries = fs.readdirSync(currentDir, { withFileTypes: true });
  
      for (const entry of entries) {
        const fullPath = join(currentDir, entry.name);
        const s3Key = currentS3Path ? `${currentS3Path}/${entry.name}` : entry.name;
  
        if (entry.isDirectory()) {
          // Recursively upload subdirectory
          await uploadDirRecursive(fullPath, s3Key);
        } else if (entry.isFile()) {
          // Upload file
          const fileContent = fs.readFileSync(fullPath);
          const blob = new Blob([fileContent], { type: 'application/octet-stream' });
          await wmillclient.HelpersService.gitRepoViewerFileUpload({
            workspace,
            fileKey: s3Key,
            requestBody: blob
          });
          console.log(`Uploaded: ${s3Key}`);
        }
      }
    }
  
    await uploadDirRecursive(directoryPath, s3BasePath);
    console.log("Directory upload completed");
  }
  
  function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise<string> {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0)
      secret_position = nargs.length - 1 + secret_position;
  
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
    console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);
  
    return new Promise((resolve, reject) => {
      const process = spawn(cmd, args);
  
      let stdout = '';
      let stderr = '';
  
      process.stdout.on('data', (data) => {
        stdout += data.toString();
      });
  
      process.stderr.on('data', (data) => {
        stderr += data.toString();
      });
  
      process.on('error', (error) => {
        let errorString = error.toString();
        if (secret) errorString = errorString.replace(secret, "***");
        console.log(`Shell command FAILED: ${cmd}`, errorString);
        const e = new Error(
          `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
        );
        reject(e);
      });
  
      process.on('close', (code) => {
        if (stdout.length > 0) {
          console.log("Shell stdout:", stdout);
        }
        if (stderr.length > 0) {
          console.log("Shell stderr:", stderr);
        }
        if (code === 0) {
          console.log(`Shell command completed successfully: ${cmd}`);
          resolve(stdout);
        } else {
          reject(new Error(`Command failed with code ${code}: ${stderr}`));
        }
      });
    });
  }
  
  // async function sh_run(
  //   secret_position: number | undefined,
  //   cmd: string,
  //   ...args: string[]
  // ) {
  //   const nargs = secret_position != undefined ? args.slice() : args;
  //   if (secret_position && secret_position < 0)
  //     secret_position = nargs.length - 1 + secret_position;
  //
  //   let secret: string | undefined = undefined;
  //   if (secret_position != undefined) {
  //     nargs[secret_position] = "***";
  //     secret = args[secret_position];
  //   }
  //
  //   console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);
  //   try {
  //     const { stdout, stderr } = await exec(`${cmd} ${args.join(" ")}`);
  //     if (stdout.length > 0) {
  //       console.log("Shell stdout:", stdout);
  //     }
  //     if (stderr.length > 0) {
  //       console.log("Shell stderr:", stderr);
  //     }
  //     console.log(`Shell command completed successfully: ${cmd}`);
  //     return stdout;
  //   } catch (error: any) {
  //     let errorString = error.toString();
  //     if (secret) errorString = errorString.replace(secret, "***");
  //     console.log(`Shell command FAILED: ${cmd}`, errorString);
  //     throw new Error(
  //       `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
  //     );
  //   }
  // }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({ job_token: jobToken }),
    });
  
    if (!response.ok)
      throw new Error(`GitHub App token error: ${response.statusText}`);
    const data = await response.json();
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    const url = new URL(gitHubUrl);
    return `https://x-access-token:${installationToken}@${url.hostname}${url.pathname}`;
  }
  

  Submitted by hugo989 26 days ago

• bun

  Edit in Windmill

  Embed

  import * as wmillclient from "windmill-client";
  import { basename, join } from "node:path";
  import { existsSync, rmSync } from "fs";
  import process from "process";
  import { spawn } from 'child_process';
  import * as fs_async from 'fs/promises';
  import * as fs from 'node:fs';
  
  type GitRepository = {
    url: string;
    branch: string;
    folder: string;
    gpg_key: any;
    is_github_app: boolean;
  };
  
  export async function main(
    resource_path: string,
    workspace: string,
    git_ssh_identity?: string[],
    commit?: string
  ) {
    let clonedRepoPath: string | undefined;
  
    try {
      console.log("Starting git clone and Blob storage upload process");
  
      // Get the git repository resource
      const repo_resource: GitRepository = await wmillclient.getResource(resource_path);
  
      const cwd = process.cwd();
  
      if (git_ssh_identity) {
        process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity)
      }
  
      // Handle GitHub App authentication if needed
      if (repo_resource.is_github_app) {
        const token = await get_gh_app_token();
        repo_resource.url = prependTokenToGitHubUrl(repo_resource.url, token);
      }
  
      process.env["HOME"] = ".";
      process.env.GIT_TERMINAL_PROMPT = "0";
  
      // Clone the repository
      const { repo_name, commitHash } = await git_clone(cwd, repo_resource, commit);
      clonedRepoPath = join(cwd, repo_name);
  
      // Remove .git directory to avoid uploading git history
      const gitDir = join(clonedRepoPath, ".git");
      if (existsSync(gitDir)) {
        rmSync(gitDir, { recursive: true, force: true });
        console.log("Removed .git directory");
      }
  
      // Upload to S3
      const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`;
      await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace);
  
      return {
        success: true,
        message: "Repository cloned and uploaded to S3 successfully",
        s3_path: s3Path,
        commit_hash: commitHash
      };
  
    } catch (error) {
      console.error("Error in git clone and upload:", error);
      throw error;
    } finally {
      // Clean up cloned repository
      if (clonedRepoPath && existsSync(clonedRepoPath)) {
        rmSync(clonedRepoPath, { recursive: true, force: true });
        console.log("Cleaned up cloned repository");
      }
    }
  }
  
  async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise<string> {
    const sshIdFiles = await Promise.all(
      git_ssh_identity.map(async (varPath, i) => {
        const filePath = join(cwd, `./ssh_id_priv_${i}`);
  
        try {
          // Get variable value using windmill
          let content = await wmillclient.getVariable(varPath);
          content += '\n';
  
          // Write file with content
          await fs_async.writeFile(filePath, content, { encoding: 'utf8' });
  
          // Set file permissions to 0o600 (read/write for owner only)
          await fs_async.chmod(filePath, 0o600);
  
          // Escape single quotes for shell command
          const escapedPath = filePath.replace(/'/g, "'\\''");
          return ` -i '${escapedPath}'`;
        } catch (error) {
          console.error(
            `Variable ${varPath} not found for git ssh identity: ${error}`
          );
          return '';
        }
      })
    );
  
    const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`;
    return gitSshCmd;
  }
  async function git_clone(
    cwd: string,
    repo_resource: GitRepository,
    commit?: string,
  ): Promise<{ repo_name: string; commitHash: string }> {
      if (commit) {
        return git_clone_at_commit(cwd, repo_resource, commit);
      } else {
        return git_clone_at_latest(cwd, repo_resource);
      }
  }
  
  async function git_clone_at_commit(
    cwd: string,
    repo_resource: GitRepository,
    commit: string,
  ): Promise<{ repo_name: string; commitHash: string }> {
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    let branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
  
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log("Fetching Azure DevOps access token...");
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
  
    const repoPath = join(cwd, repo_name);
    await fs_async.mkdir(repoPath, { recursive: true });
  
    process.chdir(repoPath);
  
    let args = ['init', '--quiet']
    if (branch) {
      args.push(`--initial-branch=${branch}`)
    }
    await runCommand(undefined, 'git', ...args);
  
    await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url);
  
    await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit);
  
    await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD');
  
    const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();
  
    // Return to original directory
    process.chdir(cwd);
  
    return { repo_name, commitHash };
  }
  
  async function git_clone_at_latest(
    cwd: string,
    repo_resource: GitRepository
  ): Promise<{ repo_name: string; commitHash: string }> {
    let repo_url = repo_resource.url;
    const subfolder = repo_resource.folder ?? "";
    let branch = repo_resource.branch ?? "";
    const repo_name = basename(repo_url, ".git");
  
    // Handle Azure DevOps token if needed
    const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
    if (azureMatch) {
      console.log("Fetching Azure DevOps access token...");
      const azureResource = await wmillclient.getResource(azureMatch.groups.url);
      const response = await fetch(
        `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
        {
          method: "POST",
          body: new URLSearchParams({
            client_id: azureResource.azureClientId,
            client_secret: azureResource.azureClientSecret,
            grant_type: "client_credentials",
            resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
          }),
        }
      );
      const { access_token } = await response.json();
      repo_url = repo_url.replace(azureMatch[0], access_token);
    }
  
    const args = ["clone", "--quiet", "--depth", "1"];
    if (subfolder !== "") args.push("--sparse");
    if (branch !== "") args.push("--branch", branch);
    args.push(repo_url, repo_name);
  
    await runCommand(-1, "git", ...args);
  
    const fullPath = join(cwd, repo_name);
    process.chdir(fullPath);
  
    if (subfolder !== "") {
      await runCommand(undefined, "git", "sparse-checkout", "add", subfolder);
      const subfolderPath = join(fullPath, subfolder);
  
      if (!existsSync(subfolderPath)) {
        throw new Error(`Subfolder ${subfolder} does not exist.`);
      }
  
      process.chdir(subfolderPath);
    }
  
    // Get the commit hash
    const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();
  
    // Return to original directory
    process.chdir(cwd);
  
    return { repo_name, commitHash };
  }
  
  async function uploadDirectoryToS3(
    directoryPath: string,
    s3BasePath: string,
    workspace: string,
  ) {
    console.log(`Uploading directory ${directoryPath} to S3 path ${s3BasePath}`);
  
    async function uploadDirRecursive(currentDir: string, currentS3Path: string) {
      const entries = fs.readdirSync(currentDir, { withFileTypes: true });
  
      for (const entry of entries) {
        const fullPath = join(currentDir, entry.name);
        const s3Key = currentS3Path ? `${currentS3Path}/${entry.name}` : entry.name;
  
        if (entry.isDirectory()) {
          // Recursively upload subdirectory
          await uploadDirRecursive(fullPath, s3Key);
        } else if (entry.isFile()) {
          // Upload file
          const fileContent = fs.readFileSync(fullPath);
          const blob = new Blob([fileContent], { type: 'application/octet-stream' });
          await wmillclient.HelpersService.gitRepoViewerFileUpload({
            workspace,
            fileKey: s3Key,
            requestBody: blob
          });
          console.log(`Uploaded: ${s3Key}`);
        }
      }
    }
  
    await uploadDirRecursive(directoryPath, s3BasePath);
    console.log("Directory upload completed");
  }
  
  function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise<string> {
    const nargs = secret_position != undefined ? args.slice() : args;
    if (secret_position && secret_position < 0)
      secret_position = nargs.length - 1 + secret_position;
  
    let secret: string | undefined = undefined;
    if (secret_position != undefined) {
      nargs[secret_position] = "***";
      secret = args[secret_position];
    }
    console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);
  
    return new Promise((resolve, reject) => {
      const process = spawn(cmd, args);
  
      let stdout = '';
      let stderr = '';
  
      process.stdout.on('data', (data) => {
        stdout += data.toString();
      });
  
      process.stderr.on('data', (data) => {
        stderr += data.toString();
      });
  
      process.on('error', (error) => {
        let errorString = error.toString();
        if (secret) errorString = errorString.replace(secret, "***");
        console.log(`Shell command FAILED: ${cmd}`, errorString);
        const e = new Error(
          `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
        );
        reject(e);
      });
  
      process.on('close', (code) => {
        if (stdout.length > 0) {
          console.log("Shell stdout:", stdout);
        }
        if (stderr.length > 0) {
          console.log("Shell stderr:", stderr);
        }
        if (code === 0) {
          console.log(`Shell command completed successfully: ${cmd}`);
          resolve(stdout);
        } else {
          reject(new Error(`Command failed with code ${code}: ${stderr}`));
        }
      });
    });
  }
  
  // async function sh_run(
  //   secret_position: number | undefined,
  //   cmd: string,
  //   ...args: string[]
  // ) {
  //   const nargs = secret_position != undefined ? args.slice() : args;
  //   if (secret_position && secret_position < 0)
  //     secret_position = nargs.length - 1 + secret_position;
  //
  //   let secret: string | undefined = undefined;
  //   if (secret_position != undefined) {
  //     nargs[secret_position] = "***";
  //     secret = args[secret_position];
  //   }
  //
  //   console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);
  //   try {
  //     const { stdout, stderr } = await exec(`${cmd} ${args.join(" ")}`);
  //     if (stdout.length > 0) {
  //       console.log("Shell stdout:", stdout);
  //     }
  //     if (stderr.length > 0) {
  //       console.log("Shell stderr:", stderr);
  //     }
  //     console.log(`Shell command completed successfully: ${cmd}`);
  //     return stdout;
  //   } catch (error: any) {
  //     let errorString = error.toString();
  //     if (secret) errorString = errorString.replace(secret, "***");
  //     console.log(`Shell command FAILED: ${cmd}`, errorString);
  //     throw new Error(
  //       `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
  //     );
  //   }
  // }
  
  async function get_gh_app_token() {
    const workspace = process.env["WM_WORKSPACE"];
    const jobToken = process.env["WM_TOKEN"];
    const baseUrl =
      process.env["BASE_INTERNAL_URL"] ??
      process.env["BASE_URL"] ??
      "http://localhost:8000";
    const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
  
    const response = await fetch(url, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${jobToken}`,
      },
      body: JSON.stringify({ job_token: jobToken }),
    });
  
    if (!response.ok)
      throw new Error(`GitHub App token error: ${response.statusText}`);
    const data = await response.json();
    return data.token;
  }
  
  function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
    const url = new URL(gitHubUrl);
    if (url.hostname !== "github.com") throw new Error("Invalid GitHub URL");
    return `https://x-access-token:${installationToken}@github.com${url.pathname}`;
  }
  

  Submitted by hugo697 185 days ago

• bun

  Edit in Windmill

  Embed

  export async function main() {
  	throw new Error("This action is not supported in the current version of windmill.")
  }
  

  Submitted by hugo697 187 days ago