logo Windmill Hub
Discord
Discussions
Windmill
Login
clone repo and upload to instance storage
One script reply has been approved by the moderators Verified

Clones a github repo defined in a git_repository resource, then uploads the files to the instance storage for them to be cached

Created by hugo697 183 days ago Picked 3 times
Submitted by hugo697 Bun
Verified 19 days ago
All edits
Permalink
1 
import * as wmillclient from "windmill-client";
2 
import { basename, join } from "node:path";
3 
import { existsSync, rmSync } from "fs";
4 
import process from "process";
5 
import { spawn } from 'child_process';
6 
import * as fs_async from 'fs/promises';
7 
import * as fs from 'node:fs';
8
9 
type GitRepository = {
10 
  url: string;
11 
  branch: string;
12 
  folder: string;
13 
  gpg_key: any;
14 
  is_github_app: boolean;
15 
};
16
17 
export async function main(
18 
  resource_path: string,
19 
  workspace: string,
20 
  git_ssh_identity?: string[],
21 
  commit?: string
22 
) {
23 
  let clonedRepoPath: string | undefined;
24
25 
  try {
26 
    console.log("Starting git clone and Blob storage upload process");
27
28 
    // Get the git repository resource
29 
    const repo_resource: GitRepository = await wmillclient.getResource(resource_path);
30
31 
    const cwd = process.cwd();
32
33 
    if (git_ssh_identity) {
34 
      process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity)
35 
    }
36
37 
    // Handle GitHub App authentication if needed
38 
    if (repo_resource.is_github_app) {
39 
      const token = await get_gh_app_token();
40 
      repo_resource.url = prependTokenToGitHubUrl(repo_resource.url, token);
41 
    }
42
43 
    process.env["HOME"] = ".";
44 
    process.env.GIT_TERMINAL_PROMPT = "0";
45
46 
    // Clone the repository
47 
    const { repo_name, commitHash } = await git_clone(cwd, repo_resource, commit);
48 
    clonedRepoPath = join(cwd, repo_name);
49
50 
    // Remove .git directory to avoid uploading git history
51 
    const gitDir = join(clonedRepoPath, ".git");
52 
    if (existsSync(gitDir)) {
53 
      rmSync(gitDir, { recursive: true, force: true });
54 
      console.log("Removed .git directory");
55 
    }
56
57 
    // Upload to S3
58 
    const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`;
59 
    await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace);
60
61 
    return {
62 
      success: true,
63 
      message: "Repository cloned and uploaded to S3 successfully",
64 
      s3_path: s3Path,
65 
      commit_hash: commitHash
66 
    };
67
68 
  } catch (error) {
69 
    console.error("Error in git clone and upload:", error);
70 
    throw error;
71 
  } finally {
72 
    // Clean up cloned repository
73 
    if (clonedRepoPath && existsSync(clonedRepoPath)) {
74 
      rmSync(clonedRepoPath, { recursive: true, force: true });
75 
      console.log("Cleaned up cloned repository");
76 
    }
77 
  }
78 
}
79
80 
async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise<string> {
81 
  const sshIdFiles = await Promise.all(
82 
    git_ssh_identity.map(async (varPath, i) => {
83 
      const filePath = join(cwd, `./ssh_id_priv_${i}`);
84
85 
      try {
86 
        // Get variable value using windmill
87 
        let content = await wmillclient.getVariable(varPath);
88 
        content += '\n';
89
90 
        // Write file with content
91 
        await fs_async.writeFile(filePath, content, { encoding: 'utf8' });
92
93 
        // Set file permissions to 0o600 (read/write for owner only)
94 
        await fs_async.chmod(filePath, 0o600);
95
96 
        // Escape single quotes for shell command
97 
        const escapedPath = filePath.replace(/'/g, "'\\''");
98 
        return ` -i '${escapedPath}'`;
99 
      } catch (error) {
100 
        console.error(
101 
          `Variable ${varPath} not found for git ssh identity: ${error}`
102 
        );
103 
        return '';
104 
      }
105 
    })
106 
  );
107
108 
  const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`;
109 
  return gitSshCmd;
110 
}
111 
async function git_clone(
112 
  cwd: string,
113 
  repo_resource: GitRepository,
114 
  commit?: string,
115 
): Promise<{ repo_name: string; commitHash: string }> {
116 
    if (commit) {
117 
      return git_clone_at_commit(cwd, repo_resource, commit);
118 
    } else {
119 
      return git_clone_at_latest(cwd, repo_resource);
120 
    }
121 
}
122
123 
async function git_clone_at_commit(
124 
  cwd: string,
125 
  repo_resource: GitRepository,
126 
  commit: string,
127 
): Promise<{ repo_name: string; commitHash: string }> {
128 
  let repo_url = repo_resource.url;
129 
  const subfolder = repo_resource.folder ?? "";
130 
  let branch = repo_resource.branch ?? "";
131 
  const repo_name = basename(repo_url, ".git");
132
133 
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
134 
  if (azureMatch) {
135 
    console.log("Fetching Azure DevOps access token...");
136 
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
137 
    const response = await fetch(
138 
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
139 
      {
140 
        method: "POST",
141 
        body: new URLSearchParams({
142 
          client_id: azureResource.azureClientId,
143 
          client_secret: azureResource.azureClientSecret,
144 
          grant_type: "client_credentials",
145 
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
146 
        }),
147 
      }
148 
    );
149 
    const { access_token } = await response.json();
150 
    repo_url = repo_url.replace(azureMatch[0], access_token);
151 
  }
152
153 
  const repoPath = join(cwd, repo_name);
154 
  await fs_async.mkdir(repoPath, { recursive: true });
155
156 
  process.chdir(repoPath);
157
158 
  let args = ['init', '--quiet']
159 
  if (branch) {
160 
    args.push(`--initial-branch=${branch}`)
161 
  }
162 
  await runCommand(undefined, 'git', ...args);
163
164 
  await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url);
165
166 
  await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit);
167
168 
  await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD');
169
170 
  const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();
171
172 
  // Return to original directory
173 
  process.chdir(cwd);
174
175 
  return { repo_name, commitHash };
176 
}
177
178 
async function git_clone_at_latest(
179 
  cwd: string,
180 
  repo_resource: GitRepository
181 
): Promise<{ repo_name: string; commitHash: string }> {
182 
  let repo_url = repo_resource.url;
183 
  const subfolder = repo_resource.folder ?? "";
184 
  let branch = repo_resource.branch ?? "";
185 
  const repo_name = basename(repo_url, ".git");
186
187 
  // Handle Azure DevOps token if needed
188 
  const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
189 
  if (azureMatch) {
190 
    console.log("Fetching Azure DevOps access token...");
191 
    const azureResource = await wmillclient.getResource(azureMatch.groups.url);
192 
    const response = await fetch(
193 
      `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
194 
      {
195 
        method: "POST",
196 
        body: new URLSearchParams({
197 
          client_id: azureResource.azureClientId,
198 
          client_secret: azureResource.azureClientSecret,
199 
          grant_type: "client_credentials",
200 
          resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
201 
        }),
202 
      }
203 
    );
204 
    const { access_token } = await response.json();
205 
    repo_url = repo_url.replace(azureMatch[0], access_token);
206 
  }
207
208 
  const args = ["clone", "--quiet", "--depth", "1"];
209 
  if (subfolder !== "") args.push("--sparse");
210 
  if (branch !== "") args.push("--branch", branch);
211 
  args.push(repo_url, repo_name);
212
213 
  await runCommand(-1, "git", ...args);
214
215 
  const fullPath = join(cwd, repo_name);
216 
  process.chdir(fullPath);
217
218 
  if (subfolder !== "") {
219 
    await runCommand(undefined, "git", "sparse-checkout", "add", subfolder);
220 
    const subfolderPath = join(fullPath, subfolder);
221
222 
    if (!existsSync(subfolderPath)) {
223 
      throw new Error(`Subfolder ${subfolder} does not exist.`);
224 
    }
225
226 
    process.chdir(subfolderPath);
227 
  }
228
229 
  // Get the commit hash
230 
  const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim();
231
232 
  // Return to original directory
233 
  process.chdir(cwd);
234
235 
  return { repo_name, commitHash };
236 
}
237
238 
async function uploadDirectoryToS3(
239 
  directoryPath: string,
240 
  s3BasePath: string,
241 
  workspace: string,
242 
) {
243 
  console.log(`Uploading directory ${directoryPath} to S3 path ${s3BasePath}`);
244
245 
  async function uploadDirRecursive(currentDir: string, currentS3Path: string) {
246 
    const entries = fs.readdirSync(currentDir, { withFileTypes: true });
247
248 
    for (const entry of entries) {
249 
      const fullPath = join(currentDir, entry.name);
250 
      const s3Key = currentS3Path ? `${currentS3Path}/${entry.name}` : entry.name;
251
252 
      if (entry.isDirectory()) {
253 
        // Recursively upload subdirectory
254 
        await uploadDirRecursive(fullPath, s3Key);
255 
      } else if (entry.isFile()) {
256 
        // Upload file
257 
        const fileContent = fs.readFileSync(fullPath);
258 
        const blob = new Blob([fileContent], { type: 'application/octet-stream' });
259 
        await wmillclient.HelpersService.gitRepoViewerFileUpload({
260 
          workspace,
261 
          fileKey: s3Key,
262 
          requestBody: blob
263 
        });
264 
        console.log(`Uploaded: ${s3Key}`);
265 
      }
266 
    }
267 
  }
268
269 
  await uploadDirRecursive(directoryPath, s3BasePath);
270 
  console.log("Directory upload completed");
271 
}
272
273 
function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise<string> {
274 
  const nargs = secret_position != undefined ? args.slice() : args;
275 
  if (secret_position && secret_position < 0)
276 
    secret_position = nargs.length - 1 + secret_position;
277
278 
  let secret: string | undefined = undefined;
279 
  if (secret_position != undefined) {
280 
    nargs[secret_position] = "***";
281 
    secret = args[secret_position];
282 
  }
283 
  console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);
284
285 
  return new Promise((resolve, reject) => {
286 
    const process = spawn(cmd, args);
287
288 
    let stdout = '';
289 
    let stderr = '';
290
291 
    process.stdout.on('data', (data) => {
292 
      stdout += data.toString();
293 
    });
294
295 
    process.stderr.on('data', (data) => {
296 
      stderr += data.toString();
297 
    });
298
299 
    process.on('error', (error) => {
300 
      let errorString = error.toString();
301 
      if (secret) errorString = errorString.replace(secret, "***");
302 
      console.log(`Shell command FAILED: ${cmd}`, errorString);
303 
      const e = new Error(
304 
        `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
305 
      );
306 
      reject(e);
307 
    });
308
309 
    process.on('close', (code) => {
310 
      if (stdout.length > 0) {
311 
        console.log("Shell stdout:", stdout);
312 
      }
313 
      if (stderr.length > 0) {
314 
        console.log("Shell stderr:", stderr);
315 
      }
316 
      if (code === 0) {
317 
        console.log(`Shell command completed successfully: ${cmd}`);
318 
        resolve(stdout);
319 
      } else {
320 
        reject(new Error(`Command failed with code ${code}: ${stderr}`));
321 
      }
322 
    });
323 
  });
324 
}
325
326 
// async function sh_run(
327 
//   secret_position: number | undefined,
328 
//   cmd: string,
329 
//   ...args: string[]
330 
// ) {
331 
//   const nargs = secret_position != undefined ? args.slice() : args;
332 
//   if (secret_position && secret_position < 0)
333 
//     secret_position = nargs.length - 1 + secret_position;
334 
//
335 
//   let secret: string | undefined = undefined;
336 
//   if (secret_position != undefined) {
337 
//     nargs[secret_position] = "***";
338 
//     secret = args[secret_position];
339 
//   }
340 
//
341 
//   console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`);
342 
//   try {
343 
//     const { stdout, stderr } = await exec(`${cmd} ${args.join(" ")}`);
344 
//     if (stdout.length > 0) {
345 
//       console.log("Shell stdout:", stdout);
346 
//     }
347 
//     if (stderr.length > 0) {
348 
//       console.log("Shell stderr:", stderr);
349 
//     }
350 
//     console.log(`Shell command completed successfully: ${cmd}`);
351 
//     return stdout;
352 
//   } catch (error: any) {
353 
//     let errorString = error.toString();
354 
//     if (secret) errorString = errorString.replace(secret, "***");
355 
//     console.log(`Shell command FAILED: ${cmd}`, errorString);
356 
//     throw new Error(
357 
//       `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}`
358 
//     );
359 
//   }
360 
// }
361
362 
async function get_gh_app_token() {
363 
  const workspace = process.env["WM_WORKSPACE"];
364 
  const jobToken = process.env["WM_TOKEN"];
365 
  const baseUrl =
366 
    process.env["BASE_INTERNAL_URL"] ??
367 
    process.env["BASE_URL"] ??
368 
    "http://localhost:8000";
369 
  const url = `${baseUrl}/api/w/${workspace}/github_app/token`;
370
371 
  const response = await fetch(url, {
372 
    method: "POST",
373 
    headers: {
374 
      "Content-Type": "application/json",
375 
      Authorization: `Bearer ${jobToken}`,
376 
    },
377 
    body: JSON.stringify({ job_token: jobToken }),
378 
  });
379
380 
  if (!response.ok) {
381 
    const errorBody = await response.text().catch(() => "");
382 
    throw new Error(`GitHub App token error (${response.status}): ${errorBody || response.statusText}`);
383 
  }
384 
  const data = await response.json();
385 
  return data.token;
386 
}
387
388 
function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) {
389 
  const url = new URL(gitHubUrl);
390 
  return `https://x-access-token:${installationToken}@${url.hostname}${url.pathname}`;
391 
}
392