import * as wmillclient from "windmill-client";
import wmill from "windmill-cli";
import { basename } from "node:path"
const util = require('util');
const exec = util.promisify(require('child_process').exec);
import process from "process";
export async function main(
workspace_id: string,
repo_url_resource_path: string,
path_type:
| "script"
| "flow"
| "app"
| "folder"
| "resource"
| "variable"
| "resourcetype"
| "schedule"
| "user"
| "group",
skip_secret = true,
path: string | undefined,
parent_path: string | undefined,
commit_msg: string,
use_individual_branch = false,
group_by_folder = false
) {
const repo_resource = await wmillclient.getResource(repo_url_resource_path);
const cwd = process.cwd();
process.env["HOME"] = "."
console.log(
`Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
);
const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
await move_to_git_branch(
workspace_id,
path_type,
path,
parent_path,
use_individual_branch,
group_by_folder
);
const subfolder = repo_resource.folder ?? "";
const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
console.log(
`Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
);
await wmill_sync_pull(
path_type,
workspace_id,
path,
parent_path,
skip_secret
);
await git_push(path, parent_path, commit_msg);
console.log("Finished syncing");
process.chdir(`${cwd}`);
}
async function git_clone(
cwd: string,
repo_resource: any,
use_individual_branch: boolean
): Promise<string> {
// TODO: handle private SSH keys as well
let repo_url = repo_resource.url;
const subfolder = repo_resource.folder ?? "";
const branch = repo_resource.branch ?? "";
const repo_name = basename(repo_url, ".git");
const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
if (azureMatch) {
console.log(
"Requires Azure DevOps service account access token, requesting..."
);
const azureResource = await wmillclient.getResource(azureMatch.groups.url);
const response = await fetch(
`https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
{
method: "POST",
body: new URLSearchParams({
client_id: azureResource.azureClientId,
client_secret: azureResource.azureClientSecret,
grant_type: "client_credentials",
resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
}),
}
);
const { access_token } = await response.json();
repo_url = repo_url.replace(azureMatch[0], access_token);
}
const args = ["clone", "--quiet", "--depth", "1"];
if (use_individual_branch) {
args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
}
if (subfolder !== "") {
args.push("--sparse");
}
if (branch !== "") {
args.push("--branch");
args.push(branch);
}
args.push(repo_url);
args.push(repo_name);
await sh_run(-1, "git", ...args);
try {
process.chdir(`${cwd}/${repo_name}`);
} catch (err) {
console.log(
`Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
);
throw err;
}
process.chdir(`${cwd}/${repo_name}`);
if (subfolder !== "") {
await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
}
try {
process.chdir(`${cwd}/${repo_name}/${subfolder}`);
} catch (err) {
console.log(
`Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
);
throw err;
}
return repo_name;
}
async function move_to_git_branch(
workspace_id: string,
path_type:
| "script"
| "flow"
| "app"
| "folder"
| "resource"
| "variable"
| "resourcetype"
| "schedule"
| "user"
| "group",
path: string | undefined,
parent_path: string | undefined,
use_individual_branch: boolean,
group_by_folder: boolean
) {
if (!use_individual_branch || path_type === "user" || path_type === "group") {
return;
}
const branchName = group_by_folder
? `wm_deploy/${workspace_id}/${(path ?? parent_path)
?.split("/")
.slice(0, 2)
.join("__")}`
: `wm_deploy/${workspace_id}/${path_type}/${(
path ?? parent_path
)?.replaceAll("/", "__")}`;
try {
await sh_run(undefined, "git", "checkout", branchName);
} catch (err) {
console.log(
`Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
);
try {
await sh_run(undefined, "git", "checkout", "-b", branchName);
await sh_run(
undefined,
"git",
"config",
"--add",
"--bool",
"push.autoSetupRemote",
"true"
);
} catch (err) {
console.log(
`Error checking out branch '${branchName}'. Error was:\n${err}`
);
throw err;
}
}
console.log(`Successfully switched to branch ${branchName}`);
}
async function git_push(
path: string | undefined,
parent_path: string | undefined,
commit_msg: string
) {
await sh_run(
undefined,
"git",
"config",
"user.email",
process.env["WM_EMAIL"] ?? ""
);
await sh_run(
undefined,
"git",
"config",
"user.name",
process.env["WM_USERNAME"] ?? ""
);
if (path !== undefined && path !== null && path !== "") {
try {
await sh_run(undefined, "git", "add", `${path}**`);
} catch (e) {
console.log(`Unable to stage files matching ${path}**, ${e}`);
}
}
if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
try {
await sh_run(undefined, "git", "add", `${parent_path}**`);
} catch (e) {
console.log(`Unable to stage files matching ${parent_path}, ${e}`);
}
}
try {
await sh_run(undefined, "git", "diff", "--cached", "--quiet");
} catch {
// git diff returns exit-code = 1 when there's at least one staged changes
await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
try {
await sh_run(undefined, "git", "push", "--porcelain");
} catch {
console.log("Could not push, trying to rebase first");
await sh_run(undefined, "git", "pull", "--rebase");
await sh_run(undefined, "git", "push", "--porcelain");
}
return;
}
console.log("No changes detected, nothing to commit. Returning...");
}
async function sh_run(
secret_position: number | undefined,
cmd: string,
...args: string[]
) {
const nargs = secret_position != undefined ? args.slice() : args;
if (secret_position && secret_position < 0) {
secret_position = nargs.length - 1 + secret_position;
}
if (secret_position != undefined) {
nargs[secret_position] = "***";
}
console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
const command = exec(`${cmd} ${args.join(" ")}`)
// new Deno.Command(cmd, {
// args: args,
// });
const { error, stdout, stderr } = await command
if (stdout.length > 0) {
console.log(stdout);
}
if (stderr.length > 0) {
console.log(stderr);
}
if (error) {
const err = `SH command '${cmd} ${nargs.join(
" "
)}' returned with error ${error}`;
throw Error(err);
}
console.log("Command successfully executed");
}
function regexFromPath(
path_type:
| "script"
| "flow"
| "app"
| "folder"
| "resource"
| "variable"
| "resourcetype"
| "schedule"
| "user"
| "group",
path: string
) {
if (path_type == "flow") {
return `${path}.flow/*`;
} if (path_type == "app") {
return `${path}.app/*`;
} else if (path_type == "folder") {
return `${path}/folder.meta.*`;
} else if (path_type == "resourcetype") {
return `${path}.resource-type.*`;
} else if (path_type == "resource") {
return `${path}.resource.*`;
} else if (path_type == "variable") {
return `${path}.variable.*`;
} else if (path_type == "schedule") {
return `${path}.schedule.*`;
} else if (path_type == "user") {
return `${path}.user.*`;
} else if (path_type == "group") {
return `${path}.group.*`;
} else {
return `${path}.*`;
}
}
async function wmill_sync_pull(
path_type:
| "script"
| "flow"
| "app"
| "folder"
| "resource"
| "variable"
| "resourcetype"
| "schedule"
| "user"
| "group",
workspace_id: string,
path: string | undefined,
parent_path: string | undefined,
skip_secret: boolean
) {
const includes = [];
if (path !== undefined && path !== null && path !== "") {
includes.push(regexFromPath(path_type, path));
}
if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
includes.push(regexFromPath(path_type, parent_path));
}
await wmill_run(
6,
"workspace",
"add",
workspace_id,
workspace_id,
process.env["BASE_URL"] + "/",
"--token",
process.env["WM_TOKEN"] ?? ""
);
console.log("Pulling workspace into git repo");
await wmill_run(
3,
"sync",
"pull",
"--token",
process.env["WM_TOKEN"] ?? "",
"--workspace",
workspace_id,
"--yes",
"--raw",
skip_secret ? "--skip-secrets" : "",
"--include-schedules",
"--include-users",
"--include-groups",
"--extra-includes",
includes.join(",")
);
}
async function wmill_run(secret_position: number, ...cmd: string[]) {
cmd = cmd.filter((elt) => elt !== "");
const cmd2 = cmd.slice();
cmd2[secret_position] = "***";
console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
await wmill.parse(cmd);
console.log("Command successfully executed");
}
Submitted by rubenfiszel 77 days ago
import * as wmillclient from "windmill-client";
import wmill from "windmill-cli";
import { basename } from "node:path"
const util = require('util');
const exec = util.promisify(require('child_process').exec);
export async function main(
workspace_id: string,
repo_url_resource_path: string,
path_type:
| "script"
| "flow"
| "app"
| "folder"
| "resource"
| "variable"
| "resourcetype"
| "schedule"
| "user"
| "group",
skip_secret = true,
path: string | undefined,
parent_path: string | undefined,
commit_msg: string,
use_individual_branch = false,
group_by_folder = false
) {
const repo_resource = await wmillclient.getResource(repo_url_resource_path);
const cwd = process.cwd();
process.env["HOME"] = "."
console.log(
`Syncing ${path_type} ${path ?? ""} with parent ${parent_path ?? ""}`
);
const repo_name = await git_clone(cwd, repo_resource, use_individual_branch);
await move_to_git_branch(
workspace_id,
path_type,
path,
parent_path,
use_individual_branch,
group_by_folder
);
const subfolder = repo_resource.folder ?? "";
const branch_or_default = repo_resource.branch ?? "<DEFAULT>";
console.log(
`Pushing to repository ${repo_name} in subfolder ${subfolder} on branch ${branch_or_default}`
);
await wmill_sync_pull(
path_type,
workspace_id,
path,
parent_path,
skip_secret
);
await git_push(path, parent_path, commit_msg);
console.log("Finished syncing");
process.chdir(`${cwd}`);
}
async function git_clone(
cwd: string,
repo_resource: any,
use_individual_branch: boolean
): Promise<string> {
// TODO: handle private SSH keys as well
let repo_url = repo_resource.url;
const subfolder = repo_resource.folder ?? "";
const branch = repo_resource.branch ?? "";
const repo_name = basename(repo_url, ".git");
const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?<url>.+)\)/);
if (azureMatch) {
console.log(
"Requires Azure DevOps service account access token, requesting..."
);
const azureResource = await wmillclient.getResource(azureMatch.groups.url);
const response = await fetch(
`https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`,
{
method: "POST",
body: new URLSearchParams({
client_id: azureResource.azureClientId,
client_secret: azureResource.azureClientSecret,
grant_type: "client_credentials",
resource: "499b84ac-1321-427f-aa17-267ca6975798/.default",
}),
}
);
const { access_token } = await response.json();
repo_url = repo_url.replace(azureMatch[0], access_token);
}
const args = ["clone", "--quiet", "--depth", "1"];
if (use_individual_branch) {
args.push("--no-single-branch"); // needed in case the asset branch already exists in the repo
}
if (subfolder !== "") {
args.push("--sparse");
}
if (branch !== "") {
args.push("--branch");
args.push(branch);
}
args.push(repo_url);
args.push(repo_name);
await sh_run(-1, "git", ...args);
try {
process.chdir(`${cwd}/${repo_name}`);
} catch (err) {
console.log(
`Error changing directory to '${cwd}/${repo_name}'. Error was:\n${err}`
);
throw err;
}
process.chdir(`${cwd}/${repo_name}`);
if (subfolder !== "") {
await sh_run(undefined, "git", "sparse-checkout", "add", subfolder);
}
try {
process.chdir(`${cwd}/${repo_name}/${subfolder}`);
} catch (err) {
console.log(
`Error changing directory to '${cwd}/${repo_name}/${subfolder}'. Error was:\n${err}`
);
throw err;
}
return repo_name;
}
async function move_to_git_branch(
workspace_id: string,
path_type:
| "script"
| "flow"
| "app"
| "folder"
| "resource"
| "variable"
| "resourcetype"
| "schedule"
| "user"
| "group",
path: string | undefined,
parent_path: string | undefined,
use_individual_branch: boolean,
group_by_folder: boolean
) {
if (!use_individual_branch || path_type === "user" || path_type === "group") {
return;
}
const branchName = group_by_folder
? `wm_deploy/${workspace_id}/${(path ?? parent_path)
?.split("/")
.slice(0, 2)
.join("__")}`
: `wm_deploy/${workspace_id}/${path_type}/${(
path ?? parent_path
)?.replaceAll("/", "__")}`;
try {
await sh_run(undefined, "git", "checkout", branchName);
} catch (err) {
console.log(
`Error checking out branch ${branchName}. It is possible it doesn't exist yet, tentatively creating it... Error was:\n${err}`
);
try {
await sh_run(undefined, "git", "checkout", "-b", branchName);
await sh_run(
undefined,
"git",
"config",
"--add",
"--bool",
"push.autoSetupRemote",
"true"
);
} catch (err) {
console.log(
`Error checking out branch '${branchName}'. Error was:\n${err}`
);
throw err;
}
}
console.log(`Successfully switched to branch ${branchName}`);
}
async function git_push(
path: string | undefined,
parent_path: string | undefined,
commit_msg: string
) {
await sh_run(
undefined,
"git",
"config",
"user.email",
process.env["WM_EMAIL"] ?? ""
);
await sh_run(
undefined,
"git",
"config",
"user.name",
process.env["WM_USERNAME"] ?? ""
);
if (path !== undefined && path !== null && path !== "") {
try {
await sh_run(undefined, "git", "add", `${path}**`);
} catch (e) {
console.log(`Unable to stage files matching ${path}**, ${e}`);
}
}
if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
try {
await sh_run(undefined, "git", "add", `${parent_path}**`);
} catch (e) {
console.log(`Unable to stage files matching ${parent_path}, ${e}`);
}
}
try {
await sh_run(undefined, "git", "diff", "--cached", "--quiet");
} catch {
// git diff returns exit-code = 1 when there's at least one staged changes
await sh_run(undefined, "git", "commit", "-m", `"${commit_msg == undefined || commit_msg == "" ? "no commit msg" : commit_msg}"`);
try {
await sh_run(undefined, "git", "push", "--porcelain");
} catch {
console.log("Could not push, trying to rebase first");
await sh_run(undefined, "git", "pull", "--rebase");
await sh_run(undefined, "git", "push", "--porcelain");
}
return;
}
console.log("No changes detected, nothing to commit. Returning...");
}
async function sh_run(
secret_position: number | undefined,
cmd: string,
...args: string[]
) {
const nargs = secret_position != undefined ? args.slice() : args;
if (secret_position && secret_position < 0) {
secret_position = nargs.length - 1 + secret_position;
}
if (secret_position != undefined) {
nargs[secret_position] = "***";
}
console.log(`Running '${cmd} ${nargs.join(" ")} ...'`);
const command = exec(`${cmd} ${args.join(" ")}`)
// new Deno.Command(cmd, {
// args: args,
// });
const { error, stdout, stderr } = await command
if (stdout.length > 0) {
console.log(stdout);
}
if (stderr.length > 0) {
console.log(stderr);
}
if (error) {
const err = `SH command '${cmd} ${nargs.join(
" "
)}' returned with error ${error}`;
throw Error(err);
}
console.log("Command successfully executed");
}
function regexFromPath(
path_type:
| "script"
| "flow"
| "app"
| "folder"
| "resource"
| "variable"
| "resourcetype"
| "schedule"
| "user"
| "group",
path: string
) {
if (path_type == "flow") {
return `${path}.flow/*`;
} if (path_type == "app") {
return `${path}.app/*`;
} else if (path_type == "folder") {
return `${path}/folder.meta.*`;
} else if (path_type == "resourcetype") {
return `${path}.resource-type.*`;
} else if (path_type == "resource") {
return `${path}.resource.*`;
} else if (path_type == "variable") {
return `${path}.variable.*`;
} else if (path_type == "schedule") {
return `${path}.schedule.*`;
} else if (path_type == "user") {
return `${path}.user.*`;
} else if (path_type == "group") {
return `${path}.group.*`;
} else {
return `${path}.*`;
}
}
async function wmill_sync_pull(
path_type:
| "script"
| "flow"
| "app"
| "folder"
| "resource"
| "variable"
| "resourcetype"
| "schedule"
| "user"
| "group",
workspace_id: string,
path: string | undefined,
parent_path: string | undefined,
skip_secret: boolean
) {
const includes = [];
if (path !== undefined && path !== null && path !== "") {
includes.push(regexFromPath(path_type, path));
}
if (parent_path !== undefined && parent_path !== null && parent_path !== "") {
includes.push(regexFromPath(path_type, parent_path));
}
await wmill_run(
6,
"workspace",
"add",
workspace_id,
workspace_id,
process.env["BASE_URL"] + "/",
"--token",
process.env["WM_TOKEN"] ?? ""
);
console.log("Pulling workspace into git repo");
await wmill_run(
3,
"sync",
"pull",
"--token",
process.env["WM_TOKEN"] ?? "",
"--workspace",
workspace_id,
"--yes",
"--raw",
skip_secret ? "--skip-secrets" : "",
"--include-schedules",
"--include-users",
"--include-groups",
"--extra-includes",
includes.join(",")
);
}
async function wmill_run(secret_position: number, ...cmd: string[]) {
cmd = cmd.filter((elt) => elt !== "");
const cmd2 = cmd.slice();
cmd2[secret_position] = "***";
console.log(`Running 'wmill ${cmd2.join(" ")} ...'`);
await wmill.parse(cmd);
console.log("Command successfully executed");
}
Submitted by rubenfiszel 88 days ago