logo Windmill Hub
Login
All items
0

Put a list of objects in a bucket

by
Published Mar 24, 2023

Uploads a list of files encoded in base64 to S3. Detects file type/extension using base64 header (e.g. "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAA..."). The output of File Input component can be used as parameter for this function.

Script s3 Verified
Use in Windmill

The script

Submitted by hugo989 Bun
Verified 6 days ago
All edits
Permalink
1 
import * as Minio from "minio@7";
2 
import mime from "mime-types@2";
3
4 
const MESSAGES_I9 = {
5 
  en: {
6 
    error_mimetype: "File type could not be identified",
7 
    error_file_upload: "Error uploading file (S3)",
8 
  },
9 
  pt_br: {
10 
    error_mimetype: "Tipo do arquivo não pôde ser identificado",
11 
    error_file_upload: "Erro ao enviar arquivo (S3)",
12 
  },
13 
};
14 
const MESSAGES = MESSAGES_I9.en;
15
16 
/****
17 
 * Returns the mimetype and extension of a base64 encoded file
18 
 * @param encoded Base64 encoded file
19 
 * @returns Object with mimetype and extension
20 
 */
21 
function base64MimeType(encoded: string) {
22 
  let result = null;
23
24 
  if (typeof encoded !== "string") {
25 
    return result;
26 
  }
27
28 
  const match = encoded.match(/data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,.*/);
29 
  if (match && match.length) {
30 
    result = {
31 
      mimetype: match[1],
32 
      extension: mime.extension(match[1]),
33 
    };
34 
  }
35
36 
  return result;
37 
}
38
39 
/****
40 
 * Uploads a list of files to S3
41 
 * @param s3 S3 resource
42 
 * @param files Array of base64 encoded files
43 
 * @returns Object with error_count and array of results (files URLs or error messages)
44 
 *
45 
 * Detects file type/extension using base64 header (e.g. "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAA...").
46 
 * The output of File Input component can be used as parameter for this function.
47 
 */
48 
type S3 = {
49 
  endPoint: string;
50 
  port: number;
51 
  useSSL: boolean;
52 
  pathStyle: boolean;
53 
  bucket: string;
54 
  accessKey: string;
55 
  secretKey: string;
56 
  region: string;
57 
};
58 
export async function main(s3: S3, files: string[]) {
59 
  if (files.length <= 0) {
60 
    return;
61 
  }
62
63 
  const s3client = new Minio.Client(s3);
64
65 
  const out: any[] = [];
66 
  let error_count = 0;
67
68 
  await Promise.all(
69 
    files.map(async (file: string) => {
70 
      try {
71 
        const mime_data = base64MimeType(file)!;
72 
        if (mime_data) {
73 
          const now = new Date();
74 
          const pad = (n: number) => String(n).padStart(2, "0");
75 
          const timestamp =
76 
            `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}` +
77 
            `-${pad(now.getHours())}-${pad(now.getMinutes())}-${pad(now.getSeconds())}`;
78 
          const file_name =
79 
            timestamp +
80 
            "-" +
81 
            Math.random().toString(36).substring(2, 8) +
82 
            "." +
83 
            mime_data.extension;
84 
          const base64Data = file.replace(/^data:\w+\/\w+;base64,/, "");
85 
          const bytes = Buffer.from(base64Data, "base64");
86
87 
          await s3client.putObject(s3.bucket, file_name, bytes, bytes.length, {
88 
            "Content-Type": mime_data.mimetype,
89 
          });
90
91 
          const url = `https://${s3["endPoint"]}/${s3["bucket"]}/${file_name}`;
92 
          out.push({
93 
            success: true,
94 
            url,
95 
          });
96 
        } else {
97 
          out.push({
98 
            success: false,
99 
            message: MESSAGES.error_mimetype,
100 
          });
101 
          error_count++;
102 
        }
103 
      } catch (error) {
104 
        out.push({
105 
          success: false,
106 
          message: `${MESSAGES.error_file_upload} (S3): ${error.message}`,
107 
        });
108 
        error_count++;
109 
      }
110 
    }),
111 
  );
112
113 
  return { error_count, files: out };
114 
}
115
Other submissions
  • Submitted by vidz657 Deno
    Created 398 days ago
    All edits
    Permalink
    1 
    import { S3Client } from "https://deno.land/x/[email protected]/mod.ts";
    2 
    import { format } from "https://deno.land/[email protected]/datetime/mod.ts";
    3 
    import { mime } from "https://deno.land/x/[email protected]/mod.ts";
    4
    5 
    const MESSAGES_I9 = {
    6 
      en: {
    7 
        error_mimetype: "File type could not be identified",
    8 
        error_file_upload: "Error uploading file (S3)",
    9 
      },
    10 
      pt_br: {
    11 
        error_mimetype: "Tipo do arquivo não pôde ser identificado",
    12 
        error_file_upload: "Erro ao enviar arquivo (S3)",
    13 
      },
    14 
    };
    15 
    const MESSAGES = MESSAGES_I9.en;
    16
    17 
    /****
    18 
     * Returns the mimetype and extension of a base64 encoded file
    19 
     * @param encoded Base64 encoded file
    20 
     * @returns Object with mimetype and extension
    21 
     */
    22 
    function base64MimeType(encoded: string) {
    23 
      let result = null;
    24
    25 
      if (typeof encoded !== "string") {
    26 
        return result;
    27 
      }
    28
    29 
      const match = encoded.match(/data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,.*/);
    30 
      if (match && match.length) {
    31 
        result = {
    32 
          mimetype: match[1],
    33 
          extension: mime.getExtension(match[1]),
    34 
        };
    35 
      }
    36
    37 
      return result;
    38 
    }
    39
    40 
    /****
    41 
     * Uploads a list of files to S3
    42 
     * @param s3 S3 resource
    43 
     * @param files Array of base64 encoded files
    44 
     * @returns Object with error_count and array of results (files URLs or error messages)
    45 
     *
    46 
     * Detects file type/extension using base64 header (e.g. "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAA...").
    47 
     * The output of File Input component can be used as parameter for this function.
    48 
     */
    49 
    type S3 = {
    50 
      endPoint: string;
    51 
      port: number;
    52 
      useSSL: boolean;
    53 
      pathStyle: boolean;
    54 
      bucket: string;
    55 
      accessKey: string;
    56 
      secretKey: string;
    57 
      region: string;
    58 
    };
    59 
    export async function main(s3: S3, files: string[]) {
    60 
      if (files.length <= 0) {
    61 
        return;
    62 
      }
    63
    64 
      const s3client = new S3Client(s3);
    65
    66 
      const out: any[] = [];
    67 
      let error_count = 0;
    68
    69 
      await Promise.all(
    70 
        files.map(async (file: string) => {
    71 
          try {
    72 
            const mime_data = base64MimeType(file)!;
    73 
            if (mime_data) {
    74 
              const file_name =
    75 
                format(new Date(), "yyyy-MM-dd-HH-mm-ss") +
    76 
                "-" +
    77 
                Math.random().toString(36).substring(2, 8) +
    78 
                "." +
    79 
                mime_data.extension;
    80 
              const base64Data = file.replace(/^data:\w+\/\w+;base64,/, "");
    81 
              const bytes = Uint8Array.from(atob(base64Data), (c) =>
    82 
                c.charCodeAt(0),
    83 
              );
    84
    85 
              await s3client.putObject(file_name, bytes, {
    86 
                metadata: {
    87 
                  "Content-Type": mime_data.mimetype,
    88 
                },
    89 
              });
    90
    91 
              const url = `https://${s3["endPoint"]}/${s3["bucket"]}/${file_name}`;
    92 
              out.push({
    93 
                success: true,
    94 
                url,
    95 
              });
    96 
            } else {
    97 
              out.push({
    98 
                success: false,
    99 
                message: MESSAGES.error_mimetype,
    100 
              });
    101 
              error_count++;
    102 
            }
    103 
          } catch (error) {
    104 
            out.push({
    105 
              success: false,
    106 
              message: `${MESSAGES.error_file_upload} (S3): ${error.message}`,
    107 
            });
    108 
            error_count++;
    109 
          }
    110 
        }),
    111 
      );
    112
    113 
      return { error_count, files: out };
    114 
    }
    115