logo Windmill Hub
Login
All items

Get recently updated objects

Trigger script to check for recently updated objects.

Script· trigger s3 Verified

by adam186 · 12/21/2022

Use in Windmill

The script

Submitted by hugo989 Bun
Verified 6 days ago
All edits
Permalink
1 
import { getState, setState } from "windmill-client@1";
2 
import * as Minio from "minio@7";
3
4 
/**
5 
 * Tested on 10 000 objects, which got processed in 2.3-2.6 seconds.
6 
 */
7 
type S3 = {
8 
  endPoint: string;
9 
  port: number;
10 
  useSSL: boolean;
11 
  pathStyle: boolean;
12 
  bucket: string;
13 
  accessKey: string;
14 
  secretKey: string;
15 
  region: string;
16 
};
17 
export async function main(s3: S3, bucket?: string) {
18 
  const client = new Minio.Client(s3);
19 
  const stream = client.listObjects(bucket || s3.bucket, "", true);
20
21 
  const lastCheck = (await getState()) || 0;
22 
  await setState(Date.now());
23 
  const newlyUpdated: any[] = [];
24
25 
  await new Promise<void>((resolve, reject) => {
26 
    stream.on("data", (obj: any) => {
27 
      const lastMod = new Date(obj.lastModified).getTime();
28 
      if (lastMod > lastCheck) {
29 
        newlyUpdated.push(obj);
30 
      }
31 
    });
32 
    stream.on("end", () => resolve());
33 
    stream.on("close", () => resolve());
34 
    stream.on("error", (err: any) => reject(err));
35 
  });
36
37 
  return newlyUpdated;
38 
}
39
Other submissions
  • Submitted by adam186 Deno
    Created 398 days ago
    All edits
    Permalink
    1 
    import {
    2 
      getState,
    3 
      setState,
    4 
    } from "https://deno.land/x/[email protected]/mod.ts";
    5 
    import { S3Client } from "https://deno.land/x/[email protected]/mod.ts";
    6
    7 
    /**
    8 
     * Tested on 10 000 objects, which got processed in 2.3-2.6 seconds.
    9 
     */
    10 
    type S3 = {
    11 
      endPoint: string;
    12 
      port: number;
    13 
      useSSL: boolean;
    14 
      pathStyle: boolean;
    15 
      bucket: string;
    16 
      accessKey: string;
    17 
      secretKey: string;
    18 
      region: string;
    19 
    };
    20 
    export async function main(s3: S3, bucket?: string) {
    21 
      const client = new S3Client(s3);
    22 
      const generator = client.listObjects({
    23 
        bucketName: bucket || undefined,
    24 
      });
    25
    26 
      const lastCheck = (await getState()) || 0;
    27 
      await setState(Date.now());
    28 
      const newlyUpdated = [];
    29
    30 
      while (true) {
    31 
        const obj = await generator.next();
    32 
        if (obj.done) break;
    33
    34 
        const lastMod = new Date(obj.value.lastModified).getTime();
    35 
        if (lastMod > lastCheck) {
    36 
          newlyUpdated.push(obj.value);
    37 
        }
    38 
      }
    39
    40 
      return newlyUpdated;
    41 
    }
    42