logo Windmill Hub
Login
All items
1

Get recently updated objects

by
Published Dec 21, 2022

Trigger script to check for recently updated objects.

Script· trigger s3 Verified
Use in Windmill

The script

Submitted by hugo989 Bun
Verified 21 days ago
All edits
Permalink
1 
import { getState, setState } from "windmill-client@1";
2 
import * as Minio from "minio@7";
3
4 
/**
5 
 * Tested on 10 000 objects, which got processed in 2.3-2.6 seconds.
6 
 */
7 
type S3 = {
8 
  endPoint: string;
9 
  port: number;
10 
  useSSL: boolean;
11 
  pathStyle: boolean;
12 
  bucket: string;
13 
  accessKey: string;
14 
  secretKey: string;
15 
  region: string;
16 
};
17 
export async function main(s3: S3, bucket?: string) {
18 
  const client = new Minio.Client(s3);
19 
  const stream = client.listObjects(bucket || s3.bucket, "", true);
20
21 
  const lastCheck = (await getState()) || 0;
22 
  await setState(Date.now());
23 
  const newlyUpdated: any[] = [];
24
25 
  await new Promise<void>((resolve, reject) => {
26 
    stream.on("data", (obj: any) => {
27 
      const lastMod = new Date(obj.lastModified).getTime();
28 
      if (lastMod > lastCheck) {
29 
        newlyUpdated.push(obj);
30 
      }
31 
    });
32 
    stream.on("end", () => resolve());
33 
    stream.on("close", () => resolve());
34 
    stream.on("error", (err: any) => reject(err));
35 
  });
36
37 
  return newlyUpdated;
38 
}
39
Other submissions
  • Submitted by adam186 Deno
    Created 413 days ago
    All edits
    Permalink
    1 
    import {
    2 
      getState,
    3 
      setState,
    4 
    } from "https://deno.land/x/[email protected]/mod.ts";
    5 
    import { S3Client } from "https://deno.land/x/[email protected]/mod.ts";
    6
    7 
    /**
    8 
     * Tested on 10 000 objects, which got processed in 2.3-2.6 seconds.
    9 
     */
    10 
    type S3 = {
    11 
      endPoint: string;
    12 
      port: number;
    13 
      useSSL: boolean;
    14 
      pathStyle: boolean;
    15 
      bucket: string;
    16 
      accessKey: string;
    17 
      secretKey: string;
    18 
      region: string;
    19 
    };
    20 
    export async function main(s3: S3, bucket?: string) {
    21 
      const client = new S3Client(s3);
    22 
      const generator = client.listObjects({
    23 
        bucketName: bucket || undefined,
    24 
      });
    25
    26 
      const lastCheck = (await getState()) || 0;
    27 
      await setState(Date.now());
    28 
      const newlyUpdated = [];
    29
    30 
      while (true) {
    31 
        const obj = await generator.next();
    32 
        if (obj.done) break;
    33
    34 
        const lastMod = new Date(obj.value.lastModified).getTime();
    35 
        if (lastMod > lastCheck) {
    36 
          newlyUpdated.push(obj.value);
    37 
        }
    38 
      }
    39
    40 
      return newlyUpdated;
    41 
    }
    42