logo Windmill Hub
Login
All items
2

RSS Feed Fetcher

by
Published Feb 6, 2023

Fetch new items from an RSS Feed

Script· trigger RSS Verified
Use in Windmill

The script

Submitted by hugo989 Bun
Verified 12 days ago
All edits
Permalink
1 
import * as wmill from "windmill-client@1";
2 
import { XMLParser } from "fast-xml-parser@4";
3
4 
const parser = new XMLParser({
5 
  ignoreAttributes: false,
6 
  attributeNamePrefix: "@",
7 
});
8
9 
type Rss = {
10 
  url: string;
11 
};
12 
export async function main(rss_feed: Rss) {
13 
  const rss_feed_url = rss_feed.url;
14 
  let feedStates = (await wmill.getInternalState()) || {};
15 
  const newestItem: Date = new Date(feedStates[rss_feed_url] || 0);
16
17 
  const items = await fetch(rss_feed_url)
18 
    .then((response) => response.text())
19 
    .then((str) => parser.parse(str))
20 
    .then((feed) => {
21 
      let items = [];
22 
      if (feed["rss"]) {
23 
        items = rss2_items(feed);
24 
      } else if (feed["feed"]) {
25 
        items = atom_items(feed);
26 
      }
27 
      const new_items = items.filter((item) => item.pubDate > newestItem);
28 
      return new_items;
29 
    });
30
31 
  if (items.length > 0) {
32 
    let newState = (await wmill.getInternalState()) || {};
33 
    const dates = items.map((item) => item.pubDate);
34 
    newState[rss_feed_url] = new Date(Math.max(...dates));
35 
    await wmill.setInternalState(newState);
36 
  }
37
38 
  return items.reverse();
39 
}
40
41 
function rss2_items(feed) {
42 
  var items = feed["rss"]["channel"]["item"] || [];
43 
  if (items.length === "undefined") {
44 
    // Single entry
45 
    items = [items];
46 
  }
47 
  return items.map((item) => ({
48 
    title: item.title,
49 
    link: item.link,
50 
    pubDate: new Date(item.pubDate),
51 
  }));
52 
}
53
54 
function atom_items(feed) {
55 
  var items = feed["feed"]["entry"] || [];
56 
  if (items.length === "undefined") {
57 
    // Single entry
58 
    items = [items];
59 
  }
60 
  return items.map((item) => ({
61 
    title: typeof item.title === "string" ? item.title : item.title["#text"],
62 
    link:
63 
      item.link instanceof Array ? item.link[0]["@href"] : item.link["@href"],
64 
    pubDate: new Date(item.updated),
65 
  }));
66 
}
67
Other submissions
  • Submitted by pirmin kalberer631 Deno
    Created 404 days ago
    All edits
    Permalink
    1 
    import * as wmill from "https://deno.land/x/[email protected]/mod.ts";
    2 
    import { parse } from "https://deno.land/x/xml/mod.ts";
    3
    4 
    type Rss = {
    5 
      url: string;
    6 
    };
    7 
    export async function main(rss_feed: Rss) {
    8 
      const rss_feed_url = rss_feed.url;
    9 
      let feedStates = (await wmill.getInternalState()) || {};
    10 
      const newestItem: Date = new Date(feedStates[rss_feed_url] || 0);
    11
    12 
      const items = await fetch(rss_feed_url)
    13 
        .then((response) => response.text())
    14 
        .then((str) => parse(str))
    15 
        .then((feed) => {
    16 
          let items = [];
    17 
          if (feed["rss"]) {
    18 
            items = rss2_items(feed);
    19 
          } else if (feed["feed"]) {
    20 
            items = atom_items(feed);
    21 
          }
    22 
          const new_items = items.filter((item) => item.pubDate > newestItem);
    23 
          return new_items;
    24 
        });
    25
    26 
      if (items.length > 0) {
    27 
        let newState = (await wmill.getInternalState()) || {};
    28 
        const dates = items.map((item) => item.pubDate);
    29 
        newState[rss_feed_url] = new Date(Math.max(...dates));
    30 
        await wmill.setInternalState(newState);
    31 
      }
    32
    33 
      return items.reverse();
    34 
    }
    35
    36 
    function rss2_items(feed) {
    37 
      var items = feed["rss"]["channel"]["item"] || [];
    38 
      if (items.length === "undefined") {
    39 
        // Single entry
    40 
        items = [items];
    41 
      }
    42 
      return items.map((item) => ({
    43 
        title: item.title,
    44 
        link: item.link,
    45 
        pubDate: new Date(item.pubDate),
    46 
      }));
    47 
    }
    48
    49 
    function atom_items(feed) {
    50 
      var items = feed["feed"]["entry"] || [];
    51 
      if (items.length === "undefined") {
    52 
        // Single entry
    53 
        items = [items];
    54 
      }
    55 
      return items.map((item) => ({
    56 
        title: typeof item.title === "string" ? item.title : item.title["#text"],
    57 
        link:
    58 
          item.link instanceof Array ? item.link[0]["@href"] : item.link["@href"],
    59 
        pubDate: new Date(item.updated),
    60 
      }));
    61 
    }
    62
  • Submitted by gabrielius.m896 Deno
    Created 500 days ago
    All edits
    Permalink
    1 
    // Based on https://hub.windmill.dev/scripts/RSS/1529/rss-feed-fetcher-RSS#1536
    2 
    import * as wmill from "https://deno.land/x/[email protected]/mod.ts";
    3 
    import { parse } from "https://deno.land/x/xml/mod.ts";
    4
    5 
    type Rss = {
    6 
      url: string;
    7 
    };
    8
    9 
    type Item = {
    10 
      title: string;
    11 
      link: string;
    12 
      date: Date;
    13 
    };
    14
    15 
    type FeedState = {
    16 
      date: Date | undefined;
    17 
      link: string | undefined;
    18 
      title: string | undefined;
    19 
    };
    20
    21 
    export async function main(
    22 
      rssFeed: Rss,
    23 
      byDate: boolean = true,
    24 
      byLink: boolean = false,
    25 
      byTitle: boolean = false,
    26 
      clearState: boolean = false,
    27 
    ) {
    28 
      const state = (await wmill.getInternalState()) || {};
    29 
      const feedState: FeedState = clearState ? {} : state[rssFeed.url] || {};
    30 
      if (!byDate) {
    31 
        feedState.date = undefined;
    32 
      }
    33 
      if (!byLink) {
    34 
        feedState.link = undefined;
    35 
      }
    36 
      if (!byTitle) {
    37 
        feedState.title = undefined;
    38 
      }
    39
    40 
      const items = await fetch(rssFeed.url)
    41 
        .then((response) => response.text())
    42 
        .then((str) => parse(str))
    43 
        .then((feed) => {
    44 
          let items = [];
    45 
          if (feed["rss"]) {
    46 
            items = rss2Items(feed);
    47 
          } else if (feed["feed"]) {
    48 
            items = atomItems(feed);
    49 
          }
    50 
          return newItemFilter(items, feedState);
    51 
        });
    52
    53 
      if (items.length > 0) {
    54 
        const newFeedState: FeedState = {
    55 
          date: byDate
    56 
            ? new Date(Math.max(...items.map((item) => item.date.getTime())))
    57 
            : undefined,
    58 
          link: byLink ? items[0].link : undefined,
    59 
          title: byTitle ? items[0].title : undefined,
    60 
        };
    61
    62 
        state[rssFeed.url] = newFeedState;
    63 
        await wmill.setInternalState(state);
    64 
      }
    65
    66 
      return items.reverse();
    67 
    }
    68
    69 
    function rss2Items(feed): Array<Item> {
    70 
      let items = feed["rss"]["channel"]["item"] || [];
    71 
      if (items.length === "undefined") {
    72 
        // Single entry
    73 
        items = [items];
    74 
      }
    75 
      return items.map((item) => ({
    76 
        title: item.title,
    77 
        link: item.link,
    78 
        date: new Date(item.pubDate),
    79 
      }));
    80 
    }
    81
    82 
    function atomItems(feed): Array<Item> {
    83 
      let items = feed["feed"]["entry"] || [];
    84 
      if (items.length === "undefined") {
    85 
        // Single entry
    86 
        items = [items];
    87 
      }
    88 
      return items.map((item) => ({
    89 
        title: typeof item.title === "string" ? item.title : item.title["#text"],
    90 
        link: item.link instanceof Array
    91 
          ? item.link[0]["@href"]
    92 
          : item.link["@href"],
    93 
        date: new Date(item.updated),
    94 
      }));
    95 
    }
    96
    97 
    function newItemFilter(items: Array<Item>, state: FeedState): Array<Item> {
    98 
      if (state.date && state.date.getTime() > 0) {
    99 
        const date = state.date;
    100 
        items = items.filter((item) => item.date > date);
    101 
      }
    102
    103 
      if (state.link) {
    104 
        const link = state.link;
    105 
        const index = items.findIndex((item) => item.link === link);
    106 
        if (index >= 0) {
    107 
          items = items.slice(0, index);
    108 
        }
    109 
      }
    110
    111 
      if (state.title) {
    112 
        const title = state.title;
    113 
        const index = items.findIndex((item) => item.title === title);
    114 
        if (index >= 0) {
    115 
          items = items.slice(0, index);
    116 
        }
    117 
      }
    118
    119 
      return items;
    120 
    }
    121