logo Windmill Hub
Login
All items

RSS Feed Fetcher

Fetch new items from an RSS Feed

Script· trigger RSS Verified

by pirmin kalberer631 · 2/6/2023

Use in Windmill

The script

Submitted by pirmin kalberer631 Deno
Verified 384 days ago
All edits
Permalink
1 
import * as wmill from "https://deno.land/x/[email protected]/mod.ts";
2 
import { parse } from "https://deno.land/x/xml/mod.ts";
3
4 
type Rss = {
5 
  url: string;
6 
};
7 
export async function main(rss_feed: Rss) {
8 
  const rss_feed_url = rss_feed.url;
9 
  let feedStates = (await wmill.getInternalState()) || {};
10 
  const newestItem: Date = new Date(feedStates[rss_feed_url] || 0);
11
12 
  const items = await fetch(rss_feed_url)
13 
    .then((response) => response.text())
14 
    .then((str) => parse(str))
15 
    .then((feed) => {
16 
      let items = [];
17 
      if (feed["rss"]) {
18 
        items = rss2_items(feed);
19 
      } else if (feed["feed"]) {
20 
        items = atom_items(feed);
21 
      }
22 
      const new_items = items.filter((item) => item.pubDate > newestItem);
23 
      return new_items;
24 
    });
25
26 
  if (items.length > 0) {
27 
    let newState = (await wmill.getInternalState()) || {};
28 
    const dates = items.map((item) => item.pubDate);
29 
    newState[rss_feed_url] = new Date(Math.max(...dates));
30 
    await wmill.setInternalState(newState);
31 
  }
32
33 
  return items.reverse();
34 
}
35
36 
function rss2_items(feed) {
37 
  var items = feed["rss"]["channel"]["item"] || [];
38 
  if (items.length === "undefined") {
39 
    // Single entry
40 
    items = [items];
41 
  }
42 
  return items.map((item) => ({
43 
    title: item.title,
44 
    link: item.link,
45 
    pubDate: new Date(item.pubDate),
46 
  }));
47 
}
48
49 
function atom_items(feed) {
50 
  var items = feed["feed"]["entry"] || [];
51 
  if (items.length === "undefined") {
52 
    // Single entry
53 
    items = [items];
54 
  }
55 
  return items.map((item) => ({
56 
    title: typeof item.title === "string" ? item.title : item.title["#text"],
57 
    link:
58 
      item.link instanceof Array ? item.link[0]["@href"] : item.link["@href"],
59 
    pubDate: new Date(item.updated),
60 
  }));
61 
}
62
Other submissions
  • Submitted by gabrielius.m896 Deno
    Created 480 days ago
    All edits
    Permalink
    1 
    // Based on https://hub.windmill.dev/scripts/RSS/1529/rss-feed-fetcher-RSS#1536
    2 
    import * as wmill from "https://deno.land/x/[email protected]/mod.ts";
    3 
    import { parse } from "https://deno.land/x/xml/mod.ts";
    4
    5 
    type Rss = {
    6 
      url: string;
    7 
    };
    8
    9 
    type Item = {
    10 
      title: string;
    11 
      link: string;
    12 
      date: Date;
    13 
    };
    14
    15 
    type FeedState = {
    16 
      date: Date | undefined;
    17 
      link: string | undefined;
    18 
      title: string | undefined;
    19 
    };
    20
    21 
    export async function main(
    22 
      rssFeed: Rss,
    23 
      byDate: boolean = true,
    24 
      byLink: boolean = false,
    25 
      byTitle: boolean = false,
    26 
      clearState: boolean = false,
    27 
    ) {
    28 
      const state = (await wmill.getInternalState()) || {};
    29 
      const feedState: FeedState = clearState ? {} : state[rssFeed.url] || {};
    30 
      if (!byDate) {
    31 
        feedState.date = undefined;
    32 
      }
    33 
      if (!byLink) {
    34 
        feedState.link = undefined;
    35 
      }
    36 
      if (!byTitle) {
    37 
        feedState.title = undefined;
    38 
      }
    39
    40 
      const items = await fetch(rssFeed.url)
    41 
        .then((response) => response.text())
    42 
        .then((str) => parse(str))
    43 
        .then((feed) => {
    44 
          let items = [];
    45 
          if (feed["rss"]) {
    46 
            items = rss2Items(feed);
    47 
          } else if (feed["feed"]) {
    48 
            items = atomItems(feed);
    49 
          }
    50 
          return newItemFilter(items, feedState);
    51 
        });
    52
    53 
      if (items.length > 0) {
    54 
        const newFeedState: FeedState = {
    55 
          date: byDate
    56 
            ? new Date(Math.max(...items.map((item) => item.date.getTime())))
    57 
            : undefined,
    58 
          link: byLink ? items[0].link : undefined,
    59 
          title: byTitle ? items[0].title : undefined,
    60 
        };
    61
    62 
        state[rssFeed.url] = newFeedState;
    63 
        await wmill.setInternalState(state);
    64 
      }
    65
    66 
      return items.reverse();
    67 
    }
    68
    69 
    function rss2Items(feed): Array<Item> {
    70 
      let items = feed["rss"]["channel"]["item"] || [];
    71 
      if (items.length === "undefined") {
    72 
        // Single entry
    73 
        items = [items];
    74 
      }
    75 
      return items.map((item) => ({
    76 
        title: item.title,
    77 
        link: item.link,
    78 
        date: new Date(item.pubDate),
    79 
      }));
    80 
    }
    81
    82 
    function atomItems(feed): Array<Item> {
    83 
      let items = feed["feed"]["entry"] || [];
    84 
      if (items.length === "undefined") {
    85 
        // Single entry
    86 
        items = [items];
    87 
      }
    88 
      return items.map((item) => ({
    89 
        title: typeof item.title === "string" ? item.title : item.title["#text"],
    90 
        link: item.link instanceof Array
    91 
          ? item.link[0]["@href"]
    92 
          : item.link["@href"],
    93 
        date: new Date(item.updated),
    94 
      }));
    95 
    }
    96
    97 
    function newItemFilter(items: Array<Item>, state: FeedState): Array<Item> {
    98 
      if (state.date && state.date.getTime() > 0) {
    99 
        const date = state.date;
    100 
        items = items.filter((item) => item.date > date);
    101 
      }
    102
    103 
      if (state.link) {
    104 
        const link = state.link;
    105 
        const index = items.findIndex((item) => item.link === link);
    106 
        if (index >= 0) {
    107 
          items = items.slice(0, index);
    108 
        }
    109 
      }
    110
    111 
      if (state.title) {
    112 
        const title = state.title;
    113 
        const index = items.findIndex((item) => item.title === title);
    114 
        if (index >= 0) {
    115 
          items = items.slice(0, index);
    116 
        }
    117 
      }
    118
    119 
      return items;
    120 
    }
    121