Script 'RSS Feed Fetcher' for RSS

RSS Feed Fetcher Trigger

Verified

Fetch new items from an RSS Feed

Created by pirmin kalberer631 832 days ago Viewed 26683 times

In Windmill, a trigger script is designed to pull data from an external source and return all the new items since the last run. It operates without resorting to external webhooks and is typically used with schedules and states to compare the current execution to the previous one.

Submitted by pirmin kalberer631 Deno

Verified 832 days ago

All edits

Permalink

import * as wmill from "https://deno.land/x/windmill@v1.85.0/mod.ts";
import { parse } from "https://deno.land/x/xml/mod.ts";

type Rss = {
  url: string;
};
export async function main(rss_feed: Rss) {
  const rss_feed_url = rss_feed.url;
  let feedStates = (await wmill.getInternalState()) || {};
  const newestItem: Date = new Date(feedStates[rss_feed_url] || 0);

  const items = await fetch(rss_feed_url)
    .then((response) => response.text())
    .then((str) => parse(str))
    .then((feed) => {
      let items = [];
      if (feed["rss"]) {
        items = rss2_items(feed);
      } else if (feed["feed"]) {
        items = atom_items(feed);
      }
      const new_items = items.filter((item) => item.pubDate > newestItem);
      return new_items;
    });

  if (items.length > 0) {
    let newState = (await wmill.getInternalState()) || {};
    const dates = items.map((item) => item.pubDate);
    newState[rss_feed_url] = new Date(Math.max(...dates));
    await wmill.setInternalState(newState);
  }

  return items.reverse();
}

function rss2_items(feed) {
  var items = feed["rss"]["channel"]["item"] || [];
  if (items.length === "undefined") {
    // Single entry
    items = [items];
  }
  return items.map((item) => ({
    title: item.title,
    link: item.link,
    pubDate: new Date(item.pubDate),
  }));
}

function atom_items(feed) {
  var items = feed["feed"]["entry"] || [];
  if (items.length === "undefined") {
    // Single entry
    items = [items];
  }
  return items.map((item) => ({
    title: typeof item.title === "string" ? item.title : item.title["#text"],
    link:
      item.link instanceof Array ? item.link[0]["@href"] : item.link["@href"],
    pubDate: new Date(item.updated),
  }));
}


Other submissions

Submitted by gabrielius.m896 Deno

Created 107 days ago

All edits

Permalink

// Based on https://hub.windmill.dev/scripts/RSS/1529/rss-feed-fetcher-RSS#1536
import * as wmill from "https://deno.land/x/windmill@v1.85.0/mod.ts";
import { parse } from "https://deno.land/x/xml/mod.ts";

type Rss = {
  url: string;
};

type Item = {
  title: string;
  link: string;
  date: Date;
};

type FeedState = {
  date: Date | undefined;
  link: string | undefined;
  title: string | undefined;
};

export async function main(
  rssFeed: Rss,
  byDate: boolean = true,
  byLink: boolean = false,
  byTitle: boolean = false,
  clearState: boolean = false,
) {
  const state = (await wmill.getInternalState()) || {};
  const feedState: FeedState = clearState ? {} : state[rssFeed.url] || {};
  if (!byDate) {
    feedState.date = undefined;
  }
  if (!byLink) {
    feedState.link = undefined;
  }
  if (!byTitle) {
    feedState.title = undefined;
  }

  const items = await fetch(rssFeed.url)
    .then((response) => response.text())
    .then((str) => parse(str))
    .then((feed) => {
      let items = [];
      if (feed["rss"]) {
        items = rss2Items(feed);
      } else if (feed["feed"]) {
        items = atomItems(feed);
      }
      return newItemFilter(items, feedState);
    });

  if (items.length > 0) {
    const newFeedState: FeedState = {
      date: byDate
        ? new Date(Math.max(...items.map((item) => item.date.getTime())))
        : undefined,
      link: byLink ? items[0].link : undefined,
      title: byTitle ? items[0].title : undefined,
    };

    state[rssFeed.url] = newFeedState;
    await wmill.setInternalState(state);
  }

  return items.reverse();
}

function rss2Items(feed): Array<Item> {
  let items = feed["rss"]["channel"]["item"] || [];
  if (items.length === "undefined") {
    // Single entry
    items = [items];
  }
  return items.map((item) => ({
    title: item.title,
    link: item.link,
    date: new Date(item.pubDate),
  }));
}

function atomItems(feed): Array<Item> {
  let items = feed["feed"]["entry"] || [];
  if (items.length === "undefined") {
    // Single entry
    items = [items];
  }
  return items.map((item) => ({
    title: typeof item.title === "string" ? item.title : item.title["#text"],
    link: item.link instanceof Array
      ? item.link[0]["@href"]
      : item.link["@href"],
    date: new Date(item.updated),
  }));
}

function newItemFilter(items: Array<Item>, state: FeedState): Array<Item> {
  if (state.date && state.date.getTime() > 0) {
    const date = state.date;
    items = items.filter((item) => item.date > date);
  }

  if (state.link) {
    const link = state.link;
    const index = items.findIndex((item) => item.link === link);
    if (index >= 0) {
      items = items.slice(0, index);
    }
  }

  if (state.title) {
    const title = state.title;
    const index = items.findIndex((item) => item.title === title);
    if (index >= 0) {
      items = items.slice(0, index);
    }
  }

  return items;
}


`1`	`import * as wmill from "https://deno.land/x/windmill@v1.85.0/mod.ts";`
`2`	`import { parse } from "https://deno.land/x/xml/mod.ts";`
`3`
`4`	`type Rss = {`
`5`	`url: string;`
`6`	`};`
`7`	`export async function main(rss_feed: Rss) {`
`8`	`const rss_feed_url = rss_feed.url;`
`9`	`let feedStates = (await wmill.getInternalState()) \|\| {};`
`10`	`const newestItem: Date = new Date(feedStates[rss_feed_url] \|\| 0);`
`11`
`12`	`const items = await fetch(rss_feed_url)`
`13`	`.then((response) => response.text())`
`14`	`.then((str) => parse(str))`
`15`	`.then((feed) => {`
`16`	`let items = [];`
`17`	`if (feed["rss"]) {`
`18`	`items = rss2_items(feed);`
`19`	`} else if (feed["feed"]) {`
`20`	`items = atom_items(feed);`
`21`	`}`
`22`	`const new_items = items.filter((item) => item.pubDate > newestItem);`
`23`	`return new_items;`
`24`	`});`
`25`
`26`	`if (items.length > 0) {`
`27`	`let newState = (await wmill.getInternalState()) \|\| {};`
`28`	`const dates = items.map((item) => item.pubDate);`
`29`	`newState[rss_feed_url] = new Date(Math.max(...dates));`
`30`	`await wmill.setInternalState(newState);`
`31`	`}`
`32`
`33`	`return items.reverse();`
`34`	`}`
`35`
`36`	`function rss2_items(feed) {`
`37`	`var items = feed["rss"]["channel"]["item"] \|\| [];`
`38`	`if (items.length === "undefined") {`
`39`	`// Single entry`
`40`	`items = [items];`
`41`	`}`
`42`	`return items.map((item) => ({`
`43`	`title: item.title,`
`44`	`link: item.link,`
`45`	`pubDate: new Date(item.pubDate),`
`46`	`}));`
`47`	`}`
`48`
`49`	`function atom_items(feed) {`
`50`	`var items = feed["feed"]["entry"] \|\| [];`
`51`	`if (items.length === "undefined") {`
`52`	`// Single entry`
`53`	`items = [items];`
`54`	`}`
`55`	`return items.map((item) => ({`
`56`	`title: typeof item.title === "string" ? item.title : item.title["#text"],`
`57`	`link:`
`58`	`item.link instanceof Array ? item.link[0]["@href"] : item.link["@href"],`
`59`	`pubDate: new Date(item.updated),`
`60`	`}));`
`61`	`}`
`62`

`1`	`// Based on https://hub.windmill.dev/scripts/RSS/1529/rss-feed-fetcher-RSS#1536`
`2`	`import * as wmill from "https://deno.land/x/windmill@v1.85.0/mod.ts";`
`3`	`import { parse } from "https://deno.land/x/xml/mod.ts";`
`4`
`5`	`type Rss = {`
`6`	`url: string;`
`7`	`};`
`8`
`9`	`type Item = {`
`10`	`title: string;`
`11`	`link: string;`
`12`	`date: Date;`
`13`	`};`
`14`
`15`	`type FeedState = {`
`16`	`date: Date \| undefined;`
`17`	`link: string \| undefined;`
`18`	`title: string \| undefined;`
`19`	`};`
`20`
`21`	`export async function main(`
`22`	`rssFeed: Rss,`
`23`	`byDate: boolean = true,`
`24`	`byLink: boolean = false,`
`25`	`byTitle: boolean = false,`
`26`	`clearState: boolean = false,`
`27`	`) {`
`28`	`const state = (await wmill.getInternalState()) \|\| {};`
`29`	`const feedState: FeedState = clearState ? {} : state[rssFeed.url] \|\| {};`
`30`	`if (!byDate) {`
`31`	`feedState.date = undefined;`
`32`	`}`
`33`	`if (!byLink) {`
`34`	`feedState.link = undefined;`
`35`	`}`
`36`	`if (!byTitle) {`
`37`	`feedState.title = undefined;`
`38`	`}`
`39`
`40`	`const items = await fetch(rssFeed.url)`
`41`	`.then((response) => response.text())`
`42`	`.then((str) => parse(str))`
`43`	`.then((feed) => {`
`44`	`let items = [];`
`45`	`if (feed["rss"]) {`
`46`	`items = rss2Items(feed);`
`47`	`} else if (feed["feed"]) {`
`48`	`items = atomItems(feed);`
`49`	`}`
`50`	`return newItemFilter(items, feedState);`
`51`	`});`
`52`
`53`	`if (items.length > 0) {`
`54`	`const newFeedState: FeedState = {`
`55`	`date: byDate`
`56`	`? new Date(Math.max(...items.map((item) => item.date.getTime())))`
`57`	`: undefined,`
`58`	`link: byLink ? items[0].link : undefined,`
`59`	`title: byTitle ? items[0].title : undefined,`
`60`	`};`
`61`
`62`	`state[rssFeed.url] = newFeedState;`
`63`	`await wmill.setInternalState(state);`
`64`	`}`
`65`
`66`	`return items.reverse();`
`67`	`}`
`68`
`69`	`function rss2Items(feed): Array<Item> {`
`70`	`let items = feed["rss"]["channel"]["item"] \|\| [];`
`71`	`if (items.length === "undefined") {`
`72`	`// Single entry`
`73`	`items = [items];`
`74`	`}`
`75`	`return items.map((item) => ({`
`76`	`title: item.title,`
`77`	`link: item.link,`
`78`	`date: new Date(item.pubDate),`
`79`	`}));`
`80`	`}`
`81`
`82`	`function atomItems(feed): Array<Item> {`
`83`	`let items = feed["feed"]["entry"] \|\| [];`
`84`	`if (items.length === "undefined") {`
`85`	`// Single entry`
`86`	`items = [items];`
`87`	`}`
`88`	`return items.map((item) => ({`
`89`	`title: typeof item.title === "string" ? item.title : item.title["#text"],`
`90`	`link: item.link instanceof Array`
`91`	`? item.link[0]["@href"]`
`92`	`: item.link["@href"],`
`93`	`date: new Date(item.updated),`
`94`	`}));`
`95`	`}`
`96`
`97`	`function newItemFilter(items: Array<Item>, state: FeedState): Array<Item> {`
`98`	`if (state.date && state.date.getTime() > 0) {`
`99`	`const date = state.date;`
`100`	`items = items.filter((item) => item.date > date);`
`101`	`}`
`102`
`103`	`if (state.link) {`
`104`	`const link = state.link;`
`105`	`const index = items.findIndex((item) => item.link === link);`
`106`	`if (index >= 0) {`
`107`	`items = items.slice(0, index);`
`108`	`}`
`109`	`}`
`110`
`111`	`if (state.title) {`
`112`	`const title = state.title;`
`113`	`const index = items.findIndex((item) => item.title === title);`
`114`	`if (index >= 0) {`
`115`	`items = items.slice(0, index);`
`116`	`}`
`117`	`}`
`118`
`119`	`return items;`
`120`	`}`
`121`