added elmostrador and interferencia. Bun compatible

.env.example

@@ -5,8 +5,10 @@ MASTODON_URL = "https://<mastodon-domain>/api/v1/"
 MASTODON_ACCESS_TOKEN = "<access-token>"
 IMG_PLACEHOLDER = "https://placehold.co/600x400"
 
-THECLINIC = "https://www.theclinic.cl/lo-ultimo/"
-LATERCERA = "https://www.latercera.com/canal/nacional/"
-EMOL = "https://www.emol.com/nacional/"
 ELCIUDADANO = "https://www.elciudadano.com/chile/"
-ELDESCONCIERTO = "https://www.eldesconcierto.cl/nacional/"
+ELDESCONCIERTO = "https://www.eldesconcierto.cl/nacional/"
+ELMOSTRADOR = "https://www.elmostrador.cl/categoria/dia/"
+EMOL = "https://www.emol.com/nacional/"
+INTERFERENCIA = "https://interferencia.cl/"
+LATERCERA = "https://www.latercera.com/canal/nacional/"
+THECLINIC = "https://www.theclinic.cl/lo-ultimo/"

package.json

@@ -31,5 +31,10 @@
     "dotenv": "^16.4.5",
     "masto": "^6.7.0",
     "redis": "^4.6.13"
-  }
+  },
+  "trustedDependencies": [
+    "aws-sdk",
+    "es5-ext",
+    "serverless"
+  ]
 }

serverless.yml

@@ -14,8 +14,10 @@ plugins:
   - serverless-dotenv-plugin
 
 functions:
-  - ${file(./src/portales/theclinic/definition.yml)}
-  - ${file(./src/portales/emol/definition.yml)}
-  - ${file(./src/portales/latercera/definition.yml)}
   - ${file(./src/portales/elciudadano/definition.yml)}
   - ${file(./src/portales/eldesconcierto/definition.yml)}
+  - ${file(./src/portales/elmostrador/definition.yml)}
+  - ${file(./src/portales/interferencia/definition.yml)}
+  - ${file(./src/portales/emol/definition.yml)}
+  - ${file(./src/portales/latercera/definition.yml)}
+  - ${file(./src/portales/theclinic/definition.yml)}

src/config.ts

@@ -6,11 +6,13 @@ const config = {
   MASTODON_ACCESS_TOKEN: process.env.MASTODON_ACCESS_TOKEN ?? "",
   IMG_PLACEHOLDER: process.env.IMG_PLACEHOLDER ?? "https://placehold.co/600x400",
   // PORTALES
-  THECLINIC: process.env.THECLINIC ?? "https://www.theclinic.cl/lo-ultimo/",
-  LATERCERA: process.env.LATERCERA ?? "https://www.latercera.com/canal/nacional/",
-  EMOL: process.env.EMOL ?? "https://www.emol.com/nacional/",
   ELCIUDADANO: process.env.ELCIUDADANO ?? "https://www.elciudadano.com/chile/",
-  ELDESCONCIERTO: process.env.ELDESCONCIERTO ?? "https://www.eldesconcierto.cl/nacional/"
+  ELDESCONCIERTO: process.env.ELDESCONCIERTO ?? "https://www.eldesconcierto.cl/nacional/",
+  ELMOSTRADOR: process.env.ELMOSTRADOR ?? "https://www.elmostrador.cl/categoria/dia/",
+  EMOL: process.env.EMOL ?? "https://www.emol.com/nacional/",
+  INTERFERENCIA: process.env.INTERFERENCIA ?? "https://interferencia.cl/",
+  LATERCERA: process.env.LATERCERA ?? "https://www.latercera.com/canal/nacional/",
+  THECLINIC: process.env.THECLINIC ?? "https://www.theclinic.cl/lo-ultimo/"
 };
 
 export default config;

src/index.ts

@@ -2,7 +2,9 @@ import { type Context } from "aws-lambda";
 
 import { handler as elciudadano } from "./portales/elciudadano/handler";
 import { handler as eldesconcierto } from "./portales/eldesconcierto/handler";
+import { handler as elmostrador } from "./portales/elmostrador/handler";
 import { handler as emol } from "./portales/emol/handler";
+import { handler as interferencia } from "./portales/interferencia/handler";
 import { handler as latercera } from "./portales/latercera/handler";
 import { handler as theclinic } from "./portales/theclinic/handler";
 
@@ -21,16 +23,14 @@ const context: Context = {
   getRemainingTimeInMillis: () => 1
 };
 
+const portals = [elciudadano, eldesconcierto, elmostrador, emol, interferencia, latercera, theclinic];
+const cooldown = 10000; // 10 seconds
+
 async function main (): Promise<void> {
-  await elciudadano(null, context, () => {});
-  await new Promise((resolve) => setTimeout(resolve, 5000));
-  await eldesconcierto(null, context, () => {});
-  await new Promise((resolve) => setTimeout(resolve, 5000));
-  await emol(null, context, () => {});
-  await new Promise((resolve) => setTimeout(resolve, 5000));
-  await latercera(null, context, () => {});
-  await new Promise((resolve) => setTimeout(resolve, 5000));
-  await theclinic(null, context, () => {});
+  for (const portal of portals) {
+    await portal(null, context, () => {});
+    await new Promise((resolve) => setTimeout(resolve, cooldown));
+  }
 }
 
 main()

src/portales/elciudadano/handler.ts

@@ -56,7 +56,7 @@ export const handler: Handler = async (event, context) => {
         message = `${message}\n${article.link}`;
       }
 
-      console.log("Sending", message);
+      console.log("\nSending", message);
 
       await mastodon.v1.statuses.create({ status: message });
       await redisClient.store(article.link, date, { EX: 60 * 60 * 24 }); // EX: 24 hrs expiration

src/portales/eldesconcierto/handler.ts

@@ -51,7 +51,7 @@ export const handler: Handler = async (event, context) => {
         continue;
       }
 
-      console.log("Sending", message);
+      console.log("\nSending", message);
 
       await mastodon.v1.statuses.create({ status: message });
       await redisClient.store(article.link, date, { EX: 60 * 60 * 24 }); // EX: 24 hrs expiration

src/portales/elmostrador/definition.yml

@@ -0,0 +1,4 @@
+elmostrador:
+  handler: ./src/portales/elmostrador/handler.handler
+  events: 
+    - schedule: rate(1 hour)

src/portales/elmostrador/handler.ts

@@ -0,0 +1,73 @@
+import type { Handler } from "aws-lambda";
+import { createRestAPIClient } from "masto";
+import "dotenv/config";
+
+import ScraperArticles from "../../utils/scraper-articles";
+import RedisClient from "../../libs/redis-client";
+
+import config from "../../config";
+import LogLevels from "../../enums/log-levels";
+import Emojis from "../../enums/emojis";
+
+export const handler: Handler = async (event, context) => {
+  const name = "El Mostrado";
+  try {
+    const redisClient = new RedisClient();
+    const mastodon = createRestAPIClient({
+      url: config.MASTODON_URL,
+      accessToken: config.MASTODON_ACCESS_TOKEN
+    });
+    const scraperArticles = new ScraperArticles(name, {
+      url: config.ELMOSTRADOR,
+      articlesSelector: "div.d-section__body div.d-tag-card",
+      titleSelector: "h4",
+      contentSelector: "h4",
+      linkSelector: "h4 a",
+      imageSelector: "",
+      authorSelector: "span.d-tag-card__subtitle",
+      dateSelector: "time"
+    });
+
+    const articles = await scraperArticles.getArticles();
+    if (config.LOG_LEVEL === LogLevels.DEBUG) {
+      console.log("Articles", articles);
+    }
+
+    let totalPublished = 0;
+    const length = articles.length;
+
+    // Order has to be reversed to appear in the correct order when posting
+    for (let i = length - 1; i >= 0; i--) {
+      const article = articles[i];
+      const exists = await redisClient.retrieve(article.link);
+      if (exists !== null) {
+        continue;
+      }
+
+      const date = new Date(Date.now()).toLocaleDateString();
+      const message = `${Emojis.NEWS} ${article.title}\n${article.link}`;
+
+      if (message.trim().length === 0) {
+        continue;
+      }
+
+      console.log("\nSending", message);
+
+      await mastodon.v1.statuses.create({ status: message });
+      await redisClient.store(article.link, date, { EX: 60 * 60 * 24 }); // EX: 24 hrs expiration
+      totalPublished++
+    }
+    console.log(`Published ${totalPublished} new articles`);
+  } catch (err: any) {
+    console.log('An error has occurred\n')
+    console.error(err.message);
+    if (config.LOG_LEVEL === LogLevels.DEBUG) {
+      console.debug("\nEvent\n");
+      console.debug(event);
+      console.debug("\nContext\n");
+      console.debug(context);
+    }
+  }
+
+  return "The End.";
+};

src/portales/emol/handler.ts

@@ -57,7 +57,7 @@ export const handler: Handler = async (event, context) => {
         mediaIds.push(media.id);
       }
 
-      console.log("Sending", message);
+      console.log("\nSending", message);
 
       await mastodon.v1.statuses.create({ status: message, mediaIds });
       await redisClient.store(article.link, date, { EX: 60 * 60 * 24 }); // EX: 24 hrs expiration

src/portales/interferencia/definition.yml

@@ -0,0 +1,4 @@
+interferencia:
+  handler: ./src/portales/interferencia/handler.handler
+  events: 
+    - schedule: rate(1 hour)

src/portales/interferencia/handler.ts

@@ -0,0 +1,79 @@
+import type { Handler } from "aws-lambda";
+import { createRestAPIClient } from "masto";
+import "dotenv/config";
+
+import ScraperArticles from "../../utils/scraper-articles";
+import RedisClient from "../../libs/redis-client";
+
+import config from "../../config";
+import LogLevels from "../../enums/log-levels";
+import Emojis from "../../enums/emojis";
+
+export const handler: Handler = async (event, context) => {
+  const name = "Interferencia";
+  const baseDomain = "https://interferencia.cl"
+  try {
+    const redisClient = new RedisClient();
+    const mastodon = createRestAPIClient({
+      url: config.MASTODON_URL,
+      accessToken: config.MASTODON_ACCESS_TOKEN
+    });
+    const scraperArticles = new ScraperArticles(name, {
+      url: config.INTERFERENCIA,
+      articlesSelector: "div.row.container:eq(6) div.row div.col-md-4, div.row.container:eq(6) div.row div.col-md-6, div.row.container:eq(6) div.row div.col-md-8",
+      titleSelector: "div.views-field-title",
+      contentSelector: "div.views-field-field-subhead div.field-content p",
+      linkSelector: "div.views-field-title a",
+      imageSelector: "",
+      authorSelector: "div.views-field-field-cover-authors div.field-content",
+      dateSelector: "div.views-field-created span.field-content"
+    });
+
+    const articles = await scraperArticles.getArticles();
+    if (config.LOG_LEVEL === LogLevels.DEBUG) {
+      console.log("Articles", articles);
+    }
+
+    let totalPublished = 0;
+    const length = articles.length;
+
+    // Order has to be reversed to appear in the correct order when posting
+    for (let i = length - 1; i >= 0; i--) {
+      const article = articles[i];
+      const exists = await redisClient.retrieve(article.link);
+      if (exists !== null) {
+        continue;
+      }
+
+      const date = new Date(Date.now()).toLocaleDateString();
+      let message = `${Emojis.NEWS} ${article.title}.\n\n${article.content}\n${baseDomain}${article.link}`;
+
+      if (message.trim().length === 0) {
+        continue;
+      }
+
+      if (message.length > 400) {
+        message = `${Emojis.NEWS} ${article.title}.\n\n${article.content}`.substring(0, 397) + "...";
+        message = `${message}\n${baseDomain}${article.link}`;
+      }
+
+      console.log("\nSending", message);
+
+      await mastodon.v1.statuses.create({ status: message });
+      await redisClient.store(article.link, date, { EX: 60 * 60 * 24 }); // EX: 24 hrs expiration
+      totalPublished++
+    }
+    console.log(`Published ${totalPublished} new articles`);
+  } catch (err: any) {
+    console.log('An error has occurred\n')
+    console.error(err.message);
+    if (config.LOG_LEVEL === LogLevels.DEBUG) {
+      console.debug("\nEvent\n");
+      console.debug(event);
+      console.debug("\nContext\n");
+      console.debug(context);
+    }
+  }
+
+  return "The End.";
+};

src/portales/latercera/handler.ts

@@ -53,7 +53,7 @@ export const handler: Handler = async (event, context) => {
         continue;
       }
 
-      console.log("Sending", message);
+      console.log("\nSending", message);
 
       await mastodon.v1.statuses.create({ status: message });
       await redisClient.store(article.link, date, { EX: 60 * 60 * 24 }); // EX: 24 hrs expiration

src/portales/theclinic/handler.ts

@@ -51,7 +51,7 @@ export const handler: Handler = async (event, context) => {
         continue;
       }
 
-      console.log("Sending", message);
+      console.log("\nSending", message);
 
       await mastodon.v1.statuses.create({ status: message });
       await redisClient.store(article.link, date, { EX: 60 * 60 * 24 }); // EX: 24 hrs expiration