Merge branch 'develop' of Mastodon/bots into main

src/index.ts

@@ -17,7 +17,7 @@ import { handler as fastcheck } from "./portales/fastcheck/handler";
 import { handler as emol } from "./portales/emol/handler";
 import { handler as interferencia } from "./portales/interferencia/handler";
 import { handler as laderasur } from "./portales/laderasur/handler";
-// import { handler as latercera } from "./portales/latercera/handler";
+import { handler as latercera } from "./portales/latercera/handler";
 // import { handler as metrodesantiago } from "./portales/metrodesantiago/handler";
 import { handler as tarreo } from "./portales/tarreo/handler";
 import { handler as theclinic } from "./portales/theclinic/handler";
@@ -53,7 +53,7 @@ const portalsHandlers = {
   // "glaciareschilenos": glaciareschilenos,
   "interferencia": interferencia,
   "laderasur": laderasur,
-  // "latercera": latercera,
+  "latercera": latercera,
   "tarreo": tarreo,
   "theclinic": theclinic
 };

src/interfaces/scaper-articles-options.ts

@@ -14,4 +14,5 @@ export interface IScraperArticlesOptions {
   scraperMethod: ScraperMethods
   cacheExpiration?: number
   hashtags?: Array<string>
+  maxArticles?: number
 };

src/portales/latercera/handler.ts

@@ -11,12 +11,15 @@ export const handler: Handler = new Portal(
   config.MASTODON_KEY_LATERCERA,
   {
     url: config.LATERCERA,
-    articlesSelector: "section.top-mainy article.card",
-    titleSelector: "div.headline",
-    linkSelector: "div.headline a",
-    linkPrefix: "https://www.latercera.cl",
+    articlesSelector: ".story-card",
+    titleSelector: ".story-card__headline",
+    linkSelector: ".story-card__headline a",
+    contentSelector: ".story-card__description",
+    imageSelector: ".story-card__image img",
+    linkPrefix: "https://www.latercera.com",
     scraperMethod: ScraperMethods.PUPPETEER,
     cacheExpiration: 60 * 60 * 24 * 5, // 5 días
+    maxArticles: 20,
     hashtags: ["LaTercera", "LT", "Noticias"]
   }
 ).getHandler();

src/portales/portal.ts

@@ -37,12 +37,28 @@ export default class Portal {
       }
 
       let totalPublished = 0;
-      const length = articles.length;
+
+      // Deduplicate articles by link within the same batch
+      const seenLinks = new Set<string>();
+      const uniqueArticles = articles.filter((article) => {
+        if (seenLinks.has(article.link)) {
+          return false;
+        }
+        seenLinks.add(article.link);
+        return true;
+      });
 
       // Order has to be reversed to appear in the correct order when posting
-      articles.reverse();
+      uniqueArticles.reverse();
+
+      const maxArticles = this._scraperArticlesOptions.maxArticles;
+      const articlesToPublish = maxArticles !== undefined && maxArticles > 0
+        ? uniqueArticles.slice(0, maxArticles)
+        : uniqueArticles;
+
+      const length = articlesToPublish.length;
       for (let i = 0; i < length; i++) {
-        const article = articles[i];
+        const article = articlesToPublish[i];
         const exists = await this._redisClient.retrieve(article.link);
 
         if (exists !== null) {