import { type Handler } from "aws-lambda"; import { createRestAPIClient, mastodon } from "masto"; import ScraperArticles from "../utils/scraper-articles"; import TagGenerator from "../utils/tag-generator"; import RedisClient from "../libs/redis-client"; import LogLevels from "../enums/log-levels"; import Emojis from "../enums/emojis"; import config from "../config"; import { type IScraperArticlesOptions } from "../interfaces/scaper-articles-options"; export default class Portal { private readonly _name: string; private readonly _scraperArticlesOptions: IScraperArticlesOptions private readonly _redisClient: RedisClient; private readonly _mastodonClient: mastodon.rest.Client private readonly _scraperArticles: ScraperArticles; private readonly _tagGenerator: TagGenerator; constructor (name: string, accessToken = config.MASTODON_TEST_ACCESS_TOKEN, scraperArticlesOptions: IScraperArticlesOptions) { this._name = name; this._scraperArticlesOptions = scraperArticlesOptions; this._scraperArticles = new ScraperArticles(this._name, this._scraperArticlesOptions); this._redisClient = new RedisClient(); this._tagGenerator = new TagGenerator(this._name); accessToken = config.DEVELOP ? config.MASTODON_TEST_ACCESS_TOKEN : accessToken; this._mastodonClient = createRestAPIClient({ url: config.MASTODON_URL, accessToken }); } public async run (event?: any, context?: any): Promise { try { const articles = await this._scraperArticles.getArticles(); if (config.LOG_LEVEL === LogLevels.DEBUG) { console.log(`${this._name} | Articles`, articles); } let totalPublished = 0; // Deduplicate articles by link within the same batch const seenLinks = new Set(); const uniqueArticles = articles.filter((article) => { if (seenLinks.has(article.link)) { return false; } seenLinks.add(article.link); return true; }); // Order has to be reversed to appear in the correct order when posting uniqueArticles.reverse(); const maxArticles = this._scraperArticlesOptions.maxArticles; const articlesToPublish = maxArticles !== undefined && maxArticles > 0 ? uniqueArticles.slice(0, maxArticles) : uniqueArticles; const length = articlesToPublish.length; for (let i = 0; i < length; i++) { const article = articlesToPublish[i]; const exists = await this._redisClient.retrieve(article.link); if (exists !== null) { continue; } // To avoid publiposts (I'm lookint at you La Tercera >:-|) if (article.link.includes("publirreportajes") || article.link.includes("publireportajes")) { continue; } if (article.title.includes("Exclusivo suscriptor")) { article.title = article.title.replace("Exclusivo suscriptor", ""); } article.title = article.title.replace(/\s+/g, " ").trim(); if (article.content !== "") { article.content = article.content.replace(/\s+/g, " ").trim(); } let message = `${Emojis.NEWS} ${article.title}`; if (article.content !== "") { message += `\n\n${article.content}`; } if (message.trim().length <= 10) { continue; } // Generating hashtags from article content using Ollama const generatedTags = await this._tagGenerator.generate(article.title, article.content); const tags = generatedTags.length > 0 ? generatedTags : (this._scraperArticlesOptions.hashtags ?? []); message += `\n${Emojis.LINK} ${article.link}`; if (tags.length > 0) { message += `\n\n${tags.map((hashtag) => `#${hashtag}`).join(" ")}`; } // Mastodon allows a maximum of 500 characters, but historically this bot limits to 400 if (message.length > 400) { const linkLine = `\n${Emojis.LINK} ${article.link}`; const tagsLine = tags.length > 0 ? `\n\n${tags.map((hashtag) => `#${hashtag}`).join(" ")}` : ""; const maxContentLength = 400 - linkLine.length - tagsLine.length; message = `${Emojis.NEWS} ${article.title}\n\n${article.content.substring(0, Math.max(0, maxContentLength - 10))}...${linkLine}${tagsLine}`; } if (this._name.toUpperCase() == "Chile Cultura".toUpperCase()) { // Icons Chilecultura let regex = new RegExp(/\n(\d{1,2} \w{3})\s*\n?\s*-\s*(\d{1,2} \w{3})/); message = message.replace("date_range", `${Emojis.CALENDAR} `); message = message.replace(regex, (match, p1, p2) => { return `${p1} - ${p2}`; }); message = message.replace("location_on", Emojis.PIN).replace(`${Emojis.PIN}\n`, `${Emojis.PIN} `); // Expresión regular para capturar el texto antes del 📆 seguido de una fecha regex = new RegExp(/(.*?)📆 (\d{1,2} \w{3}( - \d{1,2} \w{3}))?/); message = message.replace(regex, (match, p1, p2) => { const tags = p1 ? p1.split(" ").map(tag => `#${tag}`).join(" ") : ""; return `🎭 ${tags}\n📆 ${p2 ? p2 : ""}`; }); } const mediaIds: any[] = []; if (article.image !== null && article.image !== undefined) { const media = await this._mastodonClient.v2.media.create({ file: article.image, description: article.title }); mediaIds.push(media.id); } console.log(`\n${this._name} | Sending\n`, message); await this._mastodonClient.v1.statuses.create({ status: message, mediaIds }); totalPublished++ if (!config.DEVELOP) { await this._redisClient.store( article.link, new Date(Date.now()).toLocaleDateString(), { EX: this._scraperArticlesOptions.cacheExpiration ? this._scraperArticlesOptions.cacheExpiration : 60 * 60 * 24 } // EX: 24 hrs expiration ); } } console.log(`${this._name} | Published ${totalPublished} new articles`); } catch (err: any) { console.log(`${this._name} | An error has occurred\n`) console.error(err.message); if (config.LOG_LEVEL === LogLevels.DEBUG) { if (event !== undefined) { console.debug("\nEvent\n"); console.debug(event); } if (context !== undefined) { console.debug("\nContext\n"); console.debug(context); } } } console.log(`${this._name} | Finished`); } public getHandler (): Handler { return async (event, context) => { await this.run(event, context); } } }