|
@@ -38,21 +38,37 @@ export default class Portal {
|
|
|
for (let i = length - 1; i >= 0; i--) {
|
|
|
const article = articles[i];
|
|
|
const exists = await this._redisClient.retrieve(article.link);
|
|
|
+
|
|
|
if (exists !== null) {
|
|
|
continue;
|
|
|
}
|
|
|
|
|
|
- const date = new Date(Date.now()).toLocaleDateString();
|
|
|
+ // To avoid publiposts (I'm lookint at you La Tercera >:-|)
|
|
|
+ if (article.link.includes("publirreportajes")) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ if (article.title.includes("Exclusivo suscriptor")) {
|
|
|
+ article.title = article.title.replace("Exclusivo suscriptor", "");
|
|
|
+ }
|
|
|
+
|
|
|
let message = `${Emojis.NEWS} ${article.title}`;
|
|
|
if (article.content !== "") {
|
|
|
message += `\n\n${article.content}`;
|
|
|
}
|
|
|
- message += `\n${Emojis.LINK} ${article.link}`;
|
|
|
|
|
|
- if (message.trim().length === 0) {
|
|
|
+ if (message.trim().length <= 10) {
|
|
|
continue;
|
|
|
}
|
|
|
|
|
|
+ // If the message is more than 400 characters long, its very likely due to the article.content
|
|
|
+ if (message.length > 400) {
|
|
|
+ message = message.substring(0, 397) + "...";
|
|
|
+ }
|
|
|
+
|
|
|
+ message += `\n${Emojis.LINK} ${article.link}`;
|
|
|
+
|
|
|
// Adding hashtags if there is
|
|
|
if (this._scraperArticlesOptions.hashtags) {
|
|
|
message += `\n${Emojis.TAGS} ` ;
|
|
@@ -62,21 +78,6 @@ export default class Portal {
|
|
|
}
|
|
|
message.trimEnd();
|
|
|
|
|
|
- // To avoid publiposts (I'm lookint at you La Tercera >:-|)
|
|
|
- if (article.link.includes("publirreportajes")) {
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- if (article.title.includes("Exclusivo suscriptor")) {
|
|
|
- article.title = article.title.replace("Exclusivo suscriptor", "");
|
|
|
- }
|
|
|
-
|
|
|
- // If the message is more than 400 characters long, its very likely due to the article.content
|
|
|
- if (message.length > 400) {
|
|
|
- message = `${Emojis.NEWS} ${article.title}.\n\n${article.content}`.substring(0, 397) + "...";
|
|
|
- message += `\n${Emojis.LINK} ${article.link}`;
|
|
|
- }
|
|
|
-
|
|
|
if (this._name.toUpperCase() == "Chile Cultura".toUpperCase()) {
|
|
|
// Icons Chilecultura
|
|
|
let regex = new RegExp(/\n(\d{1,2} \w{3})\s*\n?\s*-\s*(\d{1,2} \w{3})/);
|
|
@@ -105,7 +106,7 @@ export default class Portal {
|
|
|
await this._mastodonClient.v1.statuses.create({ status: message, mediaIds });
|
|
|
await this._redisClient.store(
|
|
|
article.link,
|
|
|
- date,
|
|
|
+ new Date(Date.now()).toLocaleDateString(),
|
|
|
{ EX: this._scraperArticlesOptions.cacheExpiration ? this._scraperArticlesOptions.cacheExpiration : 60 * 60 * 24 } // EX: 24 hrs expiration
|
|
|
);
|
|
|
totalPublished++
|