added proxy config

.env.example

@@ -1,6 +1,7 @@
 LOG_LEVEL = "debug"
 REDIS_CONN = "redis://localhost:6379/10"
 PUPPETEER_URL = "http://localhost:8000/api/v1/visit"
+PROXY_URL = ""
 MASTODON_URL = "https://<mastodon-domain>"
 MASTODON_API_URL = "https://<mastodon-domain>/api/v1/"
 MASTODON_STREAMING_URL = "wss://<mastodon-domain>/api/v1/streaming"

src/config.ts

@@ -4,6 +4,7 @@ const config = {
   LOG_LEVEL: process.env.LOG_LEVEL ?? "debug",
   REDIS_CONN: process.env.REDIS_CONN ?? "redis://localhost:6379/10",
   PUPPETEER_URL: process.env.PUPPETEER_URL ?? "http://localhost:8000/api/v1/visit",
+  PROXY_URL: process.env.PROXY_URL,
   MASTODON_URL: process.env.MASTODON_URL ?? "https://mastodon.cl",
   MASTODON_API_URL: process.env.MASTODON_API_URL ?? "https://mastodon.cl/api/v1/",
   MASTODON_STREAMING_URL: process.env.MASTODON_STREAMING_URL ?? "wss://mastodon.cl/api/v1/streaming",

src/interfaces/scaper-articles-options.ts

@@ -15,4 +15,5 @@ export interface IScraperArticlesOptions {
   cacheExpiration?: number
   hashtags?: Array<string>
   maxArticles?: number
+  useProxy?: boolean
 };

src/interfaces/scraper-options.ts

@@ -8,4 +8,5 @@ export interface IScraperOptions {
   screenshotSelector?: string
   screenshotDelay?: number
   clickSelector?: string
+  useProxy?: boolean
 }

src/utils/scraper-articles.ts

@@ -126,7 +126,7 @@ export default class ScraperArticles {
 
     try {
       console.info("Starting scraping", this._options);
-      const response = await this._scraper.scrape({ url: this._options.url, scraperMethod: this._options.scraperMethod });
+      const response = await this._scraper.scrape({ url: this._options.url, scraperMethod: this._options.scraperMethod, useProxy: this._options.useProxy });
       const html = this._options.scraperMethod === ScraperMethods.PUPPETEER ? response.data.data.html : response.data;
 
       const $ = load(html as string);

src/utils/scraper.ts

@@ -24,6 +24,34 @@ export default class Scraper {
     }
   }
 
+  private getProxyConfig(): any {
+    const proxyUrlString = config.PROXY_URL;
+    if (!proxyUrlString || proxyUrlString.trim() === "") {
+      return undefined;
+    }
+
+    try {
+      const proxyUrl = new URL(proxyUrlString);
+      const proxyConfig: any = {
+        protocol: proxyUrl.protocol.replace(":", ""),
+        host: proxyUrl.hostname,
+        port: Number(proxyUrl.port)
+      };
+
+      if (proxyUrl.username !== "" || proxyUrl.password !== "") {
+        proxyConfig.auth = {
+          username: proxyUrl.username,
+          password: proxyUrl.password
+        };
+      }
+
+      return proxyConfig;
+    } catch (err: any) {
+      console.error(`Invalid PROXY_URL: ${err.message}`);
+      return undefined;
+    }
+  }
+
   public async scrape (options: IScraperOptions): Promise<any> {
     this._options = options;
     let response: any;
@@ -33,6 +61,8 @@ export default class Scraper {
     }
 
     try {
+      const proxyConfig = this._options.useProxy ? this.getProxyConfig() : undefined;
+
       if (this._options.scraperMethod == ScraperMethods.PUPPETEER) {
         const payload: any = {
           url: this._options.url,
@@ -52,9 +82,18 @@ export default class Scraper {
           payload.clickSelector = this._options.clickSelector;
         }
 
+        if (proxyConfig !== undefined) {
+          payload.proxy = config.PROXY_URL;
+        }
+
         response = await axios.post(config.PUPPETEER_URL, payload, { headers });
       } else {
-        response = await axios.get(this._options.url, { headers });
+        const axiosOptions: any = { headers };
+        if (proxyConfig !== undefined) {
+          axiosOptions.proxy = proxyConfig;
+        }
+
+        response = await axios.get(this._options.url, axiosOptions);
       }
       
       if (config.LOG_LEVEL === LogLevels.DEBUG) {