import axios from "axios"; import config from "../config"; import LogLevels from "../enums/log-levels"; import type { IScraperOptions } from "../interfaces/scraper-options"; export default class Scraper { private _options: IScraperOptions; public async scrape (options: IScraperOptions): Promise { this._options = options; let response: any; try { response = await axios.post(config.PUPPETEER_URL, { url: this._options.url, screenshot: config.LOG_LEVEL === LogLevels.DEBUG, incognito: this._options.incognito ?? false }, { headers: { "User-Agent": this._options.userAgent ?? "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36" } }); if (config.LOG_LEVEL === LogLevels.DEBUG) { console.debug(`From '${this._options.url}'\n`); console.debug("HTML\n") console.debug(response.data.data.html); console.debug("\n"); console.debug("Screenshot (Base64)\n") console.debug(response.data.data.screenshot); } } catch (err: any) { console.error(err.message); } return response; } public async scrapeFile (url: string): Promise { this._options = { url }; let file: File | null = null; try { const response = await fetch(url); const blob = await response.blob(); const fileUrlParts = url.split("/"); const fileName = fileUrlParts[fileUrlParts.length - 1]; file = new File([blob], fileName); } catch (err: any) { console.error(err.message); } return file; } }