crawling-service/src/services/crawlers/base/BaseCrawler.js

const { chromium } = require("playwright");
const path = require("path");
const fs = require("fs").promises;

/**
 * 基础爬虫类
 * 提供所有爬虫共用的基础功能
 */
class BaseCrawler {
  constructor(config) {
    this.config = config;
    this.browser = null;
    this.context = null;
    this.page = null;
  }

  /**
   * 初始化浏览器
   * @returns {Promise<void>}
   */
  async initBrowser() {
    // 获取截图配置
    const screenshotConfig = this.config?.common?.screenshot || {};
    const viewportConfig = screenshotConfig.viewport || { width: 1920, height: 1080 };
    
    this.browser = await chromium.launch({
      headless: false,
      args: ["--no-sandbox", "--disable-setuid-sandbox"],
    });
    this.context = await this.browser.newContext({
      locale: "ja-JP",
      userAgent:
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
      // 使用配置中的视口尺寸
      viewport: viewportConfig
    });
    this.page = await this.context.newPage();
  }

  /**
   * 关闭浏览器
   * @returns {Promise<void>}
   */
  async closeBrowser() {
    if (this.browser) {
      await this.browser.close();
      this.browser = null;
      this.context = null;
      this.page = null;
    }
  }

  /**
   * 创建截图目录
   * @returns {Promise<string>} 截图目录路径
   */
  async createScreenshotDir() {
    const dir = path.join(process.cwd(), "screenshots");
    await fs.mkdir(dir, { recursive: true });
    return dir;
  }
}

module.exports = BaseCrawler;