crawling-service/src/config/crawler.config.js

/**
 * 爬虫配置
 */
module.exports = {
  // 通用配置
  common: {
    screenshotDir: 'screenshots',
    upload: {
      url: 'https://apibase.sohomall.jp/uploaders',
      scene: 'digital-yy',
      timeout: 30000 * 5
    },
    browser: {
      headless: true,
      args: ['--no-sandbox', '--disable-setuid-sandbox']
    },
    page: {
      locale: 'ja-JP',
    },
    // 截图配置
    screenshot: {
      // 视口尺寸
      viewport: {
        width: 1920,
        height: 1080
      },
      // 截图选项
      options: {
        fullPage: false,
        quality: 60, // 图片质量 0-100
        type: 'png' // 图片格式: 'png' | 'jpeg'
      }
    }
  },

  // 亚马逊配置
  amazon: {
    selectors: {
      title: '#productTitle',
      price: 'span.a-price > span.a-offscreen',
      coupon: '.a-declarative[data-action="a-modal"], .couponLabelText',
      variants: '.a-cardui-body #twister-plus-inline-twister > .a-section'
    },
    timeouts: {
      pageLoad: 600000,
      elementWait: 100000,
      networkIdle: 50000
    },
    retry: {
      maxAttempts: 3,
      delay: 6000
    },
    headers: {
    }
  },

  // 乐天配置
  rakuten: {
    selectors: {
      title: '.normal_reserve_item_name',
      price: 'span[irc="Price"] .number-display--3s4mj',
      coupon: '',
      variants: ''
    },
    timeouts: {
      pageLoad: 60000,
      elementWait: 10000,
      networkIdle: 50000
    },
    retry: {
      maxAttempts: 3,
      delay: 6000
    },
    headers: {
      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
      'Accept-Language': 'ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7'
    }
  },

  // 雅虎配置（预留）
  yahoo: {
    selectors: {},
    timeouts: {
      pageLoad: 5000,
      elementWait: 1000,
      networkIdle: 500
    },
    retry: {
      maxAttempts: 3,
      delay: 1000
    }
  }
};