Node.js Axios代理采集

export PROXY_USER="替换为控制台代理用户名"
export PROXY_PASS="替换为控制台代理密码"
# 隧道住宅代理固定网关：residential.123proxy.cn:33000

$env:PROXY_USER="替换为控制台代理用户名"
$env:PROXY_PASS="替换为控制台代理密码"
# 隧道住宅代理固定网关：residential.123proxy.cn:33000

// requirements:
//   npm init -y
//   npm install axios cheerio
// save as scrape-axios.mjs, then run:
//   node scrape-axios.mjs
import fs from "node:fs/promises";
import http from "node:http";
import https from "node:https";
import axios from "axios";
import * as cheerio from "cheerio";

const PROXY_HOST = "residential.123proxy.cn";
const PROXY_PORT = 33000;
const PROXY_USER = process.env.PROXY_USER;
const PROXY_PASS = process.env.PROXY_PASS;
const START_URL = "https://quotes.toscrape.com/";
const MAX_PAGES = 3;

if (!PROXY_USER || !PROXY_PASS) {
  throw new Error("set PROXY_USER and PROXY_PASS first");
}

const client = axios.create({
  timeout: 30_000,
  proxy: {
    protocol: "http",
    host: PROXY_HOST,
    port: PROXY_PORT,
    auth: {
      username: PROXY_USER,
      password: PROXY_PASS,
    },
  },
  headers: {
    "User-Agent": "123Proxy-Developer-Example/1.0",
    "Accept-Language": "en-US,en;q=0.8",
  },
  httpAgent: new http.Agent({ keepAlive: true, maxSockets: 10 }),
  httpsAgent: new https.Agent({ keepAlive: true, maxSockets: 10 }),
});

function delay(milliseconds) {
  return new Promise((resolve) => setTimeout(resolve, milliseconds));
}

async function getWithRetry(url, maxAttempts = 3) {
  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
    try {
      return await client.get(url);
    } catch (error) {
      const status = error.response?.status;
      const retryable =
        status === 429
        || (status >= 500 && status <= 599)
        || error.code === "ECONNRESET"
        || error.code === "ETIMEDOUT";
      if (!retryable || attempt === maxAttempts) throw error;

      const retryAfter = Number(error.response?.headers?.["retry-after"]);
      const waitMs = Number.isFinite(retryAfter)
        ? Math.min(retryAfter * 1000, 30_000)
        : 500 * (2 ** (attempt - 1));
      await delay(waitMs + Math.floor(Math.random() * 250));
    }
  }
  throw new Error("unreachable");
}

function parsePage(html, sourceUrl) {
  const $ = cheerio.load(html);
  const records = $("div.quote")
    .map((_, node) => ({
      text: $(node).find("span.text").text().trim(),
      author: $(node).find("small.author").text().trim(),
      tags: $(node).find("a.tag")
        .map((__, tag) => $(tag).text().trim())
        .get(),
      source_url: sourceUrl,
    }))
    .get();
  const nextHref = $("li.next > a").attr("href");
  return { records, nextHref };
}

async function main() {
  const records = [];
  let url = START_URL;

  for (let page = 0; page < MAX_PAGES && url; page += 1) {
    const response = await getWithRetry(url);
    const parsed = parsePage(response.data, url);
    records.push(...parsed.records);
    url = parsed.nextHref ? new URL(parsed.nextHref, url).href : null;
    await delay(500);
  }

  const jsonLines = records.map((record) => JSON.stringify(record)).join("\n");
  await fs.writeFile("quotes-axios.jsonl", jsonLines + "\n", "utf8");
  console.log(
    "saved " + records.length + " records to quotes-axios.jsonl",
  );
}

main().catch((error) => {
  console.error(error.response?.status || error.code, error.message);
  process.exitCode = 1;
});