PHP cURL代理爬虫

export PROXY_USER="替换为控制台代理用户名"
export PROXY_PASS="替换为控制台代理密码"
# 隧道代理固定网关：proxy.123proxy.cn:36923

$env:PROXY_USER="替换为控制台代理用户名"
$env:PROXY_PASS="替换为控制台代理密码"
# 隧道代理固定网关：proxy.123proxy.cn:36923

<?php
// requirements: PHP 8+ with curl and dom extensions
$proxyHost = 'proxy.123proxy.cn';
$proxyPort = 36923;
$proxyUser = getenv('PROXY_USER');
$proxyPass = getenv('PROXY_PASS');

if (!$proxyUser || !$proxyPass) {
    throw new RuntimeException('set PROXY_USER and PROXY_PASS first');
}

function fetchHtml(
    string $url,
    string $proxyHost,
    int $proxyPort,
    string $proxyUser,
    string $proxyPass
): string {
    $curl = curl_init($url);
    curl_setopt_array($curl, [
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_PROXY => $proxyHost,
        CURLOPT_PROXYPORT => $proxyPort,
        CURLOPT_PROXYTYPE => CURLPROXY_HTTP,
        CURLOPT_PROXYUSERPWD => $proxyUser . ':' . $proxyPass,
        CURLOPT_CONNECTTIMEOUT => 15,
        CURLOPT_TIMEOUT => 30,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_FAILONERROR => true,
        CURLOPT_USERAGENT => '123Proxy-Developer-Example/1.0',
    ]);
    $body = curl_exec($curl);
    if ($body === false) {
        $error = curl_error($curl);
        curl_close($curl);
        throw new RuntimeException($error);
    }
    curl_close($curl);
    return $body;
}

$records = [];
$url = 'https://quotes.toscrape.com/';
for ($pageNumber = 1; $pageNumber <= 3; $pageNumber++) {
    $html = fetchHtml(
        $url,
        $proxyHost,
        $proxyPort,
        $proxyUser,
        $proxyPass
    );
    $document = new DOMDocument();
    @$document->loadHTML($html, LIBXML_NOERROR | LIBXML_NOWARNING);
    $xpath = new DOMXPath($document);

    foreach ($xpath->query('//div[contains(@class, "quote")]') as $quote) {
        $text = $xpath->query(
            './/span[contains(@class, "text")]',
            $quote
        )->item(0)?->textContent;
        $author = $xpath->query(
            './/small[contains(@class, "author")]',
            $quote
        )->item(0)?->textContent;
        $tags = [];
        foreach ($xpath->query(
            './/a[contains(@class, "tag")]',
            $quote
        ) as $tag) {
            $tags[] = trim($tag->textContent);
        }
        $records[] = [
            'text' => trim((string) $text),
            'author' => trim((string) $author),
            'tags' => $tags,
            'source_url' => $url,
        ];
    }

    $next = $xpath->query('//li[contains(@class, "next")]/a')
        ->item(0);
    if (!$next) {
        break;
    }
    $url = 'https://quotes.toscrape.com' .
        $next->getAttribute('href');
    usleep(500000);
}

file_put_contents(
    'quotes-php.json',
    json_encode($records, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)
);
echo 'saved ' . count($records) . " records to quotes-php.json\n";