Golang Colly代理爬虫

export PROXY_USER="替换为控制台代理用户名"
export PROXY_PASS="替换为控制台代理密码"
# 隧道代理固定网关：proxy.123proxy.cn:36923

$env:PROXY_USER="替换为控制台代理用户名"
$env:PROXY_PASS="替换为控制台代理密码"
# 隧道代理固定网关：proxy.123proxy.cn:36923

// initialize:
//   go mod init example.com/quotes-proxy
//   go get github.com/gocolly/colly/v2
// run:
//   go run .
package main

import (
    "encoding/csv"
    "fmt"
    "log"
    "net/url"
    "os"
    "strings"
    "time"

    "github.com/gocolly/colly/v2"
    collyproxy "github.com/gocolly/colly/v2/proxy"
)

func main() {
    host := "proxy.123proxy.cn"
    port := 36923
    username := os.Getenv("PROXY_USER")
    password := os.Getenv("PROXY_PASS")
    if username == "" || password == "" {
        log.Fatal("set PROXY_USER and PROXY_PASS first")
    }

    proxyURL := &url.URL{
        Scheme: "http",
        Host:   fmt.Sprintf("%s:%d", host, port),
        User:   url.UserPassword(username, password),
    }
    switcher, err := collyproxy.RoundRobinProxySwitcher(
        proxyURL.String(),
    )
    if err != nil {
        log.Fatal(err)
    }

    output, err := os.Create("quotes-colly.csv")
    if err != nil {
        log.Fatal(err)
    }
    defer output.Close()
    writer := csv.NewWriter(output)
    defer writer.Flush()
    _ = writer.Write([]string{"text", "author", "tags", "source_url"})

    collector := colly.NewCollector(
        colly.AllowedDomains("quotes.toscrape.com"),
        colly.UserAgent("123Proxy-Developer-Example/1.0"),
    )
    collector.SetProxyFunc(switcher)
    _ = collector.Limit(&colly.LimitRule{
        DomainGlob:  "*toscrape.com",
        Parallelism: 2,
        RandomDelay: 500 * time.Millisecond,
    })

    collector.OnHTML("div.quote", func(element *colly.HTMLElement) {
        tags := element.ChildTexts("a.tag")
        _ = writer.Write([]string{
            strings.TrimSpace(element.ChildText("span.text")),
            strings.TrimSpace(element.ChildText("small.author")),
            strings.Join(tags, "|"),
            element.Request.URL.String(),
        })
    })
    collector.OnHTML("li.next > a", func(element *colly.HTMLElement) {
        _ = element.Request.Visit(element.Attr("href"))
    })
    collector.OnError(func(response *colly.Response, requestErr error) {
        log.Printf(
            "request failed: url=%s status=%d error=%v",
            response.Request.URL,
            response.StatusCode,
            requestErr,
        )
    })

    if err := collector.Visit("https://quotes.toscrape.com/"); err != nil {
        log.Fatal(err)
    }
    fmt.Println("saved records to quotes-colly.csv")
}