Java Jsoup代理爬虫

export PROXY_USER="替换为控制台代理用户名"
export PROXY_PASS="替换为控制台代理密码"
# 隧道代理固定网关：proxy.123proxy.cn:36923

$env:PROXY_USER="替换为控制台代理用户名"
$env:PROXY_PASS="替换为控制台代理密码"
# 隧道代理固定网关：proxy.123proxy.cn:36923

// pom.xml dependency:
// <dependency>
//   <groupId>org.jsoup</groupId>
//   <artifactId>jsoup</artifactId>
//   <version>1.21.2</version>
// </dependency>
import java.io.BufferedWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

public final class BooksWithProxy {
    static {
        // Allow Basic proxy authentication for HTTPS CONNECT tunnels.
        System.setProperty(
            "jdk.http.auth.tunneling.disabledSchemes",
            ""
        );
    }

    private static final String PROXY_HOST = "proxy.123proxy.cn";
    private static final int PROXY_PORT = 36923;

    private static String csv(String value) {
        return "\"" + value.replace("\"", "\"\"") + "\"";
    }

    public static void main(String[] args) throws Exception {
        String proxyUser = System.getenv("PROXY_USER");
        String proxyPass = System.getenv("PROXY_PASS");
        if (proxyUser == null || proxyPass == null) {
            throw new IllegalStateException(
                "set PROXY_USER and PROXY_PASS first"
            );
        }

        Connection session = Jsoup.newSession()
            .proxy(PROXY_HOST, PROXY_PORT)
            .auth(context -> {
                if (!context.isProxy()) {
                    return null;
                }
                return context.credentials(proxyUser, proxyPass);
            })
            .userAgent("123Proxy-Developer-Example/1.0")
            .timeout(30_000);

        try (BufferedWriter output = Files.newBufferedWriter(
            Path.of("books-jsoup.csv"),
            StandardCharsets.UTF_8
        )) {
            output.write("title,price,availability,url\n");
            String pageUrl =
                "https://books.toscrape.com/catalogue/page-1.html";

            for (int pageNumber = 1; pageNumber <= 3; pageNumber++) {
                Document document = session
                    .newRequest(pageUrl)
                    .get();

                for (Element book : document.select("article.product_pod")) {
                    Element link = book.selectFirst("h3 a");
                    String title = link.attr("title");
                    String price = book.selectFirst("p.price_color").text();
                    String availability = book
                        .selectFirst("p.availability")
                        .text();
                    String detailUrl = link.absUrl("href");
                    output.write(
                        csv(title) + "," +
                        csv(price) + "," +
                        csv(availability) + "," +
                        csv(detailUrl) + "\n"
                    );
                }

                Element next = document.selectFirst("li.next > a");
                if (next == null) {
                    break;
                }
                pageUrl = next.absUrl("href");
                Thread.sleep(500);
            }
        }
        System.out.println("saved records to books-jsoup.csv");
    }
}