Прокси в Scrapy: запрос и авторизация

python -m pip install "scrapy==2.19.0"
python -c "import scrapy; print(scrapy.__version__)"

import getpass
import ipaddress
import os
from urllib.parse import quote, urlsplit

import scrapy
from scrapy.crawler import CrawlerProcess


class ProxyCheck(scrapy.Spider):
    name = "proxy_check"
    ok = False

    async def start(self):
        yield scrapy.Request(
            "https://api.portproof.org/v1/echo-ip", callback=self.parse, errback=self.failed,
            meta={"proxy": self.proxy, "handle_httpstatus_all": True},
        )

    def parse(self, response):
        if response.status != 200:
            print("HTTP response after CONNECT:", response.status)
            return
        try:
            payload = response.json()
            if not isinstance(payload, dict) or not isinstance(payload.get("ip"), str):
                raise ValueError("Expected an IP address")
            address = ipaddress.ip_address(payload["ip"])
        except (ValueError, KeyError, TypeError):
            print("Check failed: invalid echo response")
            return
        self.ok = True
        print("HTTP", response.status, "exit", address)

    def failed(self, failure):
        print("No destination response:", failure.type.__name__)


if __name__ == "__main__":
    server = os.environ.get("PROXY_SERVER", "http://gw.portproof.org:7000")
    try:
        gateway = urlsplit(server)
        valid = (
            gateway.scheme == "http" and gateway.hostname
            and gateway.port is not None and gateway.username is None
            and gateway.password is None and gateway.path in ("", "/")
            and not gateway.query and not gateway.fragment
        )
    except ValueError:
        valid = False
    if not valid:
        raise SystemExit("Use an HTTP gateway URL without credentials or a path")
    user = os.environ.get("PROXY_USER", "")
    if not user or ":" in user:
        raise SystemExit("Set PROXY_USER to your complete proxy username")
    password = getpass.getpass("Proxy password: ")
    proxy = (
        "http://" + quote(user, safe="") + ":" + quote(password, safe="")
        + "@" + gateway.netloc
    )
    process = CrawlerProcess(settings={
        "TWISTED_REACTOR_ENABLED": True,
        "DOWNLOAD_HANDLERS": {
            "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
        },
        "DOWNLOAD_VERIFY_CERTIFICATES": True,
        "CONCURRENT_REQUESTS": 1,
        "DOWNLOAD_TIMEOUT": 20,
        "DOWNLOAD_MAXSIZE": 65536,
        "RETRY_ENABLED": False,
        "REDIRECT_ENABLED": False,
        "METAREFRESH_ENABLED": False,
        "ROBOTSTXT_OBEY": False,  # This diagnostic schedules only the echo URL.
        "COOKIES_ENABLED": False,
        "LOG_ENABLED": False,
        "TELNETCONSOLE_ENABLED": False,
        "REMOTE_CONTROL_ENABLED": False,
    })
    crawler = process.create_crawler(ProxyCheck)
    process.crawl(crawler, proxy=proxy)
    process.start()
    raise SystemExit(0 if crawler.spider and crawler.spider.ok else 1)