自定义下载处理器炸了，Scrapy 只报「Unsupported URL scheme」——而且这辈子都不会再试一次

NotSupported: Unsupported URL scheme 'https': name 'foo' is not defined

self._schemes: dict[str, str | Callable[..., Any]] = {}   # 配置里声明的
self._handlers: dict[str, DownloadHandlerProtocol] = {}   # 已经建好的
self._notconfigured: dict[str, str] = {}                  # 建失败的

def _get_handler(self, scheme: str) -> DownloadHandlerProtocol | None:
    if scheme in self._handlers:
        return self._handlers[scheme]
    if scheme in self._notconfigured:      # ← 失败过就直接返回 None
        return None
    if scheme not in self._schemes:
        self._notconfigured[scheme] = "no handler available for that scheme"
        return None
    return self._load_handler(scheme)

def _load_handler(self, scheme: str, skip_lazy: bool = False):
    path = self._schemes[scheme]
    try:
        dhcls = load_object(path)
        if skip_lazy:
            if getattr(dhcls, "lazy", True):
                return None            # ← lazy 的在启动阶段直接跳过
        dh = build_from_crawler(dhcls, self._crawler)
    except NotConfigured as ex:
        self._notconfigured[scheme] = str(ex)      # 安静地记下
        return None
    except Exception as ex:
        logger.error(...)                          # 打 ERROR 但不中断
        self._notconfigured[scheme] = str(ex)
        return None
    self._handlers[scheme] = dh
    return dh

def build_from_crawler(objcls, crawler, /, *args, **kwargs):
    if hasattr(objcls, "from_crawler"):
        instance = objcls.from_crawler(crawler, *args, **kwargs)
        method_name = "from_crawler"
    else:
        instance = objcls(*args, **kwargs)      # ← 注意：不传 crawler
        method_name = "__new__"
    if instance is None:
        raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
    return instance

class Base:
    lazy = True
    async def download_request(self, request): ...
    async def close(self): ...
 
class BoomLazy(Base):
    @classmethod
    def from_crawler(cls, crawler):
        TRACE.append("尝试构造 handler")
        raise RuntimeError("构造函数炸了")
 
class BoomEager(BoomLazy):
    lazy = False                      # 只改这一个属性
 
class NotConf(Base):
    @classmethod
    def from_crawler(cls, crawler):
        raise NotConfigured("缺少某个依赖")
 
class Counting(Base):
    builds = 0
    @classmethod
    def from_crawler(cls, crawler):
        Counting.builds += 1
        raise RuntimeError(f"第 {Counting.builds} 次构造，还是炸")
 
class NoReturn(Base):
    @classmethod
    def from_crawler(cls, crawler):
        cls()                          # 建了，但没返回

TRACE: list[str] = []
 
class S(scrapy.Spider):
    @classmethod
    def from_crawler(cls, crawler, *a, **kw):
        sp = super().from_crawler(crawler, *a, **kw)
        # 必须挂绑定方法，不能挂 lambda：SignalManager 用弱引用存接收者，
        # 临时 lambda 没有别的引用，连上就被回收，信号永远不触发。
        crawler.signals.connect(sp._opened, signal=scrapy.signals.spider_opened)
        return sp
 
    def _opened(self):
        TRACE.append("spider 打开")

scrapy 2.17.0  python 3.11.13
 
──── ① lazy=True（默认）：handler 构造抛 RuntimeError
  [请求] err  NotSupported: Unsupported URL scheme 'probe': 构造函数炸了
  [时序] spider 打开 → 尝试构造 handler → 请求返回
  [日志] ERROR: Loading "<class '__main__.BoomLazy'>" for scheme "probe"
  [日志] RuntimeError: 构造函数炸了
 
──── ② lazy=False：同一个 handler，只改这一个属性
  [请求] err  NotSupported: Unsupported URL scheme 'probe': 构造函数炸了
  [时序] 尝试构造 handler → spider 打开 → 请求返回
  [日志] ERROR: Loading "<class '__main__.BoomEager'>" for scheme "probe"
  [日志] RuntimeError: 构造函数炸了
 
──── ③ 抛 NotConfigured 而不是普通异常
  [请求] err  NotSupported: Unsupported URL scheme 'probe': 缺少某个依赖
  [时序] spider 打开 → 尝试构造 handler → 请求返回
 
──── ④ 同一 scheme 连请求 3 次：失败后会重试构造吗
  [请求] err  NotSupported: Unsupported URL scheme 'probe': 第 1 次构造，还是炸
  [请求] err  NotSupported: Unsupported URL scheme 'probe': 第 1 次构造，还是炸
  [请求] err  NotSupported: Unsupported URL scheme 'probe': 第 1 次构造，还是炸
  [统计] from_crawler 实际被调用 1 次
 
──── ⑤ from_crawler 忘了 return
  [请求] err  NotSupported: Unsupported URL scheme 'probe': NoReturn.from_crawler returned None
  [日志] TypeError: NoReturn.from_crawler returned None

if scheme in self._notconfigured:
    return None

except NotConfigured as ex:
    self._notconfigured[scheme] = str(ex)     # 没有 logger.error
    return None
except Exception as ex:
    logger.error(..., exc_info=True)          # 有
    self._notconfigured[scheme] = str(ex)

handler = self._get_handler(scheme)
if not handler:
    raise NotSupported(
        f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
    )

class MyHandler:
    lazy = False        # 建不起来就别让爬虫跑起来

from scrapy import signals
from scrapy.exceptions import NotSupported
 
class HandlerHealth:
    """NotSupported 一旦出现就是配置或代码问题，不该混在普通失败里。"""
 
    @classmethod
    def from_crawler(cls, crawler):
        ext = cls()
        crawler.signals.connect(ext.closed, signal=signals.spider_closed)
        return ext              # ← 忘了 return，这个扩展自己就静默失效了
 
    def closed(self, spider):
        n = spider.crawler.stats.get_value(
            "downloader/exception_type_count/scrapy.exceptions.NotSupported", 0
        )
        if n:
            spider.logger.error(
                "本轮有 %d 个请求死于 NotSupported —— "
                "去看冒号后面那半句，多半是下载处理器没建起来", n
            )