龍魂·蜘蛛网 — 代理池管理器

#!/usr/bin/env python3 龍魂·蜘蛛网 — 代理池管理器 自动获取免费代理 → 验证可用性 → 轮换使用 → 标记失效 DNA: #龍芯⚡️2026-07-07-SPIDER-NET-PROXY-v1.0 支持: - 多源免费代理采集89ip / kuaidaili / proxylist - 并发验证最多 20 线程 - 加权轮换成功率高优先 - 自动清理过期代理 importjsonimportrandomimportthreadingimporttimefromconcurrent.futuresimportThreadPoolExecutor,as_completedfromdatetimeimportdatetime,timezonefrompathlibimportPathfromtypingimportAny,Dict,List,Optional,Set,Tuplefromurllib.parseimporturlparseimportrequestsfrombs4importBeautifulSoup# ── 路径 ────────────────────────────────────────────────ROOTPath(__file__).resolve().parents[3]PROXY_STATEROOT/L7_数据层/spider_net_proxy_pool.json# ── 常量 ────────────────────────────────────────────────PROXY_TEST_URLhttps://httpbin.org/ipPROXY_TEST_TIMEOUT8PROXY_MAX_LIFETIME1800# 30 分钟过期PROXY_MIN_COUNT5# 不足时自动补充VALIDATE_WORKERS20REQUEST_HEADERS{User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36,Accept:text/html,application/xhtmlxml,}classProxyPool:代理池管理器。 生命周期: 1. fetch() — 从免费源获取代理 2. validate_all() — 并发验证 3. get_working_proxy() — 获取可用代理 4. mark_failed() — 标记失效 5. prune() — 清理过期 def__init__(self):self._proxies:Dict[str,Dict[str,Any]]{}# proxy_url → {score, last_ok, last_fail, fail_count}self._working:List[str][]# 当前可用代理列表self._failed:Set[str]set()# 永久失效集合self._lockthreading.Lock()self._load_state()# ── 公共 API ──────────────────────────────────────deffetch(self)-List[str]:从多个免费代理源获取代理列表。raw:Set[str]set()fetchers[self._fetch_89ip,self._fetch_kuaidaili,self._fetch_proxylist,]forfetcherinfetchers:try:newfetcher()raw.update(new)exceptExceptionase:print(f ⚠️ 代理源获取失败:{e})# 过滤已失效的new_proxies[pforpinrawifpnotinself._failed]print(f 从{len(fetchers)}个源获取{len(raw)}个代理有效{len(new_proxies)}个)# 注册新代理withself._lock:forpinnew_proxies:ifpnotinself._proxies:self._proxies[p]{score:1.0,last_ok:None,last_fail:None,fail_count:0,added_at:datetime.now(timezone.utc).isoformat(),}returnnew_proxiesdefvalidate_all(self)-Tuple[int,int]:并发验证所有代理返回 (可用, 总数)。withself._lock:candidateslist(self._proxies.keys())ifnotcandidates:return0,0ok_count0withThreadPoolExecutor(max_workersVALIDATE_WORKERS)asexecutor:futures{executor.submit(self._test_one,p):pforpincandidates}forfutureinas_completed(futures):proxyfutures[future]try:iffuture.result():ok_count1exceptException:pass# 重建可用列表withself._lock:self._working[pforpinself._proxiesifself._proxies[p][fail_count]0and(time.time()-self._proxies[p].get(_last_ok_ts,0))PROXY_MAX_LIFETIME]# 按成功率排序self._working.sort(keylambdap:self._proxies[p].get(score,1.0),reverseTrue)self._save_state()returnok_count,len(candidates)defget_working_proxy(self)-Optional[str]:获取一个可用代理加权随机。withself._lock:ifnotself._working:returnNone# 加权随机score 高的更优先weights[max(0.1,self._proxies[p].get(score,1.0))forpinself._working]totalsum(weights)iftotal0:returnrandom.choice(self._working)rrandom.uniform(0,total)cum0.0fori,pinenumerate(self._working):cumweights[i]ifrcum:returnpreturnself._working[0]defmark_success(self,proxy:str)-None:标记代理使用成功加分。withself._lock:ifproxyinself._proxies:entryself._proxies[proxy]entry[score]min(3.0,entry.get(score,1.0)0.1)entry[last_ok]datetime.now(timezone.utc).isoformat()entry[_last_ok_ts]time.time()entry[fail_count]0defmark_failed(self,proxy:str)-None:标记代理失败扣分连续失败 3 次移入永久失效。withself._lock:ifproxynotinself._proxies:returnentryself._proxies[proxy]entry[fail_count]entry.get(fail_count,0)1entry[score]max(0.0,entry.get(score,1.0)-0.4)entry[last_fail]datetime.now(timezone.utc).isoformat()ifentry[fail_count]3:self._failed.add(proxy)self._proxies.pop(proxy,None)ifproxyinself._working:self._working.remove(proxy)defprune(self)-int:清理过期代理返回清理数量。nowtime.time()expired:List[str][]withself._lock:forp,entryinlist(self._proxies.items()):last_okentry.get(_last_ok_ts,0)iflast_ok0and(now-last_ok)PROXY_MAX_LIFETIME:expired.append(p)forpinexpired:self._proxies.pop(p,None)ifpinself._working:self._working.remove(p)self._save_state()returnlen(expired)defensure_min_pool(self,min_count:intPROXY_MIN_COUNT)-None:确保代理池至少有 min_count 个可用代理。withself._lock:currentlen(self._working)ifcurrentmin_count:print(f 代理池不足 ({current}/{min_count})自动补充...)self.fetch()ok,_self.validate_all()print(f ✅ 补充后可用:{ok})defstats(self)-Dict[str,Any]:代理池统计。withself._lock:return{total_registered:len(self._proxies),working:len(self._working),permanently_failed:len(self._failed),top_proxies:[{proxy:p[:30]...,score:round(self._proxies[p].get(score,0),2),fail_count:self._proxies[p].get(fail_count,0),}forpinself._working[:5]],}# ── 免费代理源 ────────────────────────────────────staticmethoddef_fetch_89ip()-Set[str]:89ip 免费代理高匿。proxies:Set[str]set()try:resprequests.get(https://www.89ip.cn/index_1.html,headersREQUEST_HEADERS,timeout10,)soupBeautifulSoup(resp.text,lxml)forrowinsoup.select(table.layui-table tbody tr):colsrow.find_all(td)iflen(cols)2:ipcols[0].get_text(stripTrue)portcols[1].get_text(stripTrue)ifipandportandport.isdigit():proxies.add(fhttp://{ip}:{port})exceptException:passreturnproxiesstaticmethoddef_fetch_kuaidaili()-Set[str]:快代理免费代理。proxies:Set[str]set()try:resprequests.get(https://www.kuaidaili.com/free/intr/1/,headersREQUEST_HEADERS,timeout10,)soupBeautifulSoup(resp.text,lxml)forrowinsoup.select(table tbody tr):colsrow.find_all(td)iflen(cols)2:ipcols[0].get_text(stripTrue)portcols[1].get_text(stripTrue)ifipandportandport.isdigit():proxies.add(fhttp://{ip}:{port})exceptException:passreturnproxiesstaticmethoddef_fetch_proxylist()-Set[str]:proxylist.geonode.com API。proxies:Set[str]set()try:resprequests.get(https://proxylist.geonode.com/api/proxy-list?limit50page1sort_bylastCheckedsort_typedescprotocolshttp,httpsanonymityLeveleliteanonymityLevelanonymous,headersREQUEST_HEADERS,timeout10,)dataresp.json()foritemindata.get(data,[])[:50]:ipitem.get(ip,)portitem.get(port,)protoitem.get(protocols,[http])[0]ifipandport:proxies.add(f{proto}://{ip}:{port})exceptException:passreturnproxies# ── 内部 ──────────────────────────────────────────def_test_one(self,proxy:str)-bool:测试单个代理是否可用。try:resprequests.get(PROXY_TEST_URL,proxies{http:proxy,https:proxy},timeoutPROXY_TEST_TIMEOUT,)ifresp.status_code200:self.mark_success(proxy)returnTrueexceptException:passself.mark_failed(proxy)returnFalsedef_save_state(self)-None:持久化代理池状态。PROXY_STATE.parent.mkdir(parentsTrue,exist_okTrue)state{proxies:{p:{score:e.get(score,1.0),fail_count:e.get(fail_count,0),last_ok:e.get(last_ok),last_fail:e.get(last_fail),added_at:e.get(added_at),}forp,einself._proxies.items()},failed:list(self._failed),updated_at:datetime.now(timezone.utc).isoformat(),}PROXY_STATE.write_text(json.dumps(state,ensure_asciiFalse,indent2),encodingutf-8)def_load_state(self)-None:从磁盘恢复代理池状态。ifnotPROXY_STATE.exists():returntry:datajson.loads(PROXY_STATE.read_text(encodingutf-8))forp,eindata.get(proxies,{}).items():self._proxies[p]{score:e.get(score,1.0),fail_count:e.get(fail_count,0),last_ok:e.get(last_ok),last_fail:e.get(last_fail),added_at:e.get(added_at),}self._failedset(data.get(failed,[]))# 重建 working 列表nowtime.time()self._working[pforp,einself._proxies.items()ife.get(fail_count,0)0and(now-e.get(_last_ok_ts,0))PROXY_MAX_LIFETIME]exceptException:pass# ═══════════════════════════════════════════════════════════# CLI 测试# ═══════════════════════════════════════════════════════════if__name____main__:poolProxyPool()print( 代理池管理器\n)print( 获取代理...)pool.fetch()print( 验证代理...)ok,totalpool.validate_all()print(f 可用:{ok}/{total})statspool.stats()print(f\n 统计:)print(f 注册:{stats[total_registered]})print(f 可用:{stats[working]})print(f 失效:{stats[permanently_failed]})print(f\n Top 代理:)forpinstats[top_proxies]:print(f{p[proxy]}(score{p[score]}, fails{p[fail_count]}))