osc-group/osc
1
1
Fork 0
Code Pull Requests Projects Wiki Activity

Update middlewares.py

Browse Source 
采集开始时显示并发线程
This commit is contained in:
yuxin-pc 2025-06-13 09:41:04 +08:00
parent 8f1999376f
commit 58bdf5cc0c
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spiders/WebsiteSpider/WebsiteSpider/middlewares.py
View File

@ -116,6 +116,8 @@ class InitDownloaderMiddleware(object):
def spider_opened(self, spider): def spider_opened(self, spider):
spider.logger.info('InitDownloaderMiddleware Enabled') spider.logger.info('InitDownloaderMiddleware Enabled')
CONCURRENT_REQUESTS = spider.settings['CONCURRENT_REQUESTS']
spider.logger.info(f'本次采集任务的并发线程数为：{CONCURRENT_REQUESTS}')
self.redis_client = redis.Redis(host=spider.settings['REDIS_HOST'], port=int(spider.settings['REDIS_PORT']), self.redis_client = redis.Redis(host=spider.settings['REDIS_HOST'], port=int(spider.settings['REDIS_PORT']),
password=spider.settings['REDIS_PWD']) password=spider.settings['REDIS_PWD'])
# 更新采集规则 # 更新采集规则