osc-group/osc
1
1
Fork 0
Code Pull Requests Projects Wiki Activity

Compare commits

base: osc-group:58bdf5cc0cd69997cd583620a94023708b800cc3
Branches Tags
osc-group:main
..
compare: osc-group:e2da209f39145a2449bf1c11e7002d0c84836e32
Branches Tags
osc-group:main

No commits in common. "58bdf5cc0cd69997cd583620a94023708b800cc3" and "e2da209f39145a2449bf1c11e7002d0c84836e32" have entirely different histories.
58bdf5cc0c ... e2da209f39

2 changed files with 15 additions and 17 deletions
Show Stats Expand all files Collapse all files

2
spiders/WebsiteSpider/WebsiteSpider/middlewares.py
View File

@ -116,8 +116,6 @@ class InitDownloaderMiddleware(object):
def spider_opened(self, spider):
spider.logger.info('InitDownloaderMiddleware Enabled')
CONCURRENT_REQUESTS = spider.settings['CONCURRENT_REQUESTS']
spider.logger.info(f'本次采集任务的并发线程数为：{CONCURRENT_REQUESTS}')
self.redis_client = redis.Redis(host=spider.settings['REDIS_HOST'], port=int(spider.settings['REDIS_PORT']),
password=spider.settings['REDIS_PWD'])
# 更新采集规则

30
spiders/WebsiteSpider/WebsiteSpider/settings.py
View File

@ -13,10 +13,10 @@ SELENIUM_DRIVER_NAME = 'firefox'
SELENIUM_DRIVER_EXECUTABLE_PATH = [
'http://154.90.40.71:28095',
'http://154.90.63.14:28095',
# 'http://156.244.20.57:28095',
'http://10.55.13.121:28095',
'http://10.55.13.108:28095',
'http://10.55.13.3:28095',
'http://156.244.20.57:28095',
# 'http://10.55.13.121:28095',
# 'http://10.55.13.108:28095',
# 'http://10.55.13.3:28095',
]
SELENIUM_DRIVER_ARGUMENTS = ['-headless'] # '--headless' if using chrome instead of firefox
SELENIUM_DRIVER_PREFERENCES = {
@ -27,10 +27,10 @@ SELENIUM_DRIVER_PREFERENCES = {
PROXY_SERVICE = 'http://107.182.191.3:6800'
PER_BATCH_IP_USE_TIMES = 5 # 代理中间件每次从ip池获取一批ip，定义这批ip使用次数，达到次数后重新从ip池获取新的一批
# REDIS_HOST = '38.54.94.107'
# REDIS_PORT = '28097'
REDIS_HOST = '10.55.13.3'
REDIS_PORT = '7379'
REDIS_HOST = '38.54.94.107'
REDIS_PORT = '28097'
# REDIS_HOST = '10.55.13.3'
# REDIS_PORT = '7379'
REDIS_PWD = 'jlkj-841-2-redis'
REDIS_PARAMS = {
'password': 'jlkj-841-2-redis',
@ -39,9 +39,9 @@ REDIS_PARAMS = {
USCARRIERS_KEY = 'USCARRIERS_ID'
ZIP_FILE_PATH = ''
# KAFKA_PROCESS_QUEUE = ['stream-protobuf']
KAFKA_PROCESS_QUEUE = ['stream-protobuf', 'stream-db']
KAFKA_SERVER = '47.113.231.200:9092'
KAFKA_PROCESS_QUEUE = ['stream-protobuf']
# KAFKA_PROCESS_QUEUE = ['stream-protobuf', 'stream-db']
KAFKA_SERVER = '38.54.125.182:9092'
RANDOMIZE_DOWNLOAD_DELAY = True
@ -61,7 +61,7 @@ CRAWL_DEEPTH = 3 # 若起始url为门户首页或板块首页，则深度至少
# Configure maximum concurrent requests performed by Scrapy (default: 16)
# 控制并发采集数量，即同时请求url数量，也可控制scrapy-redis从队列中读取url得数量
CONCURRENT_REQUESTS = 12
CONCURRENT_REQUESTS = 16
PROTO_MODULE_PATH = 'WebsiteSpider.proto.Es_pb2'
PROTO_CLASS_NAME = 'EsSets'
@ -81,9 +81,9 @@ IMG_ZIP_FILE_NAME = 'image_data_publicinfo_'
MYEXT_ENABLED = True # 开启超时关闭scrapy扩展
IDLE_NUMBER = 36 # 配置允许的空闲时长，每5秒会增加一次IDLE_NUMBER，直到增加到60，程序才会close,空闲时间=5*IDLE_NUMBER 秒
CRAWL_JOB_UPDATE_API = 'http://47.115.228.133:28081/api/open/crawljob'
WORD_BANK_QUERY_API = 'http://47.115.228.133:28081/api/open/wordBank/queryAll'
CRAWL_RULE_QUERY_API = 'http://47.115.228.133:28081/api/open/target/website/queryAll'
CRAWL_JOB_UPDATE_API = 'http://38.54.94.107:28081/api/open/crawljob'
WORD_BANK_QUERY_API = 'http://38.54.94.107:28081/api/open/wordBank/queryAll'
CRAWL_RULE_QUERY_API = 'http://38.54.94.107:28081/api/open/target/website/queryAll'
BATCH_SAVE_SIZE = 5
PROTO_SAVE_FILE_PATH = '/usr/local/spider_data'