osc-group/osc
1
1
Fork 0
Code Pull Requests Projects Wiki Activity

Update WeiboUserSpider.py

Browse Source 
批次大小和间隔时间修改
This commit is contained in:
yuxin-pc 2025-07-23 15:33:43 +08:00
parent 45110c22d3
commit 95ee8f5f59
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spiders/MediaSpiders/MediaSpiders/spiders/WeiboUserSpider.py
View File

@ -25,6 +25,8 @@ class WeiboSpider(scrapy.Spider):
'FILES_RESULT_FIELD': 'es_video', 'FILES_RESULT_FIELD': 'es_video',
'ZIP_FILE_NAME': 'image_data_publicinfo_', 'ZIP_FILE_NAME': 'image_data_publicinfo_',
'FILE_ZIP_FILE_NAME': 'image_data_plane_', 'FILE_ZIP_FILE_NAME': 'image_data_plane_',
'DOWNLOAD_DELAY': 3,
'BATCH_SAVE_SIZE': 20,
'ITEM_PIPELINES': { 'ITEM_PIPELINES': {
'scrapy.pipelines.images.ImagesPipeline': 2, 'scrapy.pipelines.images.ImagesPipeline': 2,
'scrapy.pipelines.files.FilesPipeline': 1, 'scrapy.pipelines.files.FilesPipeline': 1,