diff --git a/spiders/MediaSpiders/MediaSpiders/spiders/WeiboUserSpider.py b/spiders/MediaSpiders/MediaSpiders/spiders/WeiboUserSpider.py index 706d474..d433a4a 100644 --- a/spiders/MediaSpiders/MediaSpiders/spiders/WeiboUserSpider.py +++ b/spiders/MediaSpiders/MediaSpiders/spiders/WeiboUserSpider.py @@ -25,6 +25,8 @@ class WeiboSpider(scrapy.Spider): 'FILES_RESULT_FIELD': 'es_video', 'ZIP_FILE_NAME': 'image_data_publicinfo_', 'FILE_ZIP_FILE_NAME': 'image_data_plane_', + 'DOWNLOAD_DELAY': 3, + 'BATCH_SAVE_SIZE': 20, 'ITEM_PIPELINES': { 'scrapy.pipelines.images.ImagesPipeline': 2, 'scrapy.pipelines.files.FilesPipeline': 1,