osc-group/osc
1
1
Fork 0
Code Pull Requests Projects Wiki Activity

[twitter] 推文采集字段增加

Browse Source
This commit is contained in:
DELL 2026-01-28 15:27:33 +08:00
parent d023703622
commit 3fdd2f5473
1 changed files with 16 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
spiders/MediaSpiders/MediaSpiders/spiders/TwitterUserSpider.py
View File

@ -176,12 +176,25 @@ class TwitterSpider(scrapy.Spider):
item['es_urlname'] = 'https://x.com/%s/status/%s' % (screen_name, result['rest_id']) item['es_urlname'] = 'https://x.com/%s/status/%s' % (screen_name, result['rest_id'])
item['es_authors'] = screen_name item['es_authors'] = screen_name
item['es_extname'] = user_name item['es_extname'] = user_name
device_html = result['source']
device_type = re.search(r'>([^<]+)</a>', device_html).group(1)
legacy = result['legacy'] legacy = result['legacy']
author_full_text = legacy['full_text'] author_full_text = legacy['full_text']
created_at = legacy['created_at'] created_at = legacy['created_at']
# 评论、转发、点赞数量
item['es_commentcount'] = legacy['reply_count'] item['es_commentcount'] = legacy['reply_count']
item['es_forwardcount'] = legacy['retweet_count'] item['es_forwardcount'] = legacy['retweet_count']
item['es_likecount'] = legacy['favorite_count'] item['es_likecount'] = legacy['favorite_count']
# 评论+ 转发+ 点赞数量 TODO
interaction_count = legacy['reply_count'] + legacy['retweet_count'] + legacy['favorite_count']
# 语种
lang = legacy['lang']
# 推文话题 、 提及
topic = legacy['entities']['hashtags']
mentions = legacy['entities']['user_mentions']
item['es_lasttime'] = get_current_timestamp() item['es_lasttime'] = get_current_timestamp()
item['es_loadtime'] = get_current_timestamp() item['es_loadtime'] = get_current_timestamp()
item['es_urltime'] = get_time_stamp( item['es_urltime'] = get_time_stamp(
@ -196,13 +209,16 @@ class TwitterSpider(scrapy.Spider):
else: else:
item['es_catalog2'] = '' item['es_catalog2'] = ''
legacy = result['quoted_status_result']['result']['legacy'] legacy = result['quoted_status_result']['result']['legacy']
original_tweet = result['quoted_status_result']['result']['rest_id']
self.logger.info('采集引用推文原文信息') self.logger.info('采集引用推文原文信息')
elif 'retweeted_status_result' in legacy: elif 'retweeted_status_result' in legacy:
item['es_isrepost'] = '1' item['es_isrepost'] = '1'
legacy = legacy['retweeted_status_result']['result']['legacy'] legacy = legacy['retweeted_status_result']['result']['legacy']
original_tweet = result['retweeted_status_result']['result']['rest_id']
self.logger.info('采集转发推文原文信息') self.logger.info('采集转发推文原文信息')
else: else:
item['es_isrepost'] = '0' item['es_isrepost'] = '0'
original_tweet = ''
self.logger.info('采集原文信息') self.logger.info('采集原文信息')
item['es_urlcontent'] = legacy['full_text'] item['es_urlcontent'] = legacy['full_text']