65 lines
3.0 KiB
Python
65 lines
3.0 KiB
Python
import scrapy
|
||
|
||
|
||
class ForumspidersItem(scrapy.Item):
|
||
pass
|
||
|
||
|
||
class ZhihuItem(scrapy.Item):
|
||
# （问题） （回答） （评论）
|
||
item_id = scrapy.Field() # 随机生成ID 随机生成ID 随机生成ID
|
||
content_type = scrapy.Field() # q a c
|
||
question_id = scrapy.Field() # 问题ID 关联问题ID 关联问题ID
|
||
answer_id = scrapy.Field() # 回答ID 关联回答ID
|
||
comment_id = scrapy.Field() # 评论ID
|
||
question_title = scrapy.Field() # 问题标题 问题标题 问题标题
|
||
question_link = scrapy.Field() # 问题链接 问题链接 问题链接
|
||
answer_link = scrapy.Field() # 回答链接 回答链接
|
||
content_text = scrapy.Field() # 内容文本 内容文本 内容文本
|
||
content_html = scrapy.Field() # 内容HTML 内容HTML 内容HTML
|
||
image_urls = scrapy.Field() # 图片链接 图片链接 图片链接
|
||
author_id = scrapy.Field() # 作者ID 作者ID 作者ID
|
||
author_name = scrapy.Field() # 作者名 作者名 作者名
|
||
author_link = scrapy.Field() # 作者主页 作者主页 作者主页
|
||
upvote_num = scrapy.Field() # 好问题数 赞同数 点赞数
|
||
comment_num = scrapy.Field() # 评论数 评论数
|
||
date_created = scrapy.Field() # 发布时间戳 发布时间戳 发布时间戳
|
||
date_modified = scrapy.Field() # 修改时间戳 修改时间戳 修改时间戳
|
||
remarks = scrapy.Field() # 备注 备注 备注
|
||
meta = scrapy.Field() # 元数据 元数据 元数据
|
||
|
||
|
||
class MdcTwItem(scrapy.Item):
|
||
item_id = scrapy.Field()
|
||
author_name = scrapy.Field()
|
||
author_link = scrapy.Field()
|
||
author_info = scrapy.Field()
|
||
post_time = scrapy.Field()
|
||
post_content = scrapy.Field()
|
||
content_image_urls = scrapy.Field()
|
||
post_title = scrapy.Field()
|
||
post_id = scrapy.Field()
|
||
|
||
|
||
class EsItem(scrapy.Item):
|
||
es_sid = scrapy.Field() # sid
|
||
es_hkey = scrapy.Field() # 根据发帖内容、时间、作者等生成的 ID
|
||
es_urlname = scrapy.Field() # 发帖内容的 URL，如果是回帖则再加上 hkey
|
||
es_sitename = scrapy.Field() # 站点名称
|
||
es_srcname = scrapy.Field() # 来源站点 URL
|
||
es_urltitle = scrapy.Field() # 发帖标题（主贴的标题，跟帖采用主帖标题）
|
||
es_urltopic = scrapy.Field() # 主贴 ID，这个字段用来关联更贴和主帖
|
||
es_isrepost = scrapy.Field() # 0-主贴；1-跟帖
|
||
es_lasttime = scrapy.Field() # 更新时间
|
||
es_loadtime = scrapy.Field() # 入库时间
|
||
es_urltime = scrapy.Field() # 发布时间
|
||
es_authors = scrapy.Field() # 作者
|
||
es_urlimage = scrapy.Field() # 图片内容
|
||
es_doclength = scrapy.Field() # 贴文长度
|
||
es_content = scrapy.Field() # 发帖内容
|
||
es_urlcontent = scrapy.Field() # 发帖内容（去除 html 标签）
|
||
es_carriertype = scrapy.Field() # 赋值为 forum
|
||
es_commentcount = scrapy.Field() # 回帖数，这个字段不采，留给业务系统实时查询并赋值
|
||
es_likecount = scrapy.Field() # 点赞或赞同数
|
||
es_attachment = scrapy.Field() # 附件内容
|