ben/vath
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added forum metadata for later use

Browse Source
This commit is contained in:
eulaly
2026-05-09 00:36:30 -04:00
parent 181477bce7
commit 8f1d9e7723
2 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scraper/items.py
View File

@@ -5,6 +5,8 @@ class ForumItem(scrapy.Item):
forum_id = scrapy.Field() forum_id = scrapy.Field()
reg_title = scrapy.Field() reg_title = scrapy.Field()
reg_desc = scrapy.Field() reg_desc = scrapy.Field()
scraped_at = scrapy.Field()
forum_url = scrapy.Field()
class CommentItem(scrapy.Item): class CommentItem(scrapy.Item):

2
scraper/spiders/forum.py
View File

@@ -63,6 +63,8 @@ class ForumSpider(scrapy.Spider):
forum_id=self.forum_id, forum_id=self.forum_id,
reg_title=reg_title, reg_title=reg_title,
reg_desc=reg_desc, reg_desc=reg_desc,
scraped_at=datetime.utcnow().isoformat(),
forum_url=_view_url(self.forum_id),
) )
for page in range(2, last_page + 1): for page in range(2, last_page + 1):
yield scrapy.FormRequest( yield scrapy.FormRequest(