diff --git a/scraper/items.py b/scraper/items.py index a5e6971..d465967 100644 --- a/scraper/items.py +++ b/scraper/items.py @@ -5,6 +5,8 @@ class ForumItem(scrapy.Item): forum_id = scrapy.Field() reg_title = scrapy.Field() reg_desc = scrapy.Field() + scraped_at = scrapy.Field() + forum_url = scrapy.Field() class CommentItem(scrapy.Item): diff --git a/scraper/spiders/forum.py b/scraper/spiders/forum.py index 910ec89..4d1cf02 100644 --- a/scraper/spiders/forum.py +++ b/scraper/spiders/forum.py @@ -63,6 +63,8 @@ class ForumSpider(scrapy.Spider): forum_id=self.forum_id, reg_title=reg_title, reg_desc=reg_desc, + scraped_at=datetime.utcnow().isoformat(), + forum_url=_view_url(self.forum_id), ) for page in range(2, last_page + 1): yield scrapy.FormRequest(