feat: save data when modifyTime has changed

Author: leafcoder

covid19/settings.py

@@ -156,5 +156,6 @@
 # Setting of Crontab
 CRONJOBS = (
     # 每分钟抓取一次
-    ('*/1 * * * *', 'ncovapi.cron.crawl_dxy', [], {}, '>> %s/crontab.log' % BASE_DIR),
+    ('*/1 * * * *', 'ncovapi.cron.crawl_dxy', [], {}, '>> %s/var/logs/crontab.log' % BASE_DIR),
 )
+

spider/nCoV/pipelines.py

@@ -42,4 +42,5 @@ def process_item(self, item, spider):
             return item
 
     def close_spider(self, spider):
-        cache.set('crawled', 1)
+        if spider.crawler is not None:
+            cache.set('crawled', 1)

spider/nCoV/spiders/dxy.py

@@ -8,11 +8,14 @@
 
 import json
 import scrapy
+import logging
 from scrapy.selector import Selector
 from .. import items
 
 from django.utils.timezone import datetime, make_aware
 
+logger = logging.getLogger()
+
 class DXYSpider(scrapy.Spider):
 
     name = "dxy"
@@ -25,6 +28,27 @@ def parse(self, response):
         sel = Selector(response)
         scripts = sel.xpath('//script')
 
+        # 判断是否需要保存抓取的数据
+        statistics = self.get_dict(scripts, '#getStatisticsService')
+        createTime = make_aware(
+            datetime.fromtimestamp(statistics['createTime'] / 1000.0))
+        modifyTime = make_aware(
+            datetime.fromtimestamp(statistics['modifyTime'] / 1000.0))
+        prev_crawler = items.CrawlerItem.django_model.objects.all().order_by('-id')[1]
+        if prev_crawler.modifyTime == modifyTime:
+            logger.info('Data does not change.')
+            self.crawler.delete()
+            self.crawler = None
+            return
+        self.crawler.createTime = createTime
+        self.crawler.modifyTime = modifyTime
+        self.crawler.save()
+
+        # 统计信息
+        statistics = self.parse_statistics(statistics)
+        for item in statistics:
+            yield item
+
         # 国内数据
         provinces = self.get_list(scripts, '#getAreaStat')
         for province in provinces:
@@ -48,11 +72,6 @@ def parse(self, response):
             country.pop('provinceShortName')
             yield items.CountryItem(**country)
 
-        # 统计信息
-        statistics = self.get_statistics(scripts, '#getStatisticsService')
-        for item in statistics:
-            yield item
-
         # 时间线事件，id=“getTimelineService2” 为英文内容
         timelines = self.get_list(scripts, '#getTimelineService1')
         for item in timelines:
@@ -106,8 +125,7 @@ def parse(self, response):
                 rumor[key] = item.get(key)
             yield items.RumorItem(**rumor)
 
-    def get_statistics(self, scripts, data_id):
-        data = self.get_dict(scripts, data_id)
+    def parse_statistics(self, data):
         statistics = data['globalStatistics']
         item = {}
         for key in (
@@ -156,16 +174,10 @@ def get_statistics(self, scripts, data_id):
         }
         yield items.NoticeItem(**item)
 
-        self.crawler.createTime = make_aware(
-            datetime.fromtimestamp(data['createTime'] / 1000.0))
-        self.crawler.modifyTime = make_aware(
-            datetime.fromtimestamp(data['modifyTime'] / 1000.0))
-        self.crawler.save()
-
     def get_list(self, scripts, data_id):
         ret = scripts.css(data_id).re(r'(\[.+\])')
         return json.loads(ret[0])
 
     def get_dict(self, scripts, data_id):
         ret = scripts.css(data_id).re(r'\=\s*(\{.+\})\}catch\(e\)\{\}')
-        return json.loads(ret[0])
+        return json.loads(ret[0])