feat: add country and province daily list api

Author: leafcoder

ncovapi/models.py

@@ -194,6 +194,11 @@ class Province(models.Model):
     deadCount = models.IntegerField(default=0)
     comment = models.CharField(max_length=200)
     statisticsData = models.CharField(max_length=500)
+    dailyData = models.TextField()
+    created = models.DateTimeField(
+        '创建时间', auto_now_add=True, editable=False)
+    updated = models.DateTimeField(
+        '更新时间', auto_now=True, editable=False)
     crawler = models.ForeignKey(
         "Crawler", on_delete=models.CASCADE, related_name="provinces",
         db_column="crawlerId"
@@ -213,6 +218,10 @@ class City(models.Model):
     suspectedCount = models.IntegerField(default=0)
     curedCount = models.IntegerField(default=0)
     deadCount = models.IntegerField(default=0)
+    created = models.DateTimeField(
+        '创建时间', auto_now_add=True, editable=False)
+    updated = models.DateTimeField(
+        '更新时间', auto_now=True, editable=False)
     province = models.ForeignKey(
         "Province", on_delete=models.CASCADE, related_name="cities",
         db_column="provinceId"
@@ -254,8 +263,11 @@ class Country(models.Model):
     incrVo = models.TextField(null=True)
     sort = models.IntegerField(null=True)
     operator = models.CharField(max_length=50, null=True)
-    modifyTime = models.IntegerField(null=True)
-    createTime = models.IntegerField(null=True)
+    dailyData = models.TextField()
+    created = models.DateTimeField(
+        '创建时间', auto_now_add=True, editable=False)
+    updated = models.DateTimeField(
+        '更新时间', auto_now=True, editable=False)
     crawler = models.ForeignKey(
         "Crawler", on_delete=models.CASCADE, related_name="countries",
         db_column="countryId"

ncovapi/urls.py

@@ -16,7 +16,9 @@
     path('provinces/', views.ProvinceListView.as_view(), name='province-list'),
     path('provinces/<int:pk>/', views.ProvinceRetrieveView.as_view(), name='province-detail'),
     path('provinces/<str:provinceShortName>/', views.ProvinceRetrieveByNameView.as_view(), name='province-detail-by-name'),
+    path('provinces/<str:provinceShortName>/daily', views.ProvinceDailyListView.as_view(), name='province-daily-list'),
     path('countries/', views.CountryListView.as_view(), name='country-list'),
     path('countries/<int:pk>/', views.CountryRetrieveView.as_view(), name='country-detail'),
     path('countries/<str:countryName>/', views.CountryRetrieveByNameView.as_view(), name='country-detail-by-name'),
+    path('countries/<str:countryName>/daily', views.CountryDailyListView.as_view(), name='country-daily-list'),
 ]

ncovapi/views.py

@@ -21,6 +21,7 @@
 from .filters import CityFilter, ProvinceFilter, CountryFilter
 
 from collections import OrderedDict
+import json
 
 TIMEOUT = 60 * 60
 
@@ -29,9 +30,9 @@ class LatestStatisticsView(APIView):
     """最新统计信息"""
 
     def get_object(self):
-        try:
-            crawler = Crawler.objects.order_by('-id').first()
-        except Crawler.DoesNotExist:
+        crawler = Crawler.objects.exclude(
+            modifyTime__isnull=True).order_by('-id').first()
+        if crawler is None:
             raise Http404
         result = {}
         insts = Statistics.objects.filter(crawler=crawler).all()
@@ -120,7 +121,8 @@ class ProvinceListView(ListAPIView):
     filter_class = ProvinceFilter
 
     def get_queryset(self):
-        crawler = Crawler.objects.order_by('-id').first()
+        crawler = Crawler.objects.exclude(
+            modifyTime__isnull=True).order_by('-id').first()
         queryset = Province.objects.filter(crawler=crawler)
         return queryset
 
@@ -129,14 +131,34 @@ def dispatch(self, *args, **kwargs):
         return super(ProvinceListView, self).dispatch(*args, **kwargs)
 
 
-class ProvinceRetrieveByNameView(APIView):
-    """通过省名获取数据"""
+class ProvinceDailyListView(APIView):
+
+    """省按天返回列表"""
 
     def get_object(self, provinceShortName):
+        crawler = Crawler.objects.exclude(
+            modifyTime__isnull=True).order_by('-id').first()
         try:
-            crawler = Crawler.objects.order_by('-id').first()
-        except Crawler.DoesNotExist:
+            return Province.objects.filter(
+                crawler=crawler,
+                provinceShortName=provinceShortName).first()
+        except Province.DoesNotExist:
             raise Http404
+
+    @method_decorator(cache_page(TIMEOUT))
+    def get(self, request, provinceShortName):
+        province = self.get_object(provinceShortName)
+        result = province.dailyData
+        result = json.loads(result)
+        return Response(result)
+
+
+class ProvinceRetrieveByNameView(APIView):
+    """通过省名获取数据"""
+
+    def get_object(self, provinceShortName):
+        crawler = Crawler.objects.exclude(
+            modifyTime__isnull=True).order_by('-id').first()
         try:
             return Province.objects.filter(
                 crawler=crawler,
@@ -172,7 +194,8 @@ class CountryListView(ListAPIView):
     filter_class = CountryFilter
 
     def get_queryset(self):
-        crawler = Crawler.objects.order_by('-id').first()
+        crawler = Crawler.objects.exclude(
+            modifyTime__isnull=True).order_by('-id').first()
         queryset = Country.objects.filter(crawler=crawler)
         return queryset.all()
 
@@ -196,13 +219,30 @@ def get(self, request, pk):
         return Response(serializer.data)
 
 
-class CountryRetrieveByNameView(APIView):
+class CountryDailyListView(APIView):
 
     def get_object(self, countryName):
+        crawler = Crawler.objects.exclude(
+            modifyTime__isnull=True).order_by('-id').first()
         try:
-            crawler = Crawler.objects.order_by('-id').first()
-        except Crawler.DoesNotExist:
+            return Country.objects.filter(
+                crawler=crawler, countryName=countryName).first()
+        except Country.DoesNotExist:
             raise Http404
+
+    @method_decorator(cache_page(TIMEOUT))
+    def get(self, request, countryName):
+        country = self.get_object(countryName)
+        result = country.dailyData
+        result = json.loads(result)
+        return Response(result)
+
+
+class CountryRetrieveByNameView(APIView):
+
+    def get_object(self, countryName):
+        crawler = Crawler.objects.exclude(
+            modifyTime__isnull=True).order_by('-id').first()
         try:
             return Country.objects.filter(
                 crawler=crawler, countryName=countryName).first()
@@ -222,7 +262,8 @@ class CityListView(ListAPIView):
     filter_class = CityFilter
 
     def get_queryset(self):
-        crawler = Crawler.objects.order_by('-id').first()
+        crawler = Crawler.objects.exclude(
+            modifyTime__isnull=True).order_by('-id').first()
         queryset = City.objects.filter(crawler=crawler)
         return queryset
 
@@ -249,10 +290,8 @@ def get(self, request, pk):
 class CityRetrieveByNameView(APIView):
 
     def get_object(self, cityName):
-        try:
-            crawler = Crawler.objects.order_by('-id').first()
-        except Crawler.DoesNotExist:
-            raise Http404
+        crawler = Crawler.objects.exclude(
+            modifyTime__isnull=True).order_by('-id').first()
         try:
             return City.objects.filter(
                 crawler=crawler, cityName=cityName).first()

spider/nCoV/spiders/dxy.py

@@ -2,7 +2,7 @@
 # @Author: zhanglei3
 # @Date:   2020-04-08 09:08:13
 # @Last Modified by:   leafcoder
-# @Last Modified time: 2020-04-29 13:51:25
+# @Last Modified time: 2020-04-30 10:09:06
 
 """丁香园数据源"""
 
@@ -19,7 +19,7 @@
 class DXYSpider(scrapy.Spider):
 
     name = "dxy"
-    allowed_domains = ["ncov.dxy.cn"]
+    allowed_domains = ["ncov.dxy.cn", "file1.dxycdn.com"]
     start_urls = [
         "http://ncov.dxy.cn/ncovh5/view/pneumonia",
     ]
@@ -45,19 +45,22 @@ def parse(self, response):
         self.crawler.save()
 
         # 统计信息
-        statistics = self.parse_statistics(statistics)
+        statistics = self.explain_statistics(statistics)
         for item in statistics:
             yield item
 
         # 国内数据
         provinces = self.get_list(scripts, '#getAreaStat')
         for province in provinces:
             cities = province.pop('cities', [])
-            province = items.ProvinceItem(**province)
-            yield province
-            for city in cities:
-                location_id = province['locationId']
-                yield items.CityItem(province=location_id, **city)
+            yield scrapy.Request(
+                province['statisticsData'],
+                callback=self.parse_province_statistics_data,
+                meta={
+                    'province': province,
+                    'cities': cities
+                }
+            )
 
         # 国外数据
         countries = self.get_list(
@@ -71,8 +74,16 @@ def parse(self, response):
             country.pop('provinceId')
             country.pop('provinceName')
             country.pop('provinceShortName')
+            country.pop('modifyTime', None)
+            country.pop('createTime', None)
             country['incrVo'] = json.dumps(country['incrVo'])
-            yield items.CountryItem(**country)
+            yield scrapy.Request(
+                country['statisticsData'],
+                callback=self.parse_country_statistics_data,
+                meta={
+                    'country': country
+                }
+            )
 
         # 时间线事件，id=“getTimelineService2” 为英文内容
         timelines = self.get_list(scripts, '#getTimelineService1')
@@ -127,7 +138,25 @@ def parse(self, response):
                 rumor[key] = item.get(key)
             yield items.RumorItem(**rumor)
 
-    def parse_statistics(self, data):
+    def parse_province_statistics_data(self, response):
+        result = json.loads(response.text)
+        data = json.dumps(result['data'])
+        meta = response.meta
+        cities = meta['cities']
+        province = meta['province']
+        yield items.ProvinceItem(dailyData=data, **province)
+        for city in cities:
+            location_id = province['locationId']
+            yield items.CityItem(province=location_id, **city)
+
+    def parse_country_statistics_data(self, response):
+        result = json.loads(response.text)
+        data = json.dumps(result['data'])
+        meta = response.meta
+        country = meta['country']
+        yield items.CountryItem(dailyData=data, **country)
+
+    def explain_statistics(self, data):
         statistics = data['globalStatistics']
         item = {}
         for key in (