如何使用TastyPie API从所有对象中获取数据

Question

我的目标是从tastypie api获取所有数据。此API内部有超过1.000.000个对象！我创建了另一个Django项目和一个自定义的" product_import.py"命令，该命令应从此API中获取所有数据。

当我达到极限时，我该怎么办才能从API获取数据？我想从API ['meta'] ['Next']词典中获取数据，以继续在另一个项目中收集对象。

以下是我现在创建的示例，并被卡住了。

api.py

# coding: utf-8
from tastypie.resources import ModelResource
from tastypie import fields
from tastypie.authentication import ApiKeyAuthentication
from tastypie.authorization import DjangoAuthorization
from .models import Product

class UserguideResource(ModelResource):
    class Meta:
        resource_name = 'product'
        allowed_methods = ['get']
        fields = ['id', 'name', 'date_added', 'author']
        queryset = Product.objects.all()
        authentication = ApiKeyAuthentication()
        authorization = DjangoAuthorization()

此文件是在不同的Django项目中创建的，该项目使用上面项目的TastyPie API。我想从此API中获取所有数据。

product_import.py

from django.core.management.base import BaseCommand
import requests

class Command(BaseCommand):
    """ Import Products from API """
    def __init__(self):
        self.API_KEY = 'secret-key'
        self.API_USERNAME = 'keNzi'
        self.product_api_url = 'http://localhost:8000/api/v1/product?username={0}&api_key={1}'.format(
            self.API_USERNAME, self.API_KEY)
        self.limit = 1
        self.offset = 0
    def get_latest_product_from_api(self):
        response = requests.get(
            "{0}&limit={1}&offset={2}".format(
                self.product_api_url,
                self.limit,
                self.offset,
            )
        ).json()
        return response
    def get_next_url_from_product_list(self):
        return self.get_latest_product_from_api()['meta']['next']
    def get_next_product_list(self):
        self.next = self.get_next_url_from_product_list()
        response = requests.get(
            "{0}{1}".format(
                self.product_api_url,
                self.next,
            )
        ).json()
        return response
    def handle(self, *args, **options):
        print(self.get_latest_product_from_api())
        print(self.get_next_product_list())

print（）显示API的产品，但我想继续并收集所有数据。

Answer 1

这是一个可以正常工作的代码;）

import requests
from django.core.management.base import BaseCommand
from product.models import Product

class Command(BaseCommand):
    """ Import Products from API """
    def __init__(self):
        self.API_URL = 'http://localhost:8000/api/v1'
        self.API_USERNAME = 'keNzi'
        self.API_KEY = 'keNzi-secret-2017'   # created in /admin/tastypie/apikey/
        self.API_AUTH = '?username={0}&api_key={1}'.format(
            self.API_USERNAME, self.API_KEY)
        self.product_api_url = '{0}/product/{1}'.format(
            self.API_URL, self.API_AUTH)
        self.limit = 1
        self.offset = 0
    def get_product_from_api(self):
        response = requests.get(
            "{0}&limit={1}&offset={2}".format(
                self.product_api_url,
                self.limit,
                self.offset,
            )
        ).json()
        return response
    def get_next_url_from_product_api(self):
        return self.get_product_from_api()['meta']['next']
    def handle(self, *args, **options):
        next_url = True
        while next_url:
            for obj in self.get_product_from_api()['objects']:
                print(obj['id'], obj['file'])
                product = Product.objects.create(
                    name=obj['name'],
                    description=obj['text'],
                    created=obj['date_added'],
                )
                product.save()
                print('Product saved! ID of the new objects: ', product.id)
            self.offset += 1
            next_url = self.get_next_url_from_product_api()