Amazon Web Services - 使用 Boto3 的 DynamoDB 分页

Question

我们正在将 boto3 用于我们的 DynamoDB，我们需要对表进行全面扫描，以便根据我们需要进行分页的其他帖子来执行此操作。但是，我们无法找到分页的工作样本。这就是我们所做的。

import boto3
client_setting = boto3.client('dynamodb', region_name='ap-southeast-2')
paginator = client_setting.get_paginator('scan')
esk = {}
data = []
unconverted_ga = ourQuery(params1, params2)
    for page in unconverted_ga:
        data.append(page)
        esk = page['LastEvaluatedKey']

我们不知道如何将 esk 作为下一个查询的 ExclusiveStartKey。独占启动键参数的预期值应该是多少？我们仍然是 DynamoDB 的新手，我们需要学习很多东西，包括这一点。谢谢！

Answer 1

来自Tay B在 https://stackoverflow.com/a/38619425/3176550 的回答

import boto3
dynamodb = boto3.resource('dynamodb',
                          aws_session_token=aws_session_token,
                          aws_access_key_id=aws_access_key_id,
                          aws_secret_access_key=aws_secret_access_key,
                          region_name=region
)
table = dynamodb.Table('widgetsTableName')
response = table.scan()
data = response['Items']
while 'LastEvaluatedKey' in response:
    response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
    data.update(response['Items'])

Answer 2

经过一小时的搜索，我终于找到了更好的解决方案。对于那些不熟悉 DynamoDB 的用户，我们不应该错过这个 - http://docs.aws.amazon.com/amazondynamodb/latest/gettingstartedguide/GettingStarted.Python.04.html

from __future__ import print_function # Python 2/3 compatibility
import boto3
import json
import decimal
from boto3.dynamodb.conditions import Key, Attr
# Helper class to convert a DynamoDB item to JSON.
class DecimalEncoder(json.JSONEncoder):
    def default(self, o):
        if isinstance(o, decimal.Decimal):
            if o % 1 > 0:
                return float(o)
            else:
                return int(o)
        return super(DecimalEncoder, self).default(o)
dynamodb = boto3.resource('dynamodb', region_name='us-west-2', endpoint_url="http://localhost:8000")
table = dynamodb.Table('Movies')
fe = Key('year').between(1950, 1959)
pe = "#yr, title, info.rating"
# Expression Attribute Names for Projection Expression only.
ean = { "#yr": "year", }
esk = None

response = table.scan(
    FilterExpression=fe,
    ProjectionExpression=pe,
    ExpressionAttributeNames=ean
    )
for i in response['Items']:
    print(json.dumps(i, cls=DecimalEncoder))
// As long as LastEvaluatedKey is in response it means there are still items from the query related to the data
while 'LastEvaluatedKey' in response:
    response = table.scan(
        ProjectionExpression=pe,
        FilterExpression=fe,
        ExpressionAttributeNames= ean,
        ExclusiveStartKey=response['LastEvaluatedKey']
        )
    for i in response['Items']:
        print(json.dumps(i, cls=DecimalEncoder))

Answer 3

您可以尝试使用以下代码：

esk = None
while True:
    scan_generator = YourTableName.scan(max_results=10, exclusive_start_key=esk)
    for item in scan_generator:
        # your code for processing
    # condition to check if entire table is scanned 
    else:
        break;
    # Load the last keys
    esk = scan_generator.kwargs['exclusive_start_key'].values()

这是参考文档链接。

希望有帮助

Answer 4

有点冗长，但我喜欢它。

def fetch_from_table(last_key=None):
if last_key:
    response = table.query(
        IndexName='advertCatalogIdx',
        KeyConditionExpression=Key('sk').eq('CATALOG'),
        Limit=5,
        ExclusiveStartKey=last_key
    )
else:
    response = table.query(
        IndexName='advertCatalogIdx',
        KeyConditionExpression=Key('sk').eq('CATALOG'),
        Limit=5
    )
# print(response)
for item in response['Items']:
    print(item['address'])
    print('***************************')
return response.get('LastEvaluatedKey')

last_key = fetch_from_table()
while last_key != None:
    print("Running again : ")
    last_key = fetch_from_table(last_key)

Answer 5

import sys
import boto3
client = boto3.client('dynamodb')
marker = None
while True:
   paginator = client.get_paginator('list_tables')
   page_iterator = paginator.paginate(
       PaginationConfig={
           'MaxItems': 1000,
           'PageSize': 100,
           'StartingToken': marker})
   for page in page_iterator:
       tables=page['TableNames']
       for table in tables:
           print (table)
   try:
       marker = page['NextToken']
    except KeyError:
       sys.exit()