如何通过Lambda重命名S3中的多个输出文件

Question

S3桶中多个输出文件的重命名方法。例如，我使用

os.path.basename(keyprefix),

'w'写入模式

的文件

abc_00000.csv.gz
abc_00001.csv.gz 

我想用命名约定重命名上面的文件:

abc_{today date in YYYYMMDD format}_00.csv.gz 
abc_{today date in YYYYMMDD format}_01.csv.gz

下面是供参考的代码:

import boto3
import os
from smart_open import open
import gzip
import csv
import io
def lambda_handler(event, context):
dirpath = 'output/'
bucket = 'export'
key = 'export/_SUCCESS'
if '_SUCCESS' in key:
client = boto3.client('s3')
response = client.list_objects_v2(Bucket=bucket,Prefix=os.path.dirname(key))
for i in response['Contents']:
keyprefix = i['Key']
if 'part-' in keyprefix:
with gzip.GzipFile(fileobj=client.get_object(Bucket=bucket,Key=keyprefix)['Body']) as gzipfile, open('s3://'+bucket + '/' + dirpath + os.path.basename(keyprefix),'w') as fout :
writer = csv.writer(fout , delimiter=',')
writer.writerow(['test1','test1','test3','test4','test5','test6','test7'])        
for row in csv.reader(gzipfile.read().decode('utf-8').splitlines(), delimiter=',', quotechar='"'):
if row[5] == 'CDE':         
writer.writerow(row)

Answer 1

S3 Service Resource提供了更大的灵活性。

复制文件后，我删除旧文件，如果你想保留你可以删除删除调用。

#!/usr/bin/env python3
import boto3
import os
from datetime import datetime
prefix = 'abc_'
new_prefix = f"{prefix}{datetime.today().strftime('%Y-%m-%d')}"
suffix = 'csv.gz'
bucket_name = 'mybucketname'
def lambda_handler(event, context):
s3 = boto3.resource('s3')
bucket = s3.Bucket(bucket_name)
for obj in bucket.objects.all():
key = obj.key
path_part = os.path.dirname(key)
filename = os.path.basename(key)
copy_source = {
'Bucket': bucket_name,
'Key': key
}
if filename.startswith(prefix) and filename.endswith(suffix):
new_key = f"{new_prefix}_{key.split('_')[1]}"
full_key_with_path = os.path.join(path_part, new_key)
destination_bucket = s3.Bucket(bucket_name)
print(f'copying the object with new key : {full_key_with_path}')
destination_bucket.copy(copy_source, full_key_with_path)
print(f'deleting old key : {key}')
s3.Object(bucket_name, key).delete()

❯❯ python3 s3rename.py 
copying the object with new key : myinventorylist/2021-02-07T00-00Z/abc_2021-03-07_00000.csv.gz
deleting old key : myinventorylist/2021-02-07T00-00Z/abc_00000.csv.gz
copying the object with new key : myinventorylist/2021-02-07T00-00Z/abc_2021-03-07_00001.csv.gz
deleting old key : myinventorylist/2021-02-07T00-00Z/abc_00001.csv.gz