将文件从GCP存储推送到另一个没有sdk的存储桶

Question

我在层次结构中的gcp云存储中有文件

data/staging1 
--/temp1.json
--/temp2.json
--/temp3.json
data/staging2 
--/temp1.json
--/temp2.json
--/temp3.json

想要读取阶段1中的所有文件并移动到另一个文件夹阶段2

def getFiles(self):
filename = list(self.bucketname.list_blobs(prefix=self.prefix))
target_blob = filename[0]
print(target_blob)
filename = list(self.bucketname.list_blobs(prefix=self.prefix))
for name in filename:
local_tmp_path =name.name
local_tmp_path=self.bucketname.blob(local_tmp_path)
with open(local_tmp_path, 'r') as f:
target_blob.upload_from_file(f)
print("done")

我想循环读取所有文件。。。。当我尝试打开它时，它显示错误

with open(local_tmp_path, 'r') as f:
TypeError: expected str, bytes or os.PathLike object, not Blob
for name in filename:
local_tmp_path =name.name
local_tmp_path=self.bucketname.blob(local_tmp_path)
with open(local_tmp_path, 'r') as f:
target_blob.upload_from_file(f)

Answer 1

根据GCP官方文档，您可以通过编程将文件从一个存储桶移动/传输到另一个存储库。即使bucket在另一个项目中，过程也是相同的。

"""Command-line sample that creates a daily transfer from a standard
GCS bucket to a Nearline GCS bucket for objects untouched for 30 days.
This sample is used on this page:
https://cloud.google.com/storage/transfer/create-transfer
For more information, see README.md.
"""
import argparse
import datetime
import json
import googleapiclient.discovery

def main(description, project_id, start_date, start_time, source_bucket,
sink_bucket):
"""Create a daily transfer from Standard to Nearline Storage class."""
storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1')
# Edit this template with desired parameters.
transfer_job = {
'description': description,
'status': 'ENABLED',
'projectId': project_id,
'schedule': {
'scheduleStartDate': {
'day': start_date.day,
'month': start_date.month,
'year': start_date.year
},
'startTimeOfDay': {
'hours': start_time.hour,
'minutes': start_time.minute,
'seconds': start_time.second
}
},
'transferSpec': {
'gcsDataSource': {
'bucketName': source_bucket
},
'gcsDataSink': {
'bucketName': sink_bucket
},
'objectConditions': {
'minTimeElapsedSinceLastModification': '2592000s'  # 30 days
},
'transferOptions': {
'deleteObjectsFromSourceAfterTransfer': 'true'
}
}
}
result = storagetransfer.transferJobs().create(body=transfer_job).execute()
print('Returned transferJob: {}'.format(
json.dumps(result, indent=4)))

if __name__ == '__main__':
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('description', help='Transfer description.')
parser.add_argument('project_id', help='Your Google Cloud project ID.')
parser.add_argument('start_date', help='Date YYYY/MM/DD.')
parser.add_argument('start_time', help='UTC Time (24hr) HH:MM:SS.')
parser.add_argument('source_bucket', help='Standard GCS bucket name.')
parser.add_argument('sink_bucket', help='Nearline GCS bucket name.')
args = parser.parse_args()
start_date = datetime.datetime.strptime(args.start_date, '%Y/%m/%d')
start_time = datetime.datetime.strptime(args.start_time, '%H:%M:%S')
main(
args.description,
args.project_id,
start_date,
start_time,
args.source_bucket,
args.sink_bucket)