在IBM数据科学体验(IBM DSX)中将ZIP文件导入Python笔记本

Question

我有一个zip文件train.zip（1.1GB），我想将其导入到python笔记本上，unzip，然后着手进行。我将其作为字符串io对象导入，利用选项intert stringio对象。

from io import StringIO
import requests
import json
import pandas as pd
# @hidden_cell
# This function accesses a file in your Object Storage. The definition contains your credentials.
# You might want to remove those credentials before you share your notebook.
def get_object_storage_file_with_credentials_xxxxxx(container, filename):
    """This functions returns a StringIO object containing
    the file content from Bluemix Object Storage."""
    url1 = ''.join(['https://identity.open.softlayer.com', '/v3/auth/tokens'])
    data = {'auth': {'identity': {'methods': ['password'],
            'password': {'user': {'name': 'member_xxxxxx','domain': {'id': 'xxxxxxx'},
            'password': 'xxxxx),(xxxxx'}}}}}
    headers1 = {'Content-Type': 'application/json'}
    resp1 = requests.post(url=url1, data=json.dumps(data), headers=headers1)
    resp1_body = resp1.json()
    for e1 in resp1_body['token']['catalog']:
        if(e1['type']=='object-store'):
            for e2 in e1['endpoints']:
                        if(e2['interface']=='public'and e2['region']=='dallas'):
                            url2 = ''.join([e2['url'],'/', container, '/', filename])
    s_subject_token = resp1.headers['x-subject-token']
    headers2 = {'X-Auth-Token': s_subject_token, 'accept': 'application/json'}
    resp2 = requests.get(url=url2, headers=headers2)
    return StringIO(resp2.text)
# Your data file was loaded into a StringIO object and you can process the data.
# Please read the documentation of pandas to learn more about your possibilities to load your data.
# pandas documentation: http://pandas.pydata.org/pandas-docs/stable/io.html
data_1 = get_object_storage_file_with_credentials_20e75635ab104e58bd1a6e91635fed51('DefaultProjectxxxxxxxx', 'train.zip')

这给出了一个输出：

data_1
<_io.StringIO at 0x7f8a288cd3a8>

但是，当我尝试使用Zipfile解开它时，我会受到以下错误的问候：

from zipfile import ZipFile
file = ZipFile(data_1)
BadZipFile: File is not a zip file

如何访问IBM DSX中的文件？

Answer 1

您可以使用下面显示的功能将zip文件从对象存储中保存。credentials参数是DSX笔记本中插入代码的字典。此功能也在GIST

上

import zipfile
from io import BytesIO
import requests
import json
import pandas as pd
def get_zip_file(credentials):
    url1 = ''.join(['https://identity.open.softlayer.com', '/v3/auth/tokens']) 
    data = {'auth': {'identity': {'methods': ['password'], 'password': {'user': {'name': credentials['username'],'domain': {'id': credentials['domain_id']}, 'password': credentials['password']}}}}} 
    headers1 = {'Content-Type': 'application/json'} resp1 = requests.post(url=url1, data=json.dumps(data), headers=headers1) 
    resp1_body = resp1.json() 
    for e1 in resp1_body['token']['catalog']:
        if(e1['type']=='object-store'): 
            for e2 in e1['endpoints']:   
                if(e2['interface']=='public'and e2['region']==credentials['region']): url2 = ''.join([e2['url'],'/', credentials['container'], '/', credentials['filename']]) s_subject_token = resp1.headers['x-subject-token'] headers2 = {'X-Auth-Token': s_subject_token, 'accept': 'application/json'} 
    s_subject_token = resp1.headers['x-subject-token']
    headers2 = {'X-Auth-Token': s_subject_token, 'accept': 'application/json'}
    r = requests.get(url=url2, headers=headers2, stream=True)
    z = zipfile.ZipFile(BytesIO(r.content))
    z.extractall()# save zip contents to disk
    return(z)
z = get_zip_file(credentials)

Answer 2

提取上载的zip文件，其中包括以下代码：使用pwd获取工作目录并设置路径文件夹。

from io import BytesIO
import zipfile
zip_ref = zipfile.ZipFile(BytesIO(streaming_body_1.read()), 'r')
file_paths = zip_ref.namelist()
for path in file_paths:
    zip_ref.extract(path)