我试图从昨天刚开始的网站上抓取一些数据，但我无法从我从该网站抓取的内容中获得我想要的数据。
try:
html=urlopen('https://shopee.tw/api/v4/search/search_items?by=relevancy&keyword=%E9%81%8B%E5%8B%95%E9%9E%8B&limit=60&newest=0&order=desc&page_type=search&scenario=PAGE_GLOBAL_SEARCH&version=2')
except HTTPError as e:
print(e)
except URLError as e:
print('THe server is not found')
else:
print('worked')
bs=BeautifulSoup(html,'html5lib')
bodydata=bs.find_all('body')
如何获取bodydata中的特定词典？我试过
for x in bodydata:
print(x.get_text())
但是结果是一个字符串变量。如果身体数据看起来像这样，我如何获取特定的字典？
{"bff_meta":null,"error":null,"error_msg":null,"reserved_keyword":null,"suggestion_algorithm":null,"algorithm":"eyJzZWFyY2giOiIwLmEuNjE2N0BPUEtVVVVFSklXSVBSWEtSVVNaQUVST1ZRUFZaWE5URk1FT05SWTBEWVRZSkcxRVRHVkVTMlpKM1BRNzBaQzExNUdPMTgyIn0=","total_count":3177,"nomore":false,"items":[{"item_basic":{"itemid":10288392472,"shopid":451862210,....
基于上面的示例：
import requests
import pandas as pd
r = requests.get('https://shopee.tw/api/v4/search/search_items?by=relevancy&keyword=%E9%81%8B%E5%8B%95%E9%9E%8B&limit=60&newest=0&order=desc&page_type=search&scenario=PAGE_GLOBAL_SEARCH&version=2')
# print(r.json()) ## you can print it, look at it, extract other key/values from it, etc. For example, to extract 'items' and put them into a dataframe:
df = pd.DataFrame(r.json()['items'])
df
这会返回一个类似下面的数据帧，60行×22列：
item_basic adsid campaignid distance match_type ads_keyword deduction_info collection_id display_name campaign_stock ... itemid shopid algo_image fe_flags item_type foody_item search_item_tracking bff_item_tracking personalized_labels biz_json
0 {'itemid': 10288392472, 'shopid': 451862210, '... 42222082.0 26817031.0 None 1.0 350 鞋子正品 U1lQYVlnQUFBQUJCUkc1MI2w8H7BMT9hDxZHHknw7FqdFT... None None None ... 10288392472 451862210 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1010729,1014629],"is_on_flash_sa... None None
1 {'itemid': 15649674773, 'shopid': 479604969, '... 40526742.0 25502467.0 None 1.0 休閒白鞋 U1lQYVlnQUFBQUJCUkc1MC0e79wRsWCGLqC4Kuico0aGGn... None None None ... 15649674773 479604969 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":null,"is_on_flash_sale":false,"is... None None
2 {'itemid': 5759477451, 'shopid': 275457974, 'n... NaN NaN None NaN None None None None None ... 5759477451 275457974 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1008416,1008415,1010729,1010733,... None None
3 {'itemid': 15258032169, 'shopid': 570620255, '... NaN NaN None NaN None None None None None ... 15258032169 570620255 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1014629,1002127,1007681,1007880,... None None
4 {'itemid': 10475499089, 'shopid': 117981382, '... NaN NaN None NaN None None None None None ... 10475499089 117981382 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1010729,1011331,1011333,1011847,... None None
5 {'itemid': 13088655898, 'shopid': 342768515, '... 52175386.0 34432248.0 None 0.0 wkdaelpmissisiht U1lQYVlnQUFBQUJCUkc1MCVLBvV/hLJndlLT3P7ryGL8AC... None None None ... 13088655898 342768515 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1010445,1010729,1010736,1011292,... None None
6 {'itemid': 7765671062, 'shopid': 275457974, 'n... NaN NaN None NaN None None None None None ... 7765671062 275457974 None None 1 None {"is_ads":false,"ruleids":[1000,0,1000],"group... {"label_ids":[1008416,1008415,1010729,1010733,... None None
7 {'itemid': 10405964649, 'shopid': 892903, 'nam... NaN NaN None NaN None None None None None ... 10405964649 892903 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[16,1000140,1000138,1001499,10045... None None
8 {'itemid': 14151724538, 'shopid': 612578498, '... NaN NaN None NaN None None None None None ... 14151724538 612578498 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1012634,1013629,1013627,1013618,... None None
9 {'itemid': 16510149105, 'shopid': 107174380, '... 43690945.0 27940756.0 None 0.0 wkdaelpmissisiht U1lQYVlnQUFBQUJCUkc1MEo9yrrmUKCIs04A/Sl+dth6dx... None None None ... 16510149105 107174380 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1014629,1014914,1014915,1015983,... None None
10 {'itemid': 17537486071, 'shopid': 570620255, '... NaN NaN None NaN None None None None None ... 17537486071 570620255 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1012634,1014629,1002127,1007681,... None None
11 {'itemid': 10966656779, 'shopid': 223349546, '... NaN NaN None NaN None None None None None ... 10966656779 223349546 None None 1 None {"is_ads":false,"ruleids":[1000,1000],"groupid... {"label_ids":[1014629,1013623,1014915],"is_on_... None None
12 {'itemid': 13461133395, 'shopid': 612578498, '... NaN NaN None NaN None None None None None ... 13461133395 612578498 None None 1 None {"is_ads":false,"ruleids":[0,1000,1000],"group... {"label_ids":[1013629,1013627,1013618,1014629,... None None
13 {'itemid': 21624649473, 'shopid': 819002215, '... 52491528.0 34675541.0 None 1.0 nike U1lQYVlnQUFBQUJCUkc1MABOFQgxtYRn6dl2TJ+zhF70dU... None None None ... 21624649473 819002215 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":null,"is_on_flash_sale":false,"is... None None
14 {'itemid': 17858909983, 'shopid': 565183238, '... NaN NaN None NaN None None None None None ... 17858909983 565183238 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1014629,1012634,1014912,1007880,... None None
15 {'itemid': 10387321865, 'shopid': 117981382, '... NaN NaN None NaN None None None None None ... 10387321865 117981382 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1010729,1011331,1011333,1011847,... None None
16 {'itemid': 15982004503, 'shopid': 276970990, '... NaN NaN None NaN None None None None None ... 15982004503 276970990 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1010729,1010733,1011331,1011333,... None None
17 {'itemid': 16128180376, 'shopid': 440266562, '... 43415099.0 27730568.0 None 0.0 wkdaelpmissisiht U1lQYVlnQUFBQUJCUkc1MD90vWe3xxVRRspKmttqOU9oB+... None None None ... 16128180376 440266562 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1014629,1014589,1014914,1015986,... None None
18 {'itemid': 11801176535, 'shopid': 36011643, 'n... NaN NaN None NaN None None None None None ... 11801176535 36011643 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1010729,1011292,1011926,1012346,... None None
19 {'itemid': 15008651489, 'shopid': 223349546, '... NaN NaN None NaN None None None None None ... 15008651489 223349546 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1014629,1014915],"is_on_flash_sa... None None
20 {'itemid': 17015191947, 'shopid': 264561849, '... NaN NaN None NaN None None None None None ... 17015191947 264561849 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1012634,1012808,1013627,1014629,... None None
21 {'itemid': 20424652485, 'shopid': 819002215, '... 52490421.0 34675539.0 None 1.0 慢跑鞋 U1lQYVlnQUFBQUJCUkc1ML5FgkDFq3u0YwbmFFITddg3kW... None None None ... 20424652485 819002215 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":null,"is_on_flash_sale":false,"is... None None
22 {'itemid': 15663824790, 'shopid': 424421393, '... NaN NaN None NaN None None None None None ... 15663824790 424421393 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1006579,1006629,1007465,1010729,... None None
23 {'itemid': 9822950083, 'shopid': 2304498, 'nam... NaN NaN None NaN None None None None None ... 9822950083 2304498 None None 1 None {"is_ads":false,"ruleids":[1000,1000],"groupid... {"label_ids":[1000140,1001499,1002127,1005833,... None None
24 {'itemid': 12605212937, 'shopid': 2443283, 'na... NaN NaN None NaN None None None None None ... 12605212937 2443283 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[16,1000140,1001499,1002936,10015... None None
25 {'itemid': 17957096336, 'shopid': 752665010, '... 46234219.0 29894142.0 None 0.0 wkdaelpmissisiht U1lQYVlnQUFBQUJCUkc1MI9QfVgkJYuv+S3LdJULXcsn0T... None None None ... 17957096336 752665010 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":null,"is_on_flash_sale":false,"is... None None
26 {'itemid': 21104501514, 'shopid': 555360855, '... NaN NaN None NaN None None None None None ... 21104501514 555360855 None None 1 None {"is_ads":false,"ruleids":[1000,1000],"groupid... {"label_ids":[1010733,1011292,1011331,1011333,... None None
27 {'itemid': 11582152525, 'shopid': 607317086, '... NaN NaN None NaN None None None None None ... 11582152525 607317086 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1012634,1012806,1013629,1013627,... None None
28 {'itemid': 10955439292, 'shopid': 12616228, 'n... NaN NaN None NaN None None None None None ... 10955439292 12616228 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1010729,1010733,1011331,1011333,... None None
29 {'itemid': 14888873227, 'shopid': 230771967, '... 51925081.0 34232774.0 None 1.0 327 U1lQYVlnQUFBQUJCUkc1MLztFn/QffvV2OrivgXmINmtXs... None None None ... 14888873227 230771967 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1014629],"is_on_flash_sale":fals... None None
30 {'itemid': 5571523506, 'shopid': 275457974, 'n... NaN NaN None NaN None None None None None ... 5571523506 275457974 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1008416,1008415,1010729,1010733,... None None
31 {'itemid': 16400569451, 'shopid': 612578498, '... NaN NaN None NaN None None None None None ... 16400569451 612578498 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1012634,1013629,1013627,1013618,... None None
32 {'itemid': 5180067072, 'shopid': 326356101, 'n... NaN NaN None NaN None None None None None ... 5180067072 326356101 None None 1 None {"is_ads":false,"ruleids":[1000,0,1000],"group... {"label_ids":[1009753,1010729,1010733,1010735,... None None
33 {'itemid': 21104501514, 'shopid': 555360855, '... 48758219.0 31615286.0 None 1.0 跑鞋 U1lQYVlnQUFBQUJCUkc1MH4HwIKJa+WQUrtlpBG2L8VzNJ... None None None ... 21104501514 555360855 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1010733,1011292,1011331,1011333,... None None
34 {'itemid': 14651880343, 'shopid': 612578498, '... NaN NaN None NaN None None None None None ... 14651880343 612578498 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1012634,1013629,1013627,1013618,... None None
35 {'itemid': 10278011366, 'shopid': 365689255, '... NaN NaN None NaN None None None None None ... 10278011366 365689255 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1004778,1004779,1004780,1005131,... None None
36 {'itemid': 4065692740, 'shopid': 275457974, 'n... NaN NaN None NaN None None None None None ... 4065692740 275457974 None None 1 None {"is_ads":false,"ruleids":[1000,0,1000],"group... {"label_ids":[1008416,1008415,1010729,1010733,... None None
37 {'itemid': 13741712216, 'shopid': 479604969, '... 41765043.0 22743315.0 None 1.0 鞋 U1lQYVlnQUFBQUJCUkc1MMiNM31En9NgXKp7ZdEFgn2dmq... None None None ... 13741712216 479604969 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":null,"is_on_flash_sale":false,"is... None None
38 {'itemid': 14129123236, 'shopid': 450599940, '... NaN NaN None NaN None None None None None ... 14129123236 450599940 None None 1 None {"is_ads":false,"ruleids":[1000,0,1000],"group... {"label_ids":[1006112,1006579,1006629,1008403,... None None
39 {'itemid': 11390620224, 'shopid': 4636818, 'na... NaN NaN None NaN None None None None None ... 11390620224 4636818 None None 1 None {"is_ads":false,"ruleids":[1000,1000],"groupid... {"label_ids":[1014629,1015983,1015984,1015987,... None None
40 {'itemid': 11865538830, 'shopid': 223349546, '... NaN NaN None NaN None None None None None ... 11865538830 223349546 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1014629,1014915],"is_on_flash_sa... None None
41 {'itemid': 15663824790, 'shopid': 424421393, '... 43074045.0 27477258.0 None 1.0 運動鞋 U1lQYVlnQUFBQUJCUkc1MFTA1+Y8XGP+sTjh6Gm374Vsnk... None None None ... 15663824790 424421393 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1006579,1006629,1007465,1010729,... None None
42 {'itemid': 16447619272, 'shopid': 17527952, 'n... NaN NaN None NaN None None None None None ... 16447619272 17527952 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[16,1000140,1000138,1001499,10000... None None
43 {'itemid': 16245159961, 'shopid': 2157594, 'na... NaN NaN None NaN None None None None None ... 16245159961 2157594 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1002127,1009753,1005701,1010729,... None None
44 {'itemid': 16715057285, 'shopid': 264561849, '... NaN NaN None NaN None None None None None ... 16715057285 264561849 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1012634,1012808,1013627,1014629,... None None
45 {'itemid': 16237447945, 'shopid': 390967233, '... 44116223.0 28258393.0 None 0.0 wkdaelpmissisiht U1lQYVlnQUFBQUJCUkc1MMcvMXXAR04k6McXiNefl/J7EP... None None None ... 16237447945 390967233 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1009753,1010729,1010733,1011331,... None None
46 {'itemid': 11286328992, 'shopid': 570620255, '... NaN NaN None NaN None None None None None ... 11286328992 570620255 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1012634,1014629,1002127,1007681,... None None
47 {'itemid': 12974747270, 'shopid': 253926413, '... NaN NaN None NaN None None None None None ... 12974747270 253926413 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1004846,1004778,1004779,1004780,... None None
48 {'itemid': 7591220980, 'shopid': 290994282, 'n... NaN NaN None NaN None None None None None ... 7591220980 290994282 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1000449,1000138,1001499,1001511,... None None
49 {'itemid': 3096197829, 'shopid': 39623462, 'na... 35351777.0 21618459.0 None 1.0 工作鞋 U1lQYVlnQUFBQUJCUkc1MD90vWe3xxVRRspKmttqOU9bfO... None None None ... 3096197829 39623462 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1010729,1010734,1011331,1011333,... None None
50 {'itemid': 12139827160, 'shopid': 538703151, '... NaN NaN None NaN None None None None None ... 12139827160 538703151 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1010733,1008416,1011292,1011331,... None None
51 {'itemid': 1934878030, 'shopid': 2157594, 'nam... NaN NaN None NaN None None None None None ... 1934878030 2157594 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1002127,1009753,1005701,1010729,... None None
52 {'itemid': 16846682501, 'shopid': 342768515, '... NaN NaN None NaN None None None None None ... 16846682501 342768515 None None 1 None {"is_ads":false,"ruleids":[1000,1000],"groupid... {"label_ids":[1010445,1010729,1010736,1011292,... None None
53 {'itemid': 16846682501, 'shopid': 342768515, '... 45338925.0 29188974.0 None 1.0 運動鞋 U1lQYVlnQUFBQUJCUkc1MD90vWe3xxVRRspKmttqOU/sPA... None None None ... 16846682501 342768515 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1010445,1010729,1010736,1011292,... None None
54 {'itemid': 12180979071, 'shopid': 594162705, '... NaN NaN None NaN None None None None None ... 12180979071 594162705 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1011292,1011847,1011924,1011926,... None None
55 {'itemid': 16557416632, 'shopid': 752665010, '... NaN NaN None NaN None None None None None ... 16557416632 752665010 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1015986,1015987],"is_on_flash_sa... None None
56 {'itemid': 16928780052, 'shopid': 298455536, '... NaN NaN None NaN None None None None None ... 16928780052 298455536 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1010729,1010733,1011331,1011333,... None None
57 {'itemid': 6559220933, 'shopid': 277776420, 'n... 33063174.0 19876164.0 None 0.0 運動鞋 U1lQYVlnQUFBQUJCUkc1MAbKJeEtNVZYYisoRB+n4q+ena... None None None ... 6559220933 277776420 None None 1 None {"is_ads":true,"groupid":0,"fe_flags":"{}","it... {"label_ids":[1004779,1009692,1009505,1010713,... None None
58 {'itemid': 68282596, 'shopid': 4802609, 'name'... NaN NaN None NaN None None None None None ... 68282596 4802609 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[16,1000508,1000138,1001499,10028... None None
59 {'itemid': 16512648958, 'shopid': 264561849, '... NaN NaN None NaN None None None None None ... 16512648958 264561849 None None 1 None {"is_ads":false,"ruleids":[0],"groupid":0,"fe_... {"label_ids":[1012634,1012808,1013627,1014629,... None None