我正在使用内部S3(西部数字(来存储json文件。 尝试在jupyter实验室中通过Pyspark读取json文件。我是新来的火花。请指导
import os
os.environ['PYSPARK_SUBMIT_ARGS'] = "--packages=org.apache.hadoop:hadoop-aws:2.7.4 pyspark-shell"
# pyspark --packages com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.2
import findspark
findspark.init('/root/spark/spark-3.0.0-preview-bin-hadoop2.7')
from pyspark.sql import SparkSession
import configparser
hadoopConf = sc._jsc.hadoopConfiguration()
myAccessKey = '*************'
mySecretKey = '******'
hadoopConf.set("fs.s3a.endpoint", "https://abc.domain.com")
hadoopConf.set("fs.s3a.awsAccessKeyId", myAccessKey)
hadoopConf.set("fs.s3a.awsSecretAccessKey", mySecretKey)
df = sqlContext.read.json("s3a://bucketname/filename.json")
df.show()
- 我是否错误地传递存储桶名称，因为我看到 url 为此失败。
com.amazonaws.AmazonClientException： 无法执行 HTTP 请求： bucketname.abc.domain.com：： 名称或服务未知
---------------------------------------------------------------------------
Py4JJavaError Traceback (most recent call last)
<ipython-input-11-02e7cddb8018> in <module>
89 # df.show()
90
---> 91 df = sqlContext.read.json("s3a://bucketname/test_sjc.json")
92 df.show()
93 # s3_resource.Object(s3_bucket, first_file_name).download_file(
~/spark/spark-3.0.0-preview-bin-hadoop2.7/python/pyspark/sql/readwriter.py in json(self, path, schema, primitivesAsString, prefersDecimal, allowComments, allowUnquotedFieldNames, allowSingleQuotes, allowNumericLeadingZero, allowBackslashEscapingAnyCharacter, mode, columnNameOfCorruptRecord, dateFormat, timestampFormat, multiLine, allowUnquotedControlChars, lineSep, samplingRatio, dropFieldIfAllNull, encoding, locale)
271 path = [path]
272 if type(path) == list:
--> 273 return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))
274 elif isinstance(path, RDD):
275 def func(iterator):
~/spark/spark-3.0.0-preview-bin-hadoop2.7/python/lib/py4j-0.10.8.1-src.zip/py4j/java_gateway.py in __call__(self, *args)
1284 answer = self.gateway_client.send_command(command)
1285 return_value = get_return_value(
-> 1286 answer, self.gateway_client, self.target_id, self.name)
1287
1288 for temp_arg in temp_args:
~/spark/spark-3.0.0-preview-bin-hadoop2.7/python/pyspark/sql/utils.py in deco(*a, **kw)
96 def deco(*a, **kw):
97 try:
---> 98 return f(*a, **kw)
99 except py4j.protocol.Py4JJavaError as e:
100 converted = convert_exception(e.java_exception)
~/spark/spark-3.0.0-preview-bin-hadoop2.7/python/lib/py4j-0.10.8.1-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.n".
--> 328 format(target_id, ".", name), value)
329 else:
330 raise Py4JError(
Py4JJavaError: An error occurred while calling o196.json.
: com.amazonaws.AmazonClientException: Unable to execute HTTP request: bucketname.abc.domain.com:: Name or service not known
at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:454)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:232)
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3528)
at com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1031)
at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:994)
at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:297)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2669)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94)
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2703)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2685)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:373)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2.qualifiedPathName(FileDataSourceV2.scala:59)
at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2.$anonfun$getTableName$1(FileDataSourceV2.scala:53)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at scala.collection.TraversableLike.map(TraversableLike.scala:238)
at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2.getTableName(FileDataSourceV2.scala:53)
at org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2.getTableName$(FileDataSourceV2.scala:52)
at org.apache.spark.sql.execution.datasources.v2.json.JsonDataSourceV2.getTableName(JsonDataSourceV2.scala:26)
at org.apache.spark.sql.execution.datasources.v2.json.JsonDataSourceV2.getTable(JsonDataSourceV2.scala:34)
at org.apache.spark.sql.DataFrameReader.$anonfun$load$1(DataFrameReader.scala:220)
at scala.Option.map(Option.scala:230)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:206)
at org.apache.spark.sql.DataFrameReader.json(DataFrameReader.scala:411)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.UnknownHostException: bucketname.abc.domain.com: Name or service not known
at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:929)
at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1324)
at java.net.InetAddress.getAllByName0(InetAddress.java:1277)
at java.net.InetAddress.getAllByName(InetAddress.java:1193)
at java.net.InetAddress.getAllByName(InetAddress.java:1127)
at org.apache.http.impl.conn.SystemDefaultDnsResolver.resolve(SystemDefaultDnsResolver.java:45)
at org.apache.http.impl.conn.DefaultClientConnectionOperator.resolveHostname(DefaultClientConnectionOperator.java:263)
at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:162)
at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:326)
at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:610)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:445)
at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:835)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:384)
... 39 more
我最近遇到了类似的事情： 尝试将以下行添加到 hadoop 配置中：
hadoopConf.set("fs.s3a.path.style.access", "true");
hadoopConf.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
hadoopConf.set("fs.s3a.connection.ssl.enabled", "false");
可能会解决此错误。 希望对您有所帮助！