小贝子编程

错误:无法在运行spark-submit命令时从JAR文件加载Main Class



我为我的Scala项目创建了Jar，然后从终端运行以下命令["com.sukrit.hbase_"是包名，"Filters_Usage"是我想要运行的Scala类]

Macintosh:bin sukritmehta$ ./spark-submit --class "com.sukrit.hbase_.Filters_Usage" --master local[*] "/Users/sukritmehta/Desktop/Sukrit/Spark_Hbase/target/Spark_Hbase-0.0.1-SNAPSHOT.jar"

但是在运行这个之后，我遇到了以下错误：

20/04/24 20:53:01 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Error: Failed to load class com.sukrit.hbase_.Filters_Usage.
20/04/24 20:53:02 INFO util.ShutdownHookManager: Shutdown hook called
20/04/24 20:53:02 INFO util.ShutdownHookManager: Deleting directory /private/var/folders/d4/psn4wv8s7tjbfgt6gkt35z9c0000gq/T/spark-ae120675-a1c6-4300-997c-bd53f9f35187

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>Spark_Hbase</groupId>
<artifactId>Spark_Hbase</artifactId>
<version>0.0.1-SNAPSHOT</version>
<build>
<sourceDirectory>src</sourceDirectory>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<!-- Scala and Spark dependencies -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.11.11</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_2.11</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.4.0</version>
<scope>provided</scope>
</dependency>
<!-- <dependency> <groupId>org.codehaus.janino</groupId> <artifactId>commons-compiler</artifactId> 
<version>3.0.7</version> </dependency> <dependency> <groupId>org.apache.spark</groupId> 
<artifactId>spark-network-common_2.11</artifactId> <version>2.1.1</version> 
</dependency> -->
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.9</version>
</dependency>
<!-- <dependency> <groupId>org.mongodb.spark</groupId> <artifactId>mongo-spark-connector_2.10</artifactId> 
<version>2.1.1</version> </dependency> -->
<!-- <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-mllib_2.10</artifactId> 
<version>2.1.1</version> </dependency> -->
<!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase -->

<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>2.2.4</version>
</dependency>
<!--  <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>1.2.1</version>
</dependency>      --> 

<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-spark</artifactId>
<version>2.0.0-alpha4</version> <!-- Hortonworks Latest -->
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase-mapreduce -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-mapreduce</artifactId>
<version>2.2.4</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.6.5</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project>

过滤器_使用.scala

package com.sukrit.hbase_
import org.apache.hadoop.conf
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.HTable
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.client._
import org.apache.hadoop.hbase.{HBaseConfiguration, TableName}
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.ConnectionFactory
import org.apache.hadoop.mapred.JobConf
import org.apache.spark._
import org.apache.log4j._
import org.apache.hadoop.hbase._
import org.apache.hadoop.hbase.client.Scan
import scala.collection.JavaConverters._
import org.apache.hadoop.hbase.filter._

object Filters_Usage {

def printRow(result : Result) = {
val cells = result.rawCells();
print( Bytes.toString(result.getRow) + " : " )
for(cell <- cells){
val col_name = Bytes.toString(CellUtil.cloneQualifier(cell))
val col_value = Bytes.toString(CellUtil.cloneValue(cell))
print("(%s:%s) ".format(col_name, col_value))
}
println()
}

def main(args : Array[String]) {

Logger.getLogger("org").setLevel(Level.ERROR)
val sparkConf = new SparkConf().setAppName("Spark_Hbase_Connection").setMaster("local[*]")
val sc = new SparkContext(sparkConf)
val conf = HBaseConfiguration.create()    
conf.set("hbase.rpc.timeout", "1800000");
conf.set("hbase.client.scanner.timeout.period", "1800000");
// establish a connection
val connection:Connection = ConnectionFactory.createConnection(conf)
// Table on which different commands have to be run.
val tableName = connection.getTable(TableName.valueOf("emp"))
sc.addJar("/Users/sukritmehta/Desktop/Sukrit/Spark_Hbase/target/Spark_Hbase-0.0.1-SNAPSHOT.jar")
println("n **************** All rows in Table **************** n ")
val scan:Scan = new Scan()  
val resultScanner:ResultScanner = tableName.getScanner(scan)
resultScanner.asScala.foreach(result => {
printRow(result)
})
resultScanner.close()
println(" n n ")




// **************** Row Filters ****************
println("**************** Row Filter (for Row_keys <=5) **************** n ")
val row_filter1 = new RowFilter(CompareFilter.CompareOp.LESS_OR_EQUAL, 
new BinaryComparator(Bytes.toBytes("5")));
scan.setFilter(row_filter1)
val resultScanner1:ResultScanner = tableName.getScanner(scan)
resultScanner1.asScala.foreach(result => {
printRow(result)
})
resultScanner1.close()
println(" n n ")

println("**************** Row Filter (for Row_keys starting with 1) **************** n ")
val row_filter2 = new RowFilter(CompareFilter.CompareOp.EQUAL, 
new SubstringComparator("1"));
scan.setFilter(row_filter2)
val resultScanner2:ResultScanner = tableName.getScanner(scan)
resultScanner2.asScala.foreach(result => {
printRow(result)
})
resultScanner2.close()
println(" n n ")




// **************** Family Filters ****************
println("**************** Family Filter (for Col Family = personal data) **************** n ")
val fam_filter1 = new FamilyFilter(CompareFilter.CompareOp.EQUAL, 
new BinaryComparator(Bytes.toBytes("personal data")));
scan.setFilter(fam_filter1)
val resultScanner3:ResultScanner = tableName.getScanner(scan)
resultScanner3.asScala.foreach(result => {
printRow(result)
})
resultScanner3.close()
println(" n n ")




// **************** Qualifier Filters ****************
println("**************** Qualifier Filter (for Column = name) **************** n ")
val qual_filter1 = new QualifierFilter(CompareFilter.CompareOp.EQUAL, 
new BinaryComparator(Bytes.toBytes("name")));
scan.setFilter(qual_filter1)
val resultScanner4:ResultScanner = tableName.getScanner(scan)
resultScanner4.asScala.foreach(result => {
printRow(result)
})
resultScanner4.close()
println(" n n ")

println("**************** Qualifier Filter (Printing name corresponding to Row_Key = 5) **************** n ")
val qual_filter2 = new QualifierFilter(CompareFilter.CompareOp.EQUAL, 
new BinaryComparator(Bytes.toBytes("name")));
scan.setFilter(qual_filter2)
val get = new Get(Bytes.toBytes("5")); get.setFilter(qual_filter2);
val result = tableName.get(get); 
printRow(result)
println(" n n ")




// **************** Filter List ****************
println("**************** Filter List (print all rows between two row_Keys 4 & 7 (lexicograpphically)) **************** n ")
val ss = new Scan()
val filter1 = new RowFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, 
new BinaryComparator(Bytes.toBytes("4")));
val filter2 = new RowFilter(CompareFilter.CompareOp.LESS_OR_EQUAL, 
new BinaryComparator(Bytes.toBytes("7")));
var filterList = new FilterList()
filterList.addFilter(filter1)
filterList.addFilter(filter2)
ss.setFilter(filterList)    
val resultScanner8:ResultScanner = tableName.getScanner(ss)
resultScanner8.asScala.foreach(result => {
printRow(result)
})
resultScanner8.close()
println(" n n ")




// **************** Single Column Value Filter ****************
println("**************** Single Column Value Filter (print all rows having cities lesser than Delhi (lexicographically)) **************** n ")
val ss3 = new Scan()
val filter5 = new SingleColumnValueFilter( Bytes.toBytes("personal data"),
Bytes.toBytes("city"),
CompareFilter.CompareOp.LESS_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("Delhi")));
filter5.setFilterIfMissing(false);

ss3.setFilter(filter5)    
val resultScanner10:ResultScanner = tableName.getScanner(ss3)
resultScanner10.asScala.foreach(result => {
printRow(result)
})
resultScanner10.close()
println(" n n ")





// **************** Filter List ****************
println("**************** Filter List (print all rows between Delhi and Mumbai (lexicographically)) **************** n ")

println("****** Before filtering ****** n ")
val ss2 = new Scan()
val resultScanner9:ResultScanner = tableName.getScanner(ss2)
resultScanner9.asScala.foreach(result => {
printRow(result)
})
resultScanner9.close()
println(" n n ")

println("****** After filtering ****** n ")
val filter3 = new SingleColumnValueFilter( Bytes.toBytes("personal data"),
Bytes.toBytes("city"),
CompareFilter.CompareOp.GREATER_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("Delhi")));
filter3.setFilterIfMissing(false);
val filter4 = new SingleColumnValueFilter( Bytes.toBytes("personal data"),
Bytes.toBytes("city"),
CompareFilter.CompareOp.LESS_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("Mumbai")));
filter4.setFilterIfMissing(false);
var filterList2 = new FilterList()
filterList2.addFilter(filter3)
filterList2.addFilter(filter4)
ss2.setFilter(filterList2)    
val resultScanner6:ResultScanner = tableName.getScanner(ss2)
resultScanner6.asScala.foreach(result => {
printRow(result)
})
resultScanner6.close()
println(" n n ")





// **************** Filter List ****************
println("**************** Filter List (print all rows Having DOBs between 19770101 and 19951231 (lexicographically)) **************** n ")
println("****** Before filtering ****** n ")
val ss4 = new Scan()
val resultScanner11:ResultScanner = tableName.getScanner(ss4)
resultScanner11.asScala.foreach(result => {
printRow(result)
})
resultScanner11.close()
println(" n n ")

println("****** After filtering ****** n ")
val filter7 = new SingleColumnValueFilter( Bytes.toBytes("personal data"),
Bytes.toBytes("dob"),
CompareFilter.CompareOp.GREATER_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("19770101")));
val filter8 = new SingleColumnValueFilter( Bytes.toBytes("personal data"),
Bytes.toBytes("dob"),
CompareFilter.CompareOp.LESS_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("19951231")));
var filterList3 = new FilterList()
filterList3.addFilter(filter7)
filterList3.addFilter(filter8)
ss4.setFilter(filterList3)    
val resultScanner12:ResultScanner = tableName.getScanner(ss4)
resultScanner12.asScala.foreach(result => {
printRow(result)
})
resultScanner12.close()
println(" n n ")



}
}

如果有人能帮我解决这个问题，那就太好了。

我怀疑您的scala代码没有正确编译，您可以在pom.xml和pom.xml中添加以下内容吗；根据您的本地系统更改路径。


<build>
<directory>../target/scala-2.11</directory>
<outputDirectory>../target/scala-2.11</outputDirectory>
<sourceDirectory>../src/main/scala</sourceDirectory>
<testSourceDirectory>../src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>4.3.1</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>2.11.8</scalaVersion>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.3</version>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<finalName>${project.artifactId}</finalName>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Main-Class>client.MxClient</Main-Class>
<X-Compile-Source-JDK>${maven.compile.source}</X-Compile-Source-JDK>
<X-Compile-Target-JDK>${maven.compile.target}</X-Compile-Target-JDK>
</manifestEntries>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>
META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
</resource>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

相关内容

最新更新


热门标签：

javascript python java c# php android html jquery c++ css ios sql mysql arrays asp.net json python-3.x ruby-on-rails .net sql-server django objective-c excel regex ruby linux ajax iphone xml vba spring asp.net-mvc database wordpress string postgresql wpf windows xcode bash git oracle list vb.net multithreading eclipse algorithm macos powershell visual-studio image forms numpy scala function api selenium