小贝子编程

在使用HDP的hortonworks上运行jar会引发ClassNotFoundException



我正试图用HDP在hortonworks沙箱上运行一个简单的地图缩减程序。这是程序（取自https://github.com/jz33/Coursera-Cloud-Computing-Applications-Solution-Manual/blob/master/hw2/TitleCount.java)：

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import java.io.*;
import java.util.*;
/**
* Classic "Word Count"
 */
public class TitleCount extends Configured implements Tool {
    public static void main(String[] args) throws Exception {
       int res = ToolRunner.run(new Configuration(), new TitleCount(), args);
    System.exit(res);
}
@Override
public int run(String[] args) throws Exception {
    Job job = Job.getInstance(this.getConf(), "Title Count");
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setMapperClass(TitleCountMap.class);
    job.setReducerClass(TitleCountReduce.class);
    FileInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    job.setJarByClass(TitleCount.class);
    return job.waitForCompletion(true) ? 0 : 1;
}
public static String readHDFSFile(String path, Configuration conf) throws IOException{
    Path pt=new Path(path);
    FileSystem fs = FileSystem.get(pt.toUri(), conf);
    FSDataInputStream file = fs.open(pt);
    BufferedReader buffIn=new BufferedReader(new InputStreamReader(file));
    StringBuilder everything = new StringBuilder();
    String line;
    while( (line = buffIn.readLine()) != null) {
        everything.append(line);
        everything.append("n");
    }
    return everything.toString();
}
public static class TitleCountMap extends Mapper<Object, Text, Text, IntWritable> {
    Set<String> stopWords = new HashSet<String>();
    String delimiters;
    @Override
    protected void setup(Context context) throws IOException,InterruptedException {
        Configuration conf = context.getConfiguration();
        
        String delimitersPath = conf.get("delimiters");
        delimiters = readHDFSFile(delimitersPath, conf);
        
        String stopWordsPath = conf.get("stopwords");
        List<String> stopWordsList = Arrays.asList(readHDFSFile(stopWordsPath, conf).split("n"));
        for(String e : stopWordsList){
            stopWords.add(e);
        }
    }
    @Override
    public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
        StringTokenizer stk = new StringTokenizer(value.toString(),delimiters);
        while(stk.hasMoreTokens()){
            String e = stk.nextToken().trim().toLowerCase();
            if(stopWords.contains(e) == false){
                context.write(new Text(e),new IntWritable(1));
            }
        }
    }
}
public static class TitleCountReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
    @Override
    public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        int sum = 0;
        for(IntWritable e : values){
            sum += e.get();
        }
        context.write(key, new IntWritable(sum));
    }
   }
 }

这就是我运行它的方式：

  • git clone https://github.com/xldrx/cloudapp-mp2.git

  • cd cloudapp-mp2

  • sudo -u hdfs bash start.sh

  • cp /media/sharedFolder/TitleCount.java .

  • mkdir build

  • export HADOOP_CLASSPATH=$JAVA_HOME/lib/tools.jar

  • hadoop com.sun.tools.javac.Main TitleCount.java -d build

  • jar -cvf TitleCount.jar -C build/ ./

  • sudo -u hdfs hadoop fs -rm -r /mp2/A-output

    之后，我有了包含.class文件的构建文件夹（TitleCount.class、TitleCount$TitleCountMap.class和TitleCount$TitleCountReduce.class）以及当前文件夹cloud-mp2，其中包含TitleCount.jar文件。

  • sudo -u hdfs hadoop jar TitleCount.jar TitleCount -D stopwords=/mp2/misc/stopwords.txt -D delimiters=/mp2/misc/delimiters.txt /mp2/titles /mp2/A-output

这是我得到的错误：

错误：java.lang.RuntimeException：java.lang.ClassNotFoundException：未找到Class TitleCount$TitleCountReduce

完整跟踪：

000_0, Status : FAILED
Error: java.lang.RuntimeException: java.lang.ClassNotFoundException: Class Title
Count$TitleCountReduce not found
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2195
)
        at org.apache.hadoop.mapreduce.task.JobContextImpl.getReducerClass(JobCo
ntextImpl.java:210)
        at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:611
)
        at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInforma
tion.java:1657)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.ClassNotFoundException: Class TitleCount$TitleCountReduce n
ot found
        at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.jav
a:2101)
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2193
)
        ... 8 more
15/12/30 12:46:45 INFO mapreduce.Job: Task Id : attempt_1451479269826_0001_r_000
000_1, Status : FAILED
Error: java.lang.RuntimeException: java.lang.ClassNotFoundException: Class Title
Count$TitleCountReduce not found
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2195
)
        at org.apache.hadoop.mapreduce.task.JobContextImpl.getReducerClass(JobCo
ntextImpl.java:210)
        at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:611
)
        at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInforma
tion.java:1657)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.ClassNotFoundException: Class TitleCount$TitleCountReduce n
ot found
        at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.jav
a:2101)
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2193
)
        ... 8 more
15/12/30 12:46:57 INFO mapreduce.Job: Task Id : attempt_1451479269826_0001_r_000
000_2, Status : FAILED
Error: java.lang.RuntimeException: java.lang.ClassNotFoundException: Class Title
Count$TitleCountReduce not found
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2195
)
        at org.apache.hadoop.mapreduce.task.JobContextImpl.getReducerClass(JobCo
ntextImpl.java:210)
        at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:611
)
        at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInforma
tion.java:1657)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.ClassNotFoundException: Class TitleCount$TitleCountReduce n
ot found
        at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.jav
a:2101)
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2193
)
        ... 8 more
15/12/30 12:47:05 INFO mapreduce.Job:  map 0% reduce 100%
15/12/30 12:47:06 INFO mapreduce.Job: Job job_1451479269826_0001 failed with sta
te FAILED due to: Task failed task_1451479269826_0001_r_000000
Job failed as tasks failed. failedMaps:0 failedReduces:1
15/12/30 12:47:07 INFO mapreduce.Job: Counters: 7
        Job Counters
                Failed reduce tasks=4
                Launched reduce tasks=4
                Total time spent by all maps in occupied slots (ms)=0
                Total time spent by all reduces in occupied slots (ms)=32247
                Total time spent by all reduce tasks (ms)=32247
                Total vcore-seconds taken by all reduce tasks=32247
                Total megabyte-seconds taken by all reduce tasks=8061750

问题是我使用了sudo-u hdfs，而不是使用root用户。

我不得不添加/user/root文件夹：

sudo -u hdfs hadoop fs -mkdir /user/root

并将权限授予/user/root和/mp2:

sudo -u hdfs hadoop fs -chown root:root /user/root
sudo -u hdfs hadoop fs -chown root:root /mp2
sudo -u hdfs hadoop fs -chown root:root /mp2/*

相关内容

  • 没有找到相关文章

最新更新


热门标签：

javascript python java c# php android html jquery c++ css ios sql mysql arrays asp.net json python-3.x ruby-on-rails .net sql-server django objective-c excel regex ruby linux ajax iphone xml vba spring asp.net-mvc database wordpress string postgresql wpf windows xcode bash git oracle list vb.net multithreading eclipse algorithm macos powershell visual-studio image forms numpy scala function api selenium