小贝子编程

hadoop reducer中的空指针异常



我正面对NullPointerException与以下代码。如果有人能帮我复习一下这个程序，那就太好了。

映射器运行良好，但是，当我试图在迭代器上分割值时，我得到了一个NPE。请帮我找出我的错误。我把地图附在下面。

Toppermain.java 

package TopperPackage;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

public class TopperMain {
//hadoop jar worcount.jar ars[0] args[1]
public static void main(String[] args) throws Exception {
Job myhadoopJob = new Job();
myhadoopJob.setJarByClass(TopperMain.class);
myhadoopJob.setJobName("Finding topper based on subject");
FileInputFormat.addInputPath(myhadoopJob, new Path(args[0]));
FileOutputFormat.setOutputPath(myhadoopJob, new Path(args[1]));
myhadoopJob.setInputFormatClass(TextInputFormat.class);
myhadoopJob.setOutputFormatClass(TextOutputFormat.class);
myhadoopJob.setMapperClass(TopperMapper.class);
myhadoopJob.setReducerClass(TopperReduce.class);

myhadoopJob.setMapOutputKeyClass(Text.class);
myhadoopJob.setMapOutputValueClass(Text.class);
myhadoopJob.setOutputKeyClass(Text.class);
myhadoopJob.setOutputValueClass(Text.class);
    System.exit(myhadoopJob.waitForCompletion(true) ? 0 : 1);
}
}

TopperMapper.java 

package TopperPackage;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
/*Surender,87,60,50,50,80
Raj,80,70,80,85,60
Anten,81,60,50,70,100
Dinesh,60,90,80,80,70
Priya,80,85,91,60,75
*/
public class TopperMapper extends Mapper<LongWritable, Text, Text, Text>
{
String temp,temp2;
protected void map(LongWritable key, Text value,Context context)
    throws IOException, InterruptedException {
String record = value.toString();
String[] parts = record.split(",");
temp=parts[0];
temp2=temp+ "t" + parts[1];
context.write(new Text("Tamil"),new Text(temp2));
temp2=temp+ "t" + parts[2];
context.write(new Text("English"),new Text(temp2));
temp2=temp+ "t" + parts[3];
context.write(new Text("Maths"),new Text(temp2));
temp2=temp+ "t" + parts[4];
context.write(new Text("Science"),new Text(temp2));
temp2=temp+ "t" + parts[5];
context.write(new Text("SocialScrience"),new Text(temp2));

       }
}

TopperReduce.java 

package TopperPackage;
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class TopperReduce extends Reducer<Text, Text, Text, Text> {
int temp;
private String[] names;
private int[] marks;
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String top = "";
int count =0,topmark;
marks = null;
String befsplit;
String[] parts=null;
names = null;
for (Text t : values)
{
befsplit= t.toString();
parts = befsplit.split("t");
names[count]=parts[0];
marks[count]=Integer.parseInt(parts[1]);
count = count+1;
}
topmark=calcTopper(marks);
top=names[topmark]+ "t"+marks[topmark]  ;
context.write(new Text(key), new Text(top));
}
public int calcTopper(int[] marks)
{
int count=marks.length;
temp=((marks[1]));
int i=0;
for (i=1;i<=(count-2);i++)
{
   if(temp < marks[i+1])
   {
   temp = marks[i+1];
   }
}
return i;
}
}

错误是

cloudera@cloudera-vm:~/Jarfiles$ hadoop jar TopperMain.jar /user/cloudera/inputfiles/topper/topperinput.txt /user/cloudera/outputfiles/topper/
14/08/24 23:17:07 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
14/08/24 23:17:08 INFO input.FileInputFormat: Total input paths to process : 1
14/08/24 23:17:09 INFO mapred.JobClient: Running job: job_201408241907_0012
14/08/24 23:17:10 INFO mapred.JobClient:  map 0% reduce 0%
14/08/24 23:17:49 INFO mapred.JobClient:  map 100% reduce 0%
14/08/24 23:18:03 INFO mapred.JobClient: Task Id : attempt_201408241907_0012_r_000000_0, Status : FAILED
java.lang.NullPointerException
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:25)
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:1)
    at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
    at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:571)
    at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:413)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
    at org.apache.hadoop.mapred.Child.main(Child.java:262)
attempt_201408241907_0012_r_000000_0: log4j:WARN No appenders could be found for logger (org.apache.hadoop.hdfs.DFSClient).
attempt_201408241907_0012_r_000000_0: log4j:WARN Please initialize the log4j system properly.
14/08/24 23:18:22 INFO mapred.JobClient: Task Id : attempt_201408241907_0012_r_000000_1, Status : FAILED
java.lang.NullPointerException
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:25)
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:1)
    at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
    at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:571)
    at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:413)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
    at org.apache.hadoop.mapred.Child.main(Child.java:262)
attempt_201408241907_0012_r_000000_1: log4j:WARN No appenders could be found for logger (org.apache.hadoop.hdfs.DFSClient).
attempt_201408241907_0012_r_000000_1: log4j:WARN Please initialize the log4j system properly.

我从mapper得到预期的输出，但是reducer在分割输出并存储在变量中时抛出错误。

映射器输出为

Tamil   Surender    87
English Surender    60
Maths   Surender    50
Science Surender    50
SocialScrience  Surender    80
Tamil   Raj 80
English Raj 70
Maths   Raj 80
Science Raj 85
SocialScrience  Raj 60
Tamil   Anten   81
English Anten   60
Maths   Anten   50
Science Anten   70
SocialScrience  Anten   100
Tamil   Dinesh  60
English Dinesh  90
Maths   Dinesh  80
Science Dinesh  80
SocialScrience  Dinesh  70
Tamil   Priya   80
English Priya   85
Maths   Priya   91
Science Priya   60
SocialScrience  Priya   75

错误是由于您将标记和名称数组初始化为null而没有正确初始化它们。请使用以下减速机型号。

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class TopperReduce extends Reducer<Text, Text, Text, Text> {
    int temp;
    private String[] names = new String[10];
    private int[] marks = new int[10];
    public void reduce(Text key, Iterable<Text> values, Context context)
            throws IOException, InterruptedException {
        String top = "";
        int count = 0, topmark;
        String befsplit;
        String[] parts = null;
        for (Text t : values) {
           befsplit = t.toString();
           parts = befsplit.split("t");
           names[count] = parts[0];
           marks[count] = Integer.parseInt(parts[1]);
           count++;
        }
       topmark = calcTopper(marks);
       top = names[topmark] + "t" + marks[topmark];
        context.write(new Text(key), new Text(top));
   }
public int calcTopper(int[] marks) {
    int count = marks.length;
    int i = 0;
    int highestMArk = 0;
    int mark = 0;
    int highestMarkIndex = 0;
    for (; i < count; i++) {
        mark = marks[i];
        if (mark > highestMArk) {
            highestMarkIndex = i;
        }
    }
    return highestMarkIndex;
}
}

你指的是一个空数组变量部分，所以你得到这个错误，修改你的代码，我在下面提到它可以工作

 public class TopperReduce extends Reducer<Text, Text, Text, Text> {
int temp;
private String[] names=new String[20];
private int[] marks= new int[20];
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String top = "";
int count =0,topmark;
for (Text t : values)
{
String befsplit= t.toString();
String[] parts = befsplit.split("t");
names[count]=parts[0];
marks[count]=Integer.parseInt(parts[1]);
count = count+1;
}
topmark=calcTopper(marks);
top=names[topmark]+ "t"+marks[topmark]  ;
context.write(new Text(key), new Text(top));
}

相关内容

  • 没有找到相关文章

最新更新


热门标签：

javascript python java c# php android html jquery c++ css ios sql mysql arrays asp.net json python-3.x ruby-on-rails .net sql-server django objective-c excel regex ruby linux ajax iphone xml vba spring asp.net-mvc database wordpress string postgresql wpf windows xcode bash git oracle list vb.net multithreading eclipse algorithm macos powershell visual-studio image forms numpy scala function api selenium