HadoopImageProcessingInterface (HIPI)示例程序

Question

我正在研究HIPI并开始编写一个示例程序。

我无法执行它，因为它总是给出以下异常:

hadoop jar Desktop/edureka/workspace/jars/SampleProgramHIPI.jar hdfs:/video/sampleimages.hib hdfs:/video/sampleimages.output
15/10/16 15:59:53 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Exception in thread "main" java.lang.NoClassDefFoundError: hipi/imagebundle/mapreduce/ImageBundleInputFormat
    at SampleProgram.run(SampleProgram.java:67)
    at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
    at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84)
    at SampleProgram.main(SampleProgram.java:86)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:497)
    at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
    at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Caused by: java.lang.ClassNotFoundException: hipi.imagebundle.mapreduce.ImageBundleInputFormat
    at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
    ... 10 more

构建路径已经更新为给定的库(hipi/core/Build/libs/hipi-2.1.0.jar)。

几乎到处都检查过了!!请帮帮我。

下面是我要执行的示例程序:

package hipi.image.examples;
import hipi.imagebundle.mapreduce.ImageBundleInputFormat;
import hipi.image.FloatImage;
import hipi.image.ImageHeader;
//import org.hipi.imagebundle.mapreduce.HibInputFormat;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException; 
@SuppressWarnings("unused")
public class SampleProgram extends Configured implements Tool { 
  public static class HelloWorldMapper extends Mapper<ImageHeader, FloatImage, IntWritable, FloatImage> {
    public void map(ImageHeader key, FloatImage value, Context context) 
      throws IOException, InterruptedException {
         if (value != null && value.getWidth() > 1 && value.getHeight() > 1 && value.getBands() == 3) {           
            int w = value.getWidth();
            int h = value.getHeight();
            float[] valData = value.getData();
            float[] avgData = {0,0,0};
            for (int j = 0; j < h; j++) {
              for (int i = 0; i < w; i++) {
                avgData[0] += valData[(j*w+i)*3+0]; 
                avgData[1] += valData[(j*w+i)*3+1]; 
                avgData[2] += valData[(j*w+i)*3+2]; 
              }
            }
            FloatImage avg = new FloatImage(1, 1, 3, avgData);
            avg.scale(1.0f/(float)(w*h));          
            context.write(new IntWritable(1), avg);
          } 
    }
  }
  public static class HelloWorldReducer extends Reducer<IntWritable, FloatImage, IntWritable, Text> {
    public void reduce(IntWritable key, Iterable<FloatImage> values, Context context) 
      throws IOException, InterruptedException {
        FloatImage avg = new FloatImage(1, 1, 3);
        int total = 0;
        for (FloatImage val : values) {
          avg.add(val);
          total++;
        }
        if (total > 0) {
          avg.scale(1.0f / total);         
    float[] avgData = avg.getData();
          String result = String.format("Average pixel value: %f %f %f", avgData[0], avgData[1], avgData[2]);       
          context.write(key, new Text(result));
        }
    }
  }
  public int run(String[] args) throws Exception {
    if (args.length != 2) {
      System.out.println("Usage: helloWorld <input HIB> <output directory>");
      System.exit(0);
    }  
    Job job = Job.getInstance();
    job.setInputFormatClass(ImageBundleInputFormat.class);  
    job.setOutputFormatClass(TextOutputFormat.class);
    job.setJarByClass(SampleProgram.class);
    job.setMapperClass(HelloWorldMapper.class);
    job.setReducerClass(HelloWorldReducer.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(FloatImage.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(Text.class);    
    FileInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    boolean success = job.waitForCompletion(true);    
    return success ? 0 : 1;
  }  
  public static void main(String[] args) throws Exception {
    ToolRunner.run(new SampleProgram(), args);
    System.exit(0);
  }
}

Answer 1

请尝试使用如下所示的libjars选项。libjar将给定的jar上传到集群，然后在每个mapper、reducer实例的类路径上使它们可用。如果您想在驱动程序客户端类路径中添加额外的库(jar)，则可以使用HADOOP_CLASSPATH环境变量。

export HADOOP_CLASSPATH=hipi-0.0.1.jar
hadoop jar ~/Desktop/edureka/workspace/jars/SampleProgramHIPI.jar hipi.image.examples.SampleProgram hdfs:/video/sampleimages.hib hdfs:/video/sampleimages.output