在Oozie上运行MapReduce作业

Question

我正在尝试在Oozie上运行MapReduce作业，但它失败并被杀死。这个错误也不会显示在oozie控制台上，它只给出这个错误信息:"Map/Reduce failed, error message[]"。我也可以在哪里找到日志，它会包含确切的错误。我是新手，不知道出了什么问题。有谁能告诉我以下代码有什么问题吗?我已经疼了两天了。

这是我的MapReduce程序

package com.hadoop.mapreduce;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class AnalysePatientRecords {
    public static class SampleMap extends Mapper<LongWritable, Text, Text, Text> {
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            String[] lineElements = line.split("\|",-1);
            Text state = new Text(lineElements[11]);
            Text patientId = new Text(lineElements[0]);
            context.write(state, patientId);
        }
    }
    public static class SampleReduce extends Reducer<Text, Text, Text, IntWritable> {
        @SuppressWarnings("unused")
        public void reduce(Text value, Iterable<Text> values, Context context)
                throws IOException, InterruptedException {
            int count = 0;
            for (Text val : values) {
                count = count + 1;
            }
            context.write(value, new IntWritable(count));
        }
    }
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "patient analysis");
        job.setJarByClass(AnalysePatientRecords.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setMapperClass(SampleMap.class);
        job.setReducerClass(SampleReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

job.properties

nameNode=hdfs://localhost:54310
jobTracker=localhost:8032
queueName=default
examplesRoot=MapReduce
oozie.libpath=${nameNode}/user/${user.name}/share/lib
oozie.use.system.libpath=true
oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/map-reduce/workflow.xml
outputDir=map-reduce

workflow.xml

<workflow-app xmlns="uri:oozie:workflow:0.2" name="map-reduce-wf">
    <start to="mr-node"/>
    <action name="mr-node">
        <map-reduce>
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <prepare>
                <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/${outputDir}"/>
            </prepare>
            <configuration>
        <property>
                    <name>mapred.mapper.new-api</name>
                    <value>true</value>
                </property>
                <property>
                    <name>mapred.reducer.new-api</name>
                    <value>true</value>
                </property>
                <property>
                    <name>mapred.job.queue.name</name>
                    <value>${queueName}</value>
                </property>
                <property>
                    <name>mapreduce.map.class</name>
                    <value>AnalysePatientRecords$SampleMap</value>
                </property>
                <property>
                    <name>mapreduce.reduce.class</name>
                    <value>AnalysePatientRecords$SampleReduce</value>
                </property>
        <property>
                <name>mapred.mapoutput.key.class</name>
                <value>org.apache.hadoop.io.Text</value>
                </property>
                <property>
                        <name>mapred.mapoutput.value.class</name>
                        <value>org.apache.hadoop.io.Text</value>
                </property>
                <property>
                        <name>mapred.output.key.class</name>
                        <value>org.apache.hadoop.io.Text</value>
                </property>
                <property>
                        <name>mapred.output.value.class</name>
                        <value>org.apache.hadoop.io.IntWritable</value>
                </property>
                <property>
                    <name>mapred.map.tasks</name>
                    <value>1</value>
                </property>
                <property>
                    <name>mapred.input.dir</name>
                    <value>/user/${wf:user()}/${examplesRoot}/input-data/text</value>
                </property>
                <property>
                    <name>mapred.output.dir</name>
                    <value>/user/${wf:user()}/${examplesRoot}/output-data/${outputDir}</value>
                </property>
            </configuration>
        </map-reduce>
        <ok to="end"/>
        <error to="fail"/>
    </action>
    <kill name="fail">
        <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <end name="end"/>
</workflow-app>

我还创建了上面提到的MapReduce程序的jar，并将其放在lib文件夹中。我不知道出了什么问题，请帮帮我。请…

Answer 1

查看Oozie Hive Action Logging。正如文档中提到的——"Hive操作日志被重定向到运行Hive的Oozie Launcher map-reduce作业任务STDOUT/STDERR"。@YoungHobbit也在他们的评论中澄清了关于检查oozie启动器工作日志的问题。

您可以从Oozie web-console访问该日志。从Oozie web-console，从Hive动作弹出使用'Console URL'链接，可以通过Hadoop job-tracker web-console导航到Oozie Launcher map-reduce作业任务日志。