使用Scala将SparkRDD写入HBase表

Question

我正在尝试使用scala编写一个SparkRDD到HBase表(以前没有使用过)。整个代码是这样的:

import org.apache.hadoop.hbase.client.{HBaseAdmin, Result}
import org.apache.hadoop.hbase.{HBaseConfiguration, HTableDescriptor}
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.hadoop.hbase.io.ImmutableBytesWritable    
import scala.collection.JavaConverters._
import org.apache.hadoop.hbase.util.Bytes
import org.apache.spark._
import org.apache.hadoop.mapred.JobConf
import org.apache.spark.rdd.PairRDDFunctions
import org.apache.spark.SparkContext._
import org.apache.hadoop.mapred.Partitioner;
import org.apache.hadoop.hbase.mapred.TableOutputFormat
import org.apache.hadoop.hbase.client._
object HBaseWrite {
   def main(args: Array[String]) {
     val sparkConf = new SparkConf().setAppName("HBaseWrite").setMaster("local").set("spark.driver.allowMultipleContexts","true").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
     val sc = new SparkContext(sparkConf)
     val conf = HBaseConfiguration.create()
     val outputTable = "tablename"
     System.setProperty("user.name", "hdfs")
     System.setProperty("HADOOP_USER_NAME", "hdfs")
     conf.set("hbase.master", "localhost:60000")
     conf.setInt("timeout", 120000)
     conf.set("hbase.zookeeper.quorum", "localhost")
     conf.set("zookeeper.znode.parent", "/hbase-unsecure")
     conf.setInt("hbase.client.scanner.caching", 10000)
     sparkConf.registerKryoClasses(Array(classOf[org.apache.hadoop.hbase.client.Result]))
     val jobConfig: JobConf = new JobConf(conf,this.getClass)
     jobConfig.setOutputFormat(classOf[TableOutputFormat])
     jobConfig.set(TableOutputFormat.OUTPUT_TABLE,outputTable)
     val x = 12
     val y = 15
     val z = 25
     var newarray = Array(x,y,z)
     val newrddtohbase = sc.parallelize(newarray)
     def convert(a:Int) : Tuple2[ImmutableBytesWritable,Put] = {
          val p = new Put(Bytes.toBytes(a))
          p.add(Bytes.toBytes("columnfamily"),
          Bytes.toBytes("col_1"), Bytes.toBytes(a))
          new Tuple2[ImmutableBytesWritable,Put](new ImmutableBytesWritable(a.toString.getBytes()), p);
     }
     new PairRDDFunctions(newrddtohbase.map(convert)).saveAsHadoopDataset(jobConfig)
     sc.stop()
   }
}

我做HBaseWrite(main(Array())后得到的错误是:

org.apache.spark.SparkException: Task not serializable

我该如何完成它?

Answer 1

您在这里做错的事情是在main中定义convert如果你这样写这段代码，它可能会工作:

    object HBaseWrite {
       def main(args: Array[String]) {
         val sparkConf = new SparkConf().setAppName("HBaseWrite").setMaster("local").set("spark.driver.allowMultipleContexts","true").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
         val sc = new SparkContext(sparkConf)
         val conf = HBaseConfiguration.create()
         val outputTable = "tablename"
         System.setProperty("user.name", "hdfs")
         System.setProperty("HADOOP_USER_NAME", "hdfs")
         conf.set("hbase.master", "localhost:60000")
         conf.setInt("timeout", 120000)
         conf.set("hbase.zookeeper.quorum", "localhost")
         conf.set("zookeeper.znode.parent", "/hbase-unsecure")
         conf.setInt("hbase.client.scanner.caching", 10000)
         sparkConf.registerKryoClasses(Array(classOf[org.apache.hadoop.hbase.client.Result]))
         val jobConfig: JobConf = new JobConf(conf,this.getClass)
         jobConfig.setOutputFormat(classOf[TableOutputFormat])
         jobConfig.set(TableOutputFormat.OUTPUT_TABLE,outputTable)
         val x = 12
         val y = 15
         val z = 25
         var newarray = Array(x,y,z)
         val newrddtohbase = sc.parallelize(newarray)
         val convertFunc = convert _
         new PairRDDFunctions(newrddtohbase.map(convertFunc)).saveAsHadoopDataset(jobConfig)
         sc.stop()
       }
       def convert(a:Int) : Tuple2[ImmutableBytesWritable,Put] = {
              val p = new Put(Bytes.toBytes(a))
              p.add(Bytes.toBytes("columnfamily"),
              Bytes.toBytes("col_1"), Bytes.toBytes(a))
              new Tuple2[ImmutableBytesWritable,Put](new ImmutableBytesWritable(a.toString.getBytes()), p);
         }
    }

注::代码没有经过测试，但是应该可以工作!

Answer 2

例如，下面的方法接受Int作为参数并返回Double

var toDouble: (Int) => Double = a => {
    a.toDouble
}

您可以使用toDouble(2)，它返回2.0

与将方法转换为函数字面量的方法相同，如下所示。

val convert: (Int) => Tuple2[ImmutableBytesWritable,Put] = a => {
              val p = new Put(Bytes.toBytes(a))
              p.add(Bytes.toBytes("columnfamily"),
              Bytes.toBytes("col_1"), Bytes.toBytes(a))
              new Tuple2[ImmutableBytesWritable,Put](new ImmutableBytesWritable(a.toString.getBytes()), p);
         }