How to access HBase in bulk by Spark
This article will explain in detail how Spark accesses HBase in batches. The editor thinks it is very practical, so I share it with you as a reference. I hope you can get something after reading this article.
FileAna.scala
Object FileAna {
/ / val conf: Configuration = HBaseConfiguration.create ()
Val hdfsPath = "hdfs://master:9000"
Val hdfs = FileSystem.get (new URI (hdfsPath), new Configuration ())
Def main (args: Array [String]) {
Val conf = new SparkConf (). SetAppName ("FileAna"). SetMaster ("spark://master:7077").
Set ("spark.driver.host", "192.168.1.127").
SetJars (List ("/ home/pang/woozoomws/spark-service.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/hbase-common-1.2.2.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/hbase-client-1.2.2.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/hbase-protocol-1.2.2.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/htrace-core-3.1.0-incubating.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/hbase-server-1.2.2.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/metrics-core-2.2.0.jar"))
Val sc = new SparkContext (conf)
Val rdd = sc.textFile ("hdfs://master:9000/woozoom/msgfile.txt")
Val rdd2 = rdd.map (x = > convertToHbase (anaMavlink (x)
Val hbaseConf = HBaseConfiguration.create ()
HbaseConf.addResource ("/ home/hadoop/software/hbase-1.2.2/conf/hbase-site.xml")
Val jobConf = new JobConf (hbaseConf, this.getClass)
JobConf.setOutputFormat (classof [TableOutputFormat])
JobConf.set (TableOutputFormat.OUTPUT_TABLE, "MissionItem")
Rdd2.saveAsHadoopDataset (jobConf)
Sc.stop ()
}
Def convertScanToString (scan: Scan) = {
Val proto = ProtobufUtil.toScan (scan)
Base64.encodeBytes (proto.toByteArray)
}
Def convertToHbase (msg: MAVLinkMessage) = {
Val p = new Put (Bytes.toBytes (UUID.randomUUID (). ToString ()
If (msg.isInstanceOf [msg _ mission_item]) {
Val missionItem = msg.asInstanceOf [msg _ mission_item]
P.addColumn (Bytes.toBytes ("data"), Bytes.toBytes ("x"), Bytes.toBytes (missionItem.x))
P.addColumn (Bytes.toBytes ("data"), Bytes.toBytes ("y"), Bytes.toBytes (missionItem.y))
P.addColumn (Bytes.toBytes ("data"), Bytes.toBytes ("z"), Bytes.toBytes (missionItem.z))
}
(new ImmutableBytesWritable, p)
}
Val anaMavlink = (str: String) = > {
Val bytes = ByteAndHex.hexStringToBytes (str)
QuickParser.parse (bytes) .unpack ()
}
}
ReadHBase.scala
Object ReadHBase {
/ / val conf: Configuration = HBaseConfiguration.create ()
Val hdfsPath = "hdfs://master:9000"
Val hdfs = FileSystem.get (new URI (hdfsPath), new Configuration ())
Def main (args: Array [String]) {
Val conf = new SparkConf (). SetAppName ("FileAna"). SetMaster ("spark://master:7077").
Set ("spark.driver.host", "192.168.1.127").
SetJars (List ("/ home/pang/woozoomws/spark-service.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/hbase-common-1.2.2.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/hbase-client-1.2.2.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/hbase-protocol-1.2.2.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/htrace-core-3.1.0-incubating.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/hbase-server-1.2.2.jar"
"/ home/pang/woozoomws/spark-service/lib/hbase/metrics-core-2.2.0.jar"))
Val sc = new SparkContext (conf)
Val hbaseConf = HBaseConfiguration.create ()
HbaseConf.addResource ("/ home/hadoop/software/hbase-1.2.2/conf/hbase-site.xml")
HbaseConf.set (TableInputFormat.INPUT_TABLE, "MissionItem")
Val scan = new Scan ()
HbaseConf.set (TableInputFormat.SCAN, convertScanToString (scan))
Val readRDD = sc.newAPIHadoopRDD (hbaseConf, classOf [TableInputFormat]
ClassOf [org.apache.hadoop.hbase.io.ImmutableBytesWritable]
ClassOf [org.apache.hadoop.hbase.client.Result])
Val count = readRDD.count ()
Println ("Mission Item Count:" + count)
Sc.stop ()
}
Def convertScanToString (scan: Scan) = {
Val proto = ProtobufUtil.toScan (scan)
Base64.encodeBytes (proto.toByteArray)
}
}
This is the end of the article on "how to access HBase in bulk by Spark". I hope the above content can be of some help to you, so that you can learn more knowledge. if you think the article is good, please share it for more people to see.
