Revert "Merge pull request #596 from esjewett/master" because the
dependency on hbase introduces netty-3.2.2 which conflicts with netty-3.5.3 already in Spark. This caused multiple test failures. This reverts commit0f1b7a06e1
, reversing changes made toaacca1b8a8
.
This commit is contained in:
parent
42bbe89f04
commit
012c9e5ab0
|
@ -1,35 +0,0 @@
|
||||||
package spark.examples
|
|
||||||
|
|
||||||
import spark._
|
|
||||||
import spark.rdd.NewHadoopRDD
|
|
||||||
import org.apache.hadoop.hbase.{HBaseConfiguration, HTableDescriptor}
|
|
||||||
import org.apache.hadoop.hbase.client.HBaseAdmin
|
|
||||||
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
|
|
||||||
|
|
||||||
object HBaseTest {
|
|
||||||
def main(args: Array[String]) {
|
|
||||||
val sc = new SparkContext(args(0), "HBaseTest",
|
|
||||||
System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR")))
|
|
||||||
|
|
||||||
val conf = HBaseConfiguration.create()
|
|
||||||
|
|
||||||
// Other options for configuring scan behavior are available. More information available at
|
|
||||||
// http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/TableInputFormat.html
|
|
||||||
conf.set(TableInputFormat.INPUT_TABLE, args(1))
|
|
||||||
|
|
||||||
// Initialize hBase table if necessary
|
|
||||||
val admin = new HBaseAdmin(conf)
|
|
||||||
if(!admin.isTableAvailable(args(1))) {
|
|
||||||
val tableDesc = new HTableDescriptor(args(1))
|
|
||||||
admin.createTable(tableDesc)
|
|
||||||
}
|
|
||||||
|
|
||||||
val hBaseRDD = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat],
|
|
||||||
classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable],
|
|
||||||
classOf[org.apache.hadoop.hbase.client.Result])
|
|
||||||
|
|
||||||
hBaseRDD.count()
|
|
||||||
|
|
||||||
System.exit(0)
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -200,11 +200,7 @@ object SparkBuild extends Build {
|
||||||
|
|
||||||
def examplesSettings = sharedSettings ++ Seq(
|
def examplesSettings = sharedSettings ++ Seq(
|
||||||
name := "spark-examples",
|
name := "spark-examples",
|
||||||
resolvers ++= Seq("Apache HBase" at "https://repository.apache.org/content/repositories/releases"),
|
libraryDependencies ++= Seq("com.twitter" % "algebird-core_2.9.2" % "0.1.11")
|
||||||
libraryDependencies ++= Seq(
|
|
||||||
"com.twitter" % "algebird-core_2.9.2" % "0.1.11",
|
|
||||||
"org.apache.hbase" % "hbase" % "0.94.6"
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def bagelSettings = sharedSettings ++ Seq(name := "spark-bagel")
|
def bagelSettings = sharedSettings ++ Seq(name := "spark-bagel")
|
||||||
|
|
Loading…
Reference in a new issue