最近開始了spark的學習，在spark讀取hbase中的資料這一階段，卻出現了例外，百度了很久，都無法搞定，我用的IDE是eclipse，scala的版本是2.11.6,hbase的版本是1.2.6,spark的版本是2.1.0,hadoop的版本是2.7.3（測驗用），zookeeper我沒有用自帶的，用的是3.4.6的版本的，單獨用hbase-shell曾刪改查資料都沒有問題。相應的jar包我也從hbase中的jar添加到了spark的工程當中去。我的機子是偽分布式的，只有一臺電腦，hadoop和spark都是偽分布的。代碼如下：
object Test {
def main(args: Array[String]) {
val conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "192.168.0.102")
conf.set("hbase.zookeeper.property.clientPort", "2181")
val sc = new SparkContext(new SparkConf().setMaster("local").setAppName("Hbase-test"))
conf.set(TableInputFormat.INPUT_TABLE, "student")
val stuRDD = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat], classOf[org.apache.hadoop.hbase.io.ImmutableBytes
Writable], classOf[org.apache.hadoop.hbase.client.Result])
val count = stuRDD.count()
println("Students RDD Count:"+count)
stuRDD.cache()
stuRDD.foreach({case(_,result) => val key = Bytes.toString(result.getRow)
val name = Bytes.toString(result.getValue("info".getBytes, "name".getBytes))
val gender = Bytes.toString(result.getValue("info".getBytes,"gender".getBytes))
val age = Bytes.toString(result.getValue("info".getBytes, "age".getBytes))
println("Row Key:"+key+" Name:"+name+" Gender:"+gender+" Age:"+age)
})
}
}
相關例外如下：
省略一部分無關的，起初我覺得是我zookeeper配置出了問題，我嘗試用了hbase自帶的zookeeper，錯誤還是一樣。
17/11/03 14:30:29 INFO ZooKeeper: Initiating client connection, connectString=192.168.0.102:2181 sessionTimeout=90000 watcher=hconnection-0x5c089b2f0x0, quorum=192.168.0.102:2181, baseZNode=/hbase
17/11/03 14:30:29 INFO ClientCnxn: Opening socket connection to server bigdata3/192.168.0.102:2181. Will not attempt to authenticate using SASL (unknown error)
17/11/03 14:30:29 INFO ClientCnxn: Socket connection established to bigdata3/192.168.0.102:2181, initiating session
17/11/03 14:30:29 INFO ClientCnxn: Session establishment complete on server bigdata3/192.168.0.102:2181, sessionid = 0x15f807724d60007, negotiated timeout = 40000
17/11/03 14:30:29 INFO RegionSizeCalculator: Calculating region sizes for table "student".
17/11/03 14:31:07 INFO RpcRetryingCaller: Call exception, tries=10, retries=35, started=38458 ms ago, cancelled=false, msg=row 'student,,00000000000000' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=bigdata3,16201,1509688846997, seqNum=0
17/11/03 14:31:17 INFO RpcRetryingCaller: Call exception, tries=11, retries=35, started=48531 ms ago, cancelled=false, msg=row 'student,,00000000000000' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=bigdata3,16201,1509688846997, seqNum=0
17/11/03 14:31:17 INFO ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x15f807724d60007
17/11/03 14:31:17 INFO ZooKeeper: Session: 0x15f807724d60007 closed
17/11/03 14:31:17 INFO ClientCnxn: EventThread shut down
Exception in thread "main" org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions:
Fri Nov 03 14:31:17 CST 2017, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=68712: row 'student,,00000000000000' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=bigdata3,16201,1509688846997, seqNum=0
at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:276)
at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:210)
at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60)
at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:210)
at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:327)
at org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:302)
at org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:167)
at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:162)
at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:797)
at org.apache.hadoop.hbase.client.MetaScanner.metaScan(MetaScanner.java:193)
at org.apache.hadoop.hbase.client.MetaScanner.metaScan(MetaScanner.java:89)
at org.apache.hadoop.hbase.client.MetaScanner.allTableRegions(MetaScanner.java:324)
at org.apache.hadoop.hbase.client.HRegionLocator.getAllRegionLocations(HRegionLocator.java:89)
at org.apache.hadoop.hbase.util.RegionSizeCalculator.init(RegionSizeCalculator.java:94)
at org.apache.hadoop.hbase.util.RegionSizeCalculator.<init>(RegionSizeCalculator.java:81)
at org.apache.hadoop.hbase.mapreduce.TableInputFormatBase.getSplits(TableInputFormatBase.java:256)
at org.apache.hadoop.hbase.mapreduce.TableInputFormat.getSplits(TableInputFormat.java:239)
at org.apache.spark.rdd.NewHadoopRDD.getPartitions(NewHadoopRDD.scala:125)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:250)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958)
at org.apache.spark.rdd.RDD.count(RDD.scala:1157)
at MySecond.package1.Test$.main(Test.scala:22)
at MySecond.package1.Test.main(Test.scala)
Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=68712: row 'student,,00000000000000' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=bigdata3,16201,1509688846997, seqNum=0
at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:169)
at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:65)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.io.IOException: com.google.protobuf.ServiceException: java.lang.NoClassDefFoundError: com/yammer/metrics/core/Gauge
at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:332)
at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:408)
at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:204)
at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:65)
at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:210)
at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:364)
at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:338)
at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:136)
... 4 more
Caused by: com.google.protobuf.ServiceException: java.lang.NoClassDefFoundError: com/yammer/metrics/core/Gauge
at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:240)
at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094)
at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:400)
... 10 more
Caused by: java.lang.NoClassDefFoundError: com/yammer/metrics/core/Gauge
at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:225)
... 13 more
Caused by: java.lang.ClassNotFoundException: com.yammer.metrics.core.Gauge
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 14 more
17/11/03 14:31:17 INFO SparkContext: Invoking stop() from shutdown hook
17/11/03 14:31:17 INFO SparkUI: Stopped Spark web UI at http://192.168.0.102:4040
17/11/03 14:31:17 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
17/11/03 14:31:17 INFO MemoryStore: MemoryStore cleared
17/11/03 14:31:17 INFO BlockManager: BlockManager stopped
17/11/03 14:31:17 INFO BlockManagerMaster: BlockManagerMaster stopped
17/11/03 14:31:17 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
17/11/03 14:31:17 INFO SparkContext: Successfully stopped SparkContext
17/11/03 14:31:17 INFO ShutdownHookManager: Shutdown hook called
17/11/03 14:31:17 INFO ShutdownHookManager: Deleting directory /tmp/spark-a7296cf1-3122-4cfa-9438-1fbb775c3a48
uj5u.com熱心網友回復：
我也報這個錯，請問博主解決了嗎uj5u.com熱心網友回復：
把zookeeper最大客戶端連接數maxClientCnxns調大60=>300（默認60，改成300）試試；如果還不行，zk的最大連接等待時長也調大，應該就可以了。問題應該是讀取超時導致的轉載請註明出處，本文鏈接：https://www.uj5u.com/qita/56391.html
標籤：Spark