package com.aspire;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.HTablePool;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.ipc.HBaseClient;
@SuppressWarnings({ "unused", "deprecation" })
public class Test{
public static Configuration conf = HBaseConfiguration.create();
static {
Configuration HBASE_CONFIG = new Configuration();
//HBASE_CONFIG.set("hbase.zookeeper.quorum","ugcserver3,ugcserver4,ugcserver5");
HBASE_CONFIG.set("hbase.zookeeper.property.clientPort", "2181");
conf = HBaseConfiguration.create(HBASE_CONFIG);
}
public static void creatTable(String tableName, String[] familys)
throws Exception {
HBaseAdmin admin = new HBaseAdmin(conf);
if (admin.tableExists(tableName)) {
System.out.println("table already exists!");
} else {
System.out.println("table not exists!");
HTableDescriptor tableDesc = new HTableDescriptor(tableName);
for (int i = 0; i < familys.length; i++) {
tableDesc.addFamily(new HColumnDescriptor(familys[i]));
}
admin.createTable(tableDesc);
System.out.println("create table " + tableName + " ok");
}
}
@SuppressWarnings("deprecation")
public static void main(String[] args) throws IOException {
if (args == null || args.length < 2) {
System.out.println("error: usage <file_name> <splist_flag>...");
System.exit(1);
}
String filename=args[0];
String splitflag=args[1];
System.out.println(filename+"-----"+splitflag);
String tableName = "dmuser";
String[] fa = { "basic", "features" };
String[] cols ={"name","BRAND","PROVINCE","CITY","ADDR","SEX","AGE","GROUP_NAME","CHILD","CAR","WAP_FEATURE_STR"
};
int a = 0;
List<Put> list = new ArrayList<Put>();
try {
Test.creatTable(tableName, fa);
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("開始插入資料");
long start = System.currentTimeMillis();
HTablePool pool = new HTablePool(conf,1000);
HTableInterface table = pool.getTable(tableName);
table.setWriteBufferSize(5*1024*1024);
table.setAutoFlush(false);
File file = new File(filename);
BufferedReader reader = null;
String lineString = null;
try {
reader = new BufferedReader(new FileReader(file));
while ((lineString = reader.readLine()) != null) {
a++;
String[] lines = lineString.split(splitflag);
Put put = new Put(lines[0].getBytes());
for (int i = 1; i < cols.length; i++) {
if (cols[i].equals("WAP_FEATURE_STR") || cols[i].equals("CXP_FEATURE_STR") ||cols[i].equals("KEYWORD_STR")) {
//continue;
put.add(fa[1].getBytes(), cols[i].getBytes(), lines[i].getBytes());
}else {
put.add(fa[0].getBytes(), cols[i].getBytes(),lines[i].getBytes());
}
list.add(put);
}
if (a % 10000 == 0) {
table.put(list);
list.clear();
table.flushCommits();
System.out.println("插入條數："+a);
}
}
pool.close();
reader.close();
table.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
long stop = System.currentTimeMillis();
System.out.println("插入資料共耗時：" + (stop - start) * 1.0 / 1000 + "s");
}
}
以上代碼拋的例外為：
14/03/18 10:17:23 WARN client.HConnectionManager$HConnectionImplementation: Failed all from region=dmuser,,1395108680648.fb4da06ffd652f7435d927bdbfc6b32c., hostname=BJ-YZ-103R-63-32, port=60020
java.util.concurrent.ExecutionException: java.io.IOException: Call to BJ-YZ-103R-63-32/192.168.63.32:60020 failed on local exception: java.io.EOFException
at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:252)
at java.util.concurrent.FutureTask.get(FutureTask.java:111)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatchCallback(HConnectionManager.java:1708)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatch(HConnectionManager.java:1560)
at org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:1031)
at org.apache.hadoop.hbase.client.HTable.doPut(HTable.java:887)
at org.apache.hadoop.hbase.client.HTable.put(HTable.java:875)
at org.apache.hadoop.hbase.client.HTablePool$PooledHTable.put(HTablePool.java:400)
at com.aspire.Test.main(Test.java:123)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at org.apache.hadoop.util.RunJar.main(RunJar.java:208)
Caused by: java.io.IOException: Call to BJ-YZ-103R-63-32/192.168.63.32:60020 failed on local exception: java.io.EOFException
at org.apache.hadoop.hbase.ipc.HBaseClient.wrapException(HBaseClient.java:1047)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:1016)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:87)
at $Proxy8.multi(Unknown Source)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3$1.call(HConnectionManager.java:1537)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3$1.call(HConnectionManager.java:1535)
at org.apache.hadoop.hbase.client.ServerCallable.withoutRetries(ServerCallable.java:229)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3.call(HConnectionManager.java:1544)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3.call(HConnectionManager.java:1532)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
Caused by: java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:392)
at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.receiveResponse(HBaseClient.java:654)
at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:588)
14/03/18 10:17:24 WARN client.HConnectionManager$HConnectionImplementation: Failed all from region=dm_user_library_wide,,1395108680648.fb4da06ffd652f7435d927bdbfc6b32c., hostname=BJ-YZ-103R-63-32, port=60020
java.util.concurrent.ExecutionException: java.net.ConnectException: 拒絕連接
請大俠們指點迷津，
uj5u.com熱心網友回復：
似乎是沒有聯通啊uj5u.com熱心網友回復：
連通了。資料查到一部分才拋的例外轉載請註明出處，本文鏈接：https://www.uj5u.com/qita/110012.html
標籤：云存儲
上一篇：競品分析
下一篇：關于外網IP地址不變實作冗余備份