极客观点
项目管理
HarmonyOS
idea客户端连接hbase数据库间歇性一会可以查到数据，一会查不到数据。
zookeeper地址：hbase.zookeeper.quorum = 172.32.6.51,172.32.6.52,172.32.6.53
C:WindowsSystem32driversetchosts 路径下的host的文件中映射了对应的ip
172.32.6.51 hpt1
172.32.6.52 hpt2
172.32.6.53 hpt3
连接数据库代码：
@Slf4j
@Component
public class JavaHBaseUtil {
/**
* hbase配置信息
*/
private Configuration conf;
/**
* 本地路径
*/
@Value("${hbase.path}")
private String path;
/**
* hbase登录用户
*/
@Value("${hbase.login.user}")
public String user;
/**
* hbase密钥文件
*/
@Value("${hbase.login.keytab}")
public String keyTab;
/**
* zookeeper地址
*/
@Value("${hbase.zookeeper.quorum}")
private String hbaseZookeeperQuorum;
private Connection connection;
@Autowired
public void initConfig() throws UnsupportedEncodingException {
try {
URL resource = Thread.currentThread().getContextClassLoader().getResource("");
if (StringUtils.isBlank(path)) {
path = URLDecoder.decode(resource.getPath(), "UTF-8");
}
System.setProperty("java.security.krb5.conf", path + "krb5.conf");
// 操作系统
String osName = System.getProperty("os.name");
if (osName.toLowerCase().startsWith("windows")) {
System.setProperty("hadoop.home.dir", "D:\\hadoop\\hadoop-common-bin");
}
conf = HBaseConfiguration.create();
conf.set(HConstants.ZOOKEEPER_QUORUM, hbaseZookeeperQuorum);
conf.set("hadoop.security.authentication", "Kerberos");
conf.addResource(new Path(path + "core-site.xml"));
conf.addResource(new Path(path + "hbase-site.xml"));
conf.addResource(new Path(path + "hdfs-site.xml"));
conf.setInt("hbase.hconnection.threads.max", 512);
conf.setInt("hbase.hconnection.threads.core", 64);
conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName());
log.info("zk:{}", conf.get(HConstants.ZOOKEEPER_QUORUM));
//建立HBase连接
reloadConnection();
}catch (Exception e){
log.error("hbase加载失败，异常原因:",e);
}
}
/**
* 重新加载连接
*/
private void reloadConnection() {
UserGroupInformation.setConfiguration(conf);
try {
UserGroupInformation.loginUserFromKeytab(user, path + "/" + keyTab);
connection = ConnectionFactory.createConnection(conf);
} catch (IOException e) {
log.error("初始化加载失败，原因:", e);
}
}
/**
* 获取hbase表信息
*
* @param var1 表名
* @return 表信息
*/
private Table getTable(TableName var1) throws IOException {
try {
if (connection != null && !connection.isClosed()) {
return connection.getTable(var1);
}
} catch (IOException e) {
log.error("连接失败，进行重试");
}
reloadConnection();
return connection.getTable(var1);
}
/**
* @param tableName 表名
* @param rowName 行
* @param familyName info
* @param mapper 字段属性
* @param <T> 泛型
* @return 泛型对象
*/
public <T> T get(String tableName, String rowName, String familyName, RowMapper<T> mapper) {
Table table = null;
try {
Get get = new Get(Bytes.toBytes(rowName));
if (familyName != null) {
get.addFamily(Bytes.toBytes(familyName));
}
table = getTable(TableName.valueOf(tableName));
Result result = table.get(get);
return mapper.mapRow(result, 0);
} catch (Exception e) {
log.error("获取HBase数据失败:", e);
throw new OpenApiException(OpenApiErrorCode.MSG_OUT_0001.getErrorCode(), ErrorReportMsgEnum.MSG_OUT_0001.getMsg());
} finally {
if (table != null) {
try {
table.close();
} catch (IOException e) {
log.error("(单笔)-调用HBase查询,关闭表连接异常:{}", e.getMessage());
}
}
}
}
}
异常日志如下：
[2017-09-28 15:18:59,080] [INFO ] com.xinyan.credit.manager.HbaseManager 35 -- [6714c032-5a70-4158-9b6e-f9cd1278f645] 查询大数据开始，请求参数：ReqInfo(name=武*, idCard=null, idCard2Md5=0000007b7ea76269551b30fdc5644b94)
[2017-09-28 15:20:48,084] [INFO ] o.a.h.hbase.client.RpcRetryingCaller 147 -- [6714c032-5a70-4158-9b6e-f9cd1278f645] Call exception, tries=10, retries=35, started=108979 ms ago, cancelled=false, msg=10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=hpt1/172.32.6.51:60020] row '0000007b7ea76269551b30fdc5644b94&武*' on table 'CREDIT_MINING:Md5_dir_4' at region=CREDIT_MINING:Md5_dir_4,,1506320016641.a08703a21cc6f8bc6ca75d3445306b73., hostname=hpt1,60020,1505697770526, seqNum=473
[2017-09-28 15:21:08,237] [INFO ] o.a.h.hbase.client.RpcRetryingCaller 147 -- [6714c032-5a70-4158-9b6e-f9cd1278f645] Call exception, tries=11, retries=35, started=129132 ms ago, cancelled=false, msg=10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=hpt1/172.32.6.51:60020] row '0000007b7ea76269551b30fdc5644b94&武*' on table 'CREDIT_MINING:Md5_dir_4' at region=CREDIT_MINING:Md5_dir_4,,1506320016641.a08703a21cc6f8bc6ca75d3445306b73., hostname=hpt1,60020,1505697770526, seqNum=473
[2017-09-28 15:21:38,455] [INFO ] o.a.h.hbase.client.RpcRetryingCaller 147 -- [6714c032-5a70-4158-9b6e-f9cd1278f645] Call exception, tries=12, retries=35, started=159350 ms ago, cancelled=false, msg=10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=hpt1/172.32.6.51:60020] row '0000007b7ea76269551b30fdc5644b94&武*' on table 'CREDIT_MINING:Md5_dir_4' at region=CREDIT_MINING:Md5_dir_4,,1506320016641.a08703a21cc6f8bc6ca75d3445306b73., hostname=hpt1,60020,1505697770526, seqNum=473
[2017-09-28 15:22:08,623] [INFO ] o.a.h.hbase.client.RpcRetryingCaller 147 -- [6714c032-5a70-4158-9b6e-f9cd1278f645] Call exception, tries=13, retries=35, started=189518 ms ago, cancelled=false, msg=10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=hpt1/172.32.6.51:60020] row '0000007b7ea76269551b30fdc5644b94&武*' on table 'CREDIT_MINING:Md5_dir_4' at region=CREDIT_MINING:Md5_dir_4,,1506320016641.a08703a21cc6f8bc6ca75d3445306b73., hostname=hpt1,60020,1505697770526, seqNum=473
[2017-09-28 15:22:38,702] [INFO ] o.a.h.hbase.client.RpcRetryingCaller 147 -- [6714c032-5a70-4158-9b6e-f9cd1278f645] Call exception, tries=14, retries=35, started=219597 ms ago, cancelled=false, msg=10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=hpt1/172.32.6.51:60020] row '0000007b7ea76269551b30fdc5644b94&武*' on table 'CREDIT_MINING:Md5_dir_4' at region=CREDIT_MINING:Md5_dir_4,,1506320016641.a08703a21cc6f8bc6ca75d3445306b73., hostname=hpt1,60020,1505697770526, seqNum=473
Thu Sep 28 14:43:32 CST 2017, RpcRetryingCaller{globalStartTime=1506580237677, pause=100, retries=35}, org.apache.hadoop.net.ConnectTimeoutException: 10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=hpt1/172.32.6.51:60020]
at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159)
at org.apache.hadoop.hbase.client.HTable.get(HTable.java:864)
at org.apache.hadoop.hbase.client.HTable.get(HTable.java:830)
at com.xinyan.credit.util.JavaHBaseUtil.get(JavaHBaseUtil.java:151)
at com.xinyan.credit.manager.HbaseManager.queryARadarHBase(HbaseManager.java:40)
at com.xinyan.credit.manager.BatchRadarManager.asyncProcess(BatchRadarManager.java:264)
at com.xinyan.credit.manager.BatchRadarManager.lambda$queryRadarReport4Batch$0(BatchRadarManager.java:142)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.hadoop.net.ConnectTimeoutException: 10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=hpt1/172.32.6.51:60020]
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:534)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:495)
at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:416)
at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:722)
at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:909)
at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:873)
at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1244)
at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227)
at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.get(ClientProtos.java:35372)
at org.apache.hadoop.hbase.client.HTable$3.call(HTable.java:856)
at org.apache.hadoop.hbase.client.HTable$3.call(HTable.java:847)
at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:137)
... 11 common frames omitted
[2017-09-28 14:43:32,081] [ERROR] com.xinyan.credit.manager.HbaseManager 47 -- [f1c4aa91-1d2d-4905-be0f-9a09552a6d82] 查询异常:
com.xinyan.credit.excepiton.OpenApiException: 系统繁忙，请稍后再试
at com.xinyan.credit.util.JavaHBaseUtil.get(JavaHBaseUtil.java:155)
at com.xinyan.credit.manager.HbaseManager.queryARadarHBase(HbaseManager.java:40)
at com.xinyan.credit.manager.BatchRadarManager.asyncProcess(BatchRadarManager.java:264)
at com.xinyan.credit.manager.BatchRadarManager.lambda$queryRadarReport4Batch$0(BatchRadarManager.java:142)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
[2017-09-28 15:05:42,956] [ERROR] com.xinyan.credit.util.JavaHBaseUtil 154 -- [9769a4c0-4362-4844-b1fe-908acd6628e1] 获取HBase数据失败:
org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=35, exceptions:
java
看错误是连接超时了，你确认172.32.6.51，这台上的zk服务是好的。既然你有3台zk是否做了集群配置。如果没有请先把zk的集群配置做好，zk会自动剔除出坏的zk服务。