1.Hadoop下載好之后解壓到相應目錄:
為了方便管理,我們使用mv把名稱更改為hadoop
[root@master ~]# tar -zxvf hadoop-2.7.1.tar.gz -C /usr/local/src/
[root@master ~]# cd /usr/local/src/
[root@master src]# ls
hadoop-2.7.1 java zookeeper
[root@master src]# mv hadoop-2.7.1/ hadoop
[root@master src]# ls
hadoop java zookeeper
2.配置Hadoop的環境變數
[root@master ~]# vi /etc/profile
#hadoop
export HADOOP_HOME=/usr/local/src/hadoop
export HADOOP_PREFIX=$HADOOP_HOME
export HADOOP_INSTALL=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/bin/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_INSTALL/lib:$HADOOP_COMMON_LIB_NATIVE_DIR"
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
//生效環境變數
[root@master ~]# source /etc/profile
3.配置hadoop-env.sh組態檔
進入到hadoop/etc/hadoop下面
[root@master ~]# cd /usr/local/src/
[root@master src]# cd hadoop/etc/hadoop/
[root@master hadoop]# ls
capacity-scheduler.xml hadoop-env.sh httpfs-env.sh kms-env.sh mapred-env.sh ssl-server.xml.example
configuration.xsl hadoop-metrics2.properties httpfs-log4j.properties kms-log4j.properties mapred-queues.xml.template yarn-env.cmd
container-executor.cfg hadoop-metrics.properties httpfs-signature.secret kms-site.xml mapred-site.xml.template yarn-env.sh
core-site.xml hadoop-policy.xml httpfs-site.xml log4j.properties slaves yarn-site.xml
hadoop-env.cmd hdfs-site.xml kms-acls.xml mapred-env.cmd ssl-client.xml.example
[root@master hadoop]# vi hadoop-env.sh
//將Java的路徑修改為自己的絕對路徑
# The java implementation to use.
export JAVA_HOME=/usr/local/src/java
4.創建namenode,datanode,journalnode等存放資料的目錄
[root@master hadoop]# pwd
/usr/local/src/hadoop
[root@master hadoop]# mkdir -p tmp/hdfs/nn
[root@master hadoop]# mkdir -p tmp/hdfs/dn
[root@master hadoop]# mkdir -p tmp/hdfs/jn
[root@master hadoop]# mkdir -p tmp/logs
5.配置core-site.xml檔案
core-site.xml檔案是Hadoop 核心配置,例如HDFS、MapReduce和YARN常用的I/O設定等
[root@master hadoop]# pwd
/usr/local/src/hadoop/etc/hadoop
[root@master hadoop]# vi core-site.xml
//檔案core-site.xml的具體配置如下:
<?xml version="1.0" encoding="UTF-8"?>
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<!--指定hdfs的nameservice為mycluster-->
<property>
<name>fs.defaultFS</name>
<value>hdfs://mycluster</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/usr/local/src/hadoop/tmp</value>
</property>
<!--指定zookeeper的地址-->
<property>
<name>ha.zookeeper.quorum</name>
<value>master:2181,slave1:2181,slave2:2181</value>
</property>
<!--hadoop鏈接zookeeper的超時時長設定-->
<property>
<name>ha.zookeeper.session-timeout.ms</name>
<value>30000</value>
<description>ms</description>
</property>
<property>
<name>fs.trash.interval</name>
<value>1440</value>
</property>
</configuration>
6.配置hadoop的hdfs-site.xml檔案
hdfs-site.xml檔案是Hadoop守護行程的配置項,包括namenode、輔助namenode(即SecondNameNode)和datanode等
[root@master hadoop]# vi hdfs-site.xml
<configuration>
<!-- journalnode 集群之間通信的超時時間 -->
<property>
<name>dfs.qjournal.start-segment.timeout.ms</name>
<value>60000</value>
</property>
<property>
<name>dfs.nameservices</name>
<value>mycluster</value>
</property>
<!-- mycluster 下面有兩個 NameNode,分別是 master,slave1 -->
<property>
<name>dfs.ha.namenodes.mycluster</name>
<value>master,slave1</value>
</property>
<!-- master 的 RPC 通信地址 -->
<property>
<name>dfs.namenode.rpc-address.mycluster.master</name>
<value>master:8020</value>
</property>
<!-- slave1 的 RPC 通信地址 -->
<property>
<name>dfs.namenode.rpc-address.mycluster.slave1</name>
<value>slave1:8020</value>
</property>
<!-- master 的 http 通信地址 -->
<property>
<name>dfs.namenode.http-address.mycluster.master</name>
<value>master:50070</value>
</property>
<!-- slave1 的 http 通信地址 -->
<property>
<name>dfs.namenode.http-address.mycluster.slave1</name>
<value>slave1:50070</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://master:8485;slave1:8485;slave2:8485/mycluster</value>
</property>
<!-- 配置失敗自動切換實作方式 -->
<property>
<name>dfs.client.failover.proxy.provider.mycluster</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!-- 配置隔離機制方法,多個機制用換行分割,即每個機制暫用一行 -->
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<property>
<name>dfs.permissions.enabled</name>
<value>false</value>
</property>
<property>
<name>dfs.support.append</name>
<value>true</value>
</property>
<!-- 使用 sshfence 隔離機制時需要 ssh 免登陸 -->
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<!-- 指定副本數 -->
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>slave1:8020</value>
</property>
<!-- master 的 http 通信地址 -->
<property>
<name>dfs.namenode.http-address.mycluster.master</name>
<value>master:50070</value>
</property>
<!-- slave1 的 http 通信地址 -->
<property>
<name>dfs.namenode.http-address.mycluster.slave1</name>
<value>slave1:50070</value>
</property>
<!-- 指定 NameNode 的 edits 元資料的共享存盤位置,也就是 JournalNode 串列該 url 的配置格式:qjournal://host1:port1;host2:port2;host3:port3/journalIdjournalId 推薦>使用 nameservice,默認埠號是:8485 -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://master:8485;slave1:8485;slave2:8485/mycluster</value>
</property>
<!-- 配置失敗自動切換實作方式 -->
<property>
<name>dfs.client.failover.proxy.provider.mycluster</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!-- 配置隔離機制方法,多個機制用換行分割,即每個機制暫用一行 -->
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<property>
<name>dfs.permissions.enabled</name>
<value>false</value>
</property>
<property>
<name>dfs.support.append</name>
<value>true</value>
</property>
<!-- 使用 sshfence 隔離機制時需要 ssh 免登陸 -->
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<!-- 指定副本數 -->
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/usr/local/src/hadoop/tmp/hdfs/nn</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/usr/local/src/hadoop/tmp/hdfs/dn</value>
</property>
<!-- 指定 JournalNode 在本地磁盤存放資料的位置 -->
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/usr/local/src/hadoop/tmp/hdfs/jn</value>
</property>
<!-- 開啟 NameNode 失敗自動切換 -->
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<!-- 啟用 webhdfs -->
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
<!-- 配置 sshfence 隔離機制超時時間 -->
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
</property>
<property>
<name>ha.failover-controller.cli-check.rpc-timeout.ms</name>
<value>60000</value>
</property>
</configuration>
7.配置hadoop的mapred-site.xml檔案( MapReduce守護行程的配置項,包括作業歷史服務器 )
[root@master hadoop]# cp mapred-site.xml.template mapred-site.xml
[root@master hadoop]# vi mapred-site.xml
<configuration>
<!-- 指定 mapreduce 框架為 yarn 方式 -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<!-- 指定 mapreduce jobhistory 地址 -->
<property>
<name>mapreduce.jobhistory.address</name>
<value>master:10020</value>
</property>
<!-- 任務歷史服務器的 web 地址 -->
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>master:19888</value>
</property>
</configuration>
8.配置Hadoop的yarn-site.xml檔案( YARN守護行程的配置項,包括資源管理器、web應用代理服務器和節點管理器 )
[root@master hadoop]# vi yarn-site.xml
<!-- Site specific YARN configuration properties -->
<!-- 開啟 RM 高可用 -->
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<!-- 指定 RM 的 cluster id -->
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>yrc</value>
</property>
<!-- 指定 RM 的名字 -->
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<!-- 分別指定 RM 的地址 -->
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>master</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>slave1</value>
</property>
<!-- 指定 zk 集群地址 -->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>master:2181,slave1:2181,slave2:2181</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<property>
<name>yarn.log-aggregation.retain-seconds</name>
<value>86400</value>
</property>
<!-- 啟用自動恢復 -->
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
<!-- 制定 resourcemanager 的狀態資訊存盤在 zookeeper 集群上 -->
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
</configuration>
9.配置Hadoop的slaves組態檔(控制我們的從節點在哪里 datanode nodemanager在哪些機器上)
[root@master hadoop]# vi slaves
master
slave1
slave2
10.分發檔案到從節點
(1)分發Hadoop檔案
//分發到slave1節點
[root@master ~]# scp -r /usr/local/src/hadoop/ root@slave1:/usr/local/src/
//分發到slave2節點
[root@master ~]# scp -r /usr/local/src/hadoop/ root@slave2:/usr/local/src/
(2)分發環境變數
//分發到slave1節點
[root@master ~]# scp -r /etc/profile root@slave1:/etc/
//分發到slave2節點
[root@master ~]# scp -r /etc/profile root@slave2:/etc/
11.修改所有者和所有者組
[root@master ~]# chown hadoop:hadoop /usr/local/src/hadoop/
[root@slave1 ~]# chown hadoop:hadoop /usr/local/src/hadoop/
[root@slave2 ~]# chown hadoop:hadoop /usr/local/src/hadoop/
2.生效環境變數
[root@master ~]# su hadoop
[hadoop@master root]$ cd
[hadoop@master ~]$ source /etc/profile
[root@slave1 ]# su hadoop
[hadoop@slave1 root]$ cd
[hadoop@slave1 ~]$ source /etc/profile
[root@slave2 ~]# su hadoop
[hadoop@slave2 root]$ cd
[hadoop@slave2 ~]$ source /etc/profile
ok,到這里就完成了Hadoop HA高可用集群的配置
[hadoop@master ~]$ hadoop version
Hadoop 2.7.1
Subversion https://git-wip-us.apache.org/repos/asf/hadoop.git -r 15ecc87ccf4a0228f35af08fc56de536e6ce657a
Compiled by jenkins on 2015-06-29T06:04Z
Compiled with protoc 2.5.0
From source with checksum fc0a1a23fc1868e4d5ee7fa2b28a58a
This command was run using /usr/local/src/hadoop/share/hadoop/common/hadoop-common-2.7.1.jar
下一章講解Hadoop HA集群的啟動與測驗
轉載請註明出處,本文鏈接:https://www.uj5u.com/qita/379430.html
標籤:其他
上一篇:Sonarqube配置問題
下一篇:大資料之Spark
