大資料技術_ linux系統 之hadoop 3.1.2版本的安裝

環境版本： hadoop 3.1.2版本、CentOS7.8

1、準備階段(hduser)

使用hduser賬號登陸
上傳檔案，

解壓檔案： tar -zxvf ~/hadoop-3.1.2.tar.gz
修改檔案名為hadoop： mv hadoop-3.1.2 hadoop

備注：當我們不知道解壓后的檔案名是什么時 我們可以使用 ls 命令列出當前目錄的檔案名或檔案夾名

2、配置hadoop環境變數(hduser)

編譯 sudo vi /etc/profile 檔案，在最后加入以下內容：

export HADOOP_HOME=/home/hduser/hadoop
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
export JAVA_LIBRARY_PATH=/home/hduser/hadoop/lib/native

3、集群配置(hduser)

3.0、進入配置存放目錄

3.1、core-site.xml 配置

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://node1:9000</value>
  </property>
  <property>
<name>io.file.buffer.size</name>
<value>131072</value>
  </property>
  <property>
<name>hadoop.tmp.dir</name>
<value>file:/home/hduser/hadoop/tmp</value>
  </property>
</configuration>

3.2、hadoop-env.sh配置
在最頂上，
多加一行代碼

export JAVA_HOME=/usr/java/jdk

3.3、hdfs-site.xml 配置

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>file:/home/hduser/hadoop/dfs/name</value>
  </property>
  <property>
    <name>dfs.blocksize</name>
    <value>268435456</value>
  </property>
  <property>
    <name>dfs.namenode.handler.count</name>
    <value>100</value>
  </property>
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>file:/home/hduser/hadoop/dfs/data</value>
  </property>
  <property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>node1:50090</value>
  </property>
  <property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
  </property>
  <property>
    <name>dfs.replication</name>
    <value>2</value>
  </property>
</configuration>

3.4、mapred-site.xml 配置

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
  <property>
    <name>mapreduce.jobhistory.address</name>
    <value>node1:10020</value>
  </property>
  <property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value>node1:19888</value>
  </property>
<property>
  <name>mapreduce.map.memory.mb</name>
  <value>4096</value>
 </property>
 <property>
  <name>mapreduce.reduce.memory.mb</name>
  <value>8192</value>
 </property>
 <property>
  <name>mapreduce.map.java.opts</name>
  <value>-Xmx3072m</value>
 </property>
 <property>
  <name>mapreduce.reduce.java.opts</name>
  <value>-Xmx6144m </value>
 </property>
</configuration>

3.5、yarn-site.xml 配置

<?xml version="1.0"?>
<configuration>
  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>node1</value>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.address</name>
    <value>node1:8088</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
  </property>
  <property>
    <name>yarn.resourcemanager.address</name>
    <value>node1:8032</value>
  </property>
  <property>
    <name>yarn.resourcemanager.admin.address</name>
    <value>node1:8033</value>
  </property>
  <property>
    <name>yarn.resourcemanager.scheduler.address</name>
    <value>node1:8030</value>
  </property>
  <property>
    <name>yarn.resourcemanager.resource-tracker.address</name>
    <value>node1:8035</value>
  </property>
  <property>
  <name>yarn.application.classpath</name>
      <value>
   	   /hadoop/hduser/etc/hadoop,
           /home/hduser/hadoop/share/hadoop/common/*,
           /home/hduser/hadoop/share/hadoop/common/lib/*,
           /home/hduser/hadoop/share/hadoop/mapreduce/*,
           /home/hduser/hadoop/share/hadoop/mapreduce/lib/*,
           /home/hduser/hadoop/share/hadoop/hdfs/*,
           /home/hduser/hadoop/share/hadoop/hdfs/lib/*,
           /home/hduser/hadoop/share/hadoop/yarn/*,
           /home/hduser/hadoop/share/hadoop/yarn/lib/*
      </value>
  </property>
  <property>
     <name>mapreduce.application.classpath</name>
     <value>
   	   /hadoop/hduser/etc/hadoop,
           /home/hduser/hadoop/share/hadoop/common/*,
           /home/hduser/hadoop/share/hadoop/common/lib/*,
           /home/hduser/hadoop/share/hadoop/mapreduce/*,
           /home/hduser/hadoop/share/hadoop/mapreduce/lib/*,
           /home/hduser/hadoop/share/hadoop/hdfs/*,
           /home/hduser/hadoop/share/hadoop/hdfs/lib/*,
           /home/hduser/hadoop/share/hadoop/yarn/*,
           /home/hduser/hadoop/share/hadoop/yarn/lib/*
   </value>
 </property>
</configuration>

3.6、yarn-env.sh配置
在最頂上，
多加一行代碼

export JAVA_HOME=/usr/java/jdk

3.7、workers 配置