码上编程

外观

hadoop搭建yarn ​

bash
marped-env.sh 添加环境变量
export JAVA_HOME=/export/server/jdk
export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
export HADOOP_MAPRED_ROOT_LOGGER=info,RFA
xml
<!--mapred-site.xml-->
<properties>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
        <description>description</description>
    </property>
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>master:10020</value>
        <description>description</description>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>master:19888</value>
        <description>description</description>
    </property>
    <property>
        <name>mapreduce.jobhistory.intermediate-done-dir</name>
        <value>/data/mr-history/tmp</value>
        <description>description</description>
    </property>
    <property>
        <name>mapreduce.jobhistory.done-dir</name>
        <value>/data/mr-history/done</value>
        <description>description</description>
    </property>
    <property>
        <name>yarn.app.mapreduce.am.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    </property>
    <property>
        <name>mapreduce.map.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    </property>
    <property>
        <name>mapreduce.reduce.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    </property> 
</properties>
bash
yarn-env.sh

export JAVA_HOME=/export/server/jdk
export HADOOP_HOME=/export/server/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
# export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop
# export YARN_LOG_DIR=$HADOOP_HOME/logs/yarn
export HADOOP_LOG_DIR=$HADOOP_HOME/logs
xml
<!--yarn-site.xml 配置信息-->

<property>
	<name>yarn.log.server.url</name>
	<value>http://master:19888/jobhistory/logs</value>
	<description/>
</property>
<property>
	<name>yarn.web-proxy.address</name>
	<value>master:8089</value>
	<description>proxy server hostname and port</description>
</property>
<property>
	<name>yarn.log-aggregation-enable</name>
	<value>true</value>
	<description>Configuration to enable or disable log aggregation</description>
</property>
<property>
	<name>yarn.nodemanager.remote-app-log-dir</name>
	<value>/tmp/logs</value>
	<description>Configuration to enable or disable log aggregation</description>
</property>
<!--  Site specific YARN configuration properties  -->
<property>
	<name>yarn.resourcemanager.hostname</name>
	<value>master</value>
	<description/>
</property>
<property>
	<name>yarn.resourcemanager.scheduler.class</name>
	<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
	<description/>
</property>
<property>
	<name>yarn.nodemanager.local-dirs</name>
	<value>/data/nm-local</value>
	<description>Comma-separated list of paths on the local filesystem where intermediate data is written.</description>
</property>
<property>
	<name>yarn.nodemanager.log-dirs</name>
	<value>/data/nm-log</value>
	<description>Comma-separated list of paths on the local filesystem where logs are written.</description>
</property>
<property>
	<name>yarn.nodemanager.log.retain-seconds</name>
	<value>10800</value>
	<description>Default time (in seconds) to retain log files on the NodeManager Only applicable if log-aggregation is disabled.</description>
</property>
<property>
	<name>yarn.nodemanager.aux-services</name>
	<value>mapreduce_shuffle</value>
	<description>Shuffle service that needs to be set for Map Reduce applications.</description>
</property>
bash
分发
scp mapred-env.sh mapred-site.xml yarn-env.sh yarn-site.xml node2:`pwd`/

常用的进程启动命令如下：

•一键启动YARN集群： $HADOOP_HOME/sbin/start-yarn.sh

•会基于yarn-site.xml中配置的yarn.resourcemanager.hostname来决定在哪台机器上启动resourcemanager

•会基于workers文件配置的主机启动NodeManager

•一键停止YARN集群： $HADOOP_HOME/sbin/stop-yarn.sh

•在当前机器，单独启动或停止进程

•$HADOOP_HOME/bin/yarn --daemon start|stop resourcemanager|nodemanager|proxyserver

•start和stop决定启动和停止

•可控制resourcemanager、nodemanager、proxyserver三种进程

•历史服务器启动和停止

•$HADOOP_HOME/bin/mapred --daemon start|stop historyserver

在master服务器，以hadoop用户执行

1.首先执行：$HADOOP_HOME/sbin/start-yarn.sh，一键启动所需的:

•ResourceManager

•NodeManager

•ProxyServer（代理服务器）

2.其次执行：$HADOOP_HOME/bin/mapred --daemon start historyserver 启动:

bash
start-yarn.sh

mapred --daemon start historyserver

web页面查看master:8088
bash
stop-yarn.sh
mapred --daemon stop historyserver