手写一个YARN应用程序

1、基础配置

1.1、pom

<dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>3.3.1</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-yarn-client</artifactId>
      <version>3.3.1</version>
    </dependency>

1.2、log4j.xml

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">

    <appender name="consoleAppender" class="org.apache.log4j.ConsoleAppender">
        <param name="Target" value="System.out"/>
        <layout class="org.apache.log4j.PatternLayout">
            <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n"/>
        </layout>
    </appender>

    <root>
        <priority value="INFO"/>
        <appender-ref ref="consoleAppender"/>
    </root>

</log4j:configuration>

1.3、使用shade plugin打包

<build>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <version>3.2.4</version>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>
            </goals>
            <configuration>
              <artifactSet>
                <excludes>
                  <exclude>com.google.code.findbugs:jsr305</exclude>
                  <exclude>org.slf4j:*</exclude>
                  <exclude>log4j:*</exclude>
                  <exclude>org.apache.hadoop:*</exclude>
                </excludes>
              </artifactSet>
              <filters>
                <filter>
                  <!-- Do not copy the signatures in the META-INF folder.
                  Otherwise, this might cause SecurityExceptions when using the JAR. -->
                  <artifact>*:*</artifact>
                  <excludes>
                    <exclude>META-INF/*.SF</exclude>
                    <exclude>META-INF/*.DSA</exclude>
                    <exclude>META-INF/*.RSA</exclude>
                  </excludes>
                </filter>
              </filters>
              <transformers combine.children="append">
                <transformer
                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer">
                </transformer>
              </transformers>
            </configuration>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>

2、简单启动AppMaster

2.1、AppMaster应用程序

package com.journey.mr.yarn;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;

public class YarnTaskAppMaster {

    /**
     * YarnTaskAppMaster 程序入口
     * @param args 执行参数
     */
    public static void main(String[] args) {
        YarnTaskAppMaster master = new YarnTaskAppMaster();
        master.run();
    }

    /**
     * AppMaster 运行
     */
    public void run() {
        try {
            // 开启am-rm client，建立rm-am的通道，用于注册AM
            AMRMClientAsync amRmClient = AMRMClientAsync.createAMRMClientAsync(1000, null);
            amRmClient.init(new Configuration());
            amRmClient.start();
            String hostName = NetUtils.getHostname();
            // 注册至RM
            amRmClient.registerApplicationMaster(hostName, -1, null);
            // 运行程序
            doRun();
            // 解除注册
            amRmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "SUCCESS", null);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 实际运行程序，就一个输出
     */
    private void doRun() {
        System.out.println("HELLO YarnTask");
    }
}

2.2、编写Yarn任务提交应用程序

package com.journey.mr.yarn;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class YarnTaskSubmitClient {

    private static Logger log = LoggerFactory.getLogger(YarnTaskSubmitClient.class);

    public static void main(String[] args) {
        YarnTaskSubmitClient client = new YarnTaskSubmitClient();
        try {
            client.run();
        } catch (Exception e) {
            log.error("client run exception , please check log file.", e);
        }
    }

    /**
     * 真正运行逻辑
     *
     * @throws IOException
     * @throws YarnException
     * @throws URISyntaxException
     * @throws InterruptedException
     */
    public void run() throws IOException, YarnException, URISyntaxException, InterruptedException {
        // TODO 1、Configuration基础配置
        Configuration conf = new Configuration();
        // 设置ResourceManager所在的ip地址
        conf.set("yarn.resourcemanager.hostname", "RM地址");

        // TODO 2、创建YarnClient的连接
        // 创建YarnClient和ResourceManager进行交互
        YarnClient yarnClient = YarnClient.createYarnClient();
        // 初始配置
        yarnClient.init(conf);
        // 开启(建立连接)
        yarnClient.start();
        // 向RM发送请求创建应用
        YarnClientApplication application = yarnClient.createApplication();
        // 准备应用提交上下文(RM要求你提交的信息格式)
        ApplicationSubmissionContext applicationSubmissionContext = application.getApplicationSubmissionContext();
        // 获取分配的应用id
        ApplicationId appId = applicationSubmissionContext.getApplicationId();
        log.info("appId: {}", appId);

        // TODO 3、设置应用名称
        // 设置应用名称
        applicationSubmissionContext.setApplicationName("YarnTaskSubmit Demo");

        // TODO 4、准备程序(jar包)
        String jarPath = "/Users/xxxx/IdeaProjects/mr-demo/target/mr-demo-1.0-SNAPSHOT.jar";
        String jarName = "mr-demo-1.0-SNAPSHOT.jar";
        Map<String, LocalResource> localResources = new HashMap<String, LocalResource>() {{
            put(jarName, addLocalToHdfs(jarPath, jarName));
        }};

        // TODO 5、准备程序环境
        Map<String, String> env = new HashMap<>();
        // 任务的运行依赖jar包的准备
        StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$$())
                .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
        // yarn依赖包
        for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
            classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
            classPathEnv.append(c.trim());
        }
        env.put("CLASSPATH", classPathEnv.toString());

        // TODO 6、准备启动命令
        List<String> commands = new ArrayList<String>() {{
            // TODO 这个是单独启动AppMaster
          add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java -Xmx300m com.journey.mr.yarn.YarnTaskAppMaster");
            // TODO 这个是启动AppMaster之后，AppMaster还会再启动ChildTask
           //  add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java -Xmx300m com.journey.mr.yarn.YarnTasksAppMaster");
        }};

        // TODO 7、构造am container运行资源 + 环境 + 脚本
        ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(
                localResources,
                env,
                commands,
                null,
                null,
                null);
        // 准备am Container的运行环境
        applicationSubmissionContext.setAMContainerSpec(amContainer);

        // TODO 8、设置am程序所需资源
        int memory = 1024;
        int vCores = 2;
        applicationSubmissionContext.setResource(Resource.newInstance(memory, vCores));

        // TODO 9、提交并开始作业
        yarnClient.submitApplication(applicationSubmissionContext);

        // TODO 10、查询作业是否完成
        for (;;) {
            Thread.sleep(500);
            // TODO 跟进appId获取applicationReport
            ApplicationReport applicationReport = yarnClient.getApplicationReport(appId);
            YarnApplicationState state = applicationReport.getYarnApplicationState();
            FinalApplicationStatus status = applicationReport.getFinalApplicationStatus();
            if (state.equals(YarnApplicationState.FINISHED)) {
                if (status.equals(FinalApplicationStatus.SUCCEEDED)) {
                    log.info("成功运行完毕.");
                    break;
                } else  {
                    log.error("程序运行失败.");
                    break;
                }
            } else if (state.equals(YarnApplicationState.FAILED) || state.equals(YarnApplicationState.KILLED) ) {
                log.error("程序运行失败.");
                break;
            }
            log.info("运行中...");
        }
    }

    /**
     * 上传本地jar包到hdfs
     *
     * @param jarPath
     * @param jarName
     * @throws IOException
     */
    private LocalResource addLocalToHdfs(String jarPath, String jarName) throws IOException, URISyntaxException, InterruptedException {
        // 获取文件系统
        Configuration configuration = new Configuration();

        // NameNode的ip和端口
        FileSystem fs  = FileSystem.get(new URI("hdfs://NN_IP:8020"), configuration, "root");

        // 目标路径
        String dst = "test/" + jarName;
        Path dstPath = new Path(fs.getHomeDirectory(), dst);

        // 上传
        fs.copyFromLocalFile(new Path(jarPath), dstPath);
        FileStatus scFileStatus = fs.getFileStatus(dstPath);

        // 关闭
        fs.close();

        LocalResource scRsrc = LocalResource.newInstance(
                URL.fromURI(dstPath.toUri()),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                scFileStatus.getLen(), scFileStatus.getModificationTime());
        return scRsrc;
    }
}

2.3、打包运行

mvn clean package
直接运行 IDEA 直接运行 com.journey.mr.yarn.YarnTaskSubmitClient

3、AppMaster子任务的调度

正常分布式计算任务，都是有AppMaster和Task存在的，所以开启YARN分布式调度实例吧

3.1、编写子任务应用程序

package com.journey.mr.yarn;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.net.NetUtils;

import java.net.URI;
public class ChildTask {
    public static void main(String[] args) throws Exception {
        //获取文件系统
        Configuration configuration = new Configuration();
        //NameNode的ip和端口
        FileSystem fs  = FileSystem.get(new URI("hdfs://NN_IP:8020"), configuration, "root");
        // hostName
        String hostName = NetUtils.getHostname();
        // 创建一个文件夹
        fs.mkdirs(new Path("/child/" + hostName));
        fs.close();
    }
}

3.2、编写AppMaster应用程序

package com.journey.mr.yarn;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.URI;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class YarnTasksAppMaster {

    private static Logger log = LoggerFactory.getLogger(YarnTasksAppMaster.class);

    private static NMClientAsync nmClientAsync;

    // 充当锁
    private static Object lock = new Object();
    // 任务个数
    private static int childTaskNum = 2;
    // 已完成任务个数
    private static int childTaskCompletedNum = 0;

    private static Configuration conf;


    /**
     * YarnTasksAppMaster 程序入口
     * @param args 执行参数
     */
    public static void main(String[] args) {
        YarnTasksAppMaster master = new YarnTasksAppMaster();
        master.run();
    }

    /**
     * AppMaster 运行
     */
    public void run() {
        try {
            conf = new Configuration();
            // ResourceManager 回调处理器
            AMRMClientAsync.AbstractCallbackHandler rmCallBackHandler = new RMCallBackHandler();
            // 开启AppMaster-ResourceManager client，建立ResourceManager-AppMaster的通道，用于注册AM, allocListener负责处理AppMaster的响应
            AMRMClientAsync<AMRMClient.ContainerRequest> amRmClient = AMRMClientAsync.createAMRMClientAsync(1000, rmCallBackHandler);
            amRmClient.init(new Configuration());
            amRmClient.start();
            String hostName = NetUtils.getHostname();
            // 注册至RM
            amRmClient.registerApplicationMaster(hostName, -1, null);

            // 初始化NodeManager Client
            nmClientAsync = new NMClientAsyncImpl(new NMCallBackHandler());
            nmClientAsync.init(conf);
            nmClientAsync.start();

            // 运行程序
            doRun(amRmClient);

            // 解除注册
            amRmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "SUCCESS", null);
            // am-rm客户端关闭
            amRmClient.stop();
            // nm客户端关闭
            nmClientAsync.stop();
        } catch (Exception e) {
            log.error("yarn task application master run error.", e);
        }
    }

    /**
     * 实际运行程序，就一个输出
     */
    private void doRun(AMRMClientAsync<AMRMClient.ContainerRequest> amRmClient) throws InterruptedException {
        // 两个子任务，对应两个container
        int childTaskNum = 2;
        // 申请两个资源容器
        for (int i = 0; i < childTaskNum; i++) {
            // 向rm申请一个1M内存,1个CPU的资源容器
            int memory = 1024;
            int vCores = 1;
            AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(Resource.newInstance(memory, vCores), null, null, Priority.UNDEFINED);
            // TODO 其实这里就是向ResourceManager申请cpu和内存
            amRmClient.addContainerRequest(containerRequest);
        }
        synchronized (lock) {
            // 等待子任务完成
            lock.wait();
        }
       log.info("Yarn App Master finished.");
    }

    private class NMCallBackHandler extends NMClientAsync.AbstractCallbackHandler {
        @Override
        public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> map) {

        }

        @Override
        public void onContainerStatusReceived(ContainerId containerId, ContainerStatus containerStatus) {

        }

        @Override
        public void onContainerStopped(ContainerId containerId) {

        }

        @Override
        public void onStartContainerError(ContainerId containerId, Throwable throwable) {

        }

        @Override
        public void onContainerResourceIncreased(ContainerId containerId, Resource resource) {

        }

        @Override
        public void onContainerResourceUpdated(ContainerId containerId, Resource resource) {

        }

        @Override
        public void onGetContainerStatusError(ContainerId containerId, Throwable throwable) {

        }

        @Override
        public void onIncreaseContainerResourceError(ContainerId containerId, Throwable throwable) {

        }

        @Override
        public void onUpdateContainerResourceError(ContainerId containerId, Throwable throwable) {

        }

        @Override
        public void onStopContainerError(ContainerId containerId, Throwable throwable) {

        }
    }

    public static class RMCallBackHandler extends AMRMClientAsync.AbstractCallbackHandler {

        @Override
        public void onContainersCompleted(List<ContainerStatus> statuses) {
            for (ContainerStatus status : statuses) {
                synchronized (lock) {
                    log.info(++childTaskCompletedNum + " container completed");
                    // 子任务全部完成
                    if (childTaskCompletedNum == childTaskNum) {
                        lock.notify();
                    }
                }
            }
        }

        @Override
        public void onContainersAllocated(List<Container> containers) {
            try {
                for (Container container : containers) {
                    log.info("container allocated, Node : {}", container.getNodeHttpAddress());
                    // 构建AM<->NM客户端并开启
                    // 还是YarnClient containerLaunchContext那一套，这把直接去HDFS系统取文件，因为和YarnClient打包到一个jar上传
                    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>() {{
                        //NameNode的ip和端口
                        FileSystem fs = FileSystem.get(new URI("hdfs://NN_IP:8020"), conf, "root");
                        URI appUri = new URI("/user/root/test/mr-demo-1.0-SNAPSHOT.jar");
                        FileStatus fileStatus = fs.getFileStatus(new Path(appUri));
                        put("mr-demo-1.0-SNAPSHOT.jar", LocalResource.newInstance(
                                URL.fromURI(appUri),
                                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                                fileStatus.getLen(), fileStatus.getModificationTime()));
                    }};
                    Map<String, String> env = new HashMap<>();
                    StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$$())
                            .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
                    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
                        classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
                        classPathEnv.append(c.trim());
                    }
                    env.put("CLASSPATH", classPathEnv.toString());
                    List<String> commands = new ArrayList<String>() {{
                        // 传入ip地址作为参数
                        add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java -Xmx200m com.journey.mr.yarn.ChildTask");
                    }};
                    ContainerLaunchContext containerLaunchContext = ContainerLaunchContext.newInstance(
                            localResources, env, commands, null, null, null);
                    // TODO NodeManager节点启动container
                    nmClientAsync.startContainerAsync(container, containerLaunchContext);
                }
            } catch (Exception e) {
                log.error("container allocated error.", e);
            }
        }

        @Override
        public void onContainersUpdated(List<UpdatedContainer> list) {

        }

        @Override
        public void onShutdownRequest() {

        }

        @Override
        public void onNodesUpdated(List<NodeReport> list) {

        }

        @Override
        public float getProgress() {
            return 0;
        }

        @Override
        public void onError(Throwable throwable) {

        }
    }
}

3.3、编写Yarn任务提交应用程序

注意 : 其实这个 YarnTaskSubmitClient 应用程序和上面简单的AppMaster的YarnTaskSubmitClient基本一样的，不同的点是这个YarnTaskSubmitClient要启动的是YarnTasksAppMaster

package com.journey.mr.yarn;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class YarnTaskSubmitClient {

    private static Logger log = LoggerFactory.getLogger(YarnTaskSubmitClient.class);

    public static void main(String[] args) {
        YarnTaskSubmitClient client = new YarnTaskSubmitClient();
        try {
            client.run();
        } catch (Exception e) {
            log.error("client run exception , please check log file.", e);
        }
    }

    /**
     * 真正运行逻辑
     *
     * @throws IOException
     * @throws YarnException
     * @throws URISyntaxException
     * @throws InterruptedException
     */
    public void run() throws IOException, YarnException, URISyntaxException, InterruptedException {
        // TODO 1、Configuration基础配置
        Configuration conf = new Configuration();
        // 设置ResourceManager所在的ip地址
        conf.set("yarn.resourcemanager.hostname", "MR地址");

        // TODO 2、创建YarnClient的连接
        // 创建YarnClient和ResourceManager进行交互
        YarnClient yarnClient = YarnClient.createYarnClient();
        // 初始配置
        yarnClient.init(conf);
        // 开启(建立连接)
        yarnClient.start();
        // 向RM发送请求创建应用
        YarnClientApplication application = yarnClient.createApplication();
        // 准备应用提交上下文(RM要求你提交的信息格式)
        ApplicationSubmissionContext applicationSubmissionContext = application.getApplicationSubmissionContext();
        // 获取分配的应用id
        ApplicationId appId = applicationSubmissionContext.getApplicationId();
        log.info("appId: {}", appId);

        // TODO 3、设置应用名称
        // 设置应用名称
        applicationSubmissionContext.setApplicationName("YarnTaskSubmit Demo");

        // TODO 4、准备程序(jar包)
        String jarPath = "/Users/xxxx/IdeaProjects/mr-demo/target/mr-demo-1.0-SNAPSHOT.jar";
        String jarName = "mr-demo-1.0-SNAPSHOT.jar";
        Map<String, LocalResource> localResources = new HashMap<String, LocalResource>() {{
            put(jarName, addLocalToHdfs(jarPath, jarName));
        }};

        // TODO 5、准备程序环境
        Map<String, String> env = new HashMap<>();
        // 任务的运行依赖jar包的准备
        StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$$())
                .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
        // yarn依赖包
        for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
            classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
            classPathEnv.append(c.trim());
        }
        env.put("CLASSPATH", classPathEnv.toString());

        // TODO 6、准备启动命令
        List<String> commands = new ArrayList<String>() {{
            // TODO 这个是单独启动AppMaster
//            add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java -Xmx300m com.journey.mr.yarn.YarnTaskAppMaster");
            // TODO 这个是启动AppMaster之后，AppMaster还会再启动ChildTask
            add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java -Xmx300m com.journey.mr.yarn.YarnTasksAppMaster");
        }};

        // TODO 7、构造am container运行资源 + 环境 + 脚本
        ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(
                localResources,
                env,
                commands,
                null,
                null,
                null);
        // 准备am Container的运行环境
        applicationSubmissionContext.setAMContainerSpec(amContainer);

        // TODO 8、设置am程序所需资源
        int memory = 1024;
        int vCores = 2;
        applicationSubmissionContext.setResource(Resource.newInstance(memory, vCores));

        // TODO 9、提交并开始作业
        yarnClient.submitApplication(applicationSubmissionContext);

        // TODO 10、查询作业是否完成
        for (;;) {
            Thread.sleep(500);
            // TODO 跟进appId获取applicationReport
            ApplicationReport applicationReport = yarnClient.getApplicationReport(appId);
            YarnApplicationState state = applicationReport.getYarnApplicationState();
            FinalApplicationStatus status = applicationReport.getFinalApplicationStatus();
            if (state.equals(YarnApplicationState.FINISHED)) {
                if (status.equals(FinalApplicationStatus.SUCCEEDED)) {
                    log.info("成功运行完毕.");
                    break;
                } else  {
                    log.error("程序运行失败.");
                    break;
                }
            } else if (state.equals(YarnApplicationState.FAILED) || state.equals(YarnApplicationState.KILLED) ) {
                log.error("程序运行失败.");
                break;
            }
            log.info("运行中...");
        }
    }

    /**
     * 上传本地jar包到hdfs
     *
     * @param jarPath
     * @param jarName
     * @throws IOException
     */
    private LocalResource addLocalToHdfs(String jarPath, String jarName) throws IOException, URISyntaxException, InterruptedException {
        // 获取文件系统
        Configuration configuration = new Configuration();

        // NameNode的ip和端口
        FileSystem fs  = FileSystem.get(new URI("hdfs://NN_IP:8020"), configuration, "root");

        // 目标路径
        String dst = "test/" + jarName;
        Path dstPath = new Path(fs.getHomeDirectory(), dst);

        // 上传
        fs.copyFromLocalFile(new Path(jarPath), dstPath);
        FileStatus scFileStatus = fs.getFileStatus(dstPath);

        // 关闭
        fs.close();

        LocalResource scRsrc = LocalResource.newInstance(
                URL.fromURI(dstPath.toUri()),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                scFileStatus.getLen(), scFileStatus.getModificationTime());
        return scRsrc;
    }
}

3.4、编写Yarn任务提交应用程序

mvn clean package
直接运行 IDEA 直接运行 com.journey.mr.yarn.YarnTaskSubmitClient

如感兴趣，点赞加关注，谢谢!!!