flink 提交yarn任务

Flink提交YARN任务

Apache Flink是一个分布式流处理框架，可以在集群上运行大规模的数据处理任务。在实际应用中，我们常常需要将Flink应用程序提交到YARN集群上运行，以充分利用集群的计算资源。本文将介绍如何使用Flink提交YARN任务，并提供相应的代码示例。

准备工作

在开始之前，确保你已经完成以下准备工作：

1. 安装并配置好Flink和YARN。确保Flink和YARN集群正常运行，并且能够相互通信。

2. 编写一个简单的Flink应用程序作为示例。下面是一个简单的WordCount示例代码：

import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.utils.ParameterTool;

public class WordCount {
  public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    env.getConfig().setGlobalJobParameters(params);

    DataSet<String> text = env.readTextFile(params.get("input"));

    DataSet<Tuple2<String, Integer>> counts = text
      .flatMap(new Tokenizer())
      .groupBy(0)
      .sum(1);

    if (params.has("output")) {
      counts.writeAsCsv(params.get("output"), "\n", " ");
      env.execute("WordCount Example");
    } else {
      System.out.println("Printing result to stdout. Use --output to specify output path.");
      counts.print();
    }
  }

  public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
    @Override
    public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
      String[] words = value.toLowerCase().split("\\W+");

      for (String word : words) {
        if (word.length() > 0) {
          out.collect(new Tuple2<>(word, 1));
        }
      }
    }
  }
}

提交YARN任务

下面是提交Flink应用程序到YARN集群的步骤：

1. 创建一个YarnClusterDescriptor对象，并配置相关参数。下面是一个示例代码：

import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.client.program.ClusterClient;
import org.apache.flink.client.program.PackagedProgram;
import org.apache.flink.client.program.ProgramInvocationException;
import org.apache.flink.client.program.rest.RestClusterClient;
import org.apache.flink.client.program.rest.RestClusterClientConfiguration;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.JobManagerOptions;
import org.apache.flink.yarn.YarnClusterDescriptor;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
import org.apache.hadoop.yarn.api.records.ApplicationId;

public class YarnJobSubmitter {
  public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    env.getConfig().setGlobalJobParameters(params);

    String yarnConfDir = "/path/to/yarn/conf"; // YARN的配置文件目录
    String flinkJarPath = "/path/to/flink.jar"; // Flink应用程序的jar包路径
    String mainClass = "com.example.WordCount"; // Flink应用程序的主类

    Configuration flinkConfig = new Configuration();
    flinkConfig.setString(JobManagerOptions.ADDRESS, "localhost"); // Flink JobManager的地址
    flinkConfig.setInteger(JobManagerOptions.PORT, 8081); // Flink JobManager的端口

    YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
      flinkConfig,
      yarnConfDir,
      flinkJarPath,
      "yarn-session",
      1 // 任务的并行度
    );

    yarnClusterDescriptor.setFlinkConfiguration(env.getConfig());

    // 提交应用程序到YARN集群
    ClusterClient<ApplicationId> clusterClient = yarnClusterDescriptor.deploy();
    RestClusterClient<ApplicationId> restClusterClient = new RestClusterClient<>(
      new RestClusterClientConfiguration.Builder()
        .withRestServerEndpoint("localhost", 8081) // Flink REST服务的地址和端口
        .build(),
      clusterClient.getClusterId()
    );

    PackagedProgram packagedProgram = new PackagedProgram(
      new File(flinkJarPath),
      mainClass,
      new String[] {params.get("input"), params.get("output")}
    );

    JobID jobId;
    try {
      jobId = restClusterClient.run(packagedProgram, 1);
    } catch (ProgramInvocationException e) {