Flink Yarn Per Job - JobManger 申请 Slot

 

 

JobMaster 启动时，启动 SlotPool，向 ResourceManager 注册

 

启动SlotPool

 

JobMaster

•  

private void startJobMasterServices() throws Exception {  // 启动心跳服务：taskmanager、resourcemanager  startHeartbeatServices();
  // start the slot pool make sure the slot pool now accepts messages for this leader  //  启动 slotpool  slotPool.start(getFencingToken(), getAddress(), getMainThreadExecutor());
  //TODO: Remove once the ZooKeeperLeaderRetrieval returns the stored address upon start  // try to reconnect to previously known leader  // 连接到之前已知的 ResourceManager  reconnectToResourceManager(new FlinkException("Starting JobMaster component."));
  // job is ready to go, try to establish connection with resource manager  //   - activate leader retrieval for the resource manager  //   - on notification of the leader, the connection will be established and  //     the slot pool will start requesting slots  //  与ResourceManager建立连接，slotpool开始请求资源  resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener());}

• 启动心跳服务：taskmanager、resourcemanager

• 启动 slotpool

• 连接到之前已知的 ResourceManager

• slotpool开始请求资源

 

向RM注册

 

StandaloneLeaderRetrievalService 

•  

@Overridepublic void start(LeaderRetrievalListener listener) {  checkNotNull(listener, "Listener must not be null.");
  synchronized (startStopLock) {    checkState(!started, "StandaloneLeaderRetrievalService can only be started once.");    started = true;
    // directly notify the listener, because we already know the leading JobManager's address    listener.notifyLeaderAddress(leaderAddress, leaderId);  }}

 

ResourceManagerLeaderListener in TaskExecutor

•  

private final class ResourceManagerLeaderListener implements LeaderRetrievalListener {
  @Override  public void notifyLeaderAddress(final String leaderAddress, final UUID leaderSessionID) {    runAsync(      () -> notifyOfNewResourceManagerLeader(        leaderAddress,        ResourceManagerId.fromUuidOrNull(leaderSessionID)));    }}
private void notifyOfNewResourceManagerLeader(String newLeaderAddress, ResourceManagerId newResourceManagerId) {  resourceManagerAddress = createResourceManagerAddress(newLeaderAddress, newResourceManagerId);  reconnectToResourceManager(new FlinkException(String.format("ResourceManager leader changed to new address %s", resourceManagerAddress)));}  
private void reconnectToResourceManager(Exception cause) {  closeResourceManagerConnection(cause);  startRegistrationTimeout();  tryConnectToResourceManager();}
private void tryConnectToResourceManager() {  if (resourceManagerAddress != null) {    connectToResourceManager();  }}
private void connectToResourceManager() {... ...  resourceManagerConnection.start();}

 

RegisteredRpcConnection

•  

public void start() {  checkState(!closed, "The RPC connection is already closed");  checkState(!isConnected() && pendingRegistration == null, "The RPC connection is already started");
  //  创建注册对象  final RetryingRegistration<F, G, S> newRegistration = createNewRegistration();
  if (REGISTRATION_UPDATER.compareAndSet(this, null, newRegistration)) {    // 开始注册，注册成功之后，调用 onRegistrationSuccess()    newRegistration.startRegistration();  } else {    // concurrent start operation    newRegistration.cancel();  }}

private RetryingRegistration<F, G, S> createNewRegistration() {  RetryingRegistration<F, G, S> newRegistration = checkNotNull(generateRegistration());... ...  return newRegistration;}

• 创建注册对象

• 开始注册，注册成功之后，调用 onRegistrationSuccess()

 

SlotPool 申请 slot

 

ResourceManagerConnection in JobMaster

•  

protected void onRegistrationSuccess(final JobMasterRegistrationSuccess success) {  runAsync(() -> {    // filter out outdated connections    //noinspection ObjectEquality    if (this == resourceManagerConnection) {      establishResourceManagerConnection(success);    }  });}
private void establishResourceManagerConnection(final JobMasterRegistrationSuccess success) {    final ResourceManagerId resourceManagerId = success.getResourceManagerId();... ...    // slotpool连接到ResourceManager，请求资源    slotPool.connectToResourceManager(resourceManagerGateway);... ...}

slotpool连接到ResourceManager，请求资源

 

SlotPoolImpl

•  

public void connectToResourceManager(@Nonnull ResourceManagerGateway resourceManagerGateway) {  this.resourceManagerGateway = checkNotNull(resourceManagerGateway);
  // work on all slots waiting for this connection  for (PendingRequest pendingRequest : waitingForResourceManager.values()) {    requestSlotFromResourceManager(resourceManagerGateway, pendingRequest);  }
  // all sent off  waitingForResourceManager.clear();}
private void requestSlotFromResourceManager(    final ResourceManagerGateway resourceManagerGateway,    final PendingRequest pendingRequest) {...  ...  CompletableFuture<Acknowledge> rmResponse = resourceManagerGateway.requestSlot(    jobMasterId,    new SlotRequest(jobId, allocationId, pendingRequest.getResourceProfile(), jobManagerAddress),    rpcTimeout);... ...}

从RM中申请slot

 

ResourceManager

•  

@Overridepublic CompletableFuture<Acknowledge> requestSlot(    JobMasterId jobMasterId,    SlotRequest slotRequest,    final Time timeout) {
  JobID jobId = slotRequest.getJobId();  JobManagerRegistration jobManagerRegistration = jobManagerRegistrations.get(jobId);
  if (null != jobManagerRegistration) {    if (Objects.equals(jobMasterId, jobManagerRegistration.getJobMasterId())) {      log.info("Request slot with profile {} for job {} with allocation id {}.",        slotRequest.getResourceProfile(),        slotRequest.getJobId(),        slotRequest.getAllocationId());
      try {        // RM内部的 slotManager去向 Yarn的ResourceManager申请资源        slotManager.registerSlotRequest(slotRequest);      } catch (ResourceManagerException e) {        return FutureUtils.completedExceptionally(e);      }
      return CompletableFuture.completedFuture(Acknowledge.get());    } else {      return FutureUtils.completedExceptionally(new ResourceManagerException("The job leader's id " +        jobManagerRegistration.getJobMasterId() + " does not match the received id " + jobMasterId + '.'));    }
  } else {    return FutureUtils.completedExceptionally(new ResourceManagerException("Could not find registered job manager for job " + jobId + '.'));  }}

RM内部的 slotManager去向 Yarn的ResourceManager申请资源

 

SlotManagerImpl

•  

public boolean registerSlotRequest(SlotRequest slotRequest) throws ResourceManagerException {  checkInit();... ...     try {      internalRequestSlot(pendingSlotRequest);    } catch (ResourceManagerException e) {      // requesting the slot failed --> remove pending slot request      pendingSlotRequests.remove(slotRequest.getAllocationId());
      throw new ResourceManagerException("Could not fulfill slot request " + slotRequest.getAllocationId() + '.', e);    }    return true;  }}
private void internalRequestSlot(PendingSlotRequest pendingSlotRequest) throws ResourceManagerException {  final ResourceProfile resourceProfile = pendingSlotRequest.getResourceProfile();
  OptionalConsumer.of(findMatchingSlot(resourceProfile))    .ifPresent(taskManagerSlot -> allocateSlot(taskManagerSlot, pendingSlotRequest))    .ifNotPresent(() -> fulfillPendingSlotRequestWithPendingTaskManagerSlot(pendingSlotRequest));}
private void fulfillPendingSlotRequestWithPendingTaskManagerSlot(PendingSlotRequest pendingSlotRequest) throws ResourceManagerException {  ResourceProfile resourceProfile = pendingSlotRequest.getResourceProfile();  Optional<PendingTaskManagerSlot> pendingTaskManagerSlotOptional = findFreeMatchingPendingTaskManagerSlot(resourceProfile);
  if (!pendingTaskManagerSlotOptional.isPresent()) {    pendingTaskManagerSlotOptional = allocateResource(resourceProfile);  }... ...}

 

Flink内的RM向Yarn的RM申请资源

 

SlotManagerImpl

•  

private Optional<PendingTaskManagerSlot> allocateResource(ResourceProfile requestedSlotResourceProfile) {... ...  if (!resourceActions.allocateResource(defaultWorkerResourceSpec)) {    // resource cannot be allocated    return Optional.empty();  }... ...}

 

ResourceActionsImpl in ResourceManager

•  

public boolean allocateResource(WorkerResourceSpec workerResourceSpec) {  validateRunsInMainThread();  return startNewWorker(workerResourceSpec);}

 

ActiveResourceManager

•  

public boolean startNewWorker(WorkerResourceSpec workerResourceSpec) {  requestNewWorker(workerResourceSpec);  return true;}
// 从配置中获取 taskexecutor 配置private void requestNewWorker(WorkerResourceSpec workerResourceSpec) {  final TaskExecutorProcessSpec taskExecutorProcessSpec =      TaskExecutorProcessUtils.processSpecFromWorkerResourceSpec(flinkConfig, workerResourceSpec);  final int pendingCount = pendingWorkerCounter.increaseAndGet(workerResourceSpec);
  log.info("Requesting new worker with resource spec {}, current pending count: {}.",      workerResourceSpec,      pendingCount);// 申请资源  CompletableFuture<WorkerType> requestResourceFuture = resourceManagerDriver.requestResource(taskExecutorProcessSpec);  FutureUtils.assertNoException(      requestResourceFuture.handle((worker, exception) -> {        if (exception != null) {          final int count = pendingWorkerCounter.decreaseAndGet(workerResourceSpec);          log.warn("Failed requesting worker with resource spec {}, current pending count: {}, exception: {}",              workerResourceSpec,              count,              exception);          requestWorkerIfRequired();        } else {          final ResourceID resourceId = worker.getResourceID();          workerNodeMap.put(resourceId, worker);          currentAttemptUnregisteredWorkers.put(resourceId, workerResourceSpec);          log.info("Requested worker {} with resource spec {}.",              resourceId.getStringWithMetadata(),              workerResourceSpec);        }        return null;      }));}

• 从配置中获取 taskexecutor 配置

• 申请资源

 

YarnResourceManagerDriver

•  

public CompletableFuture<YarnWorkerNode> requestResource(TaskExecutorProcessSpec taskExecutorProcessSpec) {  checkInitialized();
  final CompletableFuture<YarnWorkerNode> requestResourceFuture = new CompletableFuture<>();
  final Optional<TaskExecutorProcessSpecContainerResourcePriorityAdapter.PriorityAndResource> priorityAndResourceOpt =    taskExecutorProcessSpecContainerResourcePriorityAdapter.getPriorityAndResource(taskExecutorProcessSpec);
  if (!priorityAndResourceOpt.isPresent()) {    requestResourceFuture.completeExceptionally(      new ResourceManagerException(        String.format("Could not compute the container Resource from the given TaskExecutorProcessSpec %s. " +            "This usually indicates the requested resource is larger than Yarn's max container resource limit.",          taskExecutorProcessSpec)));  } else {    final Priority priority = priorityAndResourceOpt.get().getPriority();    final Resource resource = priorityAndResourceOpt.get().getResource();    resourceManagerClient.addContainerRequest(getContainerRequest(resource, priority));
    // make sure we transmit the request fast and receive fast news of granted allocations    resourceManagerClient.setHeartbeatInterval(containerRequestHeartbeatIntervalMillis);
    requestResourceFutures.computeIfAbsent(taskExecutorProcessSpec, ignore -> new LinkedList<>()).add(requestResourceFuture);
    log.info("Requesting new TaskExecutor container with resource {}, priority {}.", taskExecutorProcessSpec, priority);  }
  return requestResourceFuture;}