YARN REST API 总结

文章目录

01 引言

Yarn其实是有​​REST Api​​的，很多接口无需使用爬虫去爬页面，具体的接口使用在官网可以看到。

接口地址： ​​https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html​​

API 目录索引如下：

• ​​Overview​​
• ​​Enabling CORS support​​
• ​​Cluster Information API​​
• ​​Cluster Metrics API​​
• ​​Cluster Scheduler API​​
• ​​Cluster Applications API​​
• ​​Cluster Application Statistics API​​
• ​​Cluster Application API​​
• ​​Cluster Application Attempts API​​
• ​​Containers for an Application Attempt API​​
• ​​Specific Container for an Application Attempt API​​
• ​​Cluster Nodes API​​
• ​​Cluster Node API​​
• ​​Cluster Node Update Resource API​​
• ​​Cluster Writeable APIs​​
• ​​Cluster New Application API​​
• ​​Cluster Applications API(Submit Application)​​
• ​​Cluster Application State API​​
• ​​Cluster Application Queue API​​
• ​​Cluster Application Priority API​​
• ​​Cluster Delegation Tokens API​​
• ​​Cluster Reservation API List​​
• ​​Cluster Reservation API Create​​
• ​​Cluster Reservation API Submit​​
• ​​Cluster Reservation API Update​​
• ​​Cluster Reservation API Delete​​
• ​​Cluster Application Timeouts API​​
• ​​Cluster Application Timeout API​​
• ​​Cluster Application Timeout Update API​​
• ​​Scheduler Configuration Mutation API​​
• ​​Cluster Container Signal API​​
• ​​Scheduler Activities API​​
• ​​Application Activities API​​

02 常用接口

注意​​Yarn REST API​​​同时支持返回​​JSON​​​或者​​XML​​​格式的文本，只需要在请求头声明即可，比如需要返回​​XML​​​（默认返回​​JSON​​，不需要显示声明），则请求头如下：

Accept: application/xml
GET http://rm-http-address:port/ws/v1/cluster/info

2.1 指标接口

请求地址：

GET http://rm-http-address:port/ws/v1/cluster/metrics

响应接口：

{
  "clusterMetrics":
  {
    "appsSubmitted":0,
    "appsCompleted":0,
    "appsPending":0,
    "appsRunning":0,
    "appsFailed":0,
    "appsKilled":0,
    "reservedMB":0,
    "availableMB":17408,
    "allocatedMB":0,
    "reservedVirtualCores":0,
    "availableVirtualCores":7,
    "allocatedVirtualCores":1,
    "containersAllocated":0,
    "containersReserved":0,
    "containersPending":0,
    "totalMB":17408,
    "totalVirtualCores":8,
    "totalNodes":1,
    "lostNodes":0,
    "unhealthyNodes":0,
    "decommissioningNodes":0,
    "decommissionedNodes":0,
    "rebootedNodes":0,
    "activeNodes":1,
    "shutdownNodes":0
  }
}

2.2 应用接口

2.2.1 应用列表接口

请求地址：

GET http://rm-http-address:port/ws/v1/cluster/apps

返回内容：

{
  "apps":
  {
    "app":
    [
      {
        "id": "application_1476912658570_0002",
        "user": "user2",
        "name": "word count",
        "queue": "default",
        "state": "FINISHED",
        "finalStatus": "SUCCEEDED",
        "progress": 100,
        "trackingUI": "History",
        "trackingUrl": "http://host.domain.com:8088/cluster/app/application_1476912658570_0002",
        "diagnostics": "...",
        "clusterId": 1476912658570,
        "applicationType": "MAPREDUCE",
        "applicationTags": "",
        "priority": -1,
        "startedTime": 1476913457320,
        "finishedTime": 1476913761898,
        "elapsedTime": 304578,
        "amContainerLogs": "http://host.domain.com:8042/node/containerlogs/container_1476912658570_0002_02_000001/user2",
        "amHostHttpAddress": "host.domain.com:8042",
        "allocatedMB": 0,
        "allocatedVCores": 0,
        "runningContainers": 0,
        "memorySeconds": 206464,
        "vcoreSeconds": 201,
        "queueUsagePercentage": 0,
        "clusterUsagePercentage": 0,
        "preemptedResourceMB": 0,
        "preemptedResourceVCores": 0,
        "numNonAMContainerPreempted": 0,
        "numAMContainerPreempted": 0,
        "logAggregationStatus": "DISABLED",
        "unmanagedApplication": false,
        "appNodeLabelExpression": "",
        "amNodeLabelExpression": "",
        "resourceRequests": [
        {
            "capability": {
                "memory": 4096,
                "virtualCores": 1
            },
            "nodeLabelExpression": "",
            "numContainers": 0,
            "priority": {
                "priority": 0
            },
            "relaxLocality": true,
            "resourceName": "*"
        },
        {
            "capability": {
                "memory": 4096,
                "virtualCores": 1
            },
            "nodeLabelExpression": "",
            "numContainers": 0,
            "priority": {
                "priority": 20
            },
            "relaxLocality": true,
            "resourceName": "host1.domain.com"
        },
        {
            "capability": {
                "memory": 4096,
                "virtualCores": 1
            },
            "nodeLabelExpression": "",
            "numContainers": 0,
            "priority": {
                "priority": 20
            },
            "relaxLocality": true,
            "resourceName": "host2.domain.com"
        }]
      },
      {
        "id": "application_1476912658570_0001",
        "user": "user1",
        "name": "Sleep job",
        "queue": "default",
        "state": "FINISHED",
        "finalStatus": "SUCCEEDED",
        "progress": 100,
        "trackingUI": "History",
        "trackingUrl": "http://host.domain.com:8088/cluster/app/application_1476912658570_0001",
        "diagnostics": "...",
        "clusterId": 1476912658570,
        "applicationType": "YARN",
        "applicationTags": "",
        "priority": -1,
        "startedTime": 1476913464750,
        "finishedTime": 1476913863175,
        "elapsedTime": 398425,
        "amContainerLogs": "http://host.domain.com:8042/node/containerlogs/container_1476912658570_0001_02_000001/user1",
        "amHostHttpAddress": "host.domain.com:8042",
        "allocatedMB": 0,
        "allocatedVCores": 0,
        "runningContainers": 0,
        "memorySeconds": 205410,
        "vcoreSeconds": 200,
        "queueUsagePercentage": 0,
        "clusterUsagePercentage": 0,
        "preemptedResourceMB": 0,
        "preemptedResourceVCores": 0,
        "numNonAMContainerPreempted": 0,
        "numAMContainerPreempted": 0,
        "logAggregationStatus": "DISABLED",
        "unmanagedApplication": false,
        "appNodeLabelExpression": "",
        "amNodeLabelExpression": "",
        "resourceRequests": [
        {
            "capability": {
                "memory": 4096,
                "virtualCores": 1
            },
            "nodeLabelExpression": "",
            "numContainers": 0,
            "priority": {
                "priority": 0
            },
            "relaxLocality": true,
            "resourceName": "*"
        },
        {
            "capability": {
                "memory": 4096,
                "virtualCores": 1
            },
            "nodeLabelExpression": "",
            "numContainers": 0,
            "priority": {
                "priority": 20
            },
            "relaxLocality": true,
            "resourceName": "host3.domain.com"
        },
        {
            "capability": {
                "memory": 4096,
                "virtualCores": 1
            },
            "nodeLabelExpression": "",
            "numContainers": 0,
            "priority": {
                "priority": 20
            },
            "relaxLocality": true,
            "resourceName": "host4.domain.com"
        }]
      }
    ]
  }
}

2.2.2 应用统计接口

请求地址：

GET http://rm-http-address:port/ws/v1/cluster/appstatistics?states=accepted,running,finished&applicationTypes=mapreduce

响应内容：

{
  "appStatInfo":
  {
    "statItem":
    [
       {
          "state" : "accepted",
          "type" : "mapreduce",
          "count" : 4
       },
       {
          "state" : "running",
          "type" : "mapreduce",
          "count" : 1
       },
       {
          "state" : "finished",
          "type" : "mapreduce",
          "count" : 7
       }
    ]
  }
}

2.2.3 应用详情

请求地址：

GET http://rm-http-address:port/ws/v1/cluster/apps/application_1476912658570_0002

响应内容：

{
  "app": {
    "id": "application_1476912658570_0002",
    "user": "user2",
    "name": "word count",
    "queue": "default",
    "state": "FINISHED",
    "finalStatus": "SUCCEEDED",
    "progress": 100,
    "trackingUI": "History",
    "trackingUrl": "http://host.domain.com:8088/cluster/app/application_1476912658570_0002",
    "diagnostics": "...",
    "clusterId": 1476912658570,
    "applicationType": "YARN",
    "applicationTags": "",
    "priority": -1,
    "startedTime": 1476913457320,
    "finishedTime": 1476913761898,
    "elapsedTime": 304578,
    "amContainerLogs": "http://host.domain.com:8042/node/containerlogs/container_1476912658570_0002_02_000001/dr.who",
    "amHostHttpAddress": "host.domain.com:8042",
    "allocatedMB": -1,
    "allocatedVCores": -1,
    "runningContainers": -1,
    "memorySeconds": 206464,
    "vcoreSeconds": 201,
    "queueUsagePercentage": 0,
    "clusterUsagePercentage": 0,
    "preemptedResourceMB": 0,
    "preemptedResourceVCores": 0,
    "numNonAMContainerPreempted": 0,
    "numAMContainerPreempted": 0,
    "logAggregationStatus": "DISABLED",
    "unmanagedApplication": false,
    "appNodeLabelExpression": "",
    "amNodeLabelExpression": ""
  }
}

2.3 队列接口

请求地址：

GET http://rm-http-address:port/ws/v1/cluster/apps/application_1399397633663_0003/queue

响应内容：

{
  "queue":"default"
}

2.4 Node节点接口

请求地址：

GET http://rm-http-address:port/ws/v1/cluster/nodes

响应内容：

{
  "nodes":
  {
    "node":
    [
      {
        "rack":"\/default-rack",
        "state":"RUNNING",
        "id":"host.domain.com:54158",
        "nodeHostName":"host.domain.com",
        "nodeHTTPAddress":"host.domain.com:8042",
        "lastHealthUpdate": 1476995346399,
        "version": "3.0.0-alpha2-SNAPSHOT",
        "healthReport":"",
        "numContainers":0,
        "usedMemoryMB":0,
        "availMemoryMB":8192,
        "usedVirtualCores":0,
        "availableVirtualCores":8,
        "resourceUtilization":
        {
          "nodePhysicalMemoryMB":1027,
          "nodeVirtualMemoryMB":1027,
          "nodeCPUUsage":0.016661113128066063,
          "aggregatedContainersPhysicalMemoryMB":0,
          "aggregatedContainersVirtualMemoryMB":0,
          "containersCPUUsage":0
        }
      },
      {
        "rack":"\/default-rack",
        "state":"RUNNING",
        "id":"host.domain.com:54158",
        "nodeHostName":"host.domain.com",
        "nodeHTTPAddress":"host.domain.com:8042",
        "lastHealthUpdate":1476995346399,
        "version":"3.0.0-alpha2-SNAPSHOT",
        "healthReport":"",
        "numContainers":0,
        "usedMemoryMB":0,
        "availMemoryMB":8192,
        "usedVirtualCores":0,
        "availableVirtualCores":8,
        "resourceUtilization":
        {
          "nodePhysicalMemoryMB":1027,
          "nodeVirtualMemoryMB":1027,
          "nodeCPUUsage":0.016661113128066063,
          "aggregatedContainersPhysicalMemoryMB":0,
          "aggregatedContainersVirtualMemoryMB":0,
          "containersCPUUsage":0
        }
      }
    ]
  }
}

2.5 其它接口

还有其它的接口就得自己根据实际项目情况去看了，本文不再赘述。

03 文末

本文主要整理了Yarn 相关的 ​​REST API​​，并举出了一些例子，希望能帮助到大家，谢谢大家的阅读！