elasticsearch实际总结(2)——集群状态监控

一、参考

elasticsearch 学习系列目录——更新ing

官方文档

二、环境

可以使用下面两种方式查看集群状态：

（1）利用kibana查看集群状态（推荐）

（2）利用ES的RESTful接口，查看集群状态

示例：

三、集群监控

3.1 运行状况

# 请求
GET _cluster/health
(curl -XGET "https://127.0.0.1:9200/_cluster/health?pretty"）
# 返回
{
  "cluster_name" : "prs-cluster",                       # 集群名称
  "status" : "green",                                   # 运行状态       
  "timed_out" : false,                                    
  "number_of_nodes" : 3,                                # ES实例数量（节点个数）     
  "number_of_data_nodes" : 3,                           # ES数据实例数量（数据节点个数，NTA模式下，所有节点都是数据节点）
  "active_primary_shards" : 2431,                       # 主分片数量
  "active_shards" : 2449,                               # 所有分片数量（主分片和副本分片的总和）
  "relocating_shards" : 0,                              # 重新分配的分片数量
  "initializing_shards" : 0,                            # 正在初始化的分片数量
  "unassigned_shards" : 0,                              # 未分配的分片数量
  "delayed_unassigned_shards" : 0,                      # 分配超时的分片数量
  "number_of_pending_tasks" : 0,                        # 等待执行的任务数量
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 100.0             # 活跃分片占比
}

3.2 统计数据

//请求
 
GET /_cluster/stats?human&pretty
（或者 curl -XGET "https://127.0.0.1:9200/_cluster/stats?human&pretty"）
 
//返回
 
{
  "_nodes" : {                                                                  # 节点信息
    "total" : 3,
    "successful" : 3,
    "failed" : 0
  },
  "cluster_name" : "prs-cluster",
  "cluster_uuid" : "8mmexFmsRDKwBC-zjn7HeQ",
  "timestamp" : 1569317824995,
  "status" : "green",
  "indices" : {                                                                 # 索引信息
    "count" : 287,
    "shards" : {
      "total" : 2449,
      "primaries" : 2431,
      "replication" : 0.007404360345536816,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 9,
          "avg" : 8.533101045296167
        },
        "primaries" : {
          "min" : 1,
          "max" : 9,
          "avg" : 8.470383275261325
        },
        "replication" : {
          "min" : 0.0,
          "max" : 1.0,
          "avg" : 0.0627177700348432
        }
      }
    },
    "docs" : {                                                                 # 文档数量
      "count" : 109049097,
      "deleted" : 8642044
    },
    "store" : {                                                                # 存储大小
      "size" : "86.7gb",
      "size_in_bytes" : 93118657722
    },
    "fielddata" : {
      "memory_size" : "38.4kb",
      "memory_size_in_bytes" : 39344,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "58.3mb",
      "memory_size_in_bytes" : 61235421,
      "total_count" : 3134172,
      "hit_count" : 1072635,
      "miss_count" : 2061537,
      "cache_size" : 1929,
      "cache_count" : 35436,
      "evictions" : 33507
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 949,
      "memory" : "191mb",
      "memory_in_bytes" : 200284443,
      "terms_memory" : "147.5mb",
      "terms_memory_in_bytes" : 154764435,
      "stored_fields_memory" : "24.5mb",
      "stored_fields_memory_in_bytes" : 25694104,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "1kb",
      "norms_memory_in_bytes" : 1024,
      "points_memory" : "11.9mb",
      "points_memory_in_bytes" : 12512436,
      "doc_values_memory" : "6.9mb",
      "doc_values_memory_in_bytes" : 7312444,
      "index_writer_memory" : "1.7gb",
      "index_writer_memory_in_bytes" : 1835364798,
      "version_map_memory" : "7.8mb",
      "version_map_memory_in_bytes" : 8237787,
      "fixed_bit_set" : "7.4mb",
      "fixed_bit_set_memory_in_bytes" : 7824256,
      "max_unsafe_auto_id_timestamp" : 1569283209006,
      "file_sizes" : { }
    }
  },
  "nodes" : {
    "count" : {
      "total" : 3,
      "data" : 3,
      "coordinating_only" : 0,
      "master" : 3,
      "ingest" : 3
    },
    "versions" : [
      "7.2.1"
    ],
    "os" : {
      "available_processors" : 96,
      "allocated_processors" : 96,
      "names" : [
        {
          "name" : "Linux",
          "count" : 3
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "CentOS Linux 7 (Core)",
          "count" : 3
        }
      ],
      "mem" : {                                                                # 系统内存使用情况
        "total" : "439.9gb",
        "total_in_bytes" : 472367353856,
        "free" : "7.1gb",
        "free_in_bytes" : 7696609280,
        "used" : "432.7gb",
        "used_in_bytes" : 464670744576,
        "free_percent" : 2,
        "used_percent" : 98
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 0
      },
      "open_file_descriptors" : {
        "min" : 3135,
        "max" : 3158,
        "avg" : 3147
      }
    },
    "jvm" : {
      "max_uptime" : "14.2d",
      "max_uptime_in_millis" : 1230407002,
      "versions" : [
        {
          "version" : "12.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "12.0.1+12",
          "vm_vendor" : "Oracle Corporation",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 3
        }
      ],
      "mem" : {                                                               # JVM堆内存使用情况
        "heap_used" : "44.6gb",
        "heap_used_in_bytes" : 47954012576,
        "heap_max" : "96gb",
        "heap_max_in_bytes" : 103079215104
      },
      "threads" : 944
    },
    "fs" : {                                                                  # 硬盘空间
      "total" : "8tb",
      "total_in_bytes" : 8858938466304,
      "free" : "7tb",
      "free_in_bytes" : 7759058632704,
      "available" : "6.6tb",
      "available_in_bytes" : 7308977074176
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 3
      },
      "http_types" : {
        "security4" : 3
      }
    },
    "discovery_types" : {
      "zen" : 3
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 3
      }
    ]
  }
}

3.3 线程池统计

# 请求
 
GET _nodes/stats/thread_pool
（curl  -XGET "https://127.0.0.1:9200/_nodes/stats/thread_pool?human&pretty"）
 
# 返回
{
  "_nodes" : {
    "total" : 3,
    "successful" : 3,
    "failed" : 0
  },
  "cluster_name" : "prs-cluster",
  "nodes" : {
    "ILnrMcHIToifv6hNQeswVQ" : {                                                         # 节点线程池信息
      "timestamp" : 1569322121566,
      "name" : "node2-0",
      "transport_address" : "10.0.99.9:9300",
      "host" : "10.0.99.9",
      "ip" : "10.0.99.9:9300",
      "roles" : [
        "master",
        "data",
        "ingest"
      ],
      "attributes" : {
        "xpack.installed" : "true"
      },
      "thread_pool" : {
        "analyze" : {
          "threads" : 1,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 1,
          "completed" : 1
        },
        "ccr" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "data_frame_indexing" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "fetch_shard_started" : {
          "threads" : 1,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 59,
          "completed" : 709
        },
        "fetch_shard_store" : {
          "threads" : 1,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 3,
          "completed" : 36
        },
        "flush" : {
          "threads" : 2,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 5,
          "completed" : 9544
        },
        "force_merge" : {
          "threads" : 1,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 1,
          "completed" : 330
        },
        "generic" : {
          "threads" : 23,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 58,
          "completed" : 7029690
        },
        "get" : {
          "threads" : 32,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 32,
          "completed" : 71859
        },
        "listener" : {
          "threads" : 10,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 10,
          "completed" : 12438
        },
        "management" : {
          "threads" : 5,
          "queue" : 0,
          "active" : 2,
          "rejected" : 0,
          "largest" : 5,
          "completed" : 26990256
        },
        "refresh" : {
          "threads" : 10,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 10,
          "completed" : 189074135
        },
        "rollup_indexing" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "search" : {                                                              # 检索线程池信息
          "threads" : 49,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,                                                         # 失败的线程数量
          "largest" : 49,
          "completed" : 23999947
        },
        "search_throttled" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "security-token-key" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "snapshot" : {
          "threads" : 0,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 0,
          "completed" : 0
        },
        "warmer" : {
          "threads" : 1,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,
          "largest" : 5,
          "completed" : 26971
        },
        "write" : {                                                         # 写入（索引）线程池信息
          "threads" : 32,
          "queue" : 0,
          "active" : 0,
          "rejected" : 0,                                                   # 索引线程失败数量
          "largest" : 32,
          "completed" : 2641120
        }
      }
    }
  }
}