es常见查询

ES集群管理

ES提供了一套_cat API, 可以查看ES中的各类数据.

多条件查询

1.关键词:"china"
2.发布时间:"2021-09-13 00:00:00","2021-09-22 13:42:27"
3.排除站点:aol.com
4.排除国建:中国-CHN

GET /new*/_count
{
    "query":{
        "bool":{
            "must":[
                {
                    "term":{
                        "text":{
                            "value":"china"
                        }
                    }
                   
                },{
                   "term": {
                      "mediaLevel": {
                        "value": "5"
                      }
                    }
                }
            ],
            "must_not": [
              {
                "term": {
                  "countryCode": {
                    "value": "CHN"
                  }
                }
              },
              {
                "term": {
                  "domain": {
                    "value": "aol.com"
                  }
                }
              }
            ], 
            "filter":{
                "range":{
                    "pubTime":{
                        "gte":1631462400000,
                        "lte":16322893470007
                    }
                }
            }
        }
    }
}

多条件分组查询

1.查询以上条件,并且按照sourceType进行分组:

GET /new*/_search
{
    "query":{
        "bool":{
            "must":[
                {
                    "term":{
                        "text":{
                            "value":"china"
                        }
                    }
                   
                }
            ],
            "must_not": [
              {
                "term": {
                  "countryCode": {
                    "value": "CHN"
                  }
                }
              },
              {
                "term": {
                  "domain": {
                    "value": "aol.com"
                  }
                }
              }
            ], 
            "filter":{
                "range":{
                    "pubTime":{
                        "gte":1631462400000,
                        "lte":16322893470007
                    }
                }
            }
        }
    },
    "size":0,
    "aggs": {
      "group_by_sourceType": {
        "terms": {
          "field": "sourceType",
          "order": {
            "_count": "asc"
          }
        }
      }
    }
}

查询某一个字段不为空

extend不为空

GET /social_user/_search
{
  "query": {
    "bool": {
      "must": {
        "exists": {
          "field": "extend"
        }
      }
    }
  }
}

in or not in查询

GET /news*/_search
{
  "query": {
    "terms": {
      "siteUrls": [
"https://www.bechtel.com/newsroom/releases/",
"https://www.bechtel.com/newsroom/coverage/"
  ]
  }
},
  "aggs": {
    "distinct_sourcetype": {
      "terms": {
         "field": "sourceType"
      }
    }
  },
  "stored_fields": ["sourceType","siteUrls"]
}

多条件聚合

GET /news*/_search
{
  "query": {
    "terms": {
      "siteUrls": [
"https://www.rolls-royce.com/media/press-releases.aspx"
  ]
  }
},
  "aggs": {
    "distinct_sourcetype": {
      "terms": {
         "field": "sourceType"
      }
    },
    "max_aggs":{
      "max": {
        "field": "pubTime"
      }
    },
    "min_ahhs":{
      "min": {
        "field": "pubTime"
      }
    }
  },
  "stored_fields": ["sourceType","siteUrls","pubTime"]
}

ES针对于text类型的数据进行模糊匹配查询使用:query_string

系统日志存储在es的logstash下,然后是每天生成一个日志,其对应的mapping如下:

  "logstash-2021.10.28" : {
    "mappings" : {
      "properties" : {
        "class" : {
          "type" : "keyword"
        },
        "kubernetes" : {
          "properties" : {
            "container_hash" : {
              "type" : "keyword"
            },
            "container_image" : {
              "type" : "keyword"
            },
            "container_name" : {
              "type" : "keyword"
            },
            "docker_id" : {
              "type" : "keyword"
            },
            "host" : {
              "type" : "keyword"
            },
            "namespace_name" : {
              "type" : "keyword"
            },
            "pod_id" : {
              "type" : "keyword"
            },
            "pod_name" : {
              "type" : "text"
            }
          }
        },
        "level" : {
          "type" : "keyword"
        },
        "log" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "log_flie_path" : {
          "type" : "text"
        },
        "log_time" : {
          "type" : "keyword"
        },
        "message" : {
          "type" : "text"
        },
        "source" : {
          "type" : "keyword"
        },
        "stream" : {
          "type" : "keyword"
        },
        "thread" : {
          "type" : "text"
        },
        "time" : {
          "type" : "date",
          "format" : "strict_date_optional_time_nanos"
        },
        "timestamp" : {
          "type" : "date"
        }
      }
    }
  }

查询日志如下:

GET /logstash-2021.11.10/_search
{
  "query": {
   "bool": {
     "must": [
       {
         "query_string": {
            "default_field": "kubernetes.pod_name",
           "query": "\"gtcom-governance-news-k8s-kafka-test-taskmanager\""
         }
       }
     ],
     "filter": [
       {
         "term": {
           "level": "ERROR"
         }
       }
     ]
   }
  },
  "sort": [
    {
      "timestamp": {
        "order": "desc"
      }
    }
  ],
  "_source": ["message","level","kubernetes.pod_name"]
}

根据日期聚合查询

GET /news*/_search
{
  "query": {
    "bool": {
      "filter": [
        {
          "range": {
            "createTime": {
              "gte": "2021-11-01"
            }
          }
        }
      ]
    }
  },
  "aggs": {
    "NAME": {
      "date_histogram": {
        "field": "createTime",
        "interval": "day"
      }
    }
  }
}