elasticsearch基础(4)—— 复合聚合

头像
一曲广陵散
    阅读 10 分钟
    头图

    一、参考

    elasticsearch 学习系列目录——更新ing

    Composite aggregation

    Composite aggregation ORDER BY

    二、产生原因

    当有多层聚合需要时候，使用composite聚合，可以更好的分页

    三、sources类型

    sources参数中定义了，复合聚合的组成源数据

    测试数据

    GET kibana_sample_data_logs/_search
{
  "size": 1
}

{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "kibana_sample_data_logs",
        "_type" : "_doc",
        "_id" : "4O9NX3kBTG9UhPTpZasD",
        "_score" : 1.0,
        "_source" : {
          "agent" : "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
          "bytes" : 7525,
          "clientip" : "60.103.76.51",
          "extension" : "css",
          "geo" : {
            "srcdest" : "IN:TW",
            "src" : "IN",
            "dest" : "TW",
            "coordinates" : {
              "lat" : 35.23199833,
              "lon" : -102.3990931
            }
          },
          "host" : "cdn.elastic-elastic-elastic.org",
          "index" : "kibana_sample_data_logs",
          "ip" : "60.103.76.51",
          "machine" : {
            "ram" : 2147483648,
            "os" : "ios"
          },
          "memory" : null,
          "message" : "60.103.76.51 - - [2018-08-10T10:14:00.227Z] \"GET /styles/ads.css HTTP/1.1\" 200 7525 \"-\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\"",
          "phpmemory" : null,
          "referer" : "http://twitter.com/success/sandra-magnus",
          "request" : "/styles/ads.css",
          "response" : 200,
          "tags" : [
            "success",
            "security"
          ],
          "timestamp" : "2021-05-21T10:14:00.227Z",
          "url" : "https://cdn.elastic-elastic-elastic.org/styles/ads.css",
          "utc_time" : "2021-05-21T10:14:00.227Z",
          "event" : {
            "dataset" : "sample_web_logs"
          }
        }
      }
    ]
  }
}

    3.1 terms

    普通的terms聚合

    GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "terms": {
        "field": "clientip",
        "size": 3
      }
    }
  }
}

{
  "took" : 14,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 13919,
      "buckets" : [
        {
          "key" : "30.156.16.164",
          "doc_count" : 100
        },
        {
          "key" : "164.85.94.243",
          "doc_count" : 29
        },
        {
          "key" : "50.184.59.162",
          "doc_count" : 26
        }
      ]
    }
  }
}

    composite聚合中的terms

    GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "composite": {
        "size": 3, 
        "sources": [
          {
            "clientipAggs": {
              "terms": {
                "field": "clientip",
                "order": "asc"
              }
            }
          }
        ]
      }
    }
  }
}

{
  "took" : 6,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "after_key" : {
        "clientipAggs" : "0.209.144.101"
      },
      "buckets" : [
        {
          "key" : {
            "clientipAggs" : "0.72.176.46"
          },
          "doc_count" : 14
        },
        {
          "key" : {
            "clientipAggs" : "0.207.229.147"
          },
          "doc_count" : 11
        },
        {
          "key" : {
            "clientipAggs" : "0.209.144.101"
          },
          "doc_count" : 14
        }
      ]
    }
  }
}

    3.2 histogram

    普通的histogram聚合

    GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "histogram": {
        "field": "bytes",
        "interval": 5000
      }
    }
  }
}

{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "buckets" : [
        {
          "key" : 0.0,
          "doc_count" : 6377
        },
        {
          "key" : 5000.0,
          "doc_count" : 6995
        },
        {
          "key" : 10000.0,
          "doc_count" : 375
        },
        {
          "key" : 15000.0,
          "doc_count" : 327
        }
      ]
    }
  }
}

    composite聚合中的histogram

    GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "composite": {
        "sources": [
          {
            "bytesAggs": {
              "histogram": {
                "field": "bytes",
                "interval": 5000
              }
            }
          }
        ]
      }
    }
  }
}

{
  "took" : 19,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "after_key" : {
        "bytesAggs" : 15000.0
      },
      "buckets" : [
        {
          "key" : {
            "bytesAggs" : 0.0
          },
          "doc_count" : 6377
        },
        {
          "key" : {
            "bytesAggs" : 5000.0
          },
          "doc_count" : 6995
        },
        {
          "key" : {
            "bytesAggs" : 10000.0
          },
          "doc_count" : 375
        },
        {
          "key" : {
            "bytesAggs" : 15000.0
          },
          "doc_count" : 327
        }
      ]
    }
  }
}

    3.3 date_histogram

    普通的时间聚合

    GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "date_histogram": {
        "field": "timestamp",
        "interval": "1M"
      }
    }
  }
}

{
  "took" : 5,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "buckets" : [
        {
          "key_as_string" : "2021-05-01T00:00:00.000Z",
          "key" : 1619827200000,
          "doc_count" : 6926
        },
        {
          "key_as_string" : "2021-06-01T00:00:00.000Z",
          "key" : 1622505600000,
          "doc_count" : 6943
        },
        {
          "key_as_string" : "2021-07-01T00:00:00.000Z",
          "key" : 1625097600000,
          "doc_count" : 205
        }
      ]
    }
  }
}

    composite中的date_histogram

    GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "composite": {
        "sources": [
          {
            "dateAggs": {
              "date_histogram": {
                "field": "timestamp",
                "interval": "1M"
              }
            }
          }
        ]
      }
    }
  }
}

{
  "took" : 28,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "after_key" : {
        "dateAggs" : 1625097600000
      },
      "buckets" : [
        {
          "key" : {
            "dateAggs" : 1619827200000
          },
          "doc_count" : 6926
        },
        {
          "key" : {
            "dateAggs" : 1622505600000
          },
          "doc_count" : 6943
        },
        {
          "key" : {
            "dateAggs" : 1625097600000
          },
          "doc_count" : 205
        }
      ]
    }
  }
}

    3.4 地理位置

    3.5 多种混合

    普通的混合

    GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "terms": {
        "field": "clientip",
        "size": 3
      }
    },
    "aggs2": {
      "date_histogram": {
        "field": "timestamp",
        "interval": "month"
      }
    }
  }
}

{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs2" : {
      "buckets" : [
        {
          "key_as_string" : "2021-05-01T00:00:00.000Z",
          "key" : 1619827200000,
          "doc_count" : 6926
        },
        {
          "key_as_string" : "2021-06-01T00:00:00.000Z",
          "key" : 1622505600000,
          "doc_count" : 6943
        },
        {
          "key_as_string" : "2021-07-01T00:00:00.000Z",
          "key" : 1625097600000,
          "doc_count" : 205
        }
      ]
    },
    "aggs1" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 13919,
      "buckets" : [
        {
          "key" : "30.156.16.164",
          "doc_count" : 100
        },
        {
          "key" : "164.85.94.243",
          "doc_count" : 29
        },
        {
          "key" : "50.184.59.162",
          "doc_count" : 26
        }
      ]
    }
  }
}

    composite中的混合source

    GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "composite": {
        "size": 3, 
        "sources": [
          {
            "clientipAggs": {
              "terms": {
                "field": "clientip"
              }
            }
          },
          {
            "dateAggs": {
              "date_histogram": {
                "field": "timestamp",
                "interval": "month"
              }
            }
          }
        ]
      }
    }
  }
}


{
  "took" : 6,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "after_key" : {
        "clientipAggs" : "0.207.229.147",
        "dateAggs" : 1619827200000
      },
      "buckets" : [
        {
          "key" : {
            "clientipAggs" : "0.72.176.46",
            "dateAggs" : 1619827200000
          },
          "doc_count" : 6
        },
        {
          "key" : {
            "clientipAggs" : "0.72.176.46",
            "dateAggs" : 1622505600000
          },
          "doc_count" : 8
        },
        {
          "key" : {
            "clientipAggs" : "0.207.229.147",
            "dateAggs" : 1619827200000
          },
          "doc_count" : 6
        }
      ]
    }
  }
}

    四、排序

    4.1 source只有一个元素

    如上，如果sources参数只有一个terms: clientip, 排序只能是对应的field: clientip的顺序

    注意: 无法根据terms: clientip的_count来排序

    4.2 source有多个元素

    如果sources参数有多个元素，例如:

    terms: clientip 和 date_histogram: timestamp, 排序为先按照field: clientip, 然后按照field: timestamp排序

    五、分页

    使用参数, size 和after_key 实现分页功能

    六、子聚合

    同普通的bucket聚合一样，composite聚合可以包含子聚合

    七、pipeline 聚合

    目前, composite聚合和pipeline聚合不兼容，无法一起使用

    八、提前优化

    可以在索引阶段指定sort, 查询时候排序将更有效

    elasticsearch
    一曲广陵散
    76 声望21 粉丝

    柴米油盐酱醋茶

    « 上一篇
    emacs模块(2)——hideshow 代码折叠
    下一篇 »
    emacs插件(5)——复制到剪贴板 xclip

    引用和评论

    推荐阅读
    头像
    emacs开发环境配置(4)——rust开发环境(2023更新版）

    一曲广陵散阅读 715

    头像
    ElasticSearch源码本地调试指南

    Foghost阅读 1.1k

    头像
    Elasticsearch 8.x 重要变化（qbit）

    qbit阅读 744

    头像
    一夜之间，7.0万 Star，全部清零！

    CodeSheep阅读 708

    头像
    KylinV10 ARM 版本安装 Ambari + HDP + ElasticSerach 记录

    LnEoi阅读 339

    头像
    Elasticsearch Serverless 高性价比智能日志分析关键技术解读

    阿里云大数据AI阅读 303

    头像
    Elasticsearch 查看节点状态信息（qbit）

    qbit阅读 292

    0 条评论
    得票最新