nosql aggregation - Elasticsearch中的多个group-by

Question

我需要在ES中使用3个字段进行聚合(group-by)。

我可以在1查询或我需要使用一个面+迭代每列?

谢谢

Answer 1

从ElasticSearch的1.0版本开始，新的聚合 API允许使用子聚合按多个字段分组。假设要按字段field1、field2和field3进行分组:

{
  "aggs": {
    "agg1": {
      "terms": {
        "field": "field1"
      },
      "aggs": {
        "agg2": {
          "terms": {
            "field": "field2"
          },
          "aggs": {
            "agg3": {
              "terms": {
                "field": "field3"
              }
            }
          }          
        }
      }
    }
  }
}

当然，您可以对任意多个字段执行此操作。

更新:
为了完整起见，下面是上述查询的输出。下面是用于生成聚合查询并将结果平展为字典列表的python代码。

{
  "aggregations": {
    "agg1": {
      "buckets": [{
        "doc_count": <count>,
        "key": <value of field1>,
        "agg2": {
          "buckets": [{
            "doc_count": <count>,
            "key": <value of field2>,
            "agg3": {
              "buckets": [{
                "doc_count": <count>,
                "key": <value of field3>
              },
              {
                "doc_count": <count>,
                "key": <value of field3>
              }, ...
              ]
            },
            {
            "doc_count": <count>,
            "key": <value of field2>,
            "agg3": {
              "buckets": [{
                "doc_count": <count>,
                "key": <value of field3>
              },
              {
                "doc_count": <count>,
                "key": <value of field3>
              }, ...
              ]
            }, ...
          ]
        },
        {
        "doc_count": <count>,
        "key": <value of field1>,
        "agg2": {
          "buckets": [{
            "doc_count": <count>,
            "key": <value of field2>,
            "agg3": {
              "buckets": [{
                "doc_count": <count>,
                "key": <value of field3>
              },
              {
                "doc_count": <count>,
                "key": <value of field3>
              }, ...
              ]
            },
            {
            "doc_count": <count>,
            "key": <value of field2>,
            "agg3": {
              "buckets": [{
                "doc_count": <count>,
                "key": <value of field3>
              },
              {
                "doc_count": <count>,
                "key": <value of field3>
              }, ...
              ]
            }, ...
          ]
        }, ...
      ]
    }
  }
}

下面的python代码执行给定字段列表的group-by。如果您指定include_missing=True，它还包括一些字段丢失的值组合(如果您有Elasticsearch 2.0版本，则不需要它)

def group_by(es, fields, include_missing):
    current_level_terms = {'terms': {'field': fields[0]}}
    agg_spec = {fields[0]: current_level_terms}
    if include_missing:
        current_level_missing = {'missing': {'field': fields[0]}}
        agg_spec[fields[0] + '_missing'] = current_level_missing
    for field in fields[1:]:
        next_level_terms = {'terms': {'field': field}}
        current_level_terms['aggs'] = {
            field: next_level_terms,
        }
        if include_missing:
            next_level_missing = {'missing': {'field': field}}
            current_level_terms['aggs'][field + '_missing'] = next_level_missing
            current_level_missing['aggs'] = {
                field: next_level_terms,
                field + '_missing': next_level_missing,
            }
            current_level_missing = next_level_missing
        current_level_terms = next_level_terms
    agg_result = es.search(body={'aggs': agg_spec})['aggregations']
    return get_docs_from_agg_result(agg_result, fields, include_missing)

def get_docs_from_agg_result(agg_result, fields, include_missing):
    current_field = fields[0]
    buckets = agg_result[current_field]['buckets']
    if include_missing:
        buckets.append(agg_result[(current_field + '_missing')])
    if len(fields) == 1:
        return [
            {
                current_field: bucket.get('key'),
                'doc_count': bucket['doc_count'],
            }
            for bucket in buckets if bucket['doc_count'] > 0
        ]
    result = []
    for bucket in buckets:
        records = get_docs_from_agg_result(bucket, fields[1:], include_missing)
        value = bucket.get('key')
        for record in records:
            record[current_field] = value
        result.extend(records)
    return result

Answer 2

有两种方法:

1)在单个facet中使用多个字段的结果:

单字段示例facet:

curl -X GET "http://localhost:9200/sales/order/_search?pretty=true" -d '{
  "query": {
    "query_string": {
      "query": "shohi*",
      "fields": [
        "billing_name"
      ]
    }
  },
  "facets": {
    "facet_result": {
      "terms": {
        "fields": [
          "status"
        ],
        "order": "term",
        "size": 15
      }
    }
  }
}'

在单个facet结果中包含多个字段的示例:

curl -X GET "http://localhost:9200/sales/order/_search?pretty=true" -d '{
  "query": {
    "query_string": {
      "query": "shohi*",
      "fields": [
        "billing_name"
      ]
    }
  },
  "facets": {
    "facet_result": {
      "terms": {
        "fields": [
          "status",
          "customer_gender",
          "state"
        ],
        "order": "term",
        "size": 15
      }
    }
  }
}'

2)使用多个facet结果集:

curl -X GET "http://localhost:9200/sales/order/_search?pretty=true" -d '{
  "query": {
    "query_string": {
      "query": "*",
      "fields": [
        "increment_id"
      ]
    }
  },
  "facets": {
    "status_facets": {
      "terms": {
        "fields": [
          "status"
        ],
        "size": 50,
        "order": "term"
      }
    },
    "gender_facets": {
      "terms": {
        "fields": [
          "customer_gender"
        ]
      }
    },
    "state_facets": {
      "terms": {
        "fields": [
          "state"
        ],
        ,
        "order": "term"
      }
    }
  }
}'

参考链接:http://www.elasticsearch.org/guide/reference/api/search/facets/terms-facet.html