setup:
  - skip:
      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      indices.create:
        index: store
        body:
          mappings:
            properties:
              features:
                type: keyword
              error_message:
                type: keyword
              timestamp:
                type: date
              geo_point:
                type: geo_point
              histogram:
                type: histogram

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
        Content-Type: application/json
      bulk:
        index: store
        refresh: true
        body: |
          { "index": {} }
          { "features": ["turbocharger", "electric mirror", "heated steering wheel", "matrix led", "gps", "sunroof", "autopilot", "air condition"], "error_message": "compressor low pressure", "timestamp": 1020229654000 }
          { "index": {} }
          { "features": ["pink wheels", "turbocharger", "lane assist", "matrix led", "coming home", "heated seats", "blind spot monitoring", "air condition"], "error_message": "engine overheated", "timestamp": 1020316054000 }
          { "index": {} }
          { "features": ["awd", "tow-bar", "matrix led", "gps", "sunroof", "air condition"], "error_message": "compressor low pressure", "timestamp": 1022742044000 }
          { "index": {} }
          { "features": ["rwd", "tow-bar", "turbocharger", "lane assist", "pink wheels"], "error_message": "engine overheated", "timestamp": 1021532444000 }
          { "index": {} }
          { "features": ["pink wheels", "sunroof", "air condition"], "error_message": "compressor low pressure", "timestamp": 1020575254000 }
          { "index": {} }
          { "features": ["turbocharger", "pink wheels", "air condition" ], "error_message": "engine overheated", "timestamp": 1020322844000 }
          { "index": {} }
          { "features": ["awd", "heated steering wheel", "sunroof", "air condition"], "error_message": "compressor low pressure", "timestamp": 1020748054000 }
          { "index": {} }
          { "features": ["turbocharger", "sunroof", "autopilot", "air condition" ], "error_message": "compressor low pressure", "timestamp": 1020229654000 }
          { "index": {} }
          { "features": ["rwd", "turbocharger", "autopilot", "pink wheels"], "error_message": "engine overheated", "timestamp": 1022137244000 }
          { "index": {} }
          { "features": ["fwd", "tow-bar", "turbocharger", "pink wheels", "electric mirror", "heated steering wheel", "lane assist", "sunroof", "autopilot", "air condition" ], "error_message": "low tire pressure", "timestamp": 1020920854000 }
  # same data, but stored in a flattened field
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      indices.create:
        index: store-flattened
        body:
          mappings:
            properties:
              data:
                type: flattened
              timestamp:
                type: date

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
        Content-Type: application/json
      bulk:
        index: store-flattened
        refresh: true
        body: |
          { "index": {} }
          { "data": {"features": ["turbocharger", "electric mirror", "heated steering wheel", "matrix led", "gps", "sunroof", "autopilot", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1020229654000 }
          { "index": {} }
          { "data": {"features": ["pink wheels", "turbocharger", "lane assist", "matrix led", "coming home", "heated seats", "blind spot monitoring", "air condition"], "error_message": "engine overheated"}, "timestamp": 1020316054000 }
          { "index": {} }
          { "data": {"features": ["awd", "tow-bar", "matrix led", "gps", "sunroof", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1022742044000 }
          { "index": {} }
          { "data": {"features": ["rwd", "tow-bar", "turbocharger", "lane assist", "pink wheels"], "error_message": "engine overheated"}, "timestamp": 1021532444000 }
          { "index": {} }
          { "data": {"features": ["pink wheels", "sunroof", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1020575254000 }
          { "index": {} }
          { "data": {"features": ["turbocharger", "pink wheels", "air condition" ], "error_message": "engine overheated"}, "timestamp": 1020322844000 }
          { "index": {} }
          { "data": {"features": ["awd", "heated steering wheel", "sunroof", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1020748054000 }
          { "index": {} }
          { "data": {"features": ["turbocharger", "sunroof", "autopilot", "air condition" ], "error_message": "compressor low pressure"}, "timestamp": 1020229654000 }
          { "index": {} }
          { "data": {"features": ["rwd", "turbocharger", "autopilot", "pink wheels"], "error_message": "engine overheated"}, "timestamp": 1022137244000 }
          { "index": {} }
          { "data": {"features": ["fwd", "tow-bar", "turbocharger", "pink wheels", "electric mirror", "heated steering wheel", "lane assist", "sunroof", "autopilot", "air condition" ], "error_message": "low tire pressure"}, "timestamp": 1020920854000 }
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      indices.create:
        index: unavailable-data
        body:
          mappings:
            properties:
              features:
                type: keyword
              error_message:
                type: keyword
              timestamp:
                type: date
              geo_point:
                type: geo_point
              histogram:
                type: histogram

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      cluster.health:
        wait_for_status: yellow

---
"Test frequent item sets array fields":

  - do:
      search:
        index: store
        body: >
          {
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "features"},
                    {"field": "error_message"}
                  ]
                }
              }
            }
          }
  - length: { aggregations.fi.buckets: 4 }
  - match: { aggregations.fi.buckets.0.doc_count: 5 }
  - match: { aggregations.fi.buckets.0.support: 0.5 }
  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
  - match: { aggregations.fi.buckets.1.doc_count: 4 }
  - match: { aggregations.fi.buckets.1.support: 0.4 }
  - match: { aggregations.fi.buckets.1.key.error_message: ["engine overheated"] }

---
"Test frequent item sets date format":

  - do:
      search:
        index: store
        body: >
          {
            "runtime_mappings": {
              "weekday": {
                "type": "keyword",
                "script": {
                "source": "emit(doc['timestamp'].value.format(DateTimeFormatter.ofPattern('EEE')));"
                }
              }
            },
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "features"},
                    {"field": "weekday"}
                  ]
                }
              }
            }
          }
  - length: { aggregations.fi.buckets: 3 }
  - match: { aggregations.fi.buckets.0.doc_count: 5 }
  - match: { aggregations.fi.buckets.0.support: 0.5 }
  - match: { aggregations.fi.buckets.0.key.weekday: ["Thu"] }


---
"Test frequent item sets date format 2":

  - do:
      search:
        index: store
        body: >
          {
            "runtime_mappings": {
              "weekday": {
                "type": "keyword",
                "script": {
                "source": "emit(doc['timestamp'].value.format(DateTimeFormatter.ofPattern('EEE')));"
                }
              }
            },
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 2,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "error_message"},
                    {"field": "weekday"}
                  ]
                }
              }
            }
          }
  - length: { aggregations.fi.buckets: 1 }
  - match: { aggregations.fi.buckets.0.doc_count: 4 }
  - match: { aggregations.fi.buckets.0.support: 0.4 }
  - match: { aggregations.fi.buckets.0.key.weekday: ["Thu"] }
  - match: { aggregations.fi.buckets.0.key.error_message: ["engine overheated"] }

---
"Test frequent item sets array fields profile":

  - do:
      search:
        index: store
        body: >
          {
            "profile": true,
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.2,
                  "fields": [
                    {"field": "features"},
                    {"field": "error_message"}
                  ]
                }
              }
            }
          }
  - length: { aggregations.fi.buckets: 10 }
  - match: { aggregations.fi.buckets.0.doc_count: 5 }
  - match: { aggregations.fi.buckets.0.support: 0.5 }
  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
  - match: { aggregations.fi.buckets.1.doc_count: 4 }
  - match: { aggregations.fi.buckets.1.support: 0.4 }
  - match: { aggregations.fi.buckets.1.key.error_message: ["engine overheated"] }
  - match: { aggregations.fi.buckets.1.support: 0.4 }
  - match: { aggregations.fi.profile.unique_transactions_after_prune: 10 }
  - match: { aggregations.fi.profile.unique_items_after_prune: 11 }

---
"Test frequent item sets flattened fields":

  - do:
      search:
        index: store-flattened
        body: >
          {
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "data.features"},
                    {"field": "data.error_message"}
                  ]
                }
              }
            }
          }
  - length: { aggregations.fi.buckets: 4 }
  - match: { aggregations.fi.buckets.0.doc_count: 5 }
  - match: { aggregations.fi.buckets.0.support: 0.5 }
  - match: { aggregations.fi.buckets.0.key.data\.error_message: ["compressor low pressure"] }
  - match: { aggregations.fi.buckets.1.doc_count: 4 }
  - match: { aggregations.fi.buckets.1.support: 0.4 }
  - match: { aggregations.fi.buckets.1.key.data\.error_message: ["engine overheated"] }

---
"Test frequent item sets as subagg":

  - do:
      search:
        index: store
        body: >
          {
            "size": 0,
            "aggs": {
              "filter_error": {
                "filter": {
                  "term": {
                    "error_message": "compressor low pressure"
                  }
                },
                "aggs": {
                  "fi": {
                    "frequent_item_sets": {
                      "minimum_set_size": 3,
                      "minimum_support": 0.3,
                      "fields": [
                        {"field": "features"},
                        {"field": "error_message"}
                      ]
                    }
                  }
                }
              }
            }
          }
  - length: { aggregations.filter_error.fi.buckets: 5 }
  - match: { aggregations.filter_error.fi.buckets.0.doc_count: 5 }
  - match: { aggregations.filter_error.fi.buckets.0.support: 1.0 }
  - match: { aggregations.filter_error.fi.buckets.0.key.error_message: ["compressor low pressure"] }
  - match: { aggregations.filter_error.fi.buckets.1.doc_count: 2 }
  - match: { aggregations.filter_error.fi.buckets.1.support: 0.4 }
  - match: { aggregations.filter_error.fi.buckets.0.key.error_message: ["compressor low pressure"] }

---
"Test frequent item sets as multi-bucket subagg":

  - do:
      search:
        index: store
        body: >
          {
            "size": 0,
            "aggs": {
              "weekly": {
                "date_histogram": {
                  "field": "timestamp",
                  "calendar_interval": "1w"
                },
                "aggs": {
                  "fi": {
                    "frequent_item_sets": {
                      "minimum_set_size": 3,
                      "minimum_support": 0.3,
                      "fields": [
                        {"field": "features"},
                        {"field": "error_message"}
                      ]
                    }
                  }
                }
              }
            }
          }
  - length: { aggregations.weekly.buckets: 5 }
  - match: { aggregations.weekly.buckets.0.fi.buckets.0.doc_count: 3 }
  - match: { aggregations.weekly.buckets.0.fi.buckets.0.support: 0.6 }
  - match: { aggregations.weekly.buckets.0.fi.buckets.0.key.error_message: ["compressor low pressure"] }
  - length: { aggregations.weekly.buckets.0.fi.buckets: 4 }
  - match: { aggregations.weekly.buckets.2.fi.buckets.0.doc_count: 1 }

---
"Test frequent item sets filter":

  - do:
      search:
        index: store
        body: >
          {
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "features"},
                    {"field": "error_message"}
                  ],
                  "filter": {
                    "bool": {
                      "must_not": {
                        "term": {"features": "pink wheels"}
                      }
                    }
                  }
                }
              }
            }
          }
  - length: { aggregations.fi.buckets: 1 }
  - match: { aggregations.fi.buckets.0.doc_count: 4 }
  - match: { aggregations.fi.buckets.0.support: 0.4 }
  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }

---
"Test frequent item sets exclude":

  - do:
      search:
        index: store
        body: >
          {
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "features"},
                    {
                      "field": "error_message",
                      "exclude": "engine overheated"
                    }
                  ]
                }
              }
            }
          }
  - length: { aggregations.fi.buckets: 3 }
  - match: { aggregations.fi.buckets.0.doc_count: 5 }
  - match: { aggregations.fi.buckets.0.support: 0.5 }
  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
  - match: { aggregations.fi.buckets.1.doc_count: 3 }
  - match: { aggregations.fi.buckets.1.support: 0.3 }

---
"Test frequent item sets include":

  - do:
      search:
        index: store
        body: >
          {
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "features"},
                    {
                      "field": "error_message",
                      "include": "en.*ed"
                    }
                  ]
                }
              }
            }
          }
  - length: { aggregations.fi.buckets: 3 }
  - match: { aggregations.fi.buckets.0.doc_count: 4 }
  - match: { aggregations.fi.buckets.0.support: 0.4 }
  - match: { aggregations.fi.buckets.0.key.error_message: ["engine overheated"] }

---
"Test frequent item sets unsupported types":

  - do:
      catch: /Field \[geo_point\] of type \[geo_point\] is not supported for aggregation \[frequent_item_sets\]/
      search:
        index: store
        body: >
          {
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "features"},
                    {"field": "error_message"},
                    {"field": "geo_point"}
                  ]
                }
              }
            }
          }
  - do:
      catch: /Field \[histogram\] of type \[histogram\] is not supported for aggregation \[frequent_item_sets\]/
      search:
        index: store
        body: >
          {
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "features"},
                    {"field": "error_message"},
                    {"field": "histogram"}
                  ]
                }
              }
            }
          }

---
"Test frequent item sets unsupported subaggs":
  - do:
      catch: /Aggregator \[fi\] of type \[frequent_item_sets\] cannot accept sub-aggregations/
      search:
        index: store
        body: >
          {
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "features"},
                    {"field": "error_message"}
                  ]
                },
              "aggs": {
                "sum": {
                  "sum": {
                    "field": "some_field"
                  }
                }
              }
            }
            }
          }

---
"Test deprecated frequent items":
  - skip:
      features:
        - "allowed_warnings"

  - do:
      allowed_warnings:
        - 'Deprecated field [frequent_items] used, expected [frequent_item_sets] instead'

      search:
        index: store
        body: >
          {
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_items": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "features"},
                    {"field": "error_message"}
                  ]
                }
              }
            }
          }
  - length: { aggregations.fi.buckets: 4 }
  - match: { aggregations.fi.buckets.0.doc_count: 5 }
  - match: { aggregations.fi.buckets.0.support: 0.5 }
  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
  - match: { aggregations.fi.buckets.1.doc_count: 4 }
  - match: { aggregations.fi.buckets.1.support: 0.4 }
  - match: { aggregations.fi.buckets.1.key.error_message: ["engine overheated"] }

---
"Test frequent items on empty index":
  - do:
      search:
        index: unavailable-data
        body: >
          {
            "size": 0,
            "aggs": {
              "fi": {
                "frequent_item_sets": {
                  "minimum_set_size": 3,
                  "minimum_support": 0.3,
                  "fields": [
                    {"field": "features"},
                    {"field": "error_message"}
                  ]
                }
              }
            }
          }
  - length: { aggregations.fi.buckets: 0 }
