setup:
  - do:
      indices.create:
        index: airline-data
        body:
          mappings:
            properties:
              time:
                type: date
              airline:
                type: keyword
              responsetime:
                type: float
              agg_metric:
                type: aggregate_metric_double
                metrics: [ "min", "max" ]
                default_metric: "max"
              event_rate:
                type: integer
              _isDeleted:
                type: boolean
  - do:
      index:
        index: airline-data
        id: "1"
        body: >
          {
            "time": "2017-02-18T00:00:00Z",
            "airline": "foo",
            "responsetime": 1.0,
            "event_rate": 5,
            "_isDeleted": false
          }

  - do:
      index:
        index: airline-data
        id: "2"
        body: >
          {
            "time": "2017-02-18T00:30:00Z",
            "airline": "foo",
            "responsetime": 1.0,
            "event_rate": 6,
            "_isDeleted": true
          }

  - do:
      index:
        index: airline-data
        id: "3"
        body: >
          {
            "time": "2017-02-18T01:00:00Z",
            "airline": "bar",
            "responsetime": 42.0,
            "event_rate": 8,
            "_isDeleted": false
          }

  - do:
      index:
        index: airline-data
        id: "4"
        body: >
          {
            "time": "2017-02-18T01:01:00Z",
            "airline": "foo",
            "responsetime": 42.0,
            "event_rate": 7,
            "_isDeleted": true
          }

  - do:
      indices.refresh:
        index: airline-data

---
"Test preview transform":
  - do:
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "pivot": {
              "group_by": {
                "airline": {"terms": {"field": "airline"}},
                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}},
                "time.max": {"max": {"field": "time"}},
                "time.min": {"min": {"field": "time"}}
              }
            }
          }
  - match: { preview.0.airline: foo }
  - match: { preview.0.by-hour: "2017-02-18T00:00:00.000Z" }
  - match: { preview.0.avg_response: 1.0 }
  - match: { preview.0.time.max: "2017-02-18T00:30:00.000Z" }
  - match: { preview.0.time.min: "2017-02-18T00:00:00.000Z" }
  - match: { preview.1.airline: bar }
  - match: { preview.1.by-hour: "2017-02-18T01:00:00.000Z" }
  - match: { preview.1.avg_response: 42.0 }
  - match: { preview.1.time.max: "2017-02-18T01:00:00.000Z" }
  - match: { preview.1.time.min: "2017-02-18T01:00:00.000Z" }
  - match: { preview.2.airline: foo }
  - match: { preview.2.by-hour: "2017-02-18T01:00:00.000Z" }
  - match: { preview.2.avg_response: 42.0 }
  - match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" }
  - match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" }
  - match: { generated_dest_index.mappings.properties.airline.type: "keyword" }
  - match: { generated_dest_index.mappings.properties.by-hour.type: "date" }
  - match: { generated_dest_index.mappings.properties.avg_response.type: "double" }
  - match: { generated_dest_index.mappings.properties.time\.max.type: "date" }
  - match: { generated_dest_index.mappings.properties.time\.min.type: "date" }

  - do:
      ingest.put_pipeline:
        id: "data_frame_simple_pipeline"
        body:  >
          {
            "processors": [
             {
               "set" : {
                 "field" : "my_field",
                 "value": 42
               }
              }
            ]
          }
  - match: { acknowledged: true }
  - do:
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "dest": { "pipeline": "data_frame_simple_pipeline" },
            "pivot": {
              "group_by": {
                "airline": {"terms": {"field": "airline"}},
                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}}
              }
            }
          }
  - match: { preview.0.airline: foo }
  - match: { preview.0.by-hour: "2017-02-18T00:00:00.000Z" }
  - match: { preview.0.avg_response: 1.0 }
  - match: { preview.0.my_field: 42 }
  - match: { preview.1.airline: bar }
  - match: { preview.1.by-hour: "2017-02-18T01:00:00.000Z" }
  - match: { preview.1.avg_response: 42.0 }
  - match: { preview.1.my_field: 42 }
  - match: { preview.2.airline: foo }
  - match: { preview.2.by-hour: "2017-02-18T01:00:00.000Z" }
  - match: { preview.2.avg_response: 42.0 }
  - match: { preview.2.my_field: 42 }
  - match: { generated_dest_index.mappings.properties.airline.type: "keyword" }
  - match: { generated_dest_index.mappings.properties.by-hour.type: "date" }
  - match: { generated_dest_index.mappings.properties.avg_response.type: "double" }

---
"Test preview transform with offset":
  - do:
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "pivot": {
              "group_by": {
                "airline": {"terms": {"field": "airline"}},
                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time", "offset": "-15m"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}},
                "time.max": {"max": {"field": "time"}},
                "time.min": {"min": {"field": "time"}}
              }
            }
          }
  - match: { preview.0.airline: foo }
  - match: { preview.0.by-hour: "2017-02-17T23:45:00.000Z" }
  - match: { preview.0.avg_response: 1.0 }
  - match: { preview.0.time.max: "2017-02-18T00:30:00.000Z" }
  - match: { preview.0.time.min: "2017-02-18T00:00:00.000Z" }
  - match: { preview.1.airline: bar }
  - match: { preview.1.by-hour: "2017-02-18T00:45:00.000Z" }
  - match: { preview.1.avg_response: 42.0 }
  - match: { preview.1.time.max: "2017-02-18T01:00:00.000Z" }
  - match: { preview.1.time.min: "2017-02-18T01:00:00.000Z" }
  - match: { preview.2.airline: foo }
  - match: { preview.2.by-hour: "2017-02-18T00:45:00.000Z" }
  - match: { preview.2.avg_response: 42.0 }
  - match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" }
  - match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" }
  - match: { generated_dest_index.mappings.properties.airline.type: "keyword" }
  - match: { generated_dest_index.mappings.properties.by-hour.type: "date" }
  - match: { generated_dest_index.mappings.properties.avg_response.type: "double" }
  - match: { generated_dest_index.mappings.properties.time\.max.type: "date" }
  - match: { generated_dest_index.mappings.properties.time\.min.type: "date" }

---
"Test preview transform with timeout":
  - do:
      transform.preview_transform:
        timeout: "10s"
        body: >
          {
            "source": { "index": "airline-data" },
            "pivot": {
              "group_by": {
                "airline": {"terms": {"field": "airline"}},
                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}},
                "time.max": {"max": {"field": "time"}},
                "time.min": {"min": {"field": "time"}}
              }
            }
          }

---
"Test preview transform with disabled mapping deduction":
  - do:
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "pivot": {
              "group_by": {
                "airline": {"terms": {"field": "airline"}},
                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}},
                "time.max": {"max": {"field": "time"}},
                "time.min": {"min": {"field": "time"}}
              }
            },
            "settings": {
              "deduce_mappings": false
            }
          }
  - match: { preview.0.airline: foo }
  - match: { preview.0.by-hour: "2017-02-18T00:00:00.000Z" }
  - match: { preview.0.avg_response: 1.0 }
  - match: { preview.0.time.max: "2017-02-18T00:30:00.000Z" }
  - match: { preview.0.time.min: "2017-02-18T00:00:00.000Z" }
  - match: { preview.1.airline: bar }
  - match: { preview.1.by-hour: "2017-02-18T01:00:00.000Z" }
  - match: { preview.1.avg_response: 42.0 }
  - match: { preview.1.time.max: "2017-02-18T01:00:00.000Z" }
  - match: { preview.1.time.min: "2017-02-18T01:00:00.000Z" }
  - match: { preview.2.airline: foo }
  - match: { preview.2.by-hour: "2017-02-18T01:00:00.000Z" }
  - match: { preview.2.avg_response: 42.0 }
  - match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" }
  - match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" }
  - match: { generated_dest_index.mappings.properties: {} }

---
"Test preview transform by id":
  - do:
      transform.put_transform:
        transform_id: "airline-transform"
        body: >
          {
            "source": { "index": "airline-data" },
            "dest": { "index": "dest-airline-data-by-airline-and-hour" },
            "pivot": {
              "group_by": {
                "airline": {"terms": {"field": "airline"}},
                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}},
                "time.max": {"max": {"field": "time"}},
                "time.min": {"min": {"field": "time"}}
              }
            }
          }
  - match: { acknowledged: true }

  - do:
      transform.preview_transform:
        transform_id: "airline-transform"
  - match: { preview.0.airline: foo }
  - match: { preview.0.by-hour: "2017-02-18T00:00:00.000Z" }
  - match: { preview.0.avg_response: 1.0 }
  - match: { preview.0.time.max: "2017-02-18T00:30:00.000Z" }
  - match: { preview.0.time.min: "2017-02-18T00:00:00.000Z" }
  - match: { preview.1.airline: bar }
  - match: { preview.1.by-hour: "2017-02-18T01:00:00.000Z" }
  - match: { preview.1.avg_response: 42.0 }
  - match: { preview.1.time.max: "2017-02-18T01:00:00.000Z" }
  - match: { preview.1.time.min: "2017-02-18T01:00:00.000Z" }
  - match: { preview.2.airline: foo }
  - match: { preview.2.by-hour: "2017-02-18T01:00:00.000Z" }
  - match: { preview.2.avg_response: 42.0 }
  - match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" }
  - match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" }
  - match: { generated_dest_index.mappings.properties.airline.type: "keyword" }
  - match: { generated_dest_index.mappings.properties.by-hour.type: "date" }
  - match: { generated_dest_index.mappings.properties.avg_response.type: "double" }
  - match: { generated_dest_index.mappings.properties.time\.max.type: "date" }
  - match: { generated_dest_index.mappings.properties.time\.min.type: "date" }

---
"Test preview transform latest":
  - do:
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "latest": {
              "unique_key": ["airline"],
              "sort": "time"
            }
          }
  - length: { $body: 2 }
  - match: { preview.0.airline: bar }
  - match: { preview.0.time: "2017-02-18T01:00:00Z" }
  - is_false: preview.0._isDeleted
  - match: { preview.1.airline: foo }
  - match: { preview.1.time: "2017-02-18T01:01:00Z" }
  - match: { generated_dest_index.mappings.properties: {} }
  - is_false: preview.1._isDeleted

  - do:
      ingest.put_pipeline:
        id: "transform_latest_simple_pipeline"
        body:  >
          {
            "processors": [
             {
               "set" : {
                 "field" : "my_field",
                 "value": 42
               }
              }
            ]
          }
  - match: { acknowledged: true }
  - do:
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "dest": { "pipeline": "transform_latest_simple_pipeline" },
            "latest": {
              "unique_key": ["airline"],
              "sort": "time"
            }
          }
  - length: { $body: 2 }
  - match: { preview.0.airline: bar }
  - match: { preview.0.time: "2017-02-18T01:00:00Z" }
  - match: { preview.0.my_field: 42 }
  - is_false: preview.0._isDeleted
  - match: { preview.1.airline: foo }
  - match: { preview.1.time: "2017-02-18T01:01:00Z" }
  - match: { preview.1.my_field: 42 }
  - is_false: preview.1._isDeleted
  - match: { generated_dest_index.mappings.properties: {} }

---
"Test preview non-existent transform":
  - do:
      catch: /Transform with id \[non-existent-transform\] could not be found/
      transform.preview_transform:
        transform_id: "non-existent-transform"

---
"Test preview without id and body":
  - do:
      catch: /Please provide a transform \[id\] or the config object/
      transform.preview_transform: {}

---
"Test preview with both id and body":
  - do:
      catch: /Please provide either a transform \[id\] or the config object but not both/
      transform.preview_transform:
        transform_id: "non-existent-transform"
        body: >
          {
            "source": { "index": "airline-data" },
            "dest": { "pipeline": "transform_latest_simple_pipeline" },
            "latest": {
              "unique_key": ["airline"],
              "sort": "time"
            }
          }

---
"Test preview transform with invalid config":
  - do:
      catch: /\[data_frame_terms_group\] unknown field \[not_a_terms_param\]/
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "pivot": {
              "group_by": {"airline": {"terms": {"not_a_terms_param": "airline"}}},
              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
            }
          }
---
"Test preview with non-existing source index":
  - do:
      catch: /.*Validation Failed.* no such index \[does_not_exist\]/
      transform.preview_transform:
        body: >
          {
            "source": { "index": ["airline-data", "does_not_exist"] },
            "pivot": {
              "group_by": {
                "airline": {"terms": {"field": "airline"}},
                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
            }
          }
---
"Test preview returns bad request with invalid agg":
  - do:
      catch: bad_request
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "pivot": {
              "group_by": {
                "time": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}},
                "time.min": {"min": {"field": "time"}}
              }
            }
          }

  - do:
      catch: /field \[time\] cannot be both an object and a field/
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "pivot": {
              "group_by": {
                "time": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}},
                "time.min": {"min": {"field": "time"}}
              }
            }
          }
  - do:
      catch: /field \[super_metric\] cannot be both an object and a field/
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "pivot": {
              "group_by": {
                "time": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}},
                "super_metric.time": {"min": {"field": "time"}},
                "super_metric": {
                  "scripted_metric": {
                     "init_script": "",
                     "map_script": "",
                     "combine_script": "",
                     "reduce_script": "return ['value1': 1, 'value2':2]"
                  }
                }
              }
            }
          }
---
"Test preview with missing pipeline":
  - do:
      catch: bad_request
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "dest": { "pipeline": "missing-pipeline" },
            "pivot": {
              "group_by": {
                "time": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}},
                "time.min": {"min": {"field": "time"}}
              }
            }
          }
---
"Test preview with unsupported agg":
  - do:
      catch: bad_request
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "pivot": {
              "group_by": {
                "time": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "vals": {"significant_terms": {"field":"airline"}}
              }
            }
          }
  - do:
      catch: /Unsupported aggregation type \[significant_terms\]/
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "pivot": {
              "group_by": {
                "time": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "vals": {"significant_terms": {"field":"airline"}}
              }
            }
          }

---
"Test preview transform pivot with search runtime fields":

  - do:
      transform.preview_transform:
        body:  >
          {
            "source": {
              "index": "airline-data",
              "runtime_mappings" : {
                "airline-upper": {
                  "type": "keyword",
                  "script": "emit(doc['airline'].value.toUpperCase())"
                },
                "responsetime_x_2": {
                  "type": "double",
                  "script": "emit(doc['responsetime'].value * 2.0)"
                },
                "time-5m": {
                  "type": "date",
                  "script": "emit(doc['time'].value.toInstant().minus(5, ChronoUnit.MINUTES).toEpochMilli())"
                }
              }
            },
            "pivot": {
              "group_by": {
                "airline": {"terms": {"field": "airline-upper"}},
                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime_x_2"}},
                "time.max": {"max": {"field": "time-5m"}},
                "time.min": {"min": {"field": "time-5m"}}
              }
            }
          }
  - match: { preview.0.airline: FOO }
  - match: { preview.0.by-hour: "2017-02-18T00:00:00.000Z" }
  - match: { preview.0.avg_response: 2.0 }
  - match: { preview.0.time.max: "2017-02-18T00:25:00.000Z" }
  - match: { preview.0.time.min: "2017-02-17T23:55:00.000Z" }
  - match: { preview.1.airline: BAR }
  - match: { preview.1.by-hour: "2017-02-18T01:00:00.000Z" }
  - match: { preview.1.avg_response: 84.0 }
  - match: { preview.1.time.max: "2017-02-18T00:55:00.000Z" }
  - match: { preview.1.time.min: "2017-02-18T00:55:00.000Z" }
  - match: { preview.2.airline: FOO }
  - match: { preview.2.by-hour: "2017-02-18T01:00:00.000Z" }
  - match: { preview.2.avg_response: 84.0 }
  - match: { preview.2.time.max: "2017-02-18T00:56:00.000Z" }
  - match: { preview.2.time.min: "2017-02-18T00:56:00.000Z" }
  - match: { generated_dest_index.mappings.properties.airline.type: "keyword" }
  - match: { generated_dest_index.mappings.properties.by-hour.type: "date" }
  - match: { generated_dest_index.mappings.properties.avg_response.type: "double" }
  - match: { generated_dest_index.mappings.properties.time\.max.type: "date" }
  - match: { generated_dest_index.mappings.properties.time\.min.type: "date" }

---
"Test preview transform latest with search runtime fields":

  - do:
      transform.preview_transform:
        body:  >
          {
            "source": {
              "index": "airline-data",
              "runtime_mappings" : {
                "time-5m": {
                  "type": "date",
                  "script": "emit(doc['time'].value.toInstant().minus(5, ChronoUnit.MINUTES).toEpochMilli())"
                }
              }
            },
            "latest": {
              "unique_key": ["airline"],
              "sort": "time-5m"
            }
          }
  - length: { $body: 2 }
  - match: { preview.0.airline: bar }
  - match: { preview.0.time: "2017-02-18T01:00:00Z" }
  - is_false: preview.0._isDeleted  # field with leading underscore character '_' is treated as internal and filtered out
  - is_false: preview.0.time-5m  # runtime field is not available in the resulting document, only source fields are
  - match: { preview.1.airline: foo }
  - match: { preview.1.time: "2017-02-18T01:01:00Z" }
  - is_false: preview.1._isDeleted  # field with leading underscore character '_' is treated as internal and filtered out
  - is_false: preview.1.time-5m  # runtime field is not available in the resulting document, only source fields are

---
"Test preview continuous transform with no warnings":
  - do:
      transform.preview_transform:
        body: >
          {
            "source": {
              "index": "airline-data",
              "runtime_mappings" : {
                "time-10m": {
                  "type": "date",
                  "script": "emit(doc['time'].value.toInstant().minus(10, ChronoUnit.MINUTES).toEpochMilli())"
                }
              }
            },
            "pivot": {
              "group_by": {
                "airline": {"terms": {"field": "airline"}}
              },
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}}
              }
            },
            "sync": {
              "time": {
                "field": "time"
              }
            }
          }

---
"Test preview continuous transform with warnings":
  - skip:
      features: warnings

  - do:
      warnings:
        - "all the group-by fields are script-based runtime fields, this transform might run slowly, please check your configuration."
        - "sync time field is a script-based runtime field, this transform might run slowly, please check your configuration."
      transform.preview_transform:
        body: >
          {
            "source": {
              "index": "airline-data",
              "runtime_mappings" : {
                "airline-upper": {
                  "type": "keyword",
                  "script": "emit(doc['airline'].value.toUpperCase())"
                },
                "time-10m": {
                  "type": "date",
                  "script": "emit(doc['time'].value.toInstant().minus(10, ChronoUnit.MINUTES).toEpochMilli())"
                }
              }
            },
            "pivot": {
              "group_by": {
                "airline": {"terms": {"field": "airline-upper"}}
              },
              "aggs": {
                "avg_response": {"avg": {"field": "responsetime"}}
              }
            },
            "sync": {
              "time": {
                "field": "time-10m"
              }
            }
          }

---
"Test preview of unattended transform":
  - do:
      transform.preview_transform:
        body: >
          {
            "source": { "index": "airline-data" },
            "dest": { "index": "dest-airline-data-by-airline-unattended" },
            "pivot": {
              "group_by": { "airline": {"terms": {"field": "airline"}}},
              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
            },
            "settings": {
              "unattended": true
            }
          }

---
"Test preview transform latest with unique key field of aggregate_metric_double type":
  - do:
      catch: /Field \[agg_metric\] of type \[aggregate_metric_double\] is not supported for aggregation \[terms\]/
      transform.preview_transform:
        body:  >
          {
            "source": {
              "index": "airline-data"
            },
            "latest": {
              "unique_key": ["agg_metric"],
              "sort": "time"
            }
          }
