---
"Verify data stream resolvability for xpack apis":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams available in 7.9+"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [my-template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template
        body:
          index_patterns: [logs-*]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: logs-foobar
  - is_true: acknowledged

  - do:
      indices.reload_search_analyzers:
        index: logs-foobar
  - length: { reload_details: 1}

  # this 404 is expected, because there are no snapshot indices
  # but the data stream was resolved, which is what is being tested here
  - do:
      catch: missing
      searchable_snapshots.clear_cache:
        index: logs-foobar
  - match: { error.reason: "No searchable snapshots indices found"}

  # this 404 is expected, because there are no snapshot indices
  # but the data stream was resolved, which is what is being tested here
  - do:
      catch: missing
      searchable_snapshots.stats:
        index: logs-foobar
  - match: { error.reason: "No searchable snapshots indices found"}

  - do:
      indices.delete_data_stream:
        name: logs-foobar
  - is_true: acknowledged

---
"Verify data stream resolvability in ilm explain api":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams available in 7.9+"
      test_runner_features: allowed_warnings

  - do:
      ilm.put_lifecycle:
        policy: "my_lifecycle"
        body: |
          {
            "policy": {
              "phases": {
                "warm": {
                  "min_age": "1000s",
                  "actions": {
                    "forcemerge": {
                      "max_num_segments": 10000
                    }
                  }
                },
                "hot": {
                  "min_age": "1000s",
                  "actions": { }
                }
              }
            }
          }

  - do:
      allowed_warnings:
        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
      indices.put_index_template:
        name: generic_logs_template
        body:
          index_patterns: logs-*
          data_stream: {}
          template:
            settings:
              index.lifecycle.name: "my_lifecycle"
            mappings:
              properties:
                '@timestamp':
                  type: date

  - do:
      index:
        index:  logs-foobar
        refresh: true
        body:
          foo: bar
          '@timestamp': '2020-12-12'

  # save the backing index names for later use
  - do:
      indices.get_data_stream:
        name: logs-foobar
  - set: { data_streams.0.indices.0.index_name: idx0name }

  - do:
      ilm.explain_lifecycle:
        index: logs-foobar
  - is_false: indices.logs-foobar.managed
  - is_true: indices.$idx0name.managed

  - do:
      indices.delete_data_stream:
        name: logs-foobar
  - is_true: acknowledged
---
"Verify data stream resolvability rollup":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams available in 7.9+"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [my-ds-rolluptemplate] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-ds-rolluptemplate] will take precedence during new index creation"
      indices.put_index_template:
        name: my-ds-rolluptemplate
        body:
          index_patterns: [logs-*]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: logs-ds-rollup
  - is_true: acknowledged

  - do:
      # Should not raise error
      allowed_warnings:
        - "The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information."
      rollup.get_rollup_index_caps:
        index: "logs*"
  - do:
      indices.delete_data_stream:
        name: logs-ds-rollup
  - is_true: acknowledged

---
"Verify data stream resolvability in ILM remove policy API":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams available in 7.9+"
      test_runner_features: allowed_warnings

  - do:
      ilm.put_lifecycle:
        policy: "my_moveable_timeseries_lifecycle"
        body: |
          {
            "policy": {
              "phases": {
                "warm": {
                  "min_age": "1000s",
                  "actions": {
                    "forcemerge": {
                      "max_num_segments": 10000
                    }
                  }
                },
                "hot": {
                  "min_age": "1000s",
                  "actions": { }
                }
              }
            }
          }

  - do:
      allowed_warnings:
        - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template1
        body:
          index_patterns: [simple-data-stream1]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
            settings:
              index.lifecycle.name: "my_moveable_timeseries_lifecycle"
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  # rollover data stream to create new backing index
  - do:
      indices.rollover:
        alias: "simple-data-stream1"

  - match: { old_index: "/\\.ds-simple-data-stream1-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }
  - match: { new_index: "/\\.ds-simple-data-stream1-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" }
  - match: { rolled_over: true }
  - match: { dry_run: false }

  # save the backing index names for later use
  - do:
      indices.get_data_stream:
        name: simple-data-stream1
  - set: { data_streams.0.indices.0.index_name: idx0name }
  - set: { data_streams.0.indices.1.index_name: idx1name }

  # verify policy applied to all backing indices
  - do:
      indices.get_settings:
        index: ".ds-simple-data-stream1-*"

  - match: { .$idx0name.settings.index.lifecycle.name: my_moveable_timeseries_lifecycle }
  - match: { .$idx1name.settings.index.lifecycle.name: my_moveable_timeseries_lifecycle }

  - do:
      ilm.remove_policy:
        index: "simple-data-stream1"

  - is_false: has_failures
  - length: { failed_indexes: 0 }

  # verify policy removed from all backing indices
  - do:
      indices.get_settings:
        index: ".ds-simple-data-stream1-*"

  - is_false: .$idx0name.settings.index.lifecycle
  - is_false: .$idx1name.settings.index.lifecycle

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

---
"Verify data stream resolvability for graph explore API":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams available in 7.9+"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [my-ds-rolluptemplate] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-ds-rolluptemplate] will take precedence during new index creation"
      indices.put_index_template:
        name: my-ds-rolluptemplate
        body:
          index_patterns: [simple-data-stream1]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      index:
        index:   simple-data-stream1
        id:      "1"
        op_type: create
        body:    { keys: [1,2,3], '@timestamp': '2020-12-12' }

  - do:
      index:
        index:   simple-data-stream1
        id:      "2"
        op_type: create
        body:    { keys: [4,5,6], '@timestamp': '2020-12-12' }

  - do:
      indices.refresh:
        index: simple-data-stream1

  - do:
      graph.explore:
        index:  simple-data-stream1
        body:  {"query": {"match": {"keys": 1}},"controls":{"use_significance":false},"vertices":[{"field": "keys","min_doc_count": 1}]}
  - length: {failures: 0}
  - length: {vertices: 3}

  - do:
      graph.explore:
        index: simple*
        body: { "query": { "match": { "keys": 1 } },"controls": { "use_significance": false },"vertices": [ { "field": "keys","min_doc_count": 1 } ] }
  - length: { failures: 0 }
  - length: { vertices: 3 }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

---
"Verify data stream resolvability in migrations API":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams available in 7.9+"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template1
        body:
          index_patterns: [simple-data-stream1]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      migration.deprecations:
        index: "simple-data-stream1"
  - length: { cluster_settings: 0 }
  - length: { node_settings: 0 }
  - length: { index_settings: 0 }
  - length: { ml_settings: 0 }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

---
"Verify data stream resolvability in async search":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams available in 7.9+"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template1
        body:
          index_patterns: [simple-data-stream1]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      index:
        index:  simple-data-stream1
        body:   { max: 2, '@timestamp': '2020-12-12' }

  - do:
      index:
        index:  simple-data-stream1
        body:   { max: 1, '@timestamp': '2020-12-12' }

  - do:
      index:
        index:  simple-data-stream1
        body:   { max: 3, '@timestamp': '2020-12-12' }

  - do:
      indices.refresh:
        index: simple-data-stream1

  - do:
      async_search.submit:
        index: simple-data-stream1
        batched_reduce_size: 2
        wait_for_completion_timeout: 10s
        body:
          query:
            match_all: {}
          aggs:
            max:
              max:
                field: max
          sort: max

  - is_false: id
  - match:  { is_partial:                   false }
  - length: { response.hits.hits:               3 }
  - match:  { response.hits.hits.0._source.max: 1 }
  - match:  { response.aggregations.max.value:  3.0 }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

---
"Verify data stream resolvability in rollup search":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams available in 7.9+"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template1
        body:
          index_patterns: [simple-data-stream1]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
                partition:
                  type: keyword
                price:
                  type: integer
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      index:
        index:  simple-data-stream1
        body:   { partition: a, price: 1, '@timestamp': '2020-12-12T00:00:00.000Z' }

  - do:
      index:
        index:  simple-data-stream1
        body:   { partition: a, price: 2, '@timestamp': '2020-12-12T01:00:00.000Z' }

  - do:
      index:
        index:  simple-data-stream1
        body:   { partition: b, price: 3, '@timestamp': '2020-12-12T01:00:00.000Z' }

  - do:
      indices.refresh:
        index: simple-data-stream1

  - do:
      allowed_warnings:
        - "The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information."
      rollup.rollup_search:
        index: "simple-data-stream1"
        body:
          size: 0
          aggs:
            histo:
              date_histogram:
                field: "@timestamp"
                calendar_interval: "1h"
                time_zone: "UTC"

  - length: { aggregations.histo.buckets: 2 }
  - match: { aggregations.histo.buckets.0.key_as_string: "2020-12-12T00:00:00.000Z" }
  - match: { aggregations.histo.buckets.0.doc_count: 1 }
  - match: { aggregations.histo.buckets.1.key_as_string: "2020-12-12T01:00:00.000Z" }
  - match: { aggregations.histo.buckets.1.doc_count: 2 }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

---
"Verify data stream resolvability in EQL search API with no sequences":
  - requires:
      cluster_features: ["gte_v7.9.2"]
      reason: "EQL data stream resolution only properly works from 7.9.2"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template1
        body:
          index_patterns: [simple-data-stream1]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: simple-data-stream1

  - do:
      bulk:
        refresh: true
        body:
          - create:
              _index: simple-data-stream1
              _id:    "1"
          - event:
              - category: process
            "@timestamp": 2020-02-03T12:34:56Z
            user: SYSTEM

  - do:
      eql.search:
        index: simple-data-stream1
        body:
          query: 'process where user == "SYSTEM"'

  - match: {timed_out: false}
  - match: {hits.total.value: 1}
  - match: {hits.total.relation: "eq"}
  - match: {hits.events.0._source.user: "SYSTEM"}

  - do:
      eql.search:
        index: simple-data-s*
        body:
          query: 'process where user == "SYSTEM"'

  - match: {timed_out: false}
  - match: {hits.total.value: 1}
  - match: {hits.total.relation: "eq"}
  - match: {hits.events.0._source.user: "SYSTEM"}

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

---
"Verify data stream resolvability in EQL search API with sequences":
  - requires:
      cluster_features: ["gte_v7.10.0"]
      reason: "EQL data stream resolution with queries using sequences only properly works from 7.10.0 (when point-in-time is introduced)"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template1
        body:
          index_patterns: [simple-data-stream1]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: simple-data-stream1

  - do:
      bulk:
        refresh: true
        body:
          - '{"create":{"_index":"simple-data-stream1","_id": "1"}}'
          - '{"@timestamp":"2020-02-03T12:34:56Z","event":{"category":"process"},"user":"SYSTEM"}'
          - '{"create":{"_index":"simple-data-stream1", "_id": "2"}}'
          - '{"@timestamp":"2020-02-03T12:35:00Z","event":{"category":"file"},"user":"SYSTEM","file":{"name":"sample.exe"}}'

  - do:
      eql.search:
        index: simple-data-stream1
        body:
          query: 'sequence by user [process where user == "SYSTEM"] [file where file.name == "sample.exe"]'

  - match: {timed_out: false}
  - match: {hits.total.value: 1}
  - match: {hits.total.relation: "eq"}
  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
  - match: {hits.sequences.0.events.0._id: "1"}
  - match: {hits.sequences.0.events.1._id: "2"}

  - do:
      eql.search:
        index: simple-data-s*
        body:
          query: 'sequence by user [process where user == "SYSTEM"] [file where file.name == "sample.exe"]'

  - match: {timed_out: false}
  - match: {hits.total.value: 1}
  - match: {hits.total.relation: "eq"}
  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
  - match: {hits.sequences.0.events.0._id: "1"}
  - match: {hits.sequences.0.events.1._id: "2"}

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

---
"Verify data stream resolvability in PIT searches":
  - requires:
      cluster_features: ["gte_v7.10.0"]
      reason: "Point-In-Time was introduced in 7.10.0"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template1
        body:
          index_patterns: [simple-data-stream1]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: simple-data-stream1

  - do:
      bulk:
        refresh: true
        body:
          - '{"create":{"_index":"simple-data-stream1","_id": "1"}}'
          - '{"id": 1, "foo": "bar", "age": 18, "@timestamp":"2020-02-03T12:34:56Z"}'
          - '{"create":{"_index":"simple-data-stream1", "_id": "5"}}'
          - '{"id": 5, "foo": "bar", "age": 18, "@timestamp":"2020-02-04T12:34:56Z"}'
          - '{"create":{"_index":"simple-data-stream1", "_id": "123"}}'
          - '{"id": 123, "foo": "bar", "age": 22, "@timestamp":"2020-02-05T12:34:56Z"}'

  - do:
      open_point_in_time:
        index: simple-data-s*
        keep_alive: 5m
  - set: {id: point_in_time_id}

  - do:
      search:
        body:
          size: 1
          query:
            match:
              foo: bar
          sort: [{ age: desc }, { id: desc }]
          pit:
            id: "$point_in_time_id"
            keep_alive: 1m

  - match: {hits.total.value: 3 }
  - length: {hits.hits: 1 }
  - match: {hits.hits.0._index: "/\\.ds-simple-data-stream1-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }
  - match: {hits.hits.0._id: "123" }
  - match: {hits.hits.0.sort.0: 22}
  - match: {hits.hits.0.sort.1: 123}

  - do:
      search:
        body:
          size: 1
          query:
            match:
              foo: bar
          sort: [ { age: desc }, { id: desc } ]
          search_after: [22, 123]
          pit:
            id: "$point_in_time_id"

  - match: {hits.total.value: 3}
  - length: {hits.hits: 1 }
  - match: {hits.hits.0._index: "/\\.ds-simple-data-stream1-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }
  - match: {hits.hits.0._id: "5" }
  - match: {hits.hits.0.sort.0: 18}
  - match: {hits.hits.0.sort.1: 5}

  - do:
      search:
        body:
          size: 1
          query:
            match:
              foo: bar
          sort: [{ age: desc }, { id: desc } ]
          search_after: [18, 5]
          pit:
            id: "$point_in_time_id"
            keep_alive: 1m

  - match: {hits.total.value: 3}
  - length: {hits.hits: 1 }
  - match: {hits.hits.0._index: "/\\.ds-simple-data-stream1-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }
  - match: {hits.hits.0._id: "1" }
  - match: {hits.hits.0.sort.0: 18}
  - match: {hits.hits.0.sort.1: 1}

  - do:
      search:
        body:
          size: 1
          query:
            match:
              foo: bar
          sort: [{ age: desc }, { id: desc } ]
          search_after: [18, 1]
          pit:
            id: "$point_in_time_id"
            keep_alive: 1m

  - match: {hits.total.value: 3}
  - length: {hits.hits: 0 }

  - do:
      close_point_in_time:
        body:
          id: "$point_in_time_id"
