setup:
  - skip:
      features: allowed_warnings
  - do:
      allowed_warnings:
        - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template1
        body:
          index_patterns: [simple-data-stream1]
          template:
            settings:
              index.number_of_replicas: 1
          data_stream: {}

  - do:
      allowed_warnings:
        - "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template2
        body:
          index_patterns: [simple-data-stream2]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date_nanos
          data_stream: {}

---
"Create data stream":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams only supported in 7.9+"

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.create_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

  - do:
      cluster.health:
        wait_for_status: green

  - do:
      indices.get_data_stream:
        name: "*"
  - match: { data_streams.0.name: simple-data-stream1 }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.index_name: '/\.ds-simple-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - match: { data_streams.0.status: 'GREEN' }
  - match: { data_streams.0.template: 'my-template1' }
  - match: { data_streams.0.hidden: false }
  - match: { data_streams.1.name: simple-data-stream2 }
  - match: { data_streams.1.timestamp_field.name: '@timestamp' }
  - match: { data_streams.1.generation: 1 }
  - length: { data_streams.1.indices: 1 }
  - match: { data_streams.1.indices.0.index_name: '/\.ds-simple-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - match: { data_streams.1.template: 'my-template2' }
  - match: { data_streams.1.hidden: false }

  # save the backing index names for later use
  - set: { data_streams.0.indices.0.index_name: idx0name }
  - set: { data_streams.1.indices.0.index_name: idx1name }

  - do:
      indices.get_mapping:
        index: $idx0name
        expand_wildcards: hidden
  - match: { .$idx0name.mappings.properties.@timestamp.type: 'date' }

  - do:
      indices.get_mapping:
        index: $idx1name
        expand_wildcards: hidden
  - match: { .$idx1name.mappings.properties.@timestamp.type: 'date_nanos' }

  - do:
      index:
        index:  simple-data-stream1
        body:
          '@timestamp': '2020-12-12'
          foo: bar

  - do:
      indices.refresh:
        index: simple-data-stream1

  - do:
      search:
        index: simple-data-stream1
        body: { query: { match_all: {} } }
  - length:   { hits.hits: 1  }
  - match: { hits.hits.0._index: "/\\.ds-simple-data-stream1-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }
  - match: { hits.hits.0._source.foo: 'bar' }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.delete_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

---
"Create hidden data stream":
  - requires:
      cluster_features: ["gte_v7.11.0"]
      test_runner_features: ["warnings", "headers"]
      reason: "hidden data streams only available in 7.11"
  - do:
      headers: { X-elastic-product-origin: elastic }
      allowed_warnings:
        - "index template [my-template3] has index patterns [.hidden-data-stream, hidden-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template3
        body:
          index_patterns: [ .hidden-data-stream,hidden-data-stream ]
          data_stream:
            hidden: true
  - do:
      indices.create_data_stream:
        name: hidden-data-stream
  - is_true: acknowledged

  - do:
      indices.create_data_stream:
        name: .hidden-data-stream
  - is_true: acknowledged

  - do:
      indices.get_data_stream:
        name: "*"
  - length: { data_streams: 0 }

  - do:
      indices.get_data_stream:
        name: ['*', '-ilm-history*']
        expand_wildcards: hidden
  - length: { data_streams: 2 }
  - match: { data_streams.0.name: .hidden-data-stream }
  - match: { data_streams.0.hidden: true }
  - match: { data_streams.1.name: hidden-data-stream }
  - match: { data_streams.1.hidden: true }

  - do:
      index:
        index: hidden-data-stream
        body:
          '@timestamp': '2020-12-12'
          foo: bar

  - do:
      indices.refresh:
        index: hidden-data-stream

  - do:
      search:
        index: hidden-data-*
        body: { query: { match_all: {} } }
  - length:   { hits.hits: 0  }

  - do:
      search:
        index: hidden-data-*
        expand_wildcards: open,hidden
        body: { query: { match_all: { } } }
  - length: { hits.hits: 1 }
  - match: { hits.hits.0._source.foo: 'bar' }

  - do:
      index:
        index: .hidden-data-stream
        body:
          '@timestamp': '2020-12-12'
          foo: bar

  - do:
      indices.refresh:
        index: .hidden-data-stream

  - do:
      search:
        index: .hidden-data-*
        body: { query: { match_all: { } } }
  - length: { hits.hits: 1 }
  - match: { hits.hits.0._source.foo: 'bar' }

  - do:
      indices.delete_data_stream:
        name: hidden-data-stream
  - is_true: acknowledged

  - do:
      indices.delete_data_stream:
        name: .hidden-data-stream
  - is_true: acknowledged

  - do:
      indices.delete_index_template:
        name: my-template3

---
"Create data stream with failure store":
  - requires:
      test_runner_features: [ capabilities, allowed_warnings ]
      reason: "Data stream failure stores config in templates was added in 8.18+"
      capabilities:
        - method: POST
          path: /_index_template/{template}
          capabilities: [ 'data_stream_options.failure_store' ]

  - do:
      ingest.put_pipeline:
        id: "data_stream_pipeline"
        body: >
          {
            "description": "_description",
            "processors": [
              {
                "set" : {
                  "field": "message",
                  "value" : "hello world"
                }
              }
            ]
          }
  - match: { acknowledged: true }

  - do:
      ingest.put_pipeline:
        id: "data_stream_final_pipeline"
        body: >
          {
            "description": "_description",
            "processors": [
              {
                "set" : {
                  "field": "message",
                  "value" : "goodbye world"
                }
              }
            ]
          }
  - match: { acknowledged: true }

  - do:
      allowed_warnings:
        - "index template [my-template4] has index patterns [failure-data-stream1, failure-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template4] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template4
        body:
          index_patterns: [ failure-data-stream1, failure-data-stream2 ]
          data_stream: {}
          template:
            settings:
              index:
                default_pipeline: "data_stream_pipeline"
                final_pipeline: "data_stream_final_pipeline"
            mappings:
              properties:
                '@timestamp':
                  type: date
                count:
                  type: long
            data_stream_options:
              failure_store:
                enabled: true

  - do:
      indices.create_data_stream:
        name: failure-data-stream1
  - is_true: acknowledged

  - do:
      indices.create_data_stream:
        name: failure-data-stream2
  - is_true: acknowledged

  # Initialize failure store
  - do:
      index:
        index: failure-data-stream1
        refresh: true
        body:
          '@timestamp': '2020-12-12'
          count: 'invalid value'
  # Initialize failure store
  - do:
      index:
        index: failure-data-stream2
        refresh: true
        body:
          '@timestamp': '2020-12-12'
          count: 'invalid value'

  - do:
      cluster.health:
        wait_for_status: green

  - do:
      indices.get_data_stream:
        name: "*"
  - match: { data_streams.0.name: failure-data-stream1 }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 2 }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.index_name: '/\.ds-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - match: { data_streams.0.status: 'GREEN' }
  - match: { data_streams.0.template: 'my-template4' }
  - match: { data_streams.0.hidden: false }
  - match: { data_streams.0.failure_store.enabled: true }
  - length: { data_streams.0.failure_store.indices: 1 }
  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000002/'}

  - match: { data_streams.1.name: failure-data-stream2 }
  - match: { data_streams.1.timestamp_field.name: '@timestamp' }
  - match: { data_streams.1.generation: 2 }
  - length: { data_streams.1.indices: 1 }
  - match: { data_streams.1.indices.0.index_name: '/\.ds-failure-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - match: { data_streams.1.template: 'my-template4' }
  - match: { data_streams.1.hidden: false }
  - match: { data_streams.1.failure_store.enabled: true }
  - length: { data_streams.1.failure_store.indices: 1 }
  - match: { data_streams.1.failure_store.indices.0.index_name: '/\.fs-failure-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000002/' }

  # save the backing index names for later use
  - set: { data_streams.0.indices.0.index_name: idx0name }
  - set: { data_streams.0.failure_store.indices.0.index_name: fsidx0name }
  - set: { data_streams.1.indices.0.index_name: idx1name }
  - set: { data_streams.1.failure_store.indices.0.index_name: fsidx1name }

  - do:
      indices.get_mapping:
        index: $idx0name
        expand_wildcards: hidden
  - match: { .$idx0name.mappings.properties.@timestamp.type: 'date' }

  - do:
      indices.get_mapping:
        index: $fsidx0name
        expand_wildcards: hidden
  - match: { .$fsidx0name.mappings.properties.@timestamp.type: 'date' }

  - do:
      indices.get_mapping:
        index: $idx1name
        expand_wildcards: hidden
  - match: { .$idx1name.mappings.properties.@timestamp.type: 'date' }

  - do:
      indices.get_mapping:
        index: $fsidx1name
        expand_wildcards: hidden
  - match: { .$fsidx1name.mappings.properties.@timestamp.type: 'date' }

  - do:
      indices.get_settings:
        index: $idx0name
        expand_wildcards: hidden
  - match: { .$idx0name.settings.index.default_pipeline: 'data_stream_pipeline' }
  - match: { .$idx0name.settings.index.final_pipeline: 'data_stream_final_pipeline' }

  - do:
      indices.get_settings:
        index: $fsidx0name
        expand_wildcards: hidden
  - is_false: .$fsidx0name.settings.index.default_pipeline
  - is_false: .$fsidx0name.settings.index.final_pipeline
  - is_true: .$fsidx0name.settings.index.failure_store.version

  - do:
      indices.get_settings:
        index: $idx1name
        expand_wildcards: hidden
  - match: { .$idx1name.settings.index.default_pipeline: 'data_stream_pipeline' }
  - match: { .$idx1name.settings.index.final_pipeline: 'data_stream_final_pipeline' }

  - do:
      indices.get_settings:
        index: $fsidx1name
        expand_wildcards: hidden
  - is_false: .$fsidx1name.settings.index.default_pipeline
  - is_false: .$fsidx1name.settings.index.final_pipeline
  - is_true: .$fsidx1name.settings.index.failure_store.version

  - do:
      indices.delete_data_stream:
        name: failure-data-stream1
  - is_true: acknowledged

  - do:
      indices.delete_data_stream:
        name: failure-data-stream2
  - is_true: acknowledged

  - do:
      indices.delete_index_template:
        name: my-template4
  - is_true: acknowledged

---
"Create data stream with invalid name":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams only supported in 7.9+"

  - do:
      catch: bad_request
      indices.create_data_stream:
        name: invalid-data-stream#-name

  - match: { status: 400 }
  - match: { error.root_cause.0.type: "illegal_argument_exception" }
  - match: { error.root_cause.0.reason: "data_stream [invalid-data-stream#-name] must not contain '#'" }

---
"Create existing data stream":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams only supported in 7.9+"

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      catch: bad_request
      indices.create_data_stream:
        name: simple-data-stream1

  - match: { status: 400 }
  - match: { error.root_cause.0.type: "resource_already_exists_exception" }
  - match: { error.root_cause.0.reason: "data_stream [simple-data-stream1] already exists" }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

---
"Get data stream":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams available in 7.9+"

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.create_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

  - do:
      indices.get_data_stream: {}
  - match: { data_streams.0.name: simple-data-stream1 }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }
  - match: { data_streams.1.name: simple-data-stream2 }
  - match: { data_streams.1.timestamp_field.name: '@timestamp' }
  - match: { data_streams.1.generation: 1 }

  - do:
      indices.get_data_stream:
        name: simple-data-stream1
  - match: { data_streams.0.name: simple-data-stream1 }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }

  - do:
      indices.get_data_stream:
        name: simple-data-stream*
  - match: { data_streams.0.name: simple-data-stream1 }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }
  - match: { data_streams.1.name: simple-data-stream2 }
  - match: { data_streams.1.timestamp_field.name: '@timestamp' }
  - match: { data_streams.1.generation: 1 }

  - do:
      indices.get_data_stream:
        name: nonexistent-data-stream
      catch: missing

  - match: { status: 404 }
  - match: { error.root_cause.0.type: "index_not_found_exception" }

  - do:
      indices.get_data_stream:
        name: nonexistent*
  - match: { data_streams: [] }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.delete_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

---
"Get data stream and check DSL and ILM information":
  - requires:
      cluster_features: ["gte_v8.11.0"]
      reason: "data streams DSL and ILM mixing information available in 8.11+"

  - do:
      allowed_warnings:
        - "index template [mixing-dsl-template] has index patterns [mixing-dsl-stream] matching patterns from existing older templates
        [global] with patterns (global => [*]); this template [mixing-dsl-template] will take precedence during new index creation"
      indices.put_index_template:
        name: mixing-dsl-template
        body:
          index_patterns: [mixing-dsl-stream]
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date_nanos
            lifecycle:
              data_retention: "30d"
              enabled: false
            settings:
              index.lifecycle.prefer_ilm: false
              index.lifecycle.name: "missing_ilm_policy"
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: mixing-dsl-stream
  - is_true: acknowledged

  - do:
      indices.get_data_stream:
        name: mixing-dsl-stream
  - match: { data_streams.0.name: mixing-dsl-stream }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }
  - match: { data_streams.0.ilm_policy: "missing_ilm_policy" }
  - match: { data_streams.0.prefer_ilm: false }
  - match: { data_streams.0.next_generation_managed_by: "Index Lifecycle Management" }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.prefer_ilm: false }
  - match: { data_streams.0.indices.0.ilm_policy: "missing_ilm_policy" }
  - match: { data_streams.0.indices.0.managed_by: "Index Lifecycle Management" }

  - do:
      indices.put_data_lifecycle:
        name: "*"
        body: >
          {
            "data_retention": "30d",
            "enabled": true
          }

  - is_true: acknowledged

  - do:
      indices.get_data_stream:
        name: mixing-dsl-stream
  - match: { data_streams.0.name: mixing-dsl-stream }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }
  - match: { data_streams.0.ilm_policy: "missing_ilm_policy" }
  - match: { data_streams.0.prefer_ilm: false }
  - match: { data_streams.0.next_generation_managed_by: "Data stream lifecycle" }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.prefer_ilm: false }
  - match: { data_streams.0.indices.0.ilm_policy: "missing_ilm_policy" }
  - match: { data_streams.0.indices.0.managed_by: "Data stream lifecycle" }

---
"Delete data stream with backing indices":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams only supported in 7.9+"

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.create:
        index: test_index
        body:
          settings:
            number_of_shards:   1
            number_of_replicas: 1

  # save the backing index names for later use
  - do:
      indices.get_data_stream:
        name: simple-data-stream1

  - set: { data_streams.0.indices.0.index_name: idx0name }

  - do:
      indices.get:
        index: ['.ds-simple-data-stream1-*000001', 'test_index']

  - is_true: test_index.settings
  - is_true: .$idx0name.settings

  - do:
      indices.get_data_stream: {}
  - match: { data_streams.0.name: simple-data-stream1 }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.index_name: '/\.ds-simple-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      catch: missing
      indices.get:
        index: $idx0name

---
"Delete data stream with failure store":
  - requires:
      reason: "Data stream failure stores config in templates was added in 8.18+"
      test_runner_features: [ allowed_warnings, capabilities ]
      capabilities:
        - method: POST
          path: /_index_template/{template}
          capabilities: [ 'data_stream_options.failure_store' ]

  - do:
      allowed_warnings:
        - "index template [my-template4] has index patterns [failure-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template4] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template4
        body:
          index_patterns: [ failure-data-stream1 ]
          data_stream: {}
          template:
            mappings:
              properties:
                '@timestamp':
                  type: date
                count:
                  type: long
            data_stream_options:
              failure_store:
                enabled: true

  - do:
      indices.create_data_stream:
        name: failure-data-stream1
  - is_true: acknowledged

  # Initialize failure store
  - do:
      index:
        index: failure-data-stream1
        refresh: true
        body:
          '@timestamp': '2020-12-12'
          count: 'invalid value'

  - do:
      indices.create:
        index: test_index
        body:
          settings:
            number_of_shards:   1
            number_of_replicas: 1

  # save the backing index names for later use
  - do:
      indices.get_data_stream:
        name: failure-data-stream1

  - set: { data_streams.0.indices.0.index_name: idx0name }
  - set: { data_streams.0.failure_store.indices.0.index_name: fs0name }

  - do:
      indices.get:
        index: ['.ds-failure-data-stream1-*000001', 'test_index']

  - is_true: test_index.settings
  - is_true: .$idx0name.settings

  - do:
      indices.get_data_stream: {}
  - match: { data_streams.0.name: failure-data-stream1 }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 2 }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.index_name: '/\.ds-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - length: { data_streams.0.failure_store.indices: 1 }
  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000002/' }

  - do:
      indices.delete_data_stream:
        name: failure-data-stream1
  - is_true: acknowledged

  - do:
      catch: missing
      indices.get:
        index: $idx0name

  - do:
      catch: missing
      indices.get:
        index: $fs0name

  - do:
      indices.delete_index_template:
        name: my-template4
  - is_true: acknowledged

---
"Delete data stream with failure store uninitialized":
  - requires:
      reason: "Data stream failure stores config in templates was added in 8.18+"
      test_runner_features: [ capabilities, allowed_warnings ]
      capabilities:
        - method: POST
          path: /_index_template/{template}
          capabilities: [ 'data_stream_options.failure_store' ]

  - do:
      allowed_warnings:
        - "index template [my-template4] has index patterns [failure-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template4] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template4
        body:
          index_patterns: [ failure-data-stream1 ]
          data_stream: {}
          template:
            data_stream_options:
              failure_store:
                enabled: true

  - do:
      indices.create_data_stream:
        name: failure-data-stream1
  - is_true: acknowledged

  - do:
      indices.create:
        index: test_index
        body:
          settings:
            number_of_shards:   1
            number_of_replicas: 1

  # save the backing index names for later use
  - do:
      indices.get_data_stream:
        name: failure-data-stream1

  - set: { data_streams.0.indices.0.index_name: idx0name }
  - length: { data_streams.0.failure_store.indices: 0 }

  - do:
      indices.get:
        index: ['.ds-failure-data-stream1-*000001', 'test_index']

  - is_true: test_index.settings
  - is_true: .$idx0name.settings

  - do:
      indices.get_data_stream: {}
  - match: { data_streams.0.name: failure-data-stream1 }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.index_name: '/\.ds-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - length: { data_streams.0.failure_store.indices: 0 }

  - do:
      indices.delete_data_stream:
        name: failure-data-stream1
  - is_true: acknowledged

  - do:
      catch: missing
      indices.get:
        index: $idx0name

  - do:
      indices.delete_index_template:
        name: my-template4
  - is_true: acknowledged

---
"Delete data stream missing behaviour":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams only supported in 7.9+"

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.create_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

  - do:
      indices.create:
        index: simple-data-streamz

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.delete_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

  - do:
      indices.delete_data_stream:
        name: simple-data-stream*
  - is_true: acknowledged

  - do:
      catch: missing
      indices.delete_data_stream:
        name: simple-data-stream1

  - do:
      catch: missing
      indices.delete_data_stream:
        name: simple-data-stream2

---
"Delete data stream by wildcard":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams supported only in 7.9+"

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.create_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

  - do:
      indices.delete_data_stream:
        name: no-matching-data-streams*
  - is_true: acknowledged

  - do:
      indices.get_data_stream:
        name: "*"
  - match: { data_streams.0.name: simple-data-stream1 }
  - match: { data_streams.1.name: simple-data-stream2 }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream*
  - is_true: acknowledged

  - do:
      indices.get_data_stream:
        name: "*"
  - length: { data_streams: 0 }

---
"append-only writes to backing indices prohibited":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams only supported in 7.9+"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
      indices.put_index_template:
        name: generic_logs_template
        body:
          index_patterns: logs-*
          data_stream: {}

  - do:
      index:
        index:  logs-foobar
        body:   { '@timestamp': '2020-12-12' }
  - match: { _index: "/\\.ds-logs-foobar-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }

  # save the backing index names for later use
  - do:
      indices.get_data_stream:
        name: logs-foobar
  - set: { data_streams.0.indices.0.index_name: idx0name }


  - do:
      catch: bad_request
      index:
        index:  $idx0name
        body:   { '@timestamp': '2020-12-12' }

  - do:
      bulk:
        body:
          - create:
              _index: $idx0name
          - '@timestamp': '2020-12-12'
          - index:
              _index: $idx0name
          - '@timestamp': '2020-12-12'
          - create:
              _index: logs-foobar
          - '@timestamp': '2020-12-12'
  - match: { errors: true }
  - match: { items.0.create.status: 400 }
  - match: { items.0.create.error.type: illegal_argument_exception }
  - match: { items.0.create.error.reason: "index request with op_type=create targeting backing indices is disallowed, target corresponding data stream [logs-foobar] instead" }
  - match: { items.1.index.status: 400 }
  - match: { items.1.index.error.type: illegal_argument_exception }
  - match: { items.1.index.error.reason: "index request with op_type=index and no if_primary_term and if_seq_no set targeting backing indices is disallowed, target corresponding data stream [logs-foobar] instead" }
  - match: { items.2.create.result: created }
  - match: { items.2.create._index: "/\\.ds-logs-foobar-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }

  - do:
      indices.delete_data_stream:
        name: logs-foobar
  - is_true: acknowledged

---
"Indexing a document into a data stream without a timestamp field":
  - requires:
      cluster_features: ["gte_v7.9.0"]
      reason: "data streams only supported in 7.9+"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
      indices.put_index_template:
        name: generic_logs_template
        body:
          index_patterns: logs-*
          data_stream: {}

  - do:
      catch: bad_request
      index:
        index:  logs-foobar
        body:   { foo: bar }

  - do:
      bulk:
        body:
          - create:
              _index: logs-foobar
          - foo: bar
          - create:
              _index: logs-foobar
          - '@timestamp': '2020-12-12'
          - create:
              _index: logs-foobar
          - '@timestamp': ['2020-12-12', '2022-12-12']
  - match: { errors: true }
  - match: { items.0.create.status: 400 }
  - match: { items.0.create.error.caused_by.type: illegal_argument_exception }
  - match: { items.0.create.error.caused_by.reason: "data stream timestamp field [@timestamp] is missing" }
  - match: { items.1.create.result: created }
  - match: { items.1.create._index: "/\\.ds-logs-foobar-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }
  - match: { items.2.create.status: 400 }
  - match: { items.2.create.error.caused_by.type: illegal_argument_exception }
  - match: { items.2.create.error.caused_by.reason: "data stream timestamp field [@timestamp] encountered multiple values" }

  - do:
      indices.delete_data_stream:
        name: logs-foobar
  - is_true: acknowledged

---
"Include metadata in a data stream":
  - requires:
      cluster_features: ["gte_v8.0.0"]
      reason: "re-enable in 7.11 when backported"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
      indices.put_index_template:
        name: generic_logs_template
        body:
          index_patterns: logs-*
          _meta:
            managed_by: 'core-features'
            managed: true
          template:
            settings:
              number_of_replicas: 0
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: logs-foobar
  - is_true: acknowledged

  - do:
      indices.get_data_stream:
        name: "*"
  - length: { data_streams: 1 }
  - match: { data_streams.0.name: 'logs-foobar' }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.index_name: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - match: { data_streams.0.status: 'GREEN' }
  - match: { data_streams.0.template: 'generic_logs_template' }
  - length: { data_streams.0._meta: 2 }
  - match: { data_streams.0._meta.managed: true }
  - match: { data_streams.0._meta.managed_by: 'core-features' }

---
"Create index into a namespace that is governed by a data stream template":
  - requires:
      cluster_features: ["gte_v7.9.2"]
      reason: "validation added in 7.9.2"
      test_runner_features: allowed_warnings

  - do:
      allowed_warnings:
        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
      indices.put_index_template:
        name: generic_logs_template
        body:
          index_patterns: logs-*
          data_stream: {}

  # test response status code
  - do:
      catch: bad_request
      indices.create:
        index: logs-foobar

  # test error message
  - do:
      catch: /cannot create index with name \[logs-foobar\], because it matches with template \[generic_logs_template\] that creates data streams only, use create data stream api instead/
      indices.create:
        index: logs-foobar

---
"Get data stream api check existence of replicated field":
  - requires:
      cluster_features: ["gte_v7.16.1"]
      reason: "replicated field only supported in 7.16+"

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.create_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

  - do:
      cluster.health:
        wait_for_status: green

  - do:
      indices.get_data_stream:
        name: "*"
  - match: { data_streams.0.name: simple-data-stream1 }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.index_name: '/\.ds-simple-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - match: { data_streams.0.status: 'GREEN' }
  - match: { data_streams.0.template: 'my-template1' }
  - match: { data_streams.0.hidden: false }
  - match: { data_streams.0.replicated: false }
  - match: { data_streams.1.name: simple-data-stream2 }
  - match: { data_streams.1.timestamp_field.name: '@timestamp' }
  - match: { data_streams.1.generation: 1 }
  - length: { data_streams.1.indices: 1 }
  - match: { data_streams.1.indices.0.index_name: '/\.ds-simple-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - match: { data_streams.1.template: 'my-template2' }
  - match: { data_streams.1.hidden: false }
  - match: { data_streams.1.replicated: false }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.delete_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

---
"Get data stream api check existence of allow_custom_routing field":
  - requires:
      cluster_features: ["gte_v8.0.1"]
      reason: "allow_custom_routing field only supported in 8.0+"

  - do:
      indices.create_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.create_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

  - do:
      cluster.health:
        wait_for_status: green

  - do:
      indices.get_data_stream:
        name: "*"
  - match: { data_streams.0.name: simple-data-stream1 }
  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
  - match: { data_streams.0.generation: 1 }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.index_name: '/\.ds-simple-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - match: { data_streams.0.status: 'GREEN' }
  - match: { data_streams.0.template: 'my-template1' }
  - match: { data_streams.0.hidden: false }
  - match: { data_streams.0.allow_custom_routing: false }
  - match: { data_streams.1.name: simple-data-stream2 }
  - match: { data_streams.1.timestamp_field.name: '@timestamp' }
  - match: { data_streams.1.generation: 1 }
  - length: { data_streams.1.indices: 1 }
  - match: { data_streams.1.indices.0.index_name: '/\.ds-simple-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - match: { data_streams.1.template: 'my-template2' }
  - match: { data_streams.1.hidden: false }
  - match: { data_streams.1.allow_custom_routing: false }

  - do:
      indices.delete_data_stream:
        name: simple-data-stream1
  - is_true: acknowledged

  - do:
      indices.delete_data_stream:
        name: simple-data-stream2
  - is_true: acknowledged

---
"Create data stream with match all template":
  - requires:
      cluster_features: ["gte_v8.16.0"]
      reason: "data streams supoprt for match all templates only supported in 8.16"

  - do:
      allowed_warnings:
        - "index template [match-all-template] has index patterns [*] matching patterns from existing older templates [.monitoring-logstash,.monitoring-es,.monitoring-beats,.monitoring-alerts-7,global,.monitoring-kibana] with patterns (.monitoring-logstash => [.monitoring-logstash-7-*],.monitoring-es => [.monitoring-es-7-*],.monitoring-beats => [.monitoring-beats-7-*],.monitoring-alerts-7 => [.monitoring-alerts-7],global => [*],.monitoring-kibana => [.monitoring-kibana-7-*]); this template [match-all-template] will take precedence during new index creation"
        - "index template [match-all-template] has index patterns [*] matching patterns from existing older templates [.monitoring-logstash,.monitoring-es,.monitoring-beats,.monitoring-alerts-7,.monitoring-kibana] with patterns (.monitoring-logstash => [.monitoring-logstash-7-*],.monitoring-es => [.monitoring-es-7-*],.monitoring-beats => [.monitoring-beats-7-*],.monitoring-alerts-7 => [.monitoring-alerts-7],.monitoring-kibana => [.monitoring-kibana-7-*]); this template [match-all-template] will take precedence during new index creation"
      indices.put_index_template:
        name: match-all-template
        body:
          index_patterns: [ "*" ]
          priority: 1
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: match-all-data-stream
  - is_true: acknowledged

  - do:
      cluster.health:
        wait_for_status: green

  - do:
      indices.get_data_stream:
        name: "*"
  - match: { data_streams.0.name: match-all-data-stream }
  - match: { data_streams.0.generation: 1 }
  - length: { data_streams.0.indices: 1 }
  - match: { data_streams.0.indices.0.index_name: '/\.ds-match-all-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
  - match: { data_streams.0.status: 'GREEN' }
  - match: { data_streams.0.template: 'match-all-template' }
  - match: { data_streams.0.hidden: false }

  - do:
      indices.delete_data_stream:
        name: match-all-data-stream
  - is_true: acknowledged

  - do:
      indices.delete_index_template:
        name: match-all-template
  - is_true: acknowledged

---
"Create hidden data stream with match all template":
  - requires:
      cluster_features: [ "gte_v8.16.0" ]
      reason: "data streams supoprt for match all templates only supported in 8.16"
  - do:
      allowed_warnings:
        - "index template [match-all-hidden-template] has index patterns [*] matching patterns from existing older templates [.monitoring-logstash,.monitoring-es,.monitoring-beats,.monitoring-alerts-7,global,.monitoring-kibana] with patterns (.monitoring-logstash => [.monitoring-logstash-7-*],.monitoring-es => [.monitoring-es-7-*],.monitoring-beats => [.monitoring-beats-7-*],.monitoring-alerts-7 => [.monitoring-alerts-7],global => [*],.monitoring-kibana => [.monitoring-kibana-7-*]); this template [match-all-hidden-template] will take precedence during new index creation"
        - "index template [match-all-hidden-template] has index patterns [*] matching patterns from existing older templates [.monitoring-logstash,.monitoring-es,.monitoring-beats,.monitoring-alerts-7,.monitoring-kibana] with patterns (.monitoring-logstash => [.monitoring-logstash-7-*],.monitoring-es => [.monitoring-es-7-*],.monitoring-beats => [.monitoring-beats-7-*],.monitoring-alerts-7 => [.monitoring-alerts-7],.monitoring-kibana => [.monitoring-kibana-7-*]); this template [match-all-hidden-template] will take precedence during new index creation"
      indices.put_index_template:
        name: match-all-hidden-template
        body:
          index_patterns: [ "*" ]
          priority: 1
          data_stream:
            hidden: true
  - do:
      indices.create_data_stream:
        name: match-all-hidden-data-stream
  - is_true: acknowledged

  - do:
      cluster.health:
        wait_for_status: green

  - do:
      indices.get_data_stream:
        name: "*"
  - length: { data_streams: 0 }

  - do:
      indices.get_data_stream:
        name: ['*']
        expand_wildcards: hidden
  - length: { data_streams: 1 }
  - match: { data_streams.0.name: match-all-hidden-data-stream }
  - match: { data_streams.0.hidden: true }

  - do:
      indices.delete_data_stream:
        name: match-all-hidden-data-stream
  - is_true: acknowledged

  - do:
      indices.delete_index_template:
        name: match-all-hidden-template

---
"Get maximum_timestamp for the data stream":
  - requires:
       capabilities:
         - method: GET
           path: /_data_stream
           parameters: [verbose]
       test_runner_features: [allowed_warnings, capabilities]
       reason: Verbose flag required for the test

  - do:
      allowed_warnings:
        - "index template [my-template] has index patterns [data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
      indices.put_index_template:
        name: my-template
        body:
          index_patterns: [data-*]
          data_stream: {}

  - do:
      indices.create_data_stream:
        name: data-stream1
  - is_true: acknowledged

  - do:
      indices.get_data_stream:
        name: data-stream1
        verbose: false

  - match: { data_streams.0.name: data-stream1 }
  - is_false: data_streams.0.maximum_timestamp

  - do:
      indices.get_data_stream:
        name: data-stream1
        verbose: true

  - match: { data_streams.0.name: data-stream1 }
  # 0 because no documents have been indexed yet
  - match: { data_streams.0.maximum_timestamp: 0 }

  - do:
      index:
        index: data-stream1
        refresh: true
        body:
          '@timestamp': '2020-12-12'
          foo: bar

  - do:
      indices.get_data_stream:
        name: data-stream1
        verbose: true

  - match: { data_streams.0.name: data-stream1 }
  - match: { data_streams.0.maximum_timestamp: 1607731200000 }

  - do:
      indices.delete_data_stream:
        name: data-stream1
  - is_true: acknowledged
