---
"Test get all jobs and stats given no job exists":

  - do:
      ml.get_jobs:
        job_id: "_all"
  - match: { count: 0 }
  - match: { jobs: [] }

  - do:
      ml.get_job_stats:
        job_id: "_all"
  - match: { count: 0 }
  - match: { jobs: [] }

---
"Test get jobs with expression that does not match and allow_no_match":
  - skip:
      features:
        - "warnings"

  - do:
      ml.get_jobs:
        job_id: "missing-*"
        allow_no_match: true
  - match: { count: 0 }
  - match: { jobs: [] }

---
"Test get jobs with expression that does not match and not allow_no_match":
  - skip:
      features:
        - "warnings"

  - do:
      catch: missing
      ml.get_jobs:
        job_id: "missing-*"
        allow_no_match: false

---
"Test put job deprecated bucket span":
  - skip:
      features:
        - "warnings"

  - do:
      warnings:
        - 'bucket_span 1.5d [129600s] is not an integral multiple of the number of seconds in 1d [86400s]. This is now deprecated.'
      ml.put_job:
        job_id: job-deprecated-bucket-span-1.5d
        body:  >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "36h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "time_field":"time",
                "time_format":"yyyy-MM-dd HH:mm:ssX"
            }
          }
  - match: { job_id: "job-deprecated-bucket-span-1.5d" }
  - match: { analysis_config.bucket_span: "36h" }
  - match: { analysis_limits.model_memory_limit: "1024mb" }
  - match: { analysis_limits.categorization_examples_limit: 4 }

  - do:
      warnings:
        - 'bucket_span 19m [1140s] is not an integral divisor of the number of seconds in 1d [86400s]. This is now deprecated.'
      ml.put_job:
        job_id: job-deprecated-bucket-span-19m
        body:  >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "19m",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "time_field":"time",
                "time_format":"yyyy-MM-dd HH:mm:ssX"
            }
          }
  - match: { job_id: "job-deprecated-bucket-span-19m" }
  - match: { analysis_config.bucket_span: "19m" }
  - match: { analysis_limits.model_memory_limit: "1024mb" }
  - match: { analysis_limits.categorization_examples_limit: 4 }

---
"Test job crud apis":

  - do:
      ml.put_job:
        job_id: job-crud-test-apis
        body:  >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "time_field":"time",
                "time_format":"yyyy-MM-dd HH:mm:ssX"
            }
          }
  - match: { job_id: "job-crud-test-apis" }
  - match: { analysis_limits.model_memory_limit: "1024mb" }
  - match: { analysis_limits.categorization_examples_limit: 4 }
  - is_true: create_time


  - do:
      ml.get_jobs:
        job_id: "job-crud-test-apis"
  - match: { count: 1 }
  - match: { jobs.0.job_id: "job-crud-test-apis" }
  - match: { jobs.0.analysis_limits.model_memory_limit: "1024mb" }

  - do:
      ml.delete_job:
        job_id: "job-crud-test-apis"
  - match: { acknowledged: true }

---
"Test put job with model_memory_limit as number":

  - do:
      ml.put_job:
        job_id: job-model-memory-limit-as-number
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            },
            "analysis_limits": {
                "model_memory_limit": 2048
            }
          }
  - match: { job_id: "job-model-memory-limit-as-number" }
  - match: { analysis_limits.model_memory_limit: "2048mb" }

---
"Test put job with model_prune_window":
  - skip:
      features:
        - "warnings"

  - do:
      ml.put_job:
        job_id: job-model-prune-window
        body:  >
          {
            "analysis_config" : {
                "bucket_span": "15m",
                "detectors" :[{"function":"count"}],
                "model_prune_window": "14d"
            },
            "data_description" : {
            }
          }
  - match: { job_id: "job-model-prune-window" }
  - match: { analysis_config.bucket_span: "15m" }
  - match: { analysis_config.model_prune_window: "14d" }
  - is_true: create_time

  - do:
      ml.put_job:
        job_id: job-default-model-prune-window
        body: >
          {
            "analysis_config" : {
                "bucket_span": "15m",
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            }
          }
  - match: { job_id: "job-default-model-prune-window" }
  - match: { analysis_config.bucket_span: "15m" }
  - match: { analysis_config.model_prune_window: "30d" }
  - is_true: create_time

  - do:
      warnings:
        - 'bucket_span 1.6d [144000s] is not an integral multiple of the number of seconds in 1d [86400s]. This is now deprecated.'
      ml.put_job:
        job_id: job-default-model-prune-window-40h
        body: >
          {
            "analysis_config" : {
                "bucket_span": "40h",
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            }
          }
  - match: { job_id: "job-default-model-prune-window-40h" }
  - match: { analysis_config.bucket_span: "40h" }
  - match: { analysis_config.model_prune_window: "800h" }
  - is_true: create_time

  - do:
      catch: /model_prune_window \[29m\] must be a multiple of bucket_span \[15m\]/
      ml.put_job:
        job_id: job-invalid-model-prune-window
        body:  >
          {
            "analysis_config" : {
                "bucket_span": "15m",
                "detectors" :[{"function":"count"}],
                "model_prune_window": "29m"
            },
            "data_description" : {
            }
          }

  - do:
      catch: /model_prune_window \[15m\] must be at least 2 times greater than bucket_span \[15m\]/
      ml.put_job:
        job_id: job-model-prune-window
        body:  >
          {
            "analysis_config" : {
                "bucket_span": "15m",
                "detectors" :[{"function":"count"}],
                "model_prune_window": "15m"
            },
            "data_description" : {
            }
          }

# Here we test that in the case of the job being configured with an
# unconventional bucket span (17m) that the generated model_prune_window
# is the closest multiple of the bucket span greater than or equal to 30
# days (43200m)
  - do:
      warnings:
        - 'bucket_span 17m [1020s] is not an integral divisor of the number of seconds in 1d [86400s]. This is now deprecated.'
      ml.put_job:
        job_id: job-default-model-prune-window_with_odd_bucket_span
        body: >
          {
            "analysis_config" : {
                "bucket_span": "17m",
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            }
          }
  - match: { job_id: "job-default-model-prune-window_with_odd_bucket_span" }
  - match: { analysis_config.bucket_span: "17m" }
  - match: { analysis_config.model_prune_window: "43214m" }
  - is_true: create_time

# We expect that the default model_prune_window will be set to
# 20 * bucket_span if this value is greater than 30 daya
  - do:
      ml.put_job:
        job_id: job-default-model-prune-window_with_large_bucket_span
        body: >
          {
            "analysis_config" : {
                "bucket_span": "14d",
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            }
          }
  - match: { job_id: "job-default-model-prune-window_with_large_bucket_span" }
  - match: { analysis_config.bucket_span: "14d" }
  - match: { analysis_config.model_prune_window: "280d" }
  - is_true: create_time

  - do:
      ml.put_job:
        job_id: job-default-model-prune-window_with_small_bucket_span
        body: >
          {
            "analysis_config" : {
                "bucket_span": "1s",
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            }
          }
  - match: { job_id: "job-default-model-prune-window_with_small_bucket_span" }
  - match: { analysis_config.bucket_span: "1s" }
  - match: { analysis_config.model_prune_window: "30d" }
  - is_true: create_time
---
"Test put job with model_memory_limit as string and lazy open":
  - skip:
      features: headers
  - do:
      ml.put_job:
        job_id: job-model-memory-limit-as-string
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            },
            "analysis_limits": {
                "model_memory_limit": "3000g"
            },
            "allow_lazy_open": true
          }
  - match: { job_id: "job-model-memory-limit-as-string" }
  - match: { analysis_limits.model_memory_limit: "3072000mb" }
  - is_true: create_time

  # The assumption here is that a 3000GB job will not fit on the test
  # node - increase in future if the test ever fails because of this!
  # But because the job is allowed to open lazily, opening it shouldn't
  # throw an exception - it should wait for a big enough node to be
  # added to the cluster.
  - do:
      ml.open_job:
        job_id: job-model-memory-limit-as-string
  - match: { opened: true }
  - match: { node: "" }

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.get_job_stats:
        job_id: job-model-memory-limit-as-string
  - match: {"jobs.0.state": opening}

  # Despite never being assigned to a node the job should close gracefully
  - do:
      ml.close_job:
        job_id: job-model-memory-limit-as-string
  - match: { closed: true }

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.get_job_stats:
        job_id: job-model-memory-limit-as-string
  - match: {"jobs.0.state": closed}

---
"Test get job API with non existing job id":
  - do:
      catch: missing
      ml.get_jobs:
        job_id: "non-existing"

---
"Test put job with inconsistent model snapshot settings":
  - do:
      catch: /The value of daily_model_snapshot_retention_after_days \[4\] cannot be greater than model_snapshot_retention_days \[3\]/
      ml.put_job:
        job_id: inconsistent-snapshot-settings-1
        body:  >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "time_field":"@timestamp"
            },
            "model_snapshot_retention_days": 3,
            "daily_model_snapshot_retention_after_days": 4
          }

  - do:
      catch: /daily_model_snapshot_retention_after_days cannot be less than 0. Value = -1/
      ml.put_job:
        job_id: inconsistent-snapshot-settings-2
        body:  >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "time_field":"@timestamp"
            },
            "model_snapshot_retention_days": 3,
            "daily_model_snapshot_retention_after_days": -1
          }

---
"Test put job with inconsistent body/param ids":
  - do:
      catch: /illegal_argument_exception/
      ml.put_job:
        job_id: an_id
        body:  >
          {
            "job_id":"a_different_id",
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "time_field":"time",
                "time_format":"yyyy-MM-dd HH:mm:ssX"
            }
          }

  - do:
      catch: /Inconsistent job_id; 'a_different_id' specified in the body differs from 'an_id' specified as a URL argument/
      ml.put_job:
        job_id: an_id
        body:  >
          {
            "job_id":"a_different_id",
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "3600s",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "time_field":"time",
                "time_format":"yyyy-MM-dd HH:mm:ssX"
            }
          }

---
"Test put job with id that is already taken":
  - do:
      ml.put_job:
        job_id: jobs-crud-id-already-taken
        body:  >
          {
            "job_id":"jobs-crud-id-already-taken",
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "3600000ms",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "time_field":"time",
                "time_format":"yyyy-MM-dd HH:mm:ssX"
            }
          }
  - match: { job_id: "jobs-crud-id-already-taken" }
  - is_true: create_time

  - do:
      catch: /resource_already_exists_exception/
      ml.put_job:
        job_id: jobs-crud-id-already-taken
        body:  >
          {
            "job_id":"jobs-crud-id-already-taken",
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "time_field":"time",
                "time_format":"yyyy-MM-dd HH:mm:ssX"
            }
          }
  - do:
      catch: /The job cannot be created with the Id 'jobs-crud-id-already-taken'. The Id is already used./
      ml.put_job:
        job_id: jobs-crud-id-already-taken
        body:  >
          {
            "job_id":"jobs-crud-id-already-taken",
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "time_field":"time",
                "time_format":"yyyy-MM-dd HH:mm:ssX"
            }
          }

---
"Test update job":
  - do:
      ml.put_job:
        job_id: jobs-crud-update-job
        body:  >
          {
            "description":"Pre update description",
            "analysis_config" : {
                "detectors" :[{"function":"mean","field_name":"responsetime","by_field_name":"airline"},
                    {"function":"count","by_field_name":"mlcategory"}],
                "categorization_field_name": "some_category",
                "categorization_filters" : ["cat1.*", "cat2.*"],
                "model_prune_window": "30d"
            },
            "data_description" : {
            },
            "model_plot_config": {
              "enabled": true,
              "terms": "foo,bar"
            },
            "analysis_limits": {
              "model_memory_limit": "10mb"
            },
            "renormalization_window_days": 1,
            "background_persist_interval": "2h",
            "model_snapshot_retention_days": 3,
            "daily_model_snapshot_retention_after_days": 2,
            "results_retention_days": 4,
            "custom_settings": {
              "setting1": "custom1",
              "setting2": "custom2"
            }
          }
  - match: { job_id: "jobs-crud-update-job" }
  - length: { analysis_config.categorization_analyzer.filter: 2 }
  - match: { analysis_config.categorization_analyzer.tokenizer: "ml_standard" }
  - length: { analysis_config.categorization_analyzer.char_filter: 3 }
  - match: { analysis_config.categorization_analyzer.char_filter.0: "first_line_with_letters" }
  - match: { analysis_config.categorization_analyzer.char_filter.1.pattern: "cat1.*" }
  - match: { analysis_config.categorization_analyzer.char_filter.2.pattern: "cat2.*" }
  - match: { analysis_config.bucket_span: "5m" }
  - match: { analysis_config.model_prune_window: "30d" }
  - is_true: create_time

  - do:
      ml.open_job:
        job_id: jobs-crud-update-job

  - do:
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
            "groups": ["group-1", "group-2"],
            "description":"Post update description",
            "detectors": [
              {
                "detector_index": 0,
                "custom_rules":[
                  {
                    "conditions": [
                      {
                        "applies_to": "actual",
                        "operator": "gt",
                        "value": 10
                      }
                    ]
                  }
                ]
              },
              {
                "detector_index": 1,
                "description": "updated description"
              }
            ],
            "model_prune_window": "60d",
            "model_plot_config": {
              "enabled": false,
              "terms": "foobar"
            },
            "renormalization_window_days": 10,
            "background_persist_interval": "3h",
            "model_snapshot_retention_days": 30,
            "results_retention_days": 40,
            "custom_settings": {
              "setting3": "custom3"
            }
          }
  - match: { job_id: "jobs-crud-update-job" }
  - match: { groups: ["group-1", "group-2"] }
  - match: { description: "Post update description" }
  - match: { model_plot_config.enabled: false }
  - match: { model_plot_config.terms: "foobar" }
  - match: { model_plot_config.annotations_enabled: false }
  - match: { analysis_config.detectors.0.custom_rules.0.actions: ["skip_result"] }
  - length: { analysis_config.detectors.0.custom_rules.0.conditions: 1 }
  - match: { analysis_config.detectors.0.detector_index: 0 }
  - match: { analysis_config.detectors.1.detector_description: "updated description" }
  - match: { analysis_config.detectors.1.detector_index: 1 }
  - match: { renormalization_window_days: 10 }
  - match: { background_persist_interval: "3h" }
  - match: { model_snapshot_retention_days: 30 }
  - match: { daily_model_snapshot_retention_after_days: 2 }
  - match: { results_retention_days: 40 }
  - match: { analysis_config.bucket_span: "5m" }
  - match: { analysis_config.model_prune_window: "60d" }

  - do:
      catch: "/Cannot update analysis_limits while the job is open/"
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
            "analysis_limits": {
              "model_memory_limit": "20mb"
            }
          }

  - do:
      ml.close_job:
        job_id: jobs-crud-update-job
  - match: { closed: true }

  - do:
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
            "analysis_limits": {
              "model_memory_limit": "20mb"
            }
          }
  - match: { analysis_limits.model_memory_limit: "20mb" }

  - do:
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
              "model_prune_window": "12h"
          }
  - match: { analysis_config.model_prune_window: "12h" }

  - do:
      catch: /model_prune_window \[5m\] must be at least 2 times greater than bucket_span \[5m\]/
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
              "model_prune_window": "5m"
          }

  - do:
      catch: /model_prune_window \[2m\] must be a multiple of bucket_span \[5m\]/
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
              "model_prune_window": "2m"
          }

  - do:
      catch: /\[job_update\] failed to parse field \[model_prune_window\]/
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
              "model_prune_window": "8w"
          }
  - match: { analysis_config.model_prune_window: null }

  - do:
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
            "analysis_limits": {
              "model_memory_limit": "15mb"
            }
          }
  - match: { analysis_limits.model_memory_limit: "15mb" }

  - do:
      catch: /The value of daily_model_snapshot_retention_after_days \[31\] cannot be greater than model_snapshot_retention_days \[30\]/
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
            "daily_model_snapshot_retention_after_days": 31
          }

  - do:
      catch: /The value of daily_model_snapshot_retention_after_days \[2\] cannot be greater than model_snapshot_retention_days \[1\]/
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
            "model_snapshot_retention_days": 1
          }

  - do:
      catch: bad_request
      ml.update_job:
        job_id: _all
        body:  >
          {
            "description":"Can't update all description"
          }

  - do:
      ml.put_job:
        job_id: job-crud-update-group-name-clash
        body:  >
          {
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
            }
          }

  - do:
      catch: "/job and group names must be unique/"
      ml.update_job:
        job_id: jobs-crud-update-job
        body:  >
          {
            "groups": ["job-crud-update-group-name-clash"]
          }

---
"Test delete job that does not exist":
  - do:
      catch: missing
      ml.delete_job:
        job_id: not-a-job

---
"Test delete job that is referred by a datafeed":
  - do:
      ml.put_job:
        job_id: jobs-crud-datafeed-job
        body:  >
          {
            "job_id":"jobs-crud-datafeed-job",
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "format":"xcontent",
                "time_field":"time",
                "time_format":"yyyy-MM-dd HH:mm:ssX"
            }
          }
  - match: { job_id: "jobs-crud-datafeed-job" }
  - is_true: create_time

  - do:
      ml.put_datafeed:
        datafeed_id: jobs-crud-test-datafeed-1
        body:  >
          {
            "job_id":"jobs-crud-datafeed-job",
            "indexes":["index-foo"]
          }
  - match: { datafeed_id: "jobs-crud-test-datafeed-1" }

  - do:
      ml.delete_job:
        job_id: jobs-crud-datafeed-job
  - match: { acknowledged: true }

---
"Test delete job that is opened":
  - do:
      ml.put_job:
        job_id: delete-opened-job
        body:  >
          {
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {
                "time_field":"time"
            }
          }
  - match: { job_id: "delete-opened-job" }
  - is_true: create_time

  - do:
      ml.open_job:
        job_id: delete-opened-job
  - match: { opened: true }
  - match: { node: /\S+/ }

  - do:
      catch: /Cannot delete job \[delete-opened-job\] because the job is opened/
      ml.delete_job:
        job_id: delete-opened-job

---
"Test close job":
  - skip:
      features:
        - "headers"
  - do:
      ml.put_job:
        job_id: jobs-crud-close-job
        body:  >
          {
            "job_id":"jobs-crud-close-job",
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span":"1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
                "format":"xcontent",
                "time_field":"time",
                "time_format":"epoch"
            }
          }
  - match: { job_id: "jobs-crud-close-job" }
  - is_true: create_time

  - do:
      ml.open_job:
        job_id: jobs-crud-close-job

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.get_job_stats:
        job_id: jobs-crud-close-job
  - match: {"jobs.0.state": opened}

  - do:
      ml.close_job:
        job_id: jobs-crud-close-job
  - match: { closed: true }

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.get_job_stats:
        job_id: jobs-crud-close-job
  - match: {"jobs.0.state": closed}

---
"Test closing a closed job isn't an error":
  - do:
      ml.put_job:
        job_id: jobs-crud-close-a-closed-job
        body:  >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {
                "format":"xcontent"
            }
          }
  - match: { job_id: "jobs-crud-close-a-closed-job" }
  - is_true: create_time

  - do:
      ml.open_job:
        job_id: jobs-crud-close-a-closed-job

  - do:
      ml.close_job:
        job_id: jobs-crud-close-a-closed-job
  - match: { closed: true }

  - do:
      ml.close_job:
        job_id: jobs-crud-close-a-closed-job
  - match: { closed: true }

---
"Test close all jobs":
  - do:
      ml.put_job:
        job_id: jobs-crud-close-all-1
        body:  >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {
                "format":"xcontent"
            }
          }
  - match: { job_id: "jobs-crud-close-all-1" }
  - is_true: create_time

  - do:
      ml.put_job:
        job_id: jobs-crud-close-all-2
        body:  >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {
                "format":"xcontent"
            }
          }
  - match: { job_id: "jobs-crud-close-all-2" }
  - is_true: create_time

  - do:
      ml.open_job:
        job_id: jobs-crud-close-all-2

  - do:
      ml.close_job:
        job_id: _all
  - match: { closed: true }

  - do:
      ml.get_job_stats:
        job_id: _all
  - match: { jobs.0.state: closed }
  - match: { jobs.1.state: closed }

---
"Test close jobs with expression that matches":
  - do:
      ml.put_job:
        job_id: jobs-crud-with-expression-that-matches-foo-1
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {}
          }

  - do:
      ml.put_job:
        job_id: jobs-crud-with-expression-that-matches-foo-2
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {}
          }

  - do:
      ml.put_job:
        job_id: jobs-crud-with-expression-that-matches-bar-1
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {}
          }

  - do:
      ml.open_job:
        job_id: jobs-crud-with-expression-that-matches-foo-1

  - do:
      ml.open_job:
        job_id: jobs-crud-with-expression-that-matches-foo-2

  - do:
      ml.open_job:
        job_id: jobs-crud-with-expression-that-matches-bar-1

  - do:
      ml.close_job:
        job_id: "*foo-*"
  - match: { closed: true }

  - do:
      ml.get_job_stats:
        job_id: "*foo-*"
  - match: { jobs.0.state: closed }
  - match: { jobs.1.state: closed }

  - do:
      ml.get_job_stats:
        job_id: "*bar-1"
  - match: { jobs.0.state: opened }

---
"Test close jobs with expression that does not match and allow_no_match":
  - skip:
      features:
        - "warnings"
  - do:
      ml.close_job:
        job_id: "missing-*"
        allow_no_match: true
  - match: { closed: true }

---
"Test close jobs with expression that does not match and not allow_no_match":
  - skip:
      features:
        - "warnings"
  - do:
      catch: missing
      ml.close_job:
        job_id: "missing-*"
        allow_no_match: false

---
"Test force close job":
  - skip:
      features:
        - "headers"
  - do:
      ml.put_job:
        job_id: jobs-crud-force-close-job
        body:  >
          {
            "job_id":"jobs-crud-force-close-job",
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span":"1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {
                "format":"xcontent",
                "time_field":"time",
                "time_format":"epoch"
            }
          }
  - match: { job_id: "jobs-crud-force-close-job" }
  - is_true: create_time

  - do:
      ml.open_job:
        job_id: jobs-crud-force-close-job

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.get_job_stats:
        job_id: jobs-crud-force-close-job
  - match: {"jobs.0.state": opened}

  - do:
      ml.close_job:
        job_id: jobs-crud-force-close-job
        force: true
  - match: { closed: true }

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.get_job_stats:
        job_id: jobs-crud-force-close-job
  - match: {"jobs.0.state": closed}

---
"Test force closing a closed job isn't an error":
  - do:
      ml.put_job:
        job_id: jobs-crud-close-a-closed-job
        body:  >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {
                "format":"xcontent"
            }
          }
  - match: { job_id: "jobs-crud-close-a-closed-job" }
  - is_true: create_time

  - do:
      ml.open_job:
        job_id: jobs-crud-close-a-closed-job

  - do:
      ml.close_job:
        job_id: jobs-crud-close-a-closed-job
        force: true
  - match: { closed: true }

  - do:
      ml.close_job:
        job_id: jobs-crud-close-a-closed-job
        force: true
  - match: { closed: true }

---
"Test open and close an unknown job is resource not found":
  - do:
      catch: missing
      ml.open_job:
        job_id: jobs-crud-some-missing-job-i-made-up

  - do:
      catch: missing
      ml.close_job:
        job_id: jobs-crud-some-missing-job-i-made-up

---
"Test cannot create job with model snapshot id set":

  - do:
      catch: /x_content_parse_exception/
      ml.put_job:
        job_id: has-model-snapshot-id
        body:  >
          {
            "model_snapshot_id": "wont-create-with-this-setting",
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "data_description" : {
            }
          }

---
"Test put job with time field in analysis_config":

  - do:
      catch: /illegal_argument_exception/
      ml.put_job:
        job_id: jobs-crud-time-field-in-analysis_config
        body:  >
          {
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"rare","by_field_name":"@timestamp"}]
            },
            "data_description" : {
                "time_field":"@timestamp"
            }
          }

  - do:
      catch: /data_description.time_field may not be used in the analysis_config/
      ml.put_job:
        job_id: jobs-crud-time-field-in-analysis_config
        body:  >
          {
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"mean","field_name":"time"}]
            },
            "data_description" : {}
          }

---
"Test put job with duplicate detector configurations":

  - do:
      catch: /illegal_argument_exception.*Duplicate detectors are not allowed/
      ml.put_job:
        job_id: jobs-crud-duplicate-detectors
        body:  >
          {
            "analysis_config": {
                "bucket_span": "1h",
                "detectors": [
                    {"function":"max", "field_name":"responsetime"},
                    {"function":"max", "field_name":"responsetime"}
                ]
            },
            "data_description": {
                "time_field": "@timestamp"
            }
          }

  - do:
      catch: /illegal_argument_exception.*Duplicate detectors are not allowed/
      ml.put_job:
        job_id: jobs-crud-duplicate-detectors-with-explicit-indices
        body:  >
          {
            "analysis_config": {
                "bucket_span": "1h",
                "detectors": [
                    {"function":"max", "field_name":"responsetime", "detector_index": 0},
                    {"function":"max", "field_name":"responsetime", "detector_index": 1}
                ]
            },
            "data_description": {
                "time_field": "@timestamp"
            }
          }
---
"Test put job with datafeed":
  - do:
      ml.put_job:
        job_id: jobs-crud-put-with-datafeed
        body: >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {
                "format":"xcontent"
            },
            "datafeed_config": {
              "indexes":["index-foo"]
            }
          }
  - match: { job_id: "jobs-crud-put-with-datafeed" }
  - is_true: datafeed_config
  - match: { datafeed_config.job_id: "jobs-crud-put-with-datafeed" }
  - match: { datafeed_config.datafeed_id: "jobs-crud-put-with-datafeed" }
  - is_true: datafeed_config.authorization.roles
  - is_true: create_time

  - do:
      ml.get_datafeeds:
        datafeed_id: "jobs-crud-put-with-datafeed"

  - match: { count: 1 }
  - match: { datafeeds.0.job_id: "jobs-crud-put-with-datafeed" }
  - match: { datafeeds.0.datafeed_id: "jobs-crud-put-with-datafeed" }
---
"Test get job with datafeed":
  - do:
      ml.put_job:
        job_id: jobs-crud-get-with-datafeed
        body: >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {
                "format":"xcontent"
            }
          }
  - match: { job_id: "jobs-crud-get-with-datafeed" }
  - is_true: create_time

  - do:
      ml.put_datafeed:
        datafeed_id: jobs-crud-get-with-datafeed
        body: >
          {
            "job_id":"jobs-crud-get-with-datafeed",
            "indexes":["index-foo"]
          }
  - match: { datafeed_id: "jobs-crud-get-with-datafeed" }
  - do:
      ml.get_jobs:
        job_id: jobs-crud-get-with-datafeed
  - match: { count: 1 }
  - match: { jobs.0.job_id: "jobs-crud-get-with-datafeed" }
  - match: { jobs.0.datafeed_config.datafeed_id: "jobs-crud-get-with-datafeed" }

# TODO in v8.0.0 adjust this test to handle exclude_generated behavior
  - do:
      ml.get_jobs:
        job_id: jobs-crud-get-with-datafeed
        exclude_generated: true
  - match: { count: 1 }
  - match: { jobs.0.job_id: "jobs-crud-get-with-datafeed" }
  - is_false: jobs.0.datafeed_config
---
"Test put job with datafeed with indices options in params":
  - skip:
      features: warnings
  - do:
      warnings:
        - "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices."
      ml.put_job:
        job_id: jobs-crud-put-with-datafeed-with-indices-options
        ignore_throttled: false
        body: >
          {
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "20mb"
            },
            "data_description" : {
                "format":"xcontent"
            },
            "datafeed_config": {
              "indexes":["index-foo"]
            }
          }
  - match: { job_id: "jobs-crud-put-with-datafeed-with-indices-options" }
  - match: { datafeed_config.datafeed_id: "jobs-crud-put-with-datafeed-with-indices-options" }
  - match: { datafeed_config.indices_options.ignore_throttled: false }
  - is_true: create_time

  - do:
      ml.get_datafeeds:
        datafeed_id: jobs-crud-put-with-datafeed-with-indices-options

  - match: { datafeeds.0.indices_options.ignore_throttled: false }

---
"Test max model memory limit":
  - skip:
      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      cluster.put_settings:
        body:
          persistent:
            xpack.ml.max_model_memory_limit: "9g"
  - match: {persistent.xpack.ml.max_model_memory_limit: "9g"}

  - do:
      ml.put_job:
        job_id: job-model-memory-limit-below-global-max
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            },
            "analysis_limits": {
                "model_memory_limit": "8g"
            }
          }
  - match: { job_id: "job-model-memory-limit-below-global-max" }
  - match: { analysis_limits.model_memory_limit: "8192mb" }
  - is_true: create_time

  - do:
      catch: /model_memory_limit \[10gb\] must be less than the value of the xpack.ml.max_model_memory_limit setting \[9gb\]/
      ml.put_job:
        job_id: job-model-memory-limit-above-global-max
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            },
            "analysis_limits": {
                "model_memory_limit": "10g"
            }
          }

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      cluster.put_settings:
        body:
          persistent:
            xpack.ml.max_model_memory_limit: null
  - match: {persistent: {}}

  - do:
      ml.put_job:
        job_id: job-model-memory-limit-above-removed-global-max
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            },
            "analysis_limits": {
                "model_memory_limit": "10g"
            }
          }
  - match: { job_id: "job-model-memory-limit-above-removed-global-max" }
  - match: { analysis_limits.model_memory_limit: "10240mb" }
  - is_true: create_time

---
"Test jobs with named and custom categorization_analyzer":
# Check named and custom configs can share the same index & mappings
  - do:
      ml.put_job:
        job_id: jobs-crud-named-categorization-analyzer-job
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"mean","field_name":"responsetime","by_field_name":"airline"},
                    {"function":"count","by_field_name":"mlcategory"}],
                "categorization_field_name": "some_category",
                "categorization_analyzer" : "standard"
            },
            "data_description" : {
            }
          }
  - match: { job_id: "jobs-crud-named-categorization-analyzer-job" }
  - match: { analysis_config.categorization_analyzer: "standard" }
  - is_true: create_time

  - do:
      ml.put_job:
        job_id: jobs-crud-custom-categorization-analyzer-job
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"mean","field_name":"responsetime","by_field_name":"airline"},
                    {"function":"count","by_field_name":"mlcategory"}],
                "categorization_field_name": "some_category",
                "categorization_analyzer" : {
                    "char_filter" : ["html_strip"],
                    "tokenizer" : "classic",
                    "filter" : ["stop"]
                }
            },
            "data_description" : {
            }
          }
  - match: { job_id: "jobs-crud-custom-categorization-analyzer-job" }
  - match: { analysis_config.categorization_analyzer.char_filter.0: "html_strip" }
  - match: { analysis_config.categorization_analyzer.tokenizer: "classic" }
  - match: { analysis_config.categorization_analyzer.filter.0: "stop" }
  - is_true: create_time

---
"Test job with categorization_analyzer and categorization_filters":
  - do:
      catch: /categorization_filters cannot be used with categorization_analyzer - instead specify them as pattern_replace char_filters in the analyzer/
      ml.put_job:
        job_id: jobs-crud-categorization-analyzer-job
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"mean","field_name":"responsetime","by_field_name":"airline"},
                    {"function":"count","by_field_name":"mlcategory"}],
                "categorization_field_name": "some_category",
                "categorization_analyzer" : {
                    "char_filter" : ["html_strip"],
                    "tokenizer" : "classic",
                    "filter" : ["stop"]
                },
                "categorization_filters" : ["cat1.*", "cat2.*"]
            },
            "data_description" : {
            }
          }

---
"Test job with rules":

  - do:
      ml.put_job:
        job_id: jobs-crud-rules
        body:  >
          {
            "analysis_config": {
              "detectors": [
                {
                  "function": "count",
                  "by_field_name": "country",
                  "custom_rules": [
                    {
                      "actions": ["skip_result", "skip_model_update"],
                      "scope": {
                        "country": {"filter_id": "safe_countries"}
                      },
                      "conditions": [
                        {
                          "applies_to":"actual",
                          "operator":"lt",
                          "value": 33.3
                        },
                        {
                          "applies_to":"typical",
                          "operator":"lte",
                          "value": 42.0
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            "data_description" : {}
          }

  - do:
      ml.get_jobs:
        job_id: jobs-crud-rules
  - match: { count: 1 }
  - match: {
      jobs.0.analysis_config.detectors.0.custom_rules: [
        {
          "actions": ["skip_result", "skip_model_update"],
          "scope": {
            "country": {"filter_id": "safe_countries", "filter_type": "include"}
          },
          "conditions": [
            {
              "applies_to":"actual",
              "operator":"lt",
              "value": 33.3
            },
            {
              "applies_to":"typical",
              "operator":"lte",
              "value": 42.0
            }
          ]
        }
      ]
    }

---
"Test function shortcut expansion":
  - do:
      ml.put_job:
        job_id: jobs-function-shortcut-expansion
        body:  >
          {
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"nzc","by_field_name":"airline"}]
            },
            "data_description" : {}
          }
  - match: { job_id: "jobs-function-shortcut-expansion" }
  - match: { analysis_config.detectors.0.function: "non_zero_count"}
  - is_true: create_time

---
"Test open job when persistent task allocation disabled":
  - skip:
      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      cluster.put_settings:
        body:
          persistent:
            cluster.persistent_tasks.allocation.enable: "none"
  - match: {persistent.cluster.persistent_tasks.allocation.enable: "none"}

  - do:
      ml.put_job:
        job_id: persistent-task-allocation-allowed-test
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            },
            "analysis_limits": {
                "model_memory_limit": "10m"
            }
          }
  - match: { job_id: "persistent-task-allocation-allowed-test" }
  - is_true: create_time

  - do:
      catch: /Cannot open jobs because persistent task assignment is disabled by the \[cluster.persistent_tasks.allocation.enable\] setting/
      ml.open_job:
        job_id: persistent-task-allocation-allowed-test

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      cluster.put_settings:
        body:
          persistent:
            cluster.persistent_tasks.allocation.enable: "all"
  - match: {persistent.cluster.persistent_tasks.allocation.enable: "all"}

  - do:
      ml.open_job:
        job_id: persistent-task-allocation-allowed-test
  - match: { opened: true }
  - match: { node: /\S+/ }

---
"Test reopen job resets the finished time":
  - do:
      ml.put_job:
        job_id: jobs-crud-reset-finished-time
        body:  >
          {
            "analysis_config" : {
                "detectors" :[{"function":"count"}]
            },
            "data_description" : {
            }
          }
  - match: { job_id: jobs-crud-reset-finished-time }
  - is_true: create_time

  - do:
      ml.open_job:
        job_id: jobs-crud-reset-finished-time

  - do:
      ml.get_jobs:
        job_id: jobs-crud-reset-finished-time
  - is_false: jobs.0.finished_time

  - do:
      ml.close_job:
        job_id: jobs-crud-reset-finished-time

  - do:
      ml.get_jobs:
        job_id: jobs-crud-reset-finished-time
  - is_true: jobs.0.finished_time

  - do:
      ml.open_job:
        job_id: jobs-crud-reset-finished-time

  - do:
      ml.get_jobs:
        job_id: jobs-crud-reset-finished-time
  - is_false: jobs.0.finished_time

---
"Test close job with body params":
  - skip:
      features:
       - "warnings"

  - do:
      catch: missing
      ml.close_job:
        job_id: job-that-doesnot-exist*
        body:  >
          {
            "allow_no_match" : false
          }

  - do:
      ml.close_job:
        job_id: job-that-doesnot-exist*
        body:  >
          {
            "allow_no_match" : true
          }

---
"Test job with rule referencing missing filter":

  - do:
      ml.put_job:
        job_id: jobs-crud-rule-missing-filter
        body:  >
          {
            "analysis_config": {
              "detectors": [
                {
                  "function": "count",
                  "by_field_name": "country",
                  "custom_rules": [
                    {
                      "actions": ["skip_result"],
                      "scope": {
                        "country": {"filter_id": "safe_countries"}
                      }
                    }
                  ]
                }
              ]
            },
            "data_description" : {}
          }

  - do:
      catch: /Unable to find filter \[safe_countries\]/
      ml.open_job:
        job_id: jobs-crud-rule-missing-filter
