---
setup:
- skip:
    features:
    - "headers"
    - "allowed_warnings_regex"
- do:
    indices.create:
      index: "utopia"
      body:
        mappings:
          properties:
            ml.top_classes:
              type: "nested"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- do:
    index:
      index: "utopia"
      body: "{\n  \"dataset\": \"blue\",\n  \"is_outlier\": false,\n  \"is_outlier_int\"\
        : 0,\n  \"outlier_score\": 0.0,\n  \"regression_field_act\": 10.9,\n  \"regression_field_pred\"\
        : 10.9,\n  \"classification_field_act\": \"dog\",\n  \"classification_field_pred\"\
        : \"dog\",\n  \"all_true_field\": true,\n  \"all_false_field\": false,\n \
        \ \"ml.top_classes\": [\n    {\"class_name\": \"dog\", \"class_probability\"\
        : 0.9},\n    {\"class_name\": \"cat\", \"class_probability\": 0.1}\n  ]\n\
        }\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- do:
    index:
      index: "utopia"
      body: "{\n  \"dataset\": \"blue\",\n  \"is_outlier\": false,\n  \"is_outlier_int\"\
        : 0,\n  \"outlier_score\": 0.2,\n  \"regression_field_act\": 12.0,\n  \"regression_field_pred\"\
        : 9.9,\n  \"classification_field_act\": \"cat\",\n  \"classification_field_pred\"\
        : \"cat\",\n  \"all_true_field\": true,\n  \"all_false_field\": false,\n \
        \ \"ml.top_classes\": [\n    {\"class_name\": \"cat\", \"class_probability\"\
        : 0.8},\n    {\"class_name\": \"dog\", \"class_probability\": 0.2}\n  ]\n\
        }\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- do:
    index:
      index: "utopia"
      body: "{\n  \"dataset\": \"blue\",\n  \"is_outlier\": false,\n  \"is_outlier_int\"\
        : 0,\n  \"outlier_score\": 0.3,\n  \"regression_field_act\": 20.9,\n  \"regression_field_pred\"\
        : 5.9,\n  \"classification_field_act\": \"mouse\",\n  \"classification_field_pred\"\
        : \"mouse\",\n  \"all_true_field\": true,\n  \"all_false_field\": false,\n\
        \  \"ml.top_classes\": [\n    {\"class_name\": \"cat\", \"class_probability\"\
        : 0.3},\n    {\"class_name\": \"dog\", \"class_probability\": 0.1}\n  ]\n\
        }\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- do:
    index:
      index: "utopia"
      body: "{\n  \"dataset\": \"blue\",\n  \"is_outlier\": true,\n  \"is_outlier_int\"\
        : 1,\n  \"outlier_score\": 0.3,\n  \"regression_field_act\": 11.9,\n  \"regression_field_pred\"\
        : 11.9,\n  \"classification_field_act\": \"dog\",\n  \"classification_field_pred\"\
        : \"cat\",\n  \"all_true_field\": true,\n  \"all_false_field\": false,\n \
        \ \"ml.top_classes\": [\n    {\"class_name\": \"cat\", \"class_probability\"\
        : 0.6},\n    {\"class_name\": \"dog\", \"class_probability\": 0.3}\n  ]\n\
        }\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- do:
    index:
      index: "utopia"
      body: "{\n  \"dataset\": \"green\",\n  \"is_outlier\": true,\n  \"is_outlier_int\"\
        : 1,\n  \"outlier_score\": 0.4,\n  \"regression_field_act\": 42.9,\n  \"regression_field_pred\"\
        : 42.9,\n  \"classification_field_act\": \"cat\",\n  \"classification_field_pred\"\
        : \"dog\",\n  \"all_true_field\": true,\n  \"all_false_field\": false,\n \
        \ \"ml.top_classes\": [\n    {\"class_name\": \"dog\", \"class_probability\"\
        : 0.7},\n    {\"class_name\": \"cat\", \"class_probability\": 0.3}\n  ]\n\
        }\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- do:
    index:
      index: "utopia"
      body: "{\n  \"dataset\": \"green\",\n  \"is_outlier\": true,\n  \"is_outlier_int\"\
        : 1,\n  \"outlier_score\": 0.5,\n  \"regression_field_act\": 0.42,\n  \"regression_field_pred\"\
        : 0.42,\n  \"classification_field_act\": \"dog\",\n  \"classification_field_pred\"\
        : \"dog\",\n  \"all_true_field\": true,\n  \"all_false_field\": false,\n \
        \ \"ml.top_classes\": [\n    {\"class_name\": \"dog\", \"class_probability\"\
        : 0.9},\n    {\"class_name\": \"cat\", \"class_probability\": 0.1}\n  ]\n\
        }\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- do:
    index:
      index: "utopia"
      body: "{\n  \"dataset\": \"green\",\n  \"is_outlier\": true,\n  \"is_outlier_int\"\
        : 1,\n  \"outlier_score\": 0.9,\n  \"regression_field_act\": 1.1235813,\n\
        \  \"regression_field_pred\": 1.12358,\n  \"classification_field_act\": \"\
        cat\",\n  \"classification_field_pred\": \"cat\",\n  \"all_true_field\": true,\n\
        \  \"all_false_field\": false,\n  \"ml.top_classes\": [\n    {\"class_name\"\
        : \"cat\", \"class_probability\": 0.8},\n    {\"class_name\": \"dog\", \"\
        class_probability\": 0.2}\n  ]\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- do:
    index:
      index: "utopia"
      body: "{\n  \"dataset\": \"green\",\n  \"is_outlier\": true,\n  \"is_outlier_int\"\
        : 1,\n  \"outlier_score\": 0.95,\n  \"regression_field_act\": -5.20,\n  \"\
        regression_field_pred\": -5.1,\n  \"classification_field_act\": \"mouse\"\
        ,\n  \"classification_field_pred\": \"cat\",\n  \"all_true_field\": true,\n\
        \  \"all_false_field\": false,\n  \"ml.top_classes\": [\n    {\"class_name\"\
        : \"cat\", \"class_probability\": 0.8},\n    {\"class_name\": \"dog\", \"\
        class_probability\": 0.2}\n  ]\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- do:
    index:
      index: "utopia"
      body: "{\n  \"foo\": 0.24\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- do:
    indices.refresh:
      index: "utopia"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection auc_roc:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"auc_roc\": {}\n    \
        \  }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    outlier_detection.auc_roc.value: 0.99995
- is_false: "outlier_detection.auc_roc.curve"
---
Test outlier_detection auc_roc given actual_field is int:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier_int\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"auc_roc\": {}\n    \
        \  }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    outlier_detection.auc_roc.value: 0.99995
- is_false: "outlier_detection.auc_roc.curve"
---
Test outlier_detection auc_roc include curve:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"auc_roc\": { \"include_curve\"\
        : true }\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    outlier_detection.auc_roc.value: 0.99995
- is_true: "outlier_detection.auc_roc.curve"
---
Test outlier_detection auc_roc given actual_field is always true:
- do:
    catch: "/\\[auc_roc\\] requires at least one \\[all_true_field\\] to have a different\
      \ value than \\[true\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"all_true_field\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"auc_roc\": {}\n    \
        \  }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection auc_roc given actual_field is always false:
- do:
    catch: "/\\[auc_roc\\] requires at least one \\[all_false_field\\] to have the\
      \ value \\[true\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"all_false_field\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"auc_roc\": {}\n    \
        \  }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection precision:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"precision\": { \"at\"\
        : [0, 0.5] }\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    outlier_detection:
      precision:
        "0.0": 0.625
        "0.5": 1.0
---
Test outlier_detection recall:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"recall\": { \"at\":\
        \ [0, 0.4, 0.5] }\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    outlier_detection:
      recall:
        "0.0": 1.0
        "0.4": 0.8
        "0.5": 0.6
---
Test outlier_detection confusion_matrix:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"confusion_matrix\":\
        \ { \"at\": [0, 0.3, 0.5] }\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    outlier_detection:
      confusion_matrix:
        "0.0":
          tp: 5
          fp: 3
          tn: 0
          fn: 0
        "0.3":
          tp: 5
          fp: 1
          tn: 2
          fn: 0
        "0.5":
          tp: 3
          fp: 0
          tn: 3
          fn: 2
---
Test outlier_detection with query:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"query\": { \"bool\": { \"filter\": {\
        \ \"term\": { \"dataset\": \"blue\" } } } },\n  \"evaluation\": {\n    \"\
        outlier_detection\": {\n      \"actual_field\": \"is_outlier\",\n      \"\
        predicted_probability_field\": \"outlier_score\",\n      \"metrics\": {\n\
        \        \"confusion_matrix\": { \"at\": [0.5] }\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    outlier_detection:
      confusion_matrix:
        "0.5":
          tp: 0
          fp: 0
          tn: 3
          fn: 1
---
Test outlier_detection default metrics:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\"\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- is_true: "outlier_detection.auc_roc.value"
- is_true: "outlier_detection.precision.0\\.25"
- is_true: "outlier_detection.precision.0\\.5"
- is_true: "outlier_detection.precision.0\\.75"
- is_true: "outlier_detection.recall.0\\.25"
- is_true: "outlier_detection.recall.0\\.5"
- is_true: "outlier_detection.recall.0\\.75"
- is_true: "outlier_detection.confusion_matrix.0\\.25"
- is_true: "outlier_detection.confusion_matrix.0\\.5"
- is_true: "outlier_detection.confusion_matrix.0\\.75"
---
Test given missing index:
- do:
    catch: "/Required \\[index\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"evaluation\": {\n    \"outlier_detection\": {\n      \"actual_field\"\
        : \"is_outlier\",\n      \"predicted_probability_field\": \"outlier_score\"\
        \n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test given index does not exist:
- do:
    catch: "missing"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"missing_index\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\"\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test given missing evaluation:
- do:
    catch: "/Required \\[evaluation\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"foo\"\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection given evaluation with empty metrics:
- do:
    catch: "/\\[outlier_detection\\] must have one or more metrics/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection given missing actual_field:
- do:
    catch: "/No documents found containing all the required fields \\[missing, outlier_score\\\
      ]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"missing\",\n      \"predicted_probability_field\"\
        : \"outlier_score\"\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection given missing predicted_probability_field:
- do:
    catch: "/No documents found containing all the required fields \\[is_outlier,\
      \ missing\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"missing\"\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection given precision with threshold less than zero:
- do:
    catch: "/\\[precision.at\\] values must be in \\[0.0, 1.0\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"precision\": { \"at\"\
        : [ 0.25, -0.1 ]}\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection given recall with threshold less than zero:
- do:
    catch: "/\\[recall.at\\] values must be in \\[0.0, 1.0\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"recall\": { \"at\":\
        \ [ 0.25, -0.1 ]}\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection given confusion_matrix with threshold less than zero:
- do:
    catch: "/\\[confusion_matrix.at\\] values must be in \\[0.0, 1.0\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"confusion_matrix\":\
        \ { \"at\": [ 0.25, -0.1 ]}\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection given precision with empty thresholds:
- do:
    catch: "/\\[precision.at\\] must have at least one value/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"precision\": { \"at\"\
        : []}\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection given recall with empty thresholds:
- do:
    catch: "/\\[recall.at\\] must have at least one value/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"recall\": { \"at\":\
        \ []}\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test outlier_detection given confusion_matrix with empty thresholds:
- do:
    catch: "/\\[confusion_matrix.at\\] must have at least one value/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"outlier_detection\"\
        : {\n      \"actual_field\": \"is_outlier\",\n      \"predicted_probability_field\"\
        : \"outlier_score\",\n      \"metrics\": {\n        \"confusion_matrix\":\
        \ { \"at\": []}\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test classification given evaluation with empty metrics:
- do:
    catch: "/\\[classification\\] must have one or more metrics/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"predicted_field\": \"classification_field_pred.keyword\",\n      \"metrics\"\
        : {}\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test classification auc_roc with missing class_name:
- do:
    catch: "/Failed to build \\[auc_roc\\] after last required field arrived/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"top_classes_field\": \"ml.top_classes\",\n      \"metrics\": {\n     \
        \   \"auc_roc\": {}\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test classification auc_roc given actual_field is never equal to fish:
- do:
    catch: "/\\[auc_roc\\] requires at least one \\[classification_field_act.keyword\\\
      ] to have the value \\[fish\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"top_classes_field\": \"ml.top_classes\",\n      \"metrics\": {\n     \
        \   \"auc_roc\": {\n          \"class_name\": \"fish\"\n        }\n      }\n\
        \    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test classification auc_roc given predicted_class_field is never equal to mouse:
- do:
    catch: "/\\[auc_roc\\] requires that \\[mouse\\] appears as one of the \\[ml.top_classes.class_name\\\
      ] for every document \\(appeared in 0 out of 8\\)./"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"top_classes_field\": \"ml.top_classes\",\n      \"metrics\": {\n     \
        \   \"auc_roc\": {\n          \"class_name\": \"mouse\"\n        }\n     \
        \ }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test classification auc_roc:
- skip:
    features: "close_to"
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"top_classes_field\": \"ml.top_classes\",\n      \"metrics\": {\n     \
        \   \"auc_roc\": {\n          \"class_name\": \"cat\"\n        }\n      }\n\
        \    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- close_to:
    classification.auc_roc.value:
      value: 0.8612749970197677
      error: 1.0E-11
- is_false: "classification.auc_roc.curve"
---
Test classification auc_roc with default top_classes_field:
- skip:
    features: "close_to"
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"metrics\": {\n        \"auc_roc\": {\n          \"class_name\": \"cat\"\
        \n        }\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- close_to:
    classification.auc_roc.value:
      value: 0.8612749970197677
      error: 1.0E-11
- is_false: "classification.auc_roc.curve"
---
Test classification accuracy with missing predicted_field:
- do:
    catch: "/\\[classification\\] must define \\[predicted_field\\] as required by\
      \ the following metrics \\[accuracy\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"metrics\": { \"accuracy\": {} }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test classification accuracy:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"predicted_field\": \"classification_field_pred.keyword\",\n      \"metrics\"\
        : { \"accuracy\": {} }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    classification.accuracy:
      classes:
      - class_name: "cat"
        value: 0.625
      - class_name: "dog"
        value: 0.75
      - class_name: "mouse"
        value: 0.875
      overall_accuracy: 0.625
---
Test classification precision:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"predicted_field\": \"classification_field_pred.keyword\",\n      \"metrics\"\
        : { \"precision\": {} }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    classification.precision:
      classes:
      - class_name: "cat"
        value: 0.5
      - class_name: "dog"
        value: 0.6666666666666666
      - class_name: "mouse"
        value: 1.0
      avg_precision: 0.7222222222222222
---
Test classification recall:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"predicted_field\": \"classification_field_pred.keyword\",\n      \"metrics\"\
        : { \"recall\": {} }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    classification.recall:
      classes:
      - class_name: "cat"
        value: 0.6666666666666666
      - class_name: "dog"
        value: 0.6666666666666666
      - class_name: "mouse"
        value: 0.5
      avg_recall: 0.611111111111111
---
Test classification multiclass_confusion_matrix:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"predicted_field\": \"classification_field_pred.keyword\",\n      \"metrics\"\
        : { \"multiclass_confusion_matrix\": {} }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    classification.multiclass_confusion_matrix:
      confusion_matrix:
      - actual_class: "cat"
        actual_class_doc_count: 3
        predicted_classes:
        - predicted_class: "cat"
          count: 2
        - predicted_class: "dog"
          count: 1
        - predicted_class: "mouse"
          count: 0
        other_predicted_class_doc_count: 0
      - actual_class: "dog"
        actual_class_doc_count: 3
        predicted_classes:
        - predicted_class: "cat"
          count: 1
        - predicted_class: "dog"
          count: 2
        - predicted_class: "mouse"
          count: 0
        other_predicted_class_doc_count: 0
      - actual_class: "mouse"
        actual_class_doc_count: 2
        predicted_classes:
        - predicted_class: "cat"
          count: 1
        - predicted_class: "dog"
          count: 0
        - predicted_class: "mouse"
          count: 1
        other_predicted_class_doc_count: 0
      other_actual_class_count: 0
---
Test classification multiclass_confusion_matrix with explicit size:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"predicted_field\": \"classification_field_pred.keyword\",\n      \"metrics\"\
        : { \"multiclass_confusion_matrix\": { \"size\": 2 } }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- match:
    classification.multiclass_confusion_matrix:
      confusion_matrix:
      - actual_class: "cat"
        actual_class_doc_count: 3
        predicted_classes:
        - predicted_class: "cat"
          count: 2
        - predicted_class: "dog"
          count: 1
        other_predicted_class_doc_count: 0
      - actual_class: "dog"
        actual_class_doc_count: 3
        predicted_classes:
        - predicted_class: "cat"
          count: 1
        - predicted_class: "dog"
          count: 2
        other_predicted_class_doc_count: 0
      other_actual_class_count: 1
---
Test classification with null metrics:
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"predicted_field\": \"classification_field_pred.keyword\"\n    }\n  }\n\
        }\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- is_true: "classification.multiclass_confusion_matrix"
- is_true: "classification.accuracy"
- is_true: "classification.precision"
- is_true: "classification.recall"
- is_false: "classification.auc_roc"
---
Test classification given missing actual_field:
- do:
    catch: "/No documents found containing all the required fields \\[missing, classification_field_pred.keyword\\\
      ]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"missing\",\n      \"predicted_field\": \"classification_field_pred.keyword\"\
        \n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test classification given missing predicted_field:
- do:
    catch: "/No documents found containing all the required fields \\[classification_field_act.keyword,\
      \ missing\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"predicted_field\": \"missing\"\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test classification given missing top_classes_field:
- do:
    catch: "/No documents found containing all the required fields \\[classification_field_act.keyword,\
      \ missing.class_name, missing.class_probability\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"classification\"\
        : {\n      \"actual_field\": \"classification_field_act.keyword\",\n     \
        \ \"predicted_field\": \"classification_field_pred.keyword\",\n      \"top_classes_field\"\
        : \"missing\",\n      \"metrics\": {\n        \"auc_roc\": {\n          \"\
        class_name\": \"dummy\"\n        }\n      }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test regression given evaluation with empty metrics:
- do:
    catch: "/\\[regression\\] must have one or more metrics/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"regression\"\
        : {\n      \"actual_field\": \"regression_field_act\",\n      \"predicted_field\"\
        : \"regression_field_pred\",\n      \"metrics\": { }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test regression mean_squared_error:
- skip:
    features: "close_to"
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"regression\"\
        : {\n      \"actual_field\": \"regression_field_act\",\n      \"predicted_field\"\
        : \"regression_field_pred\",\n      \"metrics\": { \"mse\": {} }\n    }\n\
        \  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- close_to:
    regression.mse.value:
      value: 28.67749840974834
      error: 1.0E-11
- is_false: "regression.msle.value"
- is_false: "regression.r_squared.value"
- is_false: "regression.huber.value"
---
Test regression mean_squared_logarithmic_error:
- skip:
    features: "close_to"
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"regression\"\
        : {\n      \"actual_field\": \"regression_field_act\",\n      \"predicted_field\"\
        : \"regression_field_pred\",\n      \"metrics\": { \"msle\": { \"offset\"\
        : 6.0 } }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- close_to:
    regression.msle.value:
      value: 0.08680568028334916
      error: 1.0E-11
- is_false: "regression.mse.value"
- is_false: "regression.r_squared.value"
- is_false: "regression.huber.value"
---
Test regression huber:
- skip:
    features: "close_to"
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"regression\"\
        : {\n      \"actual_field\": \"regression_field_act\",\n      \"predicted_field\"\
        : \"regression_field_pred\",\n      \"metrics\": { \"huber\": { \"delta\"\
        : 2.0 } }\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- close_to:
    regression.huber.value:
      value: 3.5088110471730145
      error: 1.0E-11
- is_false: "regression.msle.value"
- is_false: "regression.mse.value"
- is_false: "regression.r_squared.value"
---
Test regression r_squared:
- skip:
    features: "close_to"
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"regression\"\
        : {\n      \"actual_field\": \"regression_field_act\",\n      \"predicted_field\"\
        : \"regression_field_pred\",\n      \"metrics\": { \"r_squared\": {} }\n \
        \   }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- close_to:
    regression.r_squared.value:
      value: 0.8551031778603486
      error: 1.0E-11
- is_false: "regression.mse"
- is_false: "regression.msle.value"
- is_false: "regression.huber.value"
---
Test regression with null metrics:
- skip:
    features: "close_to"
- do:
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"regression\"\
        : {\n      \"actual_field\": \"regression_field_act\",\n      \"predicted_field\"\
        : \"regression_field_pred\"\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
- close_to:
    regression.mse.value:
      value: 28.67749840974834
      error: 1.0E-11
- close_to:
    regression.r_squared.value:
      value: 0.8551031778603486
      error: 1.0E-11
- close_to:
    regression.huber.value:
      value: 1.9205280586939963
      error: 1.0E-11
- is_false: "regression.msle.value"
---
Test regression given missing actual_field:
- do:
    catch: "/No documents found containing all the required fields \\[missing, regression_field_pred\\\
      ]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"regression\"\
        : {\n      \"actual_field\": \"missing\",\n      \"predicted_field\": \"regression_field_pred\"\
        \n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
---
Test regression given missing predicted_field:
- do:
    catch: "/No documents found containing all the required fields \\[regression_field_act,\
      \ missing\\]/"
    ml.evaluate_data_frame:
      body: "{\n  \"index\": \"utopia\",\n  \"evaluation\": {\n    \"regression\"\
        : {\n      \"actual_field\": \"regression_field_act\",\n      \"predicted_field\"\
        : \"missing\"\n    }\n  }\n}\n"
    headers:
      Content-Type: "application/vnd.elasticsearch+json;compatible-with=8"
      Accept: "application/vnd.elasticsearch+json;compatible-with=8"
    allowed_warnings_regex:
    - ".*rollup functionality will be removed in Elasticsearch.*"
    - "Index \\[\\.profiling-.*\\] name begins with a dot.* and will not be allowed\
      \ in a future Elasticsearch version."
    - "Parameter \\[default_metric\\] is deprecated and will be removed in a future\
      \ version"
