andrassy · pull · Jul 22, 2022 · Jul 22, 2022 · Jul 22, 2022 · Jul 22, 2022
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,17 @@
+# GitHub CODEOWNERS definition
+# Identify which groups will be pinged by changes to different parts of the codebase.
+# For more info, see https://help.github.com/articles/about-codeowners/
+
+# Stack Monitoring index templates
+x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-beats.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-es-mb.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-es.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-kibana.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-logstash.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json @elastic/infra-monitoring-ui
+x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @elastic/infra-monitoring-ui
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -604,7 +604,7 @@ threshold has been breached:
 
     logger.warn(
         "flood stage disk watermark [{}] exceeded on {}, all indices on this node will be marked read-only",
-        diskThresholdSettings.describeFloodStageThreshold(),
+        diskThresholdSettings.describeFloodStageThreshold(total, false),
         usage
     );
 

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.4.0
-lucene            = 9.3.0-snapshot-2d05f5c623e
+lucene            = 9.3.0-snapshot-b8d1fcfd0ec
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 18.0.2+9@f6ad4b4450fd4d298113270ec84f30ee

diff --git a/docs/changelog/88023.yaml b/docs/changelog/88023.yaml
@@ -0,0 +1,5 @@
+pr: 88023
+summary: "If signature validation fails, reload JWKs and retry if new JWKs are found"
+area: Authentication
+type: enhancement
+issues: []
diff --git a/docs/changelog/88385.yaml b/docs/changelog/88385.yaml
@@ -1,5 +1,6 @@
 pr: 88385
-summary: Test balance improves after rebalancing
+summary: Prevent re-balancing using outdated node weights in some cases
 area: Allocation
 type: bug
-issues: []
+issues:
+ - 88384
diff --git a/docs/changelog/88551.yaml b/docs/changelog/88551.yaml
@@ -0,0 +1,6 @@
+pr: 88551
+summary: "Fix: use status code 500 for aggregation reduce phase errors if no shard failed"
+area: Search
+type: bug
+issues:
+ - 20004
diff --git a/docs/changelog/88622.yaml b/docs/changelog/88622.yaml
@@ -0,0 +1,6 @@
+pr: 88622
+summary: Use origin for the client when running _features/_reset
+area: Infra/Core
+type: bug
+issues:
+ - 88617
diff --git a/docs/changelog/88642.yaml b/docs/changelog/88642.yaml
@@ -0,0 +1,5 @@
+pr: 88642
+summary: Include API key metadata in audit log when an API key is created, granted, or updated
+area: Audit
+type: enhancement
+issues: []
diff --git a/docs/changelog/88694.yaml b/docs/changelog/88694.yaml
@@ -0,0 +1,14 @@
+pr: 88694
+summary: Integrate ANN into `_search` endpoint
+area: Vector Search
+type: feature
+issues:
+ - 87625
+highlight:
+  title: Integrate ANN into `_search` endpoint
+  body: |-
+    This change adds a `knn` option to the `_search` API to support ANN
+    search. It's powered by the same Lucene ANN capabilities as the old
+    `_knn_search` endpoint. The `knn` option can be combined with other
+    search features like queries and aggregations.
+  notable: true
diff --git a/docs/changelog/88702.yaml b/docs/changelog/88702.yaml
@@ -0,0 +1,5 @@
+pr: 88702
+summary: Fix validation of `close_pit` request
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/88707.yaml b/docs/changelog/88707.yaml
@@ -0,0 +1,5 @@
+pr: 88707
+summary: Avoid capturing `SnapshotsInProgress$Entry` in queue
+area: Snapshot/Restore
+type: bug
+issues: []
diff --git a/docs/changelog/88719.yaml b/docs/changelog/88719.yaml
@@ -0,0 +1,5 @@
+pr: 88719
+summary: Convert disk watermarks to RelativeByteSizeValues
+area: Infra/Settings
+type: enhancement
+issues: []
diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc
@@ -183,8 +183,8 @@ The API returns the following result:
 The new `description` field contains human readable text that identifies the
 particular request that the task is performing such as identifying the search
 request being performed by a search task like the example above. Other kinds of
-task have different descriptions, like <<docs-reindex,`_reindex`>> which
-has the search and the destination, or <<docs-bulk,`_bulk`>> which just has the
+tasks have different descriptions, like <<docs-reindex,`_reindex`>> which
+has the source and the destination, or <<docs-bulk,`_bulk`>> which just has the
 number of requests and the destination indices. Many requests will only have an
 empty description because more detailed information about the request is not
 easily available or particularly helpful in identifying the request.

diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc
@@ -152,6 +152,7 @@ NOTE: Although they are conceptually related, the `similarity` parameter is
 different from <<text,`text`>> field <<similarity,`similarity`>> and accepts
 a distinct set of options.
 
+[[dense-vector-index-options]]
 `index_options`::
 (Optional, object)
 An optional section that configures the kNN indexing algorithm. The HNSW

diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc
@@ -72,14 +72,14 @@ Defaults to `true`. Set to `false` to disable the disk allocation decider.
 // tag::cluster-routing-watermark-low-tag[]
 `cluster.routing.allocation.disk.watermark.low` {ess-icon}::
 (<<dynamic-cluster-setting,Dynamic>>)
-Controls the low watermark for disk usage. It defaults to `85%`, meaning that {es} will not allocate shards to nodes that have more than 85% disk used. It can also be set to an absolute byte value (like `500mb`) to prevent {es} from allocating shards if less than the specified amount of space is available. This setting has no effect on the primary shards of newly-created indices but will prevent their replicas from being allocated.
+Controls the low watermark for disk usage. It defaults to `85%`, meaning that {es} will not allocate shards to nodes that have more than 85% disk used. It can alternatively be set to a ratio value, e.g., `0.85`. It can also be set to an absolute byte value (like `500mb`) to prevent {es} from allocating shards if less than the specified amount of space is available. This setting has no effect on the primary shards of newly-created indices but will prevent their replicas from being allocated.
 // end::cluster-routing-watermark-low-tag[]
 
 [[cluster-routing-watermark-high]]
 // tag::cluster-routing-watermark-high-tag[]
 `cluster.routing.allocation.disk.watermark.high` {ess-icon}::
 (<<dynamic-cluster-setting,Dynamic>>)
-Controls the high watermark. It defaults to `90%`, meaning that {es} will attempt to relocate shards away from a node whose disk usage is above 90%. It can also be set to an absolute byte value (similarly to the low watermark) to relocate shards away from a node if it has less than the specified amount of free space. This setting affects the allocation of all shards, whether previously allocated or not.
+Controls the high watermark. It defaults to `90%`, meaning that {es} will attempt to relocate shards away from a node whose disk usage is above 90%. It can alternatively be set to a ratio value, e.g., `0.9`. It can also be set to an absolute byte value (similarly to the low watermark) to relocate shards away from a node if it has less than the specified amount of free space. This setting affects the allocation of all shards, whether previously allocated or not.
 // end::cluster-routing-watermark-high-tag[]
 
 `cluster.routing.allocation.disk.watermark.enable_for_single_data_node`::
@@ -95,10 +95,10 @@ is now `true`. The setting will be removed in a future release.
 +
 --
 (<<dynamic-cluster-setting,Dynamic>>)
-Controls the flood stage watermark, which defaults to 95%. {es} enforces a read-only index block (`index.blocks.read_only_allow_delete`) on every index that has one or more shards allocated on the node, and that has at least one disk exceeding the flood stage. This setting is a last resort to prevent nodes from running out of disk space. The index block is automatically released when the disk utilization falls below the high watermark.
+Controls the flood stage watermark, which defaults to 95%. {es} enforces a read-only index block (`index.blocks.read_only_allow_delete`) on every index that has one or more shards allocated on the node, and that has at least one disk exceeding the flood stage. This setting is a last resort to prevent nodes from running out of disk space. The index block is automatically released when the disk utilization falls below the high watermark. Similarly to the low and high watermark values, it can alternatively be set to a ratio value, e.g., `0.95`, or an absolute byte value.
 
-NOTE: You cannot mix the usage of percentage values and byte values within
-these settings. Either all values are set to percentage values, or all are set to byte values. This enforcement is so that {es} can validate that the settings are internally consistent, ensuring that the low disk threshold is less than the high disk threshold, and the high disk threshold is less than the flood stage threshold.
+NOTE: You cannot mix the usage of percentage/ratio values and byte values within
+the watermark settings. Either all values are set to percentage/ratio values, or all are set to byte values. This enforcement is so that {es} can validate that the settings are internally consistent, ensuring that the low disk threshold is less than the high disk threshold, and the high disk threshold is less than the flood stage threshold.
 
 An example of resetting the read-only index block on the `my-index-000001` index:
 

diff --git a/docs/reference/search/knn-search.asciidoc b/docs/reference/search/knn-search.asciidoc
@@ -75,8 +75,8 @@ that sacrifices result accuracy for improved search speed. This means the
 results returned are not always the true _k_ closest neighbors.
 //end::hnsw-algorithm[]
 
-The kNN search API supports <<knn-search-api-filter-example, restricting the search using a filter>>.
-The search will return the top `k` documents that also match the filter query.
+The kNN search API supports restricting the search using a filter. The search
+will return the top `k` documents that also match the filter query.
 
 [[knn-search-api-path-params]]
 ==== {api-path-parms-title}
@@ -152,106 +152,3 @@ the similarity between the query and document vector. See
 * The `hits.total` object contains the total number of nearest neighbor
 candidates considered, which is `num_candidates * num_shards`. The
 `hits.total.relation` will always be `eq`, indicating an exact value.
-
-[[knn-search-api-example]]
-==== {api-examples-title}
-
-===== Basic kNN search
-
-The following requests create a `dense_vector` field with indexing enabled and
-add sample documents:
-
-[source,console]
-----
-PUT my-index
-{
-  "mappings": {
-    "properties": {
-      "image_vector": {
-        "type": "dense_vector",
-        "dims": 3,
-        "index": true,
-        "similarity": "l2_norm"
-      },
-      "name": {
-        "type": "keyword"
-      },
-      "file_type": {
-        "type": "keyword"
-      }
-    }
-  }
-}
-
-PUT my-index/_doc/1?refresh
-{
-  "image_vector" : [0.5, 0.1, 2.6],
-  "name": "moose family",
-  "file_type": "jpeg"
-}
-
-PUT my-index/_doc/2?refresh
-{
-  "image_vector" : [1.0, 0.8, -0.2],
-  "name": "alpine lake",
-  "file_type": "svg"
-}
-----
-
-[[knn-search-api-filter-example]]
-===== Filtered kNN search
-
-The next request performs a kNN search filtered by the `file_type` field:
-
-[source,console]
-----
-GET my-index/_knn_search
-{
-  "knn": {
-    "field": "image_vector",
-    "query_vector": [0.3, 0.1, 1.2],
-    "k": 5,
-    "num_candidates": 50
-  },
-  "filter": {
-    "term": {
-      "file_type": "svg"
-    }
-  },
-  "_source": ["name"]
-}
-----
-// TEST[continued]
-
-[source,console-result]
-----
-{
-  "took": 5,
-  "timed_out": false,
-  "_shards": {
-    "total": 1,
-    "successful": 1,
-    "skipped": 0,
-    "failed": 0
-  },
-  "hits": {
-    "total": {
-      "value": 1,
-      "relation": "eq"
-    },
-    "max_score": 0.2538071,
-    "hits": [
-      {
-        "_index": "my-index",
-        "_id": "2",
-        "_score": 0.2538071,
-        "_source": {
-          "name": "alpine lake"
-        }
-      }
-    ]
-  }
-}
-----
-// TESTRESPONSE[s/"took": 5/"took": $body.took/]
-// TESTRESPONSE[s/,\n      \.\.\.//]