diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000000..1f9ace96bd3ee --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,17 @@ +# GitHub CODEOWNERS definition +# Identify which groups will be pinged by changes to different parts of the codebase. +# For more info, see https://help.github.com/articles/about-codeowners/ + +# Stack Monitoring index templates +x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json @elastic/infra-monitoring-ui +x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json @elastic/infra-monitoring-ui +x-pack/plugin/core/src/main/resources/monitoring-beats.json @elastic/infra-monitoring-ui +x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json @elastic/infra-monitoring-ui +x-pack/plugin/core/src/main/resources/monitoring-es-mb.json @elastic/infra-monitoring-ui +x-pack/plugin/core/src/main/resources/monitoring-es.json @elastic/infra-monitoring-ui +x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json @elastic/infra-monitoring-ui +x-pack/plugin/core/src/main/resources/monitoring-kibana.json @elastic/infra-monitoring-ui +x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json @elastic/infra-monitoring-ui +x-pack/plugin/core/src/main/resources/monitoring-logstash.json @elastic/infra-monitoring-ui +x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json @elastic/infra-monitoring-ui +x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @elastic/infra-monitoring-ui diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 91c9fd1cc6c76..f81dc6ae1a1c8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -604,7 +604,7 @@ threshold has been breached: logger.warn( "flood stage disk watermark [{}] exceeded on {}, all indices on this node will be marked read-only", - diskThresholdSettings.describeFloodStageThreshold(), + diskThresholdSettings.describeFloodStageThreshold(total, false), usage ); diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 3e7e1981d104a..8d024789dd2f7 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 8.4.0 -lucene = 9.3.0-snapshot-2d05f5c623e +lucene = 9.3.0-snapshot-b8d1fcfd0ec bundled_jdk_vendor = openjdk bundled_jdk = 18.0.2+9@f6ad4b4450fd4d298113270ec84f30ee diff --git a/docs/changelog/88023.yaml b/docs/changelog/88023.yaml new file mode 100644 index 0000000000000..94b00865601c0 --- /dev/null +++ b/docs/changelog/88023.yaml @@ -0,0 +1,5 @@ +pr: 88023 +summary: "If signature validation fails, reload JWKs and retry if new JWKs are found" +area: Authentication +type: enhancement +issues: [] diff --git a/docs/changelog/88385.yaml b/docs/changelog/88385.yaml index d1ebace3c4857..f3715950263d8 100644 --- a/docs/changelog/88385.yaml +++ b/docs/changelog/88385.yaml @@ -1,5 +1,6 @@ pr: 88385 -summary: Test balance improves after rebalancing +summary: Prevent re-balancing using outdated node weights in some cases area: Allocation type: bug -issues: [] +issues: + - 88384 diff --git a/docs/changelog/88551.yaml b/docs/changelog/88551.yaml new file mode 100644 index 0000000000000..16cac9193eaa6 --- /dev/null +++ b/docs/changelog/88551.yaml @@ -0,0 +1,6 @@ +pr: 88551 +summary: "Fix: use status code 500 for aggregation reduce phase errors if no shard failed" +area: Search +type: bug +issues: + - 20004 diff --git a/docs/changelog/88622.yaml b/docs/changelog/88622.yaml new file mode 100644 index 0000000000000..62c4080a69b01 --- /dev/null +++ b/docs/changelog/88622.yaml @@ -0,0 +1,6 @@ +pr: 88622 +summary: Use origin for the client when running _features/_reset +area: Infra/Core +type: bug +issues: + - 88617 diff --git a/docs/changelog/88642.yaml b/docs/changelog/88642.yaml new file mode 100644 index 0000000000000..967ed0ea16d6c --- /dev/null +++ b/docs/changelog/88642.yaml @@ -0,0 +1,5 @@ +pr: 88642 +summary: Include API key metadata in audit log when an API key is created, granted, or updated +area: Audit +type: enhancement +issues: [] diff --git a/docs/changelog/88694.yaml b/docs/changelog/88694.yaml new file mode 100644 index 0000000000000..6e7bbc6cafed9 --- /dev/null +++ b/docs/changelog/88694.yaml @@ -0,0 +1,14 @@ +pr: 88694 +summary: Integrate ANN into `_search` endpoint +area: Vector Search +type: feature +issues: + - 87625 +highlight: + title: Integrate ANN into `_search` endpoint + body: |- + This change adds a `knn` option to the `_search` API to support ANN + search. It's powered by the same Lucene ANN capabilities as the old + `_knn_search` endpoint. The `knn` option can be combined with other + search features like queries and aggregations. + notable: true diff --git a/docs/changelog/88702.yaml b/docs/changelog/88702.yaml new file mode 100644 index 0000000000000..50baa7e6c3c5a --- /dev/null +++ b/docs/changelog/88702.yaml @@ -0,0 +1,5 @@ +pr: 88702 +summary: Fix validation of `close_pit` request +area: Search +type: bug +issues: [] diff --git a/docs/changelog/88707.yaml b/docs/changelog/88707.yaml new file mode 100644 index 0000000000000..b40fc0d878471 --- /dev/null +++ b/docs/changelog/88707.yaml @@ -0,0 +1,5 @@ +pr: 88707 +summary: Avoid capturing `SnapshotsInProgress$Entry` in queue +area: Snapshot/Restore +type: bug +issues: [] diff --git a/docs/changelog/88719.yaml b/docs/changelog/88719.yaml new file mode 100644 index 0000000000000..374f253ab694b --- /dev/null +++ b/docs/changelog/88719.yaml @@ -0,0 +1,5 @@ +pr: 88719 +summary: Convert disk watermarks to RelativeByteSizeValues +area: Infra/Settings +type: enhancement +issues: [] diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc index 073e9f3ba9ed2..d5191264e1ef3 100644 --- a/docs/reference/cluster/tasks.asciidoc +++ b/docs/reference/cluster/tasks.asciidoc @@ -183,8 +183,8 @@ The API returns the following result: The new `description` field contains human readable text that identifies the particular request that the task is performing such as identifying the search request being performed by a search task like the example above. Other kinds of -task have different descriptions, like <> which -has the search and the destination, or <> which just has the +tasks have different descriptions, like <> which +has the source and the destination, or <> which just has the number of requests and the destination indices. Many requests will only have an empty description because more detailed information about the request is not easily available or particularly helpful in identifying the request. diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 1e0d8e781288b..b30723638c910 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -152,6 +152,7 @@ NOTE: Although they are conceptually related, the `similarity` parameter is different from <> field <> and accepts a distinct set of options. +[[dense-vector-index-options]] `index_options`:: (Optional, object) An optional section that configures the kNN indexing algorithm. The HNSW diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc index 11d94b5c59c43..fa0afe6e4aa5f 100644 --- a/docs/reference/modules/cluster/disk_allocator.asciidoc +++ b/docs/reference/modules/cluster/disk_allocator.asciidoc @@ -72,14 +72,14 @@ Defaults to `true`. Set to `false` to disable the disk allocation decider. // tag::cluster-routing-watermark-low-tag[] `cluster.routing.allocation.disk.watermark.low` {ess-icon}:: (<>) -Controls the low watermark for disk usage. It defaults to `85%`, meaning that {es} will not allocate shards to nodes that have more than 85% disk used. It can also be set to an absolute byte value (like `500mb`) to prevent {es} from allocating shards if less than the specified amount of space is available. This setting has no effect on the primary shards of newly-created indices but will prevent their replicas from being allocated. +Controls the low watermark for disk usage. It defaults to `85%`, meaning that {es} will not allocate shards to nodes that have more than 85% disk used. It can alternatively be set to a ratio value, e.g., `0.85`. It can also be set to an absolute byte value (like `500mb`) to prevent {es} from allocating shards if less than the specified amount of space is available. This setting has no effect on the primary shards of newly-created indices but will prevent their replicas from being allocated. // end::cluster-routing-watermark-low-tag[] [[cluster-routing-watermark-high]] // tag::cluster-routing-watermark-high-tag[] `cluster.routing.allocation.disk.watermark.high` {ess-icon}:: (<>) -Controls the high watermark. It defaults to `90%`, meaning that {es} will attempt to relocate shards away from a node whose disk usage is above 90%. It can also be set to an absolute byte value (similarly to the low watermark) to relocate shards away from a node if it has less than the specified amount of free space. This setting affects the allocation of all shards, whether previously allocated or not. +Controls the high watermark. It defaults to `90%`, meaning that {es} will attempt to relocate shards away from a node whose disk usage is above 90%. It can alternatively be set to a ratio value, e.g., `0.9`. It can also be set to an absolute byte value (similarly to the low watermark) to relocate shards away from a node if it has less than the specified amount of free space. This setting affects the allocation of all shards, whether previously allocated or not. // end::cluster-routing-watermark-high-tag[] `cluster.routing.allocation.disk.watermark.enable_for_single_data_node`:: @@ -95,10 +95,10 @@ is now `true`. The setting will be removed in a future release. + -- (<>) -Controls the flood stage watermark, which defaults to 95%. {es} enforces a read-only index block (`index.blocks.read_only_allow_delete`) on every index that has one or more shards allocated on the node, and that has at least one disk exceeding the flood stage. This setting is a last resort to prevent nodes from running out of disk space. The index block is automatically released when the disk utilization falls below the high watermark. +Controls the flood stage watermark, which defaults to 95%. {es} enforces a read-only index block (`index.blocks.read_only_allow_delete`) on every index that has one or more shards allocated on the node, and that has at least one disk exceeding the flood stage. This setting is a last resort to prevent nodes from running out of disk space. The index block is automatically released when the disk utilization falls below the high watermark. Similarly to the low and high watermark values, it can alternatively be set to a ratio value, e.g., `0.95`, or an absolute byte value. -NOTE: You cannot mix the usage of percentage values and byte values within -these settings. Either all values are set to percentage values, or all are set to byte values. This enforcement is so that {es} can validate that the settings are internally consistent, ensuring that the low disk threshold is less than the high disk threshold, and the high disk threshold is less than the flood stage threshold. +NOTE: You cannot mix the usage of percentage/ratio values and byte values within +the watermark settings. Either all values are set to percentage/ratio values, or all are set to byte values. This enforcement is so that {es} can validate that the settings are internally consistent, ensuring that the low disk threshold is less than the high disk threshold, and the high disk threshold is less than the flood stage threshold. An example of resetting the read-only index block on the `my-index-000001` index: diff --git a/docs/reference/search/knn-search.asciidoc b/docs/reference/search/knn-search.asciidoc index 32d69e6d64ff0..a4b1f9c00b3be 100644 --- a/docs/reference/search/knn-search.asciidoc +++ b/docs/reference/search/knn-search.asciidoc @@ -75,8 +75,8 @@ that sacrifices result accuracy for improved search speed. This means the results returned are not always the true _k_ closest neighbors. //end::hnsw-algorithm[] -The kNN search API supports <>. -The search will return the top `k` documents that also match the filter query. +The kNN search API supports restricting the search using a filter. The search +will return the top `k` documents that also match the filter query. [[knn-search-api-path-params]] ==== {api-path-parms-title} @@ -152,106 +152,3 @@ the similarity between the query and document vector. See * The `hits.total` object contains the total number of nearest neighbor candidates considered, which is `num_candidates * num_shards`. The `hits.total.relation` will always be `eq`, indicating an exact value. - -[[knn-search-api-example]] -==== {api-examples-title} - -===== Basic kNN search - -The following requests create a `dense_vector` field with indexing enabled and -add sample documents: - -[source,console] ----- -PUT my-index -{ - "mappings": { - "properties": { - "image_vector": { - "type": "dense_vector", - "dims": 3, - "index": true, - "similarity": "l2_norm" - }, - "name": { - "type": "keyword" - }, - "file_type": { - "type": "keyword" - } - } - } -} - -PUT my-index/_doc/1?refresh -{ - "image_vector" : [0.5, 0.1, 2.6], - "name": "moose family", - "file_type": "jpeg" -} - -PUT my-index/_doc/2?refresh -{ - "image_vector" : [1.0, 0.8, -0.2], - "name": "alpine lake", - "file_type": "svg" -} ----- - -[[knn-search-api-filter-example]] -===== Filtered kNN search - -The next request performs a kNN search filtered by the `file_type` field: - -[source,console] ----- -GET my-index/_knn_search -{ - "knn": { - "field": "image_vector", - "query_vector": [0.3, 0.1, 1.2], - "k": 5, - "num_candidates": 50 - }, - "filter": { - "term": { - "file_type": "svg" - } - }, - "_source": ["name"] -} ----- -// TEST[continued] - -[source,console-result] ----- -{ - "took": 5, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": 0.2538071, - "hits": [ - { - "_index": "my-index", - "_id": "2", - "_score": 0.2538071, - "_source": { - "name": "alpine lake" - } - } - ] - } -} ----- -// TESTRESPONSE[s/"took": 5/"took": $body.took/] -// TESTRESPONSE[s/,\n \.\.\.//] diff --git a/docs/reference/search/search-your-data/knn-search.asciidoc b/docs/reference/search/search-your-data/knn-search.asciidoc index 51ed5a0df10c8..c16fbff4cff41 100644 --- a/docs/reference/search/search-your-data/knn-search.asciidoc +++ b/docs/reference/search/search-your-data/knn-search.asciidoc @@ -40,11 +40,12 @@ based on a similarity metric, the better its match. {es} supports two methods for kNN search: -* experimental:[] <> using the kNN search API - * <> using a `script_score` query with a vector function +* experimental:[] <> using the `knn` search +option + In most cases, you'll want to use approximate kNN. Approximate kNN offers lower latency at the cost of slower indexing and imperfect accuracy. @@ -56,13 +57,96 @@ to limit the number of matching documents passed to the function. If you filter your data to a small subset of documents, you can get good search performance using this approach. +[discrete] +[[exact-knn]] +=== Exact kNN + +To run an exact kNN search, use a `script_score` query with a vector function. + +. Explicitly map one or more `dense_vector` fields. If you don't intend to use +the field for approximate kNN, omit the `index` mapping option or set it to +`false`. This can significantly improve indexing speed. ++ +[source,console] +---- +PUT product-index +{ + "mappings": { + "properties": { + "product-vector": { + "type": "dense_vector", + "dims": 5, + "index": false + }, + "price": { + "type": "long" + } + } + } +} +---- + +. Index your data. ++ +[source,console] +---- +POST product-index/_bulk?refresh=true +{ "index": { "_id": "1" } } +{ "product-vector": [230.0, 300.33, -34.8988, 15.555, -200.0], "price": 1599 } +{ "index": { "_id": "2" } } +{ "product-vector": [-0.5, 100.0, -13.0, 14.8, -156.0], "price": 799 } +{ "index": { "_id": "3" } } +{ "product-vector": [0.5, 111.3, -13.0, 14.8, -156.0], "price": 1099 } +... +---- +//TEST[continued] +//TEST[s/\.\.\.//] + +. Use the <> to run a `script_score` query containing +a <>. ++ +TIP: To limit the number of matched documents passed to the vector function, we +recommend you specify a filter query in the `script_score.query` parameter. If +needed, you can use a <> in this +parameter to match all documents. However, matching all documents can +significantly increase search latency. ++ +[source,console] +---- +POST product-index/_search +{ + "query": { + "script_score": { + "query" : { + "bool" : { + "filter" : { + "range" : { + "price" : { + "gte": 1000 + } + } + } + } + }, + "script": { + "source": "cosineSimilarity(params.queryVector, 'product-vector') + 1.0", + "params": { + "queryVector": [-0.5, 90.0, -10, 14.8, -156.0] + } + } + } + } +} +---- +//TEST[continued] + [discrete] [[approximate-knn]] === Approximate kNN experimental::[] -To run an approximate kNN search, use the <> +To run an approximate kNN search, use the <> to search a `dense_vector` field with indexing enabled. . Explicitly map one or more `dense_vector` fields. Approximate kNN search @@ -78,17 +162,20 @@ parameter documentation. [source,console] ---- -PUT my-approx-knn-index +PUT image-index { "mappings": { "properties": { - "my-image-vector": { + "image-vector": { "type": "dense_vector", - "dims": 5, + "dims": 3, "index": true, "similarity": "l2_norm" }, - "my-tag": { + "title": { + "type": "text" + }, + "file-type": { "type": "keyword" } } @@ -101,40 +188,42 @@ PUT my-approx-knn-index + [source,console] ---- -POST my-approx-knn-index/_bulk?refresh=true +POST image-index/_bulk?refresh=true { "index": { "_id": "1" } } -{ "my-image-vector": [230.0, 300.33, -34.8988, 15.555, -200.0], "my-tag": "cow.jpg" } +{ "image-vector": [1, 5, -20], "title": "moose family", "file-type": "jpg" } { "index": { "_id": "2" } } -{ "my-image-vector": [-0.5, 100.0, -13.0, 14.8, -156.0], "my-tag": "moose.jpg" } +{ "image-vector": [42, 8, -15], "title": "alpine lake", "file-type": "png" } { "index": { "_id": "3" } } -{ "my-image-vector": [0.5, 111.3, -13.0, 14.8, -156.0], "my-tag": "rabbit.jpg" } +{ "image-vector": [15, 11, 23], "title": "full moon", "file-type": "jpg" } ... ---- //TEST[continued] //TEST[s/\.\.\.//] -. Run the search using the <>. +. Run the search using the <>. + [source,console] ---- -GET my-approx-knn-index/_knn_search +POST image-index/_search { "knn": { - "field": "my-image-vector", - "query_vector": [-0.5, 90.0, -10, 14.8, -156.0], + "field": "image-vector", + "query_vector": [-5, 9, -12], "k": 10, "num_candidates": 100 }, - "fields": [ - "my-image-vector", - "my-tag" - ] + "fields": [ "title", "file-type" ] } ---- //TEST[continued] // TEST[s/"k": 10/"k": 3/] // TEST[s/"num_candidates": 100/"num_candidates": 3/] +The <> is determined by +the similarity between the query and document vector. See +<> for more information on how kNN +search scores are computed. + NOTE: Support for approximate kNN search was added in version 8.0. Before this, `dense_vector` fields did not support enabling `index` in the mapping. If you created an index prior to 8.0 containing `dense_vector` fields, then to @@ -159,6 +248,112 @@ search has a higher probability of finding the true `k` top nearest neighbors. Similarly, you can decrease `num_candidates` for faster searches with potentially less accurate results. +[discrete] +[[knn-search-filter-example]] +==== Filtered kNN search + +The kNN search API supports restricting the search using a filter. The search +will return the top `k` documents that also match the filter query. + +The following request performs an approximate kNN search filtered by the +`file-type` field: + +[source,console] +---- +POST image-index/_search +{ + "knn": { + "field": "image-vector", + "query_vector": [54, 10, -2], + "k": 5, + "num_candidates": 50, + "filter": { + "term": { + "file-type": "png" + } + } + }, + "fields": ["title"], + "_source": false +} +---- +// TEST[continued] + +[source,console-result] +---- +{ + "took": 5, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 0.003144654, + "hits": [ + { + "_index": "image-index", + "_id": "2", + "_score": 0.003144654, + "fields": { + "title": ["alpine lake"] + } + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 5/"took": $body.took/] +// TESTRESPONSE[s/,\n \.\.\.//] + +[discrete] +==== Combine approximate kNN and a query + +You can perform 'hybrid retrieval' by providing both the +<> and a <>: + +[source,console] +---- +POST image-index/_search +{ + "query": { + "match": { + "title": { + "query": "mountain lake", + "boost": 0.9 + } + } + }, + "knn": { + "field": "image-vector", + "query_vector": [54, 10, -2], + "k": 5, + "num_candidates": 50, + "boost": 0.1 + }, + "size": 10 +} +---- +// TEST[continued] + +This search finds the global top `k = 5` vector matches, combines them with the matches from the `match` query, and +finally returns the 10 top-scoring results. The `knn` and `query` matches are combined through a disjunction, as if you +took a boolean 'or' between them. The top `k` vector results represent the global nearest neighbors across all index +shards. + +The score of each hit is the sum of the `knn` and `query` scores. You can specify a `boost` value to give a weight to +each score in the sum. In the example above, the scores will be calculated as + +``` +score = 0.9 * match_score + 0.1 * knn_score +``` + [discrete] [[knn-indexing-considerations]] ==== Indexing considerations @@ -176,97 +371,52 @@ all-inclusive HNSW graph. When there are multiple segments, kNN search must check several smaller HNSW graphs as it searches each segment after another. You should only force merge an index if it is no longer being written to. -[discrete] -[[approximate-knn-limitations]] -==== Limitations for approximate kNN search +The HNSW algorithm has index-time parameters that trade off between the cost of +building the graph, search speed, and accuracy. When setting up the +`dense_vector` mapping, you can use the <> +argument to adjust these parameters: -* You can't run an approximate kNN search on a <>. - -* You can't run an approximate kNN search on a `dense_vector` field within a -<> mapping. - -* {blank} -include::{es-repo-dir}/search/knn-search.asciidoc[tag=hnsw-algorithm] - -[discrete] -[[exact-knn]] -=== Exact kNN - -To run an exact kNN search, use a `script_score` query with a vector function. - -. Explicitly map one or more `dense_vector` fields. If you don't intend to use -the field for approximate kNN, omit the `index` mapping option or set it to -`false`. This can significantly improve indexing speed. -+ [source,console] ---- -PUT my-exact-knn-index +PUT image-index { "mappings": { "properties": { - "my-product-vector": { + "image-vector": { "type": "dense_vector", - "dims": 5, - "index": false - }, - "my-price": { - "type": "long" + "dims": 3, + "index": true, + "similarity": "l2_norm", + "index_options": { + "type": "hnsw", + "m": 32, + "ef_construction": 100 + } } } } } ---- -. Index your data. -+ -[source,console] ----- -POST my-exact-knn-index/_bulk?refresh=true -{ "index": { "_id": "1" } } -{ "my-product-vector": [230.0, 300.33, -34.8988, 15.555, -200.0], "my-price": 1599 } -{ "index": { "_id": "2" } } -{ "my-product-vector": [-0.5, 100.0, -13.0, 14.8, -156.0], "my-price": 799 } -{ "index": { "_id": "3" } } -{ "my-product-vector": [0.5, 111.3, -13.0, 14.8, -156.0], "my-price": 1099 } -... ----- -//TEST[continued] -//TEST[s/\.\.\.//] +[discrete] +[[approximate-knn-limitations]] +==== Limitations for approximate kNN search + +* You can't run an approximate kNN search on a <>. +Running a kNN search against a filtered alias may incorrectly result in fewer +than `k` hits. + +* You can't run an approximate kNN search on a `dense_vector` field within a +<> mapping. + +* When using kNN search in <>, the <> +option is not supported. + +* {blank} +include::{es-repo-dir}/search/knn-search.asciidoc[tag=hnsw-algorithm] + +NOTE: Approximate kNN search always uses the +<> search type in order to gather +the global top `k` matches across shards. You cannot set the +`search_type` explicitly when running kNN search. -. Use the <> to run a `script_score` query containing -a <>. -+ -TIP: To limit the number of matched documents passed to the vector function, we -recommend you specify a filter query in the `script_score.query` parameter. If -needed, you can use a <> in this -parameter to match all documents. However, matching all documents can -significantly increase search latency. -+ -[source,console] ----- -GET my-exact-knn-index/_search -{ - "query": { - "script_score": { - "query" : { - "bool" : { - "filter" : { - "range" : { - "my-price" : { - "gte": 1000 - } - } - } - } - }, - "script": { - "source": "cosineSimilarity(params.queryVector, 'my-product-vector') + 1.0", - "params": { - "queryVector": [-0.5, 90.0, -10, 14.8, -156.0] - } - } - } - } -} ----- -//TEST[continued] diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 21a7366165d54..9211cc86930d9 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -480,6 +480,40 @@ A boost value greater than `1.0` increases the score. A boost value between `0` and `1.0` decreases the score. ==== +experimental::[] +[[search-api-knn]] +`knn`:: +(Optional, object) Defines the <> to +run. ++ +.Properties of `knn` object +[%collapsible%open] +==== +`field`:: +(Required, string) The name of the vector field to search against. Must be a +<>. + +`query_vector`:: +(Required, array of floats) Query vector. Must have the same number of +dimensions as the vector field you are searching against. + +`k`:: +(Required, integer) Number of nearest neighbors to return as top hits. This +value must be less than `num_candidates`. + +`num_candidates`:: +(Required, integer) The number of nearest neighbor candidates to consider per +shard. Cannot exceed 10,000. {es} collects `num_candidates` results from each +shard, then merges them to find the top `k` results. Increasing +`num_candidates` tends to improve the accuracy of the final `k` results. + +`filter`:: +(Optional, <>) Query to filter the documents that +can match. The kNN search will return the top `k` documents that also match +this filter. The value can be a single query or a list of queries. If `filter` +is not provided, all documents are allowed to match. +==== + [[search-api-min-score]] `min_score`:: (Optional, float) diff --git a/modules/lang-expression/licenses/lucene-codecs-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/modules/lang-expression/licenses/lucene-codecs-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index f164594b811ce..0000000000000 --- a/modules/lang-expression/licenses/lucene-codecs-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a2b0a4e1d763c5575f289ef7b4d8f109e7ee895f \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-codecs-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/modules/lang-expression/licenses/lucene-codecs-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..47b8df46111d4 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-codecs-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +f9d91e4de3468b4c513a82a3d20d9d19137c4311 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 98efd688cf3ac..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -37603372705778c2c755d055ac72f835af0ebe51 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..544a44a26debb --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +b146dc1d898b3f638328a4d6a64f68cfede251ec \ No newline at end of file diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index 83e4bd059d3a0..20cc16f27fd4a 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -93,7 +93,7 @@ public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, b } else { throw new IllegalArgumentException("Malformed search template"); } - RestSearchAction.checkRestTotalHits(restRequest, searchRequest); + RestSearchAction.validateSearchRequest(restRequest, searchRequest); } ); return multiRequest; diff --git a/modules/legacy-geo/licenses/lucene-spatial-extras-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/modules/legacy-geo/licenses/lucene-spatial-extras-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 75358a122c5aa..0000000000000 --- a/modules/legacy-geo/licenses/lucene-spatial-extras-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -899069409ff5a448c641700a0371ce792a0635b1 \ No newline at end of file diff --git a/modules/legacy-geo/licenses/lucene-spatial-extras-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/modules/legacy-geo/licenses/lucene-spatial-extras-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..5e9c50d838196 --- /dev/null +++ b/modules/legacy-geo/licenses/lucene-spatial-extras-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +f87c4435a856c612a5799fa89397364a7b2d6f7e \ No newline at end of file diff --git a/modules/legacy-geo/licenses/lucene-spatial3d-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/modules/legacy-geo/licenses/lucene-spatial3d-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 6017afff2e6cc..0000000000000 --- a/modules/legacy-geo/licenses/lucene-spatial3d-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3416a5a2becfd987049453c939afa2624c3db6c7 \ No newline at end of file diff --git a/modules/legacy-geo/licenses/lucene-spatial3d-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/modules/legacy-geo/licenses/lucene-spatial3d-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..6c74a9716f82f --- /dev/null +++ b/modules/legacy-geo/licenses/lucene-spatial3d-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +110e8b2e5bced4b8f482ac58a2cf9cd64591b028 \ No newline at end of file diff --git a/modules/repository-s3/build.gradle b/modules/repository-s3/build.gradle index 999fff7023477..787ddddb81f20 100644 --- a/modules/repository-s3/build.gradle +++ b/modules/repository-s3/build.gradle @@ -75,6 +75,20 @@ esplugin.bundleSpec.from('config/repository-s3') { into 'config' } +def testRepositoryCreds = tasks.register("testRepositoryCreds", Test) { + include '**/RepositoryCredentialsTests.class' + systemProperty 'es.allow_insecure_settings', 'true' +} + +tasks.named('check').configure { + dependsOn(testRepositoryCreds) +} + +tasks.named('test').configure { + // this is tested explicitly in separate test tasks + exclude '**/RepositoryCredentialsTests.class' +} + boolean useFixture = false def fixtureAddress = { fixture, name, port -> diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index ebdd8f9eaf326..7d1b495a0f008 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -263,6 +263,12 @@ S3ClientSettings refine(Settings repositorySettings) { normalizedSettings, disableChunkedEncoding ); + final S3BasicCredentials newCredentials; + if (checkDeprecatedCredentials(repositorySettings)) { + newCredentials = loadDeprecatedCredentials(repositorySettings); + } else { + newCredentials = credentials; + } final String newRegion = getRepoSettingOrDefault(REGION, normalizedSettings, region); final String newSignerOverride = getRepoSettingOrDefault(SIGNER_OVERRIDE, normalizedSettings, signerOverride); if (Objects.equals(endpoint, newEndpoint) @@ -272,6 +278,7 @@ S3ClientSettings refine(Settings repositorySettings) { && newReadTimeoutMillis == readTimeoutMillis && maxRetries == newMaxRetries && newThrottleRetries == throttleRetries + && Objects.equals(credentials, newCredentials) && newPathStyleAccess == pathStyleAccess && newDisableChunkedEncoding == disableChunkedEncoding && Objects.equals(region, newRegion) @@ -279,7 +286,7 @@ S3ClientSettings refine(Settings repositorySettings) { return this; } return new S3ClientSettings( - credentials, + newCredentials, newEndpoint, newProtocol, newProxyHost, @@ -315,6 +322,41 @@ static Map load(Settings settings) { return Collections.unmodifiableMap(clients); } + static boolean checkDeprecatedCredentials(Settings repositorySettings) { + if (S3Repository.ACCESS_KEY_SETTING.exists(repositorySettings)) { + if (S3Repository.SECRET_KEY_SETTING.exists(repositorySettings) == false) { + throw new IllegalArgumentException( + "Repository setting [" + + S3Repository.ACCESS_KEY_SETTING.getKey() + + " must be accompanied by setting [" + + S3Repository.SECRET_KEY_SETTING.getKey() + + "]" + ); + } + return true; + } else if (S3Repository.SECRET_KEY_SETTING.exists(repositorySettings)) { + throw new IllegalArgumentException( + "Repository setting [" + + S3Repository.SECRET_KEY_SETTING.getKey() + + " must be accompanied by setting [" + + S3Repository.ACCESS_KEY_SETTING.getKey() + + "]" + ); + } + return false; + } + + // backcompat for reading keys out of repository settings (clusterState) + private static S3BasicCredentials loadDeprecatedCredentials(Settings repositorySettings) { + assert checkDeprecatedCredentials(repositorySettings); + try ( + SecureString key = S3Repository.ACCESS_KEY_SETTING.get(repositorySettings); + SecureString secret = S3Repository.SECRET_KEY_SETTING.get(repositorySettings) + ) { + return new S3BasicCredentials(key.toString(), secret.toString()); + } + } + private static S3BasicCredentials loadCredentials(Settings settings, String clientName) { try ( SecureString accessKey = getConfigValue(settings, clientName, ACCESS_KEY_SETTING); diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index a1a4a52b2a279..ac0140e932b9b 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -18,6 +18,10 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -57,9 +61,16 @@ */ class S3Repository extends MeteredBlobStoreRepository { private static final Logger logger = LogManager.getLogger(S3Repository.class); + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(logger.getName()); static final String TYPE = "s3"; + /** The access key to authenticate with s3. This setting is insecure because cluster settings are stored in cluster state */ + static final Setting ACCESS_KEY_SETTING = SecureSetting.insecureString("access_key"); + + /** The secret key to authenticate with s3. This setting is insecure because cluster settings are stored in cluster state */ + static final Setting SECRET_KEY_SETTING = SecureSetting.insecureString("secret_key"); + /** * Default is to use 100MB (S3 defaults) for heaps above 2GB and 5% of * the available memory for smaller heaps. @@ -233,6 +244,16 @@ class S3Repository extends MeteredBlobStoreRepository { this.storageClass = STORAGE_CLASS_SETTING.get(metadata.settings()); this.cannedACL = CANNED_ACL_SETTING.get(metadata.settings()); + if (S3ClientSettings.checkDeprecatedCredentials(metadata.settings())) { + // provided repository settings + deprecationLogger.critical( + DeprecationCategory.SECURITY, + "s3_repository_secret_settings", + "Using s3 access/secret key from repository settings. Instead " + + "store these in named clients and the elasticsearch keystore for secure settings." + ); + } + coolDown = COOLDOWN_PERIOD.get(metadata.settings()); logger.debug( diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index e400654dbd9ff..3cc7f4a7297af 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -141,7 +141,9 @@ public List> getSettings() { S3ClientSettings.USE_THROTTLE_RETRIES_SETTING, S3ClientSettings.USE_PATH_STYLE_ACCESS, S3ClientSettings.SIGNER_OVERRIDE, - S3ClientSettings.REGION + S3ClientSettings.REGION, + S3Repository.ACCESS_KEY_SETTING, + S3Repository.SECRET_KEY_SETTING ); } diff --git a/modules/repository-s3/src/main/plugin-metadata/plugin-security.policy b/modules/repository-s3/src/main/plugin-metadata/plugin-security.policy index e38621c8a8378..c071b62b2164e 100644 --- a/modules/repository-s3/src/main/plugin-metadata/plugin-security.policy +++ b/modules/repository-s3/src/main/plugin-metadata/plugin-security.policy @@ -28,5 +28,6 @@ grant { // s3 client opens socket connections for to access repository permission java.net.SocketPermission "*", "connect"; - + // only for tests : org.elasticsearch.repositories.s3.S3RepositoryPlugin + permission java.util.PropertyPermission "es.allow_insecure_settings", "read,write"; }; diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java index 82f9b2b0688ea..1cddd9ee471cb 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java @@ -14,31 +14,54 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.rest.AbstractRestChannel; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Collection; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.repositories.s3.S3ClientSettings.ACCESS_KEY_SETTING; import static org.elasticsearch.repositories.s3.S3ClientSettings.SECRET_KEY_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +@SuppressForbidden(reason = "test requires to set a System property to allow insecure settings when running in IDE") public class RepositoryCredentialsTests extends ESSingleNodeTestCase { + static { + AccessController.doPrivileged((PrivilegedAction) () -> { + // required for client settings overwriting when running in IDE + System.setProperty("es.allow_insecure_settings", "true"); + return null; + }); + } + @Override protected Collection> getPlugins() { return List.of(ProxyS3RepositoryPlugin.class); @@ -60,11 +83,51 @@ protected Settings nodeSettings() { return Settings.builder().setSecureSettings(secureSettings).put(super.nodeSettings()).build(); } + public void testRepositoryCredentialsOverrideSecureCredentials() { + final String repositoryName = "repo-creds-override"; + final Settings.Builder repositorySettings = Settings.builder() + // repository settings for credentials override node secure settings + .put(S3Repository.ACCESS_KEY_SETTING.getKey(), "insecure_aws_key") + .put(S3Repository.SECRET_KEY_SETTING.getKey(), "insecure_aws_secret"); + + final String clientName = randomFrom("default", "other", null); + if (clientName != null) { + repositorySettings.put(S3Repository.CLIENT_NAME.getKey(), clientName); + } + createRepository(repositoryName, repositorySettings.build()); + + final RepositoriesService repositories = getInstanceFromNode(RepositoriesService.class); + assertThat(repositories.repository(repositoryName), notNullValue()); + assertThat(repositories.repository(repositoryName), instanceOf(S3Repository.class)); + + final S3Repository repository = (S3Repository) repositories.repository(repositoryName); + final AmazonS3 client = repository.createBlobStore().clientReference().client(); + assertThat(client, instanceOf(ProxyS3RepositoryPlugin.ClientAndCredentials.class)); + + final AWSCredentials credentials = ((ProxyS3RepositoryPlugin.ClientAndCredentials) client).credentials.getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("insecure_aws_key")); + assertThat(credentials.getAWSSecretKey(), is("insecure_aws_secret")); + + assertCriticalWarnings( + "[secret_key] setting was deprecated in Elasticsearch and will be removed in a future release.", + "Using s3 access/secret key from repository settings. Instead store these in named clients and" + + " the elasticsearch keystore for secure settings.", + "[access_key] setting was deprecated in Elasticsearch and will be removed in a future release." + ); + } + public void testReinitSecureCredentials() { final String clientName = randomFrom("default", "other"); final Settings.Builder repositorySettings = Settings.builder(); - repositorySettings.put(S3Repository.CLIENT_NAME.getKey(), clientName); + final boolean hasInsecureSettings = randomBoolean(); + if (hasInsecureSettings) { + // repository settings for credentials override node secure settings + repositorySettings.put(S3Repository.ACCESS_KEY_SETTING.getKey(), "insecure_aws_key"); + repositorySettings.put(S3Repository.SECRET_KEY_SETTING.getKey(), "insecure_aws_secret"); + } else { + repositorySettings.put(S3Repository.CLIENT_NAME.getKey(), clientName); + } final String repositoryName = "repo-reinit-creds"; createRepository(repositoryName, repositorySettings.build()); @@ -79,7 +142,10 @@ public void testReinitSecureCredentials() { assertThat(client, instanceOf(ProxyS3RepositoryPlugin.ClientAndCredentials.class)); final AWSCredentials credentials = ((ProxyS3RepositoryPlugin.ClientAndCredentials) client).credentials.getCredentials(); - if ("other".equals(clientName)) { + if (hasInsecureSettings) { + assertThat(credentials.getAWSAccessKeyId(), is("insecure_aws_key")); + assertThat(credentials.getAWSSecretKey(), is("insecure_aws_secret")); + } else if ("other".equals(clientName)) { assertThat(credentials.getAWSAccessKeyId(), is("secure_other_key")); assertThat(credentials.getAWSSecretKey(), is("secure_other_secret")); } else { @@ -98,7 +164,10 @@ public void testReinitSecureCredentials() { plugin.reload(newSettings); // check the not-yet-closed client reference still has the same credentials - if ("other".equals(clientName)) { + if (hasInsecureSettings) { + assertThat(credentials.getAWSAccessKeyId(), is("insecure_aws_key")); + assertThat(credentials.getAWSSecretKey(), is("insecure_aws_secret")); + } else if ("other".equals(clientName)) { assertThat(credentials.getAWSAccessKeyId(), is("secure_other_key")); assertThat(credentials.getAWSSecretKey(), is("secure_other_secret")); } else { @@ -113,11 +182,66 @@ public void testReinitSecureCredentials() { assertThat(client, instanceOf(ProxyS3RepositoryPlugin.ClientAndCredentials.class)); final AWSCredentials newCredentials = ((ProxyS3RepositoryPlugin.ClientAndCredentials) client).credentials.getCredentials(); - assertThat(newCredentials.getAWSAccessKeyId(), is("new_secret_aws_key")); - assertThat(newCredentials.getAWSSecretKey(), is("new_secret_aws_secret")); + if (hasInsecureSettings) { + assertThat(newCredentials.getAWSAccessKeyId(), is("insecure_aws_key")); + assertThat(newCredentials.getAWSSecretKey(), is("insecure_aws_secret")); + } else { + assertThat(newCredentials.getAWSAccessKeyId(), is("new_secret_aws_key")); + assertThat(newCredentials.getAWSSecretKey(), is("new_secret_aws_secret")); + } + } + + if (hasInsecureSettings) { + assertCriticalWarnings( + "[secret_key] setting was deprecated in Elasticsearch and will be removed in a future release.", + "Using s3 access/secret key from repository settings. Instead store these in named clients and" + + " the elasticsearch keystore for secure settings.", + "[access_key] setting was deprecated in Elasticsearch and will be removed in a future release." + ); } } + public void testInsecureRepositoryCredentials() throws Exception { + final String repositoryName = "repo-insecure-creds"; + createRepository( + repositoryName, + Settings.builder() + .put(S3Repository.ACCESS_KEY_SETTING.getKey(), "insecure_aws_key") + .put(S3Repository.SECRET_KEY_SETTING.getKey(), "insecure_aws_secret") + .build() + ); + + final RestRequest fakeRestRequest = new FakeRestRequest(); + fakeRestRequest.params().put("repository", repositoryName); + final RestGetRepositoriesAction action = new RestGetRepositoriesAction(getInstanceFromNode(SettingsFilter.class)); + + final CountDownLatch latch = new CountDownLatch(1); + final AtomicReference error = new AtomicReference<>(); + action.handleRequest(fakeRestRequest, new AbstractRestChannel(fakeRestRequest, true) { + @Override + public void sendResponse(RestResponse response) { + try { + String responseAsString = response.content().utf8ToString(); + assertThat(responseAsString, containsString(repositoryName)); + assertThat(responseAsString, not(containsString("insecure_"))); + } catch (final AssertionError ex) { + error.set(ex); + } + latch.countDown(); + } + }, getInstanceFromNode(NodeClient.class)); + + latch.await(); + if (error.get() != null) { + throw error.get(); + } + + assertWarnings( + "Using s3 access/secret key from repository settings. Instead store these in named clients and" + + " the elasticsearch keystore for secure settings." + ); + } + private void createRepository(final String name, final Settings repositorySettings) { assertAcked( client().admin() diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java index 1f0efd43a8bc1..550b8811a4da8 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java @@ -151,7 +151,7 @@ public void testClearVotingTombstonesWaitingForRemoval() throws Exception { final Response response = restClient.performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); assertThat(response.getEntity().getContentLength(), is(0L)); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeToWithdraw)); + internalCluster().stopNode(nodeToWithdraw); Response deleteResponse = restClient.performRequest(new Request("DELETE", "/_cluster/voting_config_exclusions")); assertThat(deleteResponse.getStatusLine().getStatusCode(), is(200)); assertThat(deleteResponse.getEntity().getContentLength(), is(0L)); @@ -167,8 +167,8 @@ public void testRemoveTwoNodesAtOnce() throws Exception { final Response response = restClient.performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); assertThat(response.getEntity().getContentLength(), is(0L)); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(0))); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(2))); + internalCluster().stopNode(nodes.get(0)); + internalCluster().stopNode(nodes.get(2)); ensureStableCluster(1); } } diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java index 8ab50fa7b6a45..7e56e0aa2d7c8 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockLogAppender; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.transport.TcpTransport; @@ -110,7 +109,7 @@ public void testConnectionLogging() throws IOException { ); final String nodeName = internalCluster().startNode(); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeName)); + internalCluster().stopNode(nodeName); appender.assertAllExpectationsMatched(); } diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 73d5dfdcae436..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -87772b1d9907c63287d7bc79cf8fa3e9a46c9c77 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..ceea8ba4f6855 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +501aa4f0028424a994b06627f30ffb36150ffbe2 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 7b95a9396dd9e..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a58272a5eb0137dbcb030a7f2cd7925dd717c3e8 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..929be5cd0d86f --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +a4a84f37391ab5da0697ba6344555b633aa4bacd \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 0e5e293c5bba6..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e15673b088608c7b7e2b5ab491665f9ea92941b2 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..60c85b324c183 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +012f177949d83aa7bdf26c309f5569f67d1c65b5 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index f818c60390a7e..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f997bf7bdd07a5684e017294764b0a2ca398f948 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..782b48c8fd4df --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +792f50d6cd8b75c277c514f2f6e9914572942dfe \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 6153f690cb542..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7cdabbc7114cb7f2925407dd7e18fe6e36fbfc02 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..4b9ceb4a4581f --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +b1488267195c87749dcc42de6b2f665d24ff8d9e \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 0e796b94b617a..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -56cd7a7ae2f9db55fc4150f60745f900a1018e35 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..45ccdbf538570 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +c78968b087eaf2a95ed3b67540efc32455bab84d \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 751392734f742..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d8a80d858ca656735e3f3e0e3803f9130a504f58 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..91d9d4c9452b2 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +978ee14dad7edab6384d04655ce1db219547b6d8 \ No newline at end of file diff --git a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java index 4fea260d49378..f55b350520106 100644 --- a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java +++ b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.function.Supplier; +import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; @@ -73,7 +74,7 @@ public Collection getSystemIndexDescriptors(Settings sett .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") .build() ) - .setOrigin("net-new") + .setOrigin(TASKS_ORIGIN) .setVersionMetaKey("version") .setPrimaryIndex(".net-new-system-index-" + Version.CURRENT.major) .build(), @@ -81,7 +82,7 @@ public Collection getSystemIndexDescriptors(Settings sett .setIndexPattern(INTERNAL_UNMANAGED_INDEX_NAME) .setDescription("internal unmanaged system index") .setType(SystemIndexDescriptor.Type.INTERNAL_UNMANAGED) - .setOrigin("qa") + .setOrigin(TASKS_ORIGIN) .setAliasName(".internal-unmanaged-alias") .build(), SystemIndexDescriptor.builder() @@ -96,7 +97,7 @@ public Collection getSystemIndexDescriptors(Settings sett .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") .build() ) - .setOrigin("qa") + .setOrigin(TASKS_ORIGIN) .setVersionMetaKey("version") .setPrimaryIndex(".internal-managed-index-" + Version.CURRENT.major) .setAliasName(".internal-managed-alias") diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml index 70156fccf9211..f2c5d635c1cc6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml @@ -21,6 +21,7 @@ setup: - do: index: index: test + id: "1" body: name: cow.jpg vector: [230.0, 300.33, -34.8988, 15.555, -200.0] @@ -45,9 +46,12 @@ setup: indices.refresh: {} --- -"Basic kNN search": +"kNN search only": + - skip: + version: ' - 8.3.99' + reason: 'kNN added to search endpoint in 8.4' - do: - knn_search: + search: index: test body: fields: [ "name" ] @@ -63,8 +67,118 @@ setup: - match: {hits.hits.1._id: "3"} - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} +--- +"kNN search plus query": + - skip: + version: ' - 8.3.99' + reason: 'kNN added to search endpoint in 8.4' + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + query: + term: + name: cow.jpg + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0.fields.name.0: "cow.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} + --- "kNN search with filter": + - skip: + version: ' - 8.3.99' + reason: 'kNN added to search endpoint in 8.4' + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + + filter: + term: + name: "rabbit.jpg" + + - match: {hits.total.value: 1} + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + filter: + - term: + name: "rabbit.jpg" + - term: + _id: 2 + + - match: {hits.total.value: 0} + +--- +"kNN search with explicit search_type": + - skip: + version: ' - 8.3.99' + reason: 'kNN added to search endpoint in 8.4' + - do: + catch: bad_request + search: + index: test + search_type: query_then_fetch + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.root_cause.0.reason: "cannot set [search_type] when using [knn] search, since the search type is determined automatically" } + +--- +"kNN search in _knn_search endpoint": + - do: + knn_search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} + +--- +"kNN search with filter in _knn_search endpoint": - skip: version: ' - 8.1.99' reason: 'kNN with filtering added in 8.2' @@ -106,9 +220,12 @@ setup: --- "Test nonexistent field": + - skip: + version: ' - 8.3.99' + reason: 'kNN added to search endpoint in 8.4' - do: catch: bad_request - knn_search: + search: index: test body: fields: [ "name" ] @@ -122,6 +239,9 @@ setup: --- "Direct kNN queries are disallowed": + - skip: + version: ' - 8.3.99' + reason: 'error message changed in 8.4' - do: catch: bad_request search: @@ -133,4 +253,4 @@ setup: query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] num_candidates: 1 - match: { error.root_cause.0.type: "illegal_argument_exception" } - - match: { error.root_cause.0.reason: "[knn] queries cannot be provided directly, use the [_knn_search] endpoint instead" } + - match: { error.root_cause.0.reason: "[knn] queries cannot be provided directly, use the [knn] body parameter instead" } diff --git a/server/licenses/lucene-analysis-common-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-analysis-common-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index b7d65fa66ed37..0000000000000 --- a/server/licenses/lucene-analysis-common-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ef6b338e156eacb5ae63daa9638899b4b0c44c15 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-analysis-common-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..6782780d6cbd4 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +7a154a194ea505d27b538270ee2db2b5a4a38371 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-backward-codecs-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 1c06267cd06de..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -25886039976045fb8f9859e12f525eedbb64860a \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-backward-codecs-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..220d2b83dacd9 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +3ad36d2a32c1dda37040cdfed9dcdf294b8f3b7c \ No newline at end of file diff --git a/server/licenses/lucene-core-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-core-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index f524e2a76baff..0000000000000 --- a/server/licenses/lucene-core-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -98b0f69f9d285dcf822bcd90974e8aa4972187b9 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-core-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..1acc580cf4a7e --- /dev/null +++ b/server/licenses/lucene-core-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +e41aa9fe38033e61da13fe420aa6e9400f467dd8 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-grouping-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 9cdedb97f47da..0000000000000 --- a/server/licenses/lucene-grouping-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6f6126445ac9b86ef93c6ab26907a69931ada790 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-grouping-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..25f07a5af5a69 --- /dev/null +++ b/server/licenses/lucene-grouping-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +71cd063e306af5acf1cef0492eebbbf000e6a6ce \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-highlighter-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 3f84d24e52870..0000000000000 --- a/server/licenses/lucene-highlighter-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dd6fafd8701b01a3f6279447f7d7c410b1230c13 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-highlighter-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..263f6dd6b208e --- /dev/null +++ b/server/licenses/lucene-highlighter-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +adc913180fac1b221f57288661f069cb7a240127 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-join-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 970cfe6d7f5be..0000000000000 --- a/server/licenses/lucene-join-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2bc4615335ba485b2152cd10d21ca0cb768b651e \ No newline at end of file diff --git a/server/licenses/lucene-join-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-join-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..638bf0f37a91f --- /dev/null +++ b/server/licenses/lucene-join-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +3618de63e62d734ab1892ff446ae4f5ef866bee6 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-memory-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index e02a9fb3c97c8..0000000000000 --- a/server/licenses/lucene-memory-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3972925a8a1e304e803491881e67c12cbcda4617 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-memory-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..3ca4420b49396 --- /dev/null +++ b/server/licenses/lucene-memory-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +07e0de548fc392428545db40192280b4f83daf4f \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-misc-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 212df59af0045..0000000000000 --- a/server/licenses/lucene-misc-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -18c2adc5e45d08d65e64d31402219b67d7bc5c61 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-misc-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..09ecd52494738 --- /dev/null +++ b/server/licenses/lucene-misc-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +ab308291a7dd5ec9988a229dc8e7c27fc2bb5409 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-queries-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 7b42f54005970..0000000000000 --- a/server/licenses/lucene-queries-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bc9e4af7d35799ba941993be0242101f79719cc6 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-queries-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..8d2959d64aac3 --- /dev/null +++ b/server/licenses/lucene-queries-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +8b5804be2c87d995c5255ff1ad739052fc243661 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-queryparser-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 9521e69459042..0000000000000 --- a/server/licenses/lucene-queryparser-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f62b8f566e37fd6cf150f13d866fcc646c210e35 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-queryparser-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..0be8f71b787a2 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +a55d8a68cccaaf4af5a973c4332519d3eb477068 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-sandbox-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 82549c3474cd4..0000000000000 --- a/server/licenses/lucene-sandbox-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -081ff7330c13bfcf0110e7140badfc8be3df910c \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-sandbox-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..da0e369ccba29 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +0a6005b6b9b09b1da1c3c74558693824f429e55d \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/server/licenses/lucene-suggest-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 133379a6f6abd..0000000000000 --- a/server/licenses/lucene-suggest-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e3240b6ab35e61a947d94978697816d102b1bd13 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/server/licenses/lucene-suggest-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..38a6a7ca5e787 --- /dev/null +++ b/server/licenses/lucene-suggest-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +d4c5418c469be74cc5df3427ac07386598c18882 \ No newline at end of file diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java index fa1aeb43fb401..e8411b5124113 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java @@ -32,7 +32,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -71,7 +70,7 @@ public void testUnassignedPrimaryWithExistingIndex() throws Exception { prepareIndex(1, 0); logger.info("--> stopping the node with the primary"); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName())); + internalCluster().stopNode(primaryNodeName()); ensureStableCluster(1); refreshClusterInfo(); @@ -150,7 +149,7 @@ public void testUnassignedReplicaDelayedAllocation() throws Exception { prepareIndex(1, 1); logger.info("--> stopping the node with the replica"); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode().getName())); + internalCluster().stopNode(replicaNode().getName()); ensureStableCluster(2); refreshClusterInfo(); assertBusy(() -> @@ -291,8 +290,8 @@ public void testUnassignedReplicaWithPriorCopy() throws Exception { logger.info("--> shutting down all nodes except the one that holds the primary"); Settings node0DataPathSettings = internalCluster().dataPathSettings(nodes.get(0)); Settings node1DataPathSettings = internalCluster().dataPathSettings(nodes.get(1)); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(0))); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1))); + internalCluster().stopNode(nodes.get(0)); + internalCluster().stopNode(nodes.get(1)); ensureStableCluster(1); logger.info("--> setting allocation filtering to only allow allocation on the currently running node"); @@ -1080,7 +1079,7 @@ public void testCannotAllocateStaleReplicaExplanation() throws Exception { assertThat(primaryNodeName(), equalTo(primaryNode)); logger.info("--> stop node with the replica shard"); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode)); + internalCluster().stopNode(replicaNode); final IndexMetadata.State indexState = randomIndexState(); if (indexState == IndexMetadata.State.OPEN) { @@ -1102,7 +1101,7 @@ public void testCannotAllocateStaleReplicaExplanation() throws Exception { } logger.info("--> stop the node with the primary"); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode)); + internalCluster().stopNode(primaryNode); logger.info("--> restart the node with the stale replica"); String restartedNode = internalCluster().startDataOnlyNode(replicaDataPathSettings); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java index 2daba2d99195e..34bbd9d20eb2e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java @@ -42,7 +42,6 @@ import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.SendRequestTransportException; import org.elasticsearch.transport.Transport; @@ -294,7 +293,7 @@ public void testCancelOrphanedTasks() throws Exception { client(nodeWithRootTask).execute(TransportTestAction.ACTION, rootRequest); allowPartialRequest(rootRequest); try { - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithRootTask)); + internalCluster().stopNode(nodeWithRootTask); assertBusy(() -> { for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { for (CancellableTask task : transportService.getTaskManager().getCancellableTasks().values()) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index 0611294e4b11b..6b487a3c72f7a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -47,7 +47,6 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentType; @@ -668,7 +667,7 @@ public void testShrinkThenSplitWithFailedNode() throws Exception { ensureGreen(); final int nodeCount = cluster().size(); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(shrinkNode)); + internalCluster().stopNode(shrinkNode); ensureStableCluster(nodeCount - 1); // demonstrate that the index.routing.allocation.initial_recovery setting from the shrink doesn't carry over into the split index, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java index 9e18b87b5461e..53b0f604f15a0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.ESIntegTestCase; import java.util.HashSet; @@ -42,6 +43,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.in; @@ -418,6 +420,12 @@ public void testPITTiebreak() throws Exception { } } + public void testCloseInvalidPointInTime() { + expectThrows(Exception.class, () -> client().execute(ClosePointInTimeAction.INSTANCE, new ClosePointInTimeRequest("")).actionGet()); + List tasks = client().admin().cluster().prepareListTasks().setActions(ClosePointInTimeAction.NAME).get().getTasks(); + assertThat(tasks, empty()); + } + @SuppressWarnings({ "rawtypes", "unchecked" }) private void assertPagination(PointInTimeBuilder pit, int expectedNumDocs, int size, SortBuilder... sorts) throws Exception { Set seen = new HashSet<>(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java index 243f32ffd0b00..6e1d561e51bc9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; @@ -224,7 +223,7 @@ public void testRetryOnStoppedTransportService() throws Exception { throw new AssertionError(e); } }); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + internalCluster().stopNode(primary); assertTrue(doneLatch.await(10, TimeUnit.SECONDS)); if (response.get() instanceof Exception) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index 9b08c56b21fd2..43ee9b5d44ad6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.test.transport.MockTransportService; @@ -133,7 +132,7 @@ public void testTwoNodesNoMasterBlock() throws Exception { client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(otherNode)).get(); logger.info("--> stop master node, no master block should appear"); Settings masterDataPathSettings = internalCluster().dataPathSettings(masterNode); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNode)); + internalCluster().stopNode(masterNode); assertBusy(() -> { ClusterState clusterState = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); @@ -186,7 +185,7 @@ public void testTwoNodesNoMasterBlock() throws Exception { client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(masterNode)).get(); logger.info("--> stop non-master node, no master block should appear"); Settings otherNodeDataPathSettings = internalCluster().dataPathSettings(otherNode); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(otherNode)); + internalCluster().stopNode(otherNode); assertBusy(() -> { ClusterState state1 = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index 8859bf4da4cdd..149148d14549f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -17,7 +17,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.InternalTestCluster; import java.io.IOException; @@ -258,7 +257,7 @@ public void testElectOnlyBetweenMasterNodes() throws Exception { equalTo(nextMasterEligableNodeName) ); }); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodeName)); + internalCluster().stopNode(masterNodeName); assertThat( internalCluster().nonMasterClient() .admin() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java index 96cd0e85db42e..01c0428cf2802 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java @@ -45,7 +45,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockLogAppender; import java.nio.file.Path; @@ -247,7 +246,7 @@ public void testDelayWithALargeAmountOfShards() throws Exception { ensureGreen(TimeValue.timeValueMinutes(1)); logger.info("--> stopping node1"); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_1)); + internalCluster().stopNode(node_1); // This might run slowly on older hardware ensureGreen(TimeValue.timeValueMinutes(2)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java index d177d3da83f07..46ace8905185b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java @@ -238,8 +238,8 @@ public void test3MasterNodes2Failed() throws Exception { Settings dataNodeDataPathSettings = internalCluster().dataPathSettings(dataNode); logger.info("--> stop 2nd and 3d master eligible node"); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(1))); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(2))); + internalCluster().stopNode(masterNodes.get(1)); + internalCluster().stopNode(masterNodes.get(2)); logger.info("--> ensure NO_MASTER_BLOCK on data-only node"); assertBusy(() -> { @@ -262,7 +262,7 @@ public void test3MasterNodes2Failed() throws Exception { logger.info("--> stop 1st master-eligible node and data-only node"); NodeEnvironment nodeEnvironment = internalCluster().getAnyMasterNodeInstance(NodeEnvironment.class); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(0))); + internalCluster().stopNode(masterNodes.get(0)); assertBusy(() -> internalCluster().getInstance(GatewayMetaState.class, dataNode).allPendingAsyncStatesWritten()); internalCluster().stopRandomDataNode(); @@ -351,7 +351,7 @@ public void testNoInitialBootstrapAfterDetach() throws Exception { ClusterState state = internalCluster().client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); assertTrue(state.blocks().hasGlobalBlockWithId(NoMasterBlockService.NO_MASTER_BLOCK_ID)); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node)); + internalCluster().stopNode(node); } public void testCanRunUnsafeBootstrapAfterErroneousDetachWithoutLoosingMetadata() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java index d6b1d7456eef8..5d4b48dffece3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java @@ -32,7 +32,6 @@ import org.elasticsearch.test.DummyShardLock; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.transport.MockTransportService; import java.io.IOException; @@ -98,13 +97,13 @@ public void testFailedRecoveryOnAllocateStalePrimaryRequiresAnotherAllocateStale Settings node1DataPathSettings = internalCluster().dataPathSettings(node1); Settings node2DataPathSettings = internalCluster().dataPathSettings(node2); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1)); + internalCluster().stopNode(node1); // index more docs to node2 that marks node1 as stale int numExtraDocs = indexDocs(indexName, "foo", "bar2"); assertHitCount(client(node2).prepareSearch(indexName).setQuery(matchAllQuery()).get(), numDocs + numExtraDocs); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node2)); + internalCluster().stopNode(node2); // create fake corrupted marker on node1 putFakeCorruptionMarker(indexSettings, shardId, indexPath); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java index d434c9b3bf2a0..7bf202d64fbe0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; import java.util.Collections; import java.util.List; @@ -39,7 +38,7 @@ public void testNoDelayedTimeout() throws Exception { ).get(); ensureGreen("test"); indexRandomData(); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(findNodeWithShard())); + internalCluster().stopNode(findNodeWithShard()); assertThat(client().admin().cluster().prepareHealth().get().getDelayedUnassignedShards(), equalTo(0)); ensureGreen("test"); } @@ -62,7 +61,7 @@ public void testDelayedAllocationNodeLeavesAndComesBack() throws Exception { indexRandomData(); String nodeWithShard = findNodeWithShard(); Settings nodeWithShardDataPathSettings = internalCluster().dataPathSettings(nodeWithShard); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithShard)); + internalCluster().stopNode(nodeWithShard); assertBusy( () -> assertThat( client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, @@ -88,7 +87,7 @@ public void testDelayedAllocationTimesOut() throws Exception { ).get(); ensureGreen("test"); indexRandomData(); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(findNodeWithShard())); + internalCluster().stopNode(findNodeWithShard()); ensureGreen("test"); internalCluster().startNode(); // do a second round with longer delay to make sure it happens @@ -101,7 +100,7 @@ public void testDelayedAllocationTimesOut() throws Exception { ) .get() ); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(findNodeWithShard())); + internalCluster().stopNode(findNodeWithShard()); ensureGreen("test"); } @@ -120,7 +119,7 @@ public void testDelayedAllocationChangeWithSettingTo100ms() throws Exception { ).get(); ensureGreen("test"); indexRandomData(); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(findNodeWithShard())); + internalCluster().stopNode(findNodeWithShard()); assertBusy( () -> assertThat( client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, @@ -156,7 +155,7 @@ public void testDelayedAllocationChangeWithSettingTo0() throws Exception { ).get(); ensureGreen("test"); indexRandomData(); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(findNodeWithShard())); + internalCluster().stopNode(findNodeWithShard()); assertBusy( () -> assertThat( client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java index 35bbf18358344..8f691be4b4d93 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java @@ -156,7 +156,7 @@ private Settings createStaleReplicaScenario(String master) throws Exception { logger.info("--> shut down node that has new acknowledged document"); final Settings inSyncDataPathSettings = internalCluster().dataPathSettings(replicaNode); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode)); + internalCluster().stopNode(replicaNode); ensureStableCluster(1, master); @@ -214,7 +214,7 @@ public void testFailedAllocationOfStalePrimaryToDataNodeWithNoData() throws Exce String dataNodeWithNoShardCopy = internalCluster().startNode(); ensureStableCluster(2); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNodeWithShardCopy)); + internalCluster().stopNode(dataNodeWithShardCopy); ensureStableCluster(1); assertThat( client().admin() @@ -487,7 +487,7 @@ public void testDoNotRemoveAllocationIdOnNodeLeave() throws Exception { String replicaNode = internalCluster().startDataOnlyNode(Settings.EMPTY); ensureGreen("test"); final Settings inSyncDataPathSettings = internalCluster().dataPathSettings(replicaNode); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode)); + internalCluster().stopNode(replicaNode); ensureYellow("test"); assertEquals(2, client().admin().cluster().prepareState().get().getState().metadata().index("test").inSyncAllocationIds(0).size()); internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() { @@ -527,7 +527,7 @@ public void testRemoveAllocationIdOnWriteAfterNodeLeave() throws Exception { String replicaNode = internalCluster().startDataOnlyNode(Settings.EMPTY); ensureGreen("test"); final Settings inSyncDataPathSettings = internalCluster().dataPathSettings(replicaNode); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode)); + internalCluster().stopNode(replicaNode); ensureYellow("test"); assertEquals(2, client().admin().cluster().prepareState().get().getState().metadata().index("test").inSyncAllocationIds(0).size()); logger.info("--> indexing..."); @@ -570,8 +570,8 @@ public void testNotWaitForQuorumCopies() throws Exception { ensureGreen("test"); client().prepareIndex("test").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); logger.info("--> removing 2 nodes from cluster"); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1), nodes.get(2))); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1), nodes.get(2))); + internalCluster().stopNode(nodes.get(1)); + internalCluster().stopNode(nodes.get(2)); internalCluster().restartRandomDataNode(); logger.info("--> checking that index still gets allocated with only 1 shard copy being available"); ensureYellow("test"); @@ -671,7 +671,7 @@ public void testPrimaryReplicaResyncFailed() throws Exception { internalCluster().setDisruptionScheme(partition); logger.info("--> isolating some replicas during primary-replica resync"); partition.startDisrupting(); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(oldPrimary)); + internalCluster().stopNode(oldPrimary); // Checks that we fails replicas in one side but not mark them as stale. assertBusy(() -> { ClusterState state = client(master).admin().cluster().prepareState().get().getState(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java index 68a025a90a868..0483d8ba0e903 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java @@ -442,7 +442,7 @@ public boolean validateClusterForming() { .isTimedOut() ); }, 30, TimeUnit.SECONDS); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode)); // otherwise we will fail during clean-up + internalCluster().stopNode(dataNode); // otherwise we will fail during clean-up } /** diff --git a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java index 4bc9ddbccdc4a..7d9ef2a105e1e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java @@ -251,8 +251,8 @@ public void testFailsToStartOnDataPathsFromMultipleNodes() throws IOException { final List allDataPaths = new ArrayList<>(node0DataPaths); allDataPaths.addAll(node1DataPaths); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1))); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(0))); + internalCluster().stopNode(nodes.get(1)); + internalCluster().stopNode(nodes.get(0)); CorruptStateException corruptStateException = expectThrows( CorruptStateException.class, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java index 51780b375b883..597c030e86fb5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; @@ -97,7 +96,7 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception { ); } ensureActivePeerRecoveryRetentionLeasesAdvanced(indexName); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithReplica)); + internalCluster().stopNode(nodeWithReplica); if (randomBoolean()) { client().admin().indices().prepareForceMerge(indexName).setFlush(true).get(); } @@ -170,7 +169,7 @@ public void testRecentPrimaryInformation() throws Exception { randomBoolean(), IntStream.range(0, between(10, 100)).mapToObj(n -> client().prepareIndex(indexName).setSource("f", "v")).toList() ); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithReplica)); + internalCluster().stopNode(nodeWithReplica); if (randomBoolean()) { indexRandom( randomBoolean(), @@ -321,7 +320,7 @@ public void testPreferCopyWithHighestMatchingOperations() throws Exception { client().admin().indices().prepareFlush(indexName).get(); String nodeWithLowerMatching = randomFrom(internalCluster().nodesInclude(indexName)); Settings nodeWithLowerMatchingSettings = internalCluster().dataPathSettings(nodeWithLowerMatching); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithLowerMatching)); + internalCluster().stopNode(nodeWithLowerMatching); ensureGreen(indexName); indexRandom( @@ -333,7 +332,7 @@ public void testPreferCopyWithHighestMatchingOperations() throws Exception { ensureActivePeerRecoveryRetentionLeasesAdvanced(indexName); String nodeWithHigherMatching = randomFrom(internalCluster().nodesInclude(indexName)); Settings nodeWithHigherMatchingSettings = internalCluster().dataPathSettings(nodeWithHigherMatching); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithHigherMatching)); + internalCluster().stopNode(nodeWithHigherMatching); if (usually()) { indexRandom( randomBoolean(), diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java index 199b1d2504f79..74eb70f4453e3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java @@ -34,7 +34,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -177,7 +176,7 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception { } ensureGlobalCheckpointAdvancedAndSynced(indexName); syncFlush(indexName); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithReplica)); + internalCluster().stopNode(nodeWithReplica); // Wait until the peer recovery retention leases of the offline node are expired assertBusy(() -> { for (ShardStats shardStats : client().admin().indices().prepareStats(indexName).get().getShards()) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/HealthMetadataServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/HealthMetadataServiceIT.java index 26119bb174eb1..1f97804efd56c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/HealthMetadataServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/HealthMetadataServiceIT.java @@ -138,7 +138,10 @@ private Settings createWatermarkSettings(String highWatermark) { DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), percentageMode ? "95%" : "1b" ) - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_SETTING.getKey(), percentageMode ? "95%" : "5b") + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), + percentageMode ? "95%" : "5b" + ) .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING.getKey(), "5b") .build(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java index cdd6d4d95d8f9..1f9276cb5643d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java @@ -331,7 +331,7 @@ public void testCorruptTranslogTruncation() throws Exception { final Settings node2PathSettings = internalCluster().dataPathSettings(node2); // shut down the replica node to be tested later - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node2)); + internalCluster().stopNode(node2); final Path translogDir = getPathToShardData(indexName, ShardPath.TRANSLOG_FOLDER_NAME); final Path indexDir = getPathToShardData(indexName, ShardPath.INDEX_FOLDER_NAME); @@ -631,7 +631,7 @@ public void testResolvePath() throws Exception { ); environmentByNodeName.put(nodeName, environment); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeName)); + internalCluster().stopNode(nodeName); logger.info(" -- stopped {}", nodeName); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java index e9b84453594a8..9657fde955dc0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java @@ -20,7 +20,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.transport.TransportService; import java.util.Set; @@ -172,7 +171,7 @@ public void testIndexLifecycleActionsWith11Shards1Backup() throws Exception { logger.info("Closing server1"); // kill the first server - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(server_1)); + internalCluster().stopNode(server_1); // verify health logger.info("Running Cluster Health"); clusterHealth = client().admin() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index fe0b53654f8c1..d5385eee26469 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -348,7 +348,7 @@ public void testReplicaRecovery() throws Exception { assertRecoveryState(nodeBRecoveryState, 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeA, nodeB); validateIndexRecoveryState(nodeBRecoveryState.getIndex()); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA)); + internalCluster().stopNode(nodeA); if (closedIndex) { assertAcked(client().admin().indices().prepareOpen(INDEX_NAME)); @@ -640,7 +640,7 @@ public void testRerouteRecovery() throws Exception { if (randomBoolean()) { // shutdown node with relocation source of replica shard and check if recovery continues - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA)); + internalCluster().stopNode(nodeA); ensureStableCluster(2); response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet(); @@ -848,10 +848,10 @@ public void testHistoryRetention() throws Exception { String firstNodeToStop = randomFrom(internalCluster().getNodeNames()); Settings firstNodeToStopDataPathSettings = internalCluster().dataPathSettings(firstNodeToStop); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(firstNodeToStop)); + internalCluster().stopNode(firstNodeToStop); String secondNodeToStop = randomFrom(internalCluster().getNodeNames()); Settings secondNodeToStopDataPathSettings = internalCluster().dataPathSettings(secondNodeToStop); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(secondNodeToStop)); + internalCluster().stopNode(secondNodeToStop); final long desyncNanoTime = System.nanoTime(); // noinspection StatementWithEmptyBody diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java index 0ebbc393f37d8..52b7e3f00c9f8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.test.BackgroundIndexer; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; import java.util.Locale; @@ -62,7 +61,7 @@ public void testPromoteReplicaToPrimary() throws Exception { final DiscoveryNode randomNode = state.nodes().resolveNode(primaryShard.currentNodeId()); // stop the random data node, all remaining shards are promoted to primaries - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(randomNode.getName())); + internalCluster().stopNode(randomNode.getName()); ensureYellowAndNoInitializingShards(indexName); state = client(internalCluster().getMasterName()).admin().cluster().prepareState().get().getState(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java index bfa21bb07ae23..41f040834f199 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java @@ -38,7 +38,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.disruption.BlockClusterStateProcessing; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.ConnectTransportException; @@ -288,7 +287,7 @@ public void testShardsCleanup() throws Exception { Path server2Shard = shardDirectory(node_2, index, 0); logger.info("--> stopping node {}", node_2); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_2)); + internalCluster().stopNode(node_2); logger.info("--> running cluster_health"); clusterHealth = client().admin() @@ -364,8 +363,8 @@ public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception { List nodesToShutDown = randomSubsetOf(2, node1, node2, node3); Settings node1DataPathSettings = internalCluster().dataPathSettings(nodesToShutDown.get(0)); Settings node2DataPathSettings = internalCluster().dataPathSettings(nodesToShutDown.get(1)); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodesToShutDown.get(0))); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodesToShutDown.get(1))); + internalCluster().stopNode(nodesToShutDown.get(0)); + internalCluster().stopNode(nodesToShutDown.get(1)); logger.debug("--> verifying index is red"); ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java index 9c3902d501c87..857ff11d5422a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; import java.nio.file.Files; import java.nio.file.Path; @@ -245,7 +244,7 @@ public void testListenersInvokedWhenIndexIsDangling() throws Exception { final String stoppedNode = randomFrom(shardsByNodes.keySet()); final Settings stoppedNodeDataPathSettings = internalCluster().dataPathSettings(stoppedNode); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(stoppedNode)); + internalCluster().stopNode(stoppedNode); ensureStableCluster(3 + 1, masterNode); assertAcked(client().admin().indices().prepareDelete(indexName)); @@ -291,7 +290,7 @@ public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception { } logger.debug("--> stopping data node [{}], the data left on disk will be injected as left-overs in a newer data node", dataNode); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode)); + internalCluster().stopNode(dataNode); ensureStableCluster(1, masterNode); logger.debug("--> deleting leftover indices"); diff --git a/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeRequest.java b/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeRequest.java index d355e222ec37a..d97cddaf533e4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeRequest.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ValidateActions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -41,7 +42,7 @@ public String getId() { @Override public ActionRequestValidationException validate() { if (Strings.isEmpty(id)) { - throw new IllegalArgumentException("reader id must be specified"); + return ValidateActions.addValidationError("id is empty", null); } return null; } diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java index c2ef4bbd03f31..5e187ad249189 100644 --- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java @@ -7,15 +7,23 @@ */ package org.elasticsearch.action.search; +import org.apache.lucene.search.ScoreDoc; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.dfs.AggregatedDfs; +import org.elasticsearch.search.dfs.DfsKnnResults; import org.elasticsearch.search.dfs.DfsSearchResult; +import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.query.QuerySearchRequest; import org.elasticsearch.search.query.QuerySearchResult; +import org.elasticsearch.search.vectors.KnnScoreDocQueryBuilder; import org.elasticsearch.transport.Transport; import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.function.Function; @@ -30,6 +38,7 @@ final class DfsQueryPhase extends SearchPhase { private final QueryPhaseResultConsumer queryResult; private final List searchResults; private final AggregatedDfs dfs; + private final DfsKnnResults knnResults; private final Function, SearchPhase> nextPhaseFactory; private final SearchPhaseContext context; private final SearchTransportService searchTransportService; @@ -38,6 +47,7 @@ final class DfsQueryPhase extends SearchPhase { DfsQueryPhase( List searchResults, AggregatedDfs dfs, + DfsKnnResults knnResults, QueryPhaseResultConsumer queryResult, Function, SearchPhase> nextPhaseFactory, SearchPhaseContext context @@ -47,6 +57,7 @@ final class DfsQueryPhase extends SearchPhase { this.queryResult = queryResult; this.searchResults = searchResults; this.dfs = dfs; + this.knnResults = knnResults; this.nextPhaseFactory = nextPhaseFactory; this.context = context; this.searchTransportService = context.getSearchTransport(); @@ -66,13 +77,15 @@ public void run() throws IOException { () -> context.executeNextPhase(this, nextPhaseFactory.apply(queryResult)), context ); + for (final DfsSearchResult dfsResult : searchResults) { final SearchShardTarget shardTarget = dfsResult.getSearchShardTarget(); Transport.Connection connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); + ShardSearchRequest shardRequest = rewriteShardSearchRequest(dfsResult.getShardSearchRequest()); QuerySearchRequest querySearchRequest = new QuerySearchRequest( context.getOriginalIndices(dfsResult.getShardIndex()), dfsResult.getContextId(), - dfsResult.getShardSearchRequest(), + shardRequest, dfs ); final int shardIndex = dfsResult.getShardIndex(); @@ -115,4 +128,30 @@ public void onFailure(Exception exception) { ); } } + + private ShardSearchRequest rewriteShardSearchRequest(ShardSearchRequest request) { + SearchSourceBuilder source = request.source(); + if (source == null || source.knnSearch() == null) { + return request; + } + + List scoreDocs = new ArrayList<>(); + for (ScoreDoc scoreDoc : knnResults.scoreDocs()) { + if (scoreDoc.shardIndex == request.shardRequestIndex()) { + scoreDocs.add(scoreDoc); + } + } + scoreDocs.sort(Comparator.comparingInt(scoreDoc -> scoreDoc.doc)); + KnnScoreDocQueryBuilder knnQuery = new KnnScoreDocQueryBuilder(scoreDocs.toArray(new ScoreDoc[0])); + + SearchSourceBuilder newSource = source.shallowCopy().knnSearch(null); + if (source.query() == null) { + newSource.query(knnQuery); + } else { + newSource.query(new BoolQueryBuilder().should(knnQuery).should(source.query())); + } + + request.source(newSource); + return request; + } } diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java index 27042d29b524a..6d48b9429b6e6 100644 --- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java @@ -102,7 +102,9 @@ private void innerRun() throws Exception { final boolean isScrollSearch = context.getRequest().scroll() != null; final List phaseResults = queryResults.asList(); final SearchPhaseController.ReducedQueryPhase reducedQueryPhase = resultConsumer.reduce(); - final boolean queryAndFetchOptimization = queryResults.length() == 1; + // Usually when there is a single shard, we force the search type QUERY_THEN_FETCH. But when there's kNN, we might + // still use DFS_QUERY_THEN_FETCH, which does not perform the "query and fetch" optimization during the query phase. + final boolean queryAndFetchOptimization = queryResults.length() == 1 && context.getRequest().hasKnnSearch() == false; final Runnable finishPhase = () -> moveToNextPhase( queryResults, reducedQueryPhase, diff --git a/server/src/main/java/org/elasticsearch/action/search/ReduceSearchPhaseException.java b/server/src/main/java/org/elasticsearch/action/search/ReduceSearchPhaseException.java index 2df8eaff55511..94e545c472624 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ReduceSearchPhaseException.java +++ b/server/src/main/java/org/elasticsearch/action/search/ReduceSearchPhaseException.java @@ -8,7 +8,9 @@ package org.elasticsearch.action.search; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.rest.RestStatus; import java.io.IOException; @@ -27,4 +29,13 @@ public ReduceSearchPhaseException(String phaseName, String msg, Throwable cause, public ReduceSearchPhaseException(StreamInput in) throws IOException { super(in); } + + @Override + public RestStatus status() { + final ShardSearchFailure[] shardFailures = shardFailures(); + if (shardFailures.length == 0) { + return getCause() == null ? RestStatus.INTERNAL_SERVER_ERROR : ExceptionsHelper.status(getCause()); + } + return super.status(); + } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java index 8a6d7b5bfbe4c..9e04c26b42eb5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.dfs.AggregatedDfs; +import org.elasticsearch.search.dfs.DfsKnnResults; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.transport.Transport; @@ -86,10 +87,12 @@ protected void executePhaseOnShard( protected SearchPhase getNextPhase(final SearchPhaseResults results, SearchPhaseContext context) { final List dfsSearchResults = results.getAtomicArray().asList(); final AggregatedDfs aggregatedDfs = SearchPhaseController.aggregateDfs(dfsSearchResults); + final DfsKnnResults mergedKnnResults = SearchPhaseController.mergeKnnResults(getRequest(), dfsSearchResults); return new DfsQueryPhase( dfsSearchResults, aggregatedDfs, + mergedKnnResults, queryPhaseResultConsumer, (queryResults) -> new FetchSearchPhase(queryResults, aggregatedDfs, context), context diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index aa44ab318b6f1..7dc1ddf0bba05 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -34,6 +34,7 @@ import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.dfs.AggregatedDfs; +import org.elasticsearch.search.dfs.DfsKnnResults; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.internal.InternalSearchResponse; @@ -45,6 +46,7 @@ import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.Suggest.Suggestion; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; +import org.elasticsearch.search.vectors.KnnSearchBuilder; import java.util.ArrayList; import java.util.Collection; @@ -126,6 +128,28 @@ public static AggregatedDfs aggregateDfs(Collection results) { return new AggregatedDfs(termStatistics, fieldStatistics, aggMaxDoc); } + public static DfsKnnResults mergeKnnResults(SearchRequest request, List dfsSearchResults) { + if (request.hasKnnSearch() == false) { + return null; + } + + List topDocs = new ArrayList<>(); + for (DfsSearchResult dfsSearchResult : dfsSearchResults) { + if (dfsSearchResult.knnResults() != null) { + ScoreDoc[] scoreDocs = dfsSearchResult.knnResults().scoreDocs(); + TotalHits totalHits = new TotalHits(scoreDocs.length, Relation.EQUAL_TO); + + TopDocs shardTopDocs = new TopDocs(totalHits, scoreDocs); + setShardIndex(shardTopDocs, dfsSearchResult.getShardIndex()); + topDocs.add(shardTopDocs); + } + } + + KnnSearchBuilder knnSearch = request.source().knnSearch(); + TopDocs mergedTopDocs = TopDocs.merge(knnSearch.k(), topDocs.toArray(new TopDocs[0])); + return new DfsKnnResults(mergedTopDocs.scoreDocs); + } + /** * Returns a score doc array of top N search docs across all shards, followed by top suggest docs for each * named completion suggestion across all shards. If more than one named completion suggestion is specified in the diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java index f28a0ceae87f7..b4d619bec1b49 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java @@ -75,7 +75,7 @@ public RestStatus status() { RestStatus status = shardFailures[0].status(); if (shardFailures.length > 1) { for (int i = 1; i < shardFailures.length; i++) { - if (shardFailures[i].status().getStatus() >= 500) { + if (shardFailures[i].status().getStatus() >= RestStatus.INTERNAL_SERVER_ERROR.getStatus()) { status = shardFailures[i].status(); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 0ec66549e43b2..87bef61239445 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -739,6 +739,13 @@ public boolean isSuggestOnly() { return source != null && source.isSuggestOnly(); } + /** + * @return true if the request contains kNN search + */ + public boolean hasKnnSearch() { + return source != null && source.knnSearch() != null; + } + public int resolveTrackTotalHitsUpTo() { return resolveTrackTotalHitsUpTo(scroll, source); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index b8ca2898e3e71..79a2d0d811ec3 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -28,6 +28,7 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.vectors.KnnSearchBuilder; import java.util.Arrays; import java.util.List; @@ -172,6 +173,15 @@ public SearchRequestBuilder setPostFilter(QueryBuilder postFilter) { return this; } + /** + * Defines a kNN search. If a query is also provided, the kNN hits + * are combined with the query hits. + */ + public SearchRequestBuilder setKnnSearch(KnnSearchBuilder knnSearch) { + sourceBuilder().knnSearch(knnSearch); + return this; + } + /** * Sets the minimum score below which docs will be filtered out. */ diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 6c78fa06b3bfa..7647258e565c2 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -478,6 +478,25 @@ private void executeRequest( Rewriteable.rewriteAndFetch(original, searchService.getRewriteContext(timeProvider::absoluteStartMillis), rewriteListener); } + static void adjustSearchType(SearchRequest searchRequest, boolean singleShard) { + // optimize search type for cases where there is only one shard group to search on + if (singleShard) { + // if we only have one group, then we always want Q_T_F, no need for DFS, and no need to do THEN since we hit one shard + searchRequest.searchType(QUERY_THEN_FETCH); + } + + // if there's only suggest, disable request cache and always use QUERY_THEN_FETCH + if (searchRequest.isSuggestOnly()) { + searchRequest.requestCache(false); + searchRequest.searchType(QUERY_THEN_FETCH); + } + + // if there's a kNN search, always use DFS_QUERY_THEN_FETCH + if (searchRequest.hasKnnSearch()) { + searchRequest.searchType(DFS_QUERY_THEN_FETCH); + } + } + static boolean shouldMinimizeRoundtrips(SearchRequest searchRequest) { if (searchRequest.isCcsMinimizeRoundtrips() == false) { return false; @@ -491,6 +510,9 @@ static boolean shouldMinimizeRoundtrips(SearchRequest searchRequest) { if (searchRequest.searchType() == DFS_QUERY_THEN_FETCH) { return false; } + if (searchRequest.hasKnnSearch()) { + return false; + } SearchSourceBuilder source = searchRequest.source(); return source == null || source.collapse() == null @@ -985,20 +1007,8 @@ private void executeSearch( Map concreteIndexBoosts = resolveIndexBoosts(searchRequest, clusterState); - // optimize search type for cases where there is only one shard group to search on - if (shardIterators.size() == 1) { - // if we only have one group, then we always want Q_T_F, no need for DFS, and no need to do THEN since we hit one shard - searchRequest.searchType(QUERY_THEN_FETCH); - } - if (searchRequest.isSuggestOnly()) { - // disable request cache if we have only suggest - searchRequest.requestCache(false); - switch (searchRequest.searchType()) { - case DFS_QUERY_THEN_FETCH -> - // convert to Q_T_F if we have only suggest - searchRequest.searchType(QUERY_THEN_FETCH); - } - } + adjustSearchType(searchRequest, shardIterators.size() == 1); + final DiscoveryNodes nodes = clusterState.nodes(); BiFunction connectionLookup = buildConnectionLookup( searchRequest.getLocalClusterAlias(), diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java index b29a3748c7879..de39117529647 100644 --- a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java @@ -40,12 +40,19 @@ protected TransportAction(String actionName, ActionFilters actionFilters, TaskMa * Use this method when the transport action should continue to run in the context of the current task */ public final void execute(Task task, Request request, ActionListener listener) { - ActionRequestValidationException validationException = request.validate(); + final ActionRequestValidationException validationException; + try { + validationException = request.validate(); + } catch (Exception e) { + assert false : new AssertionError("validating of request [" + request + "] threw exception", e); + logger.warn("validating of request [" + request + "] threw exception", e); + listener.onFailure(e); + return; + } if (validationException != null) { listener.onFailure(validationException); return; } - if (task != null && request.getShouldStoreResult()) { listener = new TaskResultStoringActionListener<>(taskManager, task, listener); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java index 04e47b4a58b30..f603578da343a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java @@ -166,14 +166,13 @@ public void onNewInfo(ClusterInfo info) { final String node = entry.getKey(); final DiskUsage usage = entry.getValue(); final RoutingNode routingNode = routingNodes.node(node); + final ByteSizeValue total = ByteSizeValue.ofBytes(usage.getTotalBytes()); if (isDedicatedFrozenNode(routingNode)) { - ByteSizeValue total = ByteSizeValue.ofBytes(usage.getTotalBytes()); - long frozenFloodStageThreshold = diskThresholdSettings.getFreeBytesThresholdFrozenFloodStage(total).getBytes(); - if (usage.getFreeBytes() < frozenFloodStageThreshold) { + if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFrozenFloodStage(total).getBytes()) { logger.warn( "flood stage disk watermark [{}] exceeded on {}", - diskThresholdSettings.describeFrozenFloodStageThreshold(total), + diskThresholdSettings.describeFrozenFloodStageThreshold(total, false), usage ); } @@ -182,9 +181,7 @@ public void onNewInfo(ClusterInfo info) { continue; } - if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage().getBytes() - || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdFloodStage()) { - + if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage(total).getBytes()) { nodesOverLowThreshold.add(node); nodesOverHighThreshold.add(node); nodesOverHighThresholdAndRelocating.remove(node); @@ -199,16 +196,14 @@ public void onNewInfo(ClusterInfo info) { logger.warn( "flood stage disk watermark [{}] exceeded on {}, all indices on this node will be marked read-only", - diskThresholdSettings.describeFloodStageThreshold(), + diskThresholdSettings.describeFloodStageThreshold(total, false), usage ); continue; } - if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() - || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) { - + if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHighStage(total).getBytes()) { if (routingNode != null) { // might be temporarily null if the ClusterInfoService and the ClusterService are out of step for (ShardRouting routing : routingNode) { String indexName = routing.index().getName(); @@ -226,9 +221,7 @@ public void onNewInfo(ClusterInfo info) { Math.max(0L, usage.getFreeBytes() - reservedSpace) ); - if (usageWithReservedSpace.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() - || usageWithReservedSpace.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) { - + if (usageWithReservedSpace.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHighStage(total).getBytes()) { nodesOverLowThreshold.add(node); nodesOverHighThreshold.add(node); @@ -245,61 +238,57 @@ public void onNewInfo(ClusterInfo info) { ); } - } else if (usageWithReservedSpace.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdLow().getBytes() - || usageWithReservedSpace.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdLow()) { + } else if (usageWithReservedSpace.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdLowStage(total).getBytes()) { + nodesOverHighThresholdAndRelocating.remove(node); + + final boolean wasUnderLowThreshold = nodesOverLowThreshold.add(node); + final boolean wasOverHighThreshold = nodesOverHighThreshold.remove(node); + assert (wasUnderLowThreshold && wasOverHighThreshold) == false; + + if (wasUnderLowThreshold) { + logger.info( + "low disk watermark [{}] exceeded on {}, replicas will not be assigned to this node", + diskThresholdSettings.describeLowThreshold(total, false), + usage + ); + } else if (wasOverHighThreshold) { + logger.info( + "high disk watermark [{}] no longer exceeded on {}, but low disk watermark [{}] is still exceeded", + diskThresholdSettings.describeHighThreshold(total, false), + usage, + diskThresholdSettings.describeLowThreshold(total, false) + ); + } - nodesOverHighThresholdAndRelocating.remove(node); + } else { + nodesOverHighThresholdAndRelocating.remove(node); - final boolean wasUnderLowThreshold = nodesOverLowThreshold.add(node); - final boolean wasOverHighThreshold = nodesOverHighThreshold.remove(node); - assert (wasUnderLowThreshold && wasOverHighThreshold) == false; + if (nodesOverLowThreshold.contains(node)) { + // The node has previously been over the low watermark, but is no longer, so it may be possible to allocate more + // shards if we reroute now. + if (lastRunTimeMillis.get() <= currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) { + reroute = true; + explanation = "one or more nodes has gone under the high or low watermark"; + nodesOverLowThreshold.remove(node); + nodesOverHighThreshold.remove(node); - if (wasUnderLowThreshold) { logger.info( - "low disk watermark [{}] exceeded on {}, replicas will not be assigned to this node", - diskThresholdSettings.describeLowThreshold(), + "low disk watermark [{}] no longer exceeded on {}", + diskThresholdSettings.describeLowThreshold(total, false), usage ); - } else if (wasOverHighThreshold) { - logger.info( - "high disk watermark [{}] no longer exceeded on {}, but low disk watermark [{}] is still exceeded", - diskThresholdSettings.describeHighThreshold(), - usage, - diskThresholdSettings.describeLowThreshold() - ); - } - - } else { - nodesOverHighThresholdAndRelocating.remove(node); - - if (nodesOverLowThreshold.contains(node)) { - // The node has previously been over the low watermark, but is no longer, so it may be possible to allocate more - // shards - // if we reroute now. - if (lastRunTimeMillis.get() <= currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) { - reroute = true; - explanation = "one or more nodes has gone under the high or low watermark"; - nodesOverLowThreshold.remove(node); - nodesOverHighThreshold.remove(node); - - logger.info( - "low disk watermark [{}] no longer exceeded on {}", - diskThresholdSettings.describeLowThreshold(), - usage - ); - - } else { - logger.debug( - "{} has gone below a disk threshold, but an automatic reroute has occurred " - + "in the last [{}], skipping reroute", - node, - diskThresholdSettings.getRerouteInterval() - ); - } + } else { + logger.debug( + "{} has gone below a disk threshold, but an automatic reroute has occurred " + + "in the last [{}], skipping reroute", + node, + diskThresholdSettings.getRerouteInterval() + ); } - } + + } } final ActionListener listener = new GroupedActionListener<>(ActionListener.wrap(this::checkFinished), 3); @@ -325,16 +314,15 @@ public void onNewInfo(ClusterInfo info) { usageIncludingRelocations = diskUsage; relocatingShardsSize = 0L; } + final ByteSizeValue total = ByteSizeValue.ofBytes(usageIncludingRelocations.getTotalBytes()); - if (usageIncludingRelocations.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() - || usageIncludingRelocations.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) { - + if (usageIncludingRelocations.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHighStage(total).getBytes()) { nodesOverHighThresholdAndRelocating.remove(diskUsage.getNodeId()); logger.warn( "high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; " + "currently relocating away shards totalling [{}] bytes; the node is expected to continue to exceed " + "the high disk watermark when these relocations are complete", - diskThresholdSettings.describeHighThreshold(), + diskThresholdSettings.describeHighThreshold(total, false), diskUsage, -relocatingShardsSize ); @@ -343,7 +331,7 @@ public void onNewInfo(ClusterInfo info) { "high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; " + "currently relocating away shards totalling [{}] bytes; the node is expected to be below the high " + "disk watermark when these relocations are complete", - diskThresholdSettings.describeHighThreshold(), + diskThresholdSettings.describeHighThreshold(total, false), diskUsage, -relocatingShardsSize ); @@ -351,7 +339,7 @@ public void onNewInfo(ClusterInfo info) { logger.debug( "high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; " + "currently relocating away shards totalling [{}] bytes", - diskThresholdSettings.describeHighThreshold(), + diskThresholdSettings.describeHighThreshold(total, false), diskUsage, -relocatingShardsSize ); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java index 93ccc8ec446f0..9c74bf01c49e5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java @@ -8,20 +8,17 @@ package org.elasticsearch.cluster.routing.allocation; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.RatioValue; import org.elasticsearch.common.unit.RelativeByteSizeValue; +import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; /** @@ -34,31 +31,31 @@ public class DiskThresholdSettings { Setting.Property.OperatorDynamic, Setting.Property.NodeScope ); - public static final Setting CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING = new Setting<>( + public static final Setting CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING = new Setting<>( "cluster.routing.allocation.disk.watermark.low", "85%", - (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.low"), - new LowDiskWatermarkValidator(), + (s) -> RelativeByteSizeValue.parseRelativeByteSizeValue(s, "cluster.routing.allocation.disk.watermark.low"), + new WatermarkValidator(), Setting.Property.Dynamic, Setting.Property.NodeScope ); - public static final Setting CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING = new Setting<>( + public static final Setting CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING = new Setting<>( "cluster.routing.allocation.disk.watermark.high", "90%", - (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.high"), - new HighDiskWatermarkValidator(), + (s) -> RelativeByteSizeValue.parseRelativeByteSizeValue(s, "cluster.routing.allocation.disk.watermark.high"), + new WatermarkValidator(), Setting.Property.Dynamic, Setting.Property.NodeScope ); - public static final Setting CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING = new Setting<>( + public static final Setting CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING = new Setting<>( "cluster.routing.allocation.disk.watermark.flood_stage", "95%", - (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.flood_stage"), - new FloodStageValidator(), + (s) -> RelativeByteSizeValue.parseRelativeByteSizeValue(s, "cluster.routing.allocation.disk.watermark.flood_stage"), + new WatermarkValidator(), Setting.Property.Dynamic, Setting.Property.NodeScope ); - public static final Setting CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_SETTING = new Setting<>( + public static final Setting CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING = new Setting<>( "cluster.routing.allocation.disk.watermark.flood_stage.frozen", "95%", (s) -> RelativeByteSizeValue.parseRelativeByteSizeValue(s, "cluster.routing.allocation.disk.watermark.flood_stage.frozen"), @@ -68,7 +65,7 @@ public class DiskThresholdSettings { public static final Setting CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING = new Setting<>( "cluster.routing.allocation.disk.watermark.flood_stage.frozen.max_headroom", (settings) -> { - if (CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_SETTING.exists(settings)) { + if (CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.exists(settings)) { return "-1"; } else { return "20GB"; @@ -85,18 +82,13 @@ public class DiskThresholdSettings { Setting.Property.NodeScope ); - private volatile String lowWatermarkRaw; - private volatile String highWatermarkRaw; - private volatile Double freeDiskThresholdLow; - private volatile Double freeDiskThresholdHigh; - private volatile ByteSizeValue freeBytesThresholdLow; - private volatile ByteSizeValue freeBytesThresholdHigh; + private volatile RelativeByteSizeValue lowStageWatermark; + private volatile RelativeByteSizeValue highStageWatermark; + private volatile RelativeByteSizeValue floodStageWatermark; + private volatile RelativeByteSizeValue frozenFloodStageWatermark; + private volatile ByteSizeValue frozenFloodStageMaxHeadroom; private volatile boolean enabled; private volatile TimeValue rerouteInterval; - private volatile Double freeDiskThresholdFloodStage; - private volatile ByteSizeValue freeBytesThresholdFloodStage; - private volatile RelativeByteSizeValue frozenFloodStage; - private volatile ByteSizeValue frozenFloodStageMaxHeadroom; static { assert Version.CURRENT.major == Version.V_7_0_0.major + 1; // this check is unnecessary in v9 @@ -108,20 +100,23 @@ public class DiskThresholdSettings { } public DiskThresholdSettings(Settings settings, ClusterSettings clusterSettings) { - final String lowWatermark = CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.get(settings); - final String highWatermark = CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.get(settings); - final String floodStage = CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.get(settings); - setHighWatermark(highWatermark); - setLowWatermark(lowWatermark); - setFloodStage(floodStage); - setFrozenFloodStage(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_SETTING.get(settings)); + setLowWatermark(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.get(settings)); + setHighWatermark(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.get(settings)); + setFloodStageWatermark(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.get(settings)); + setFrozenFloodStageWatermark(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.get(settings)); setFrozenFloodStageMaxHeadroom(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING.get(settings)); this.rerouteInterval = CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.get(settings); this.enabled = CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.get(settings); clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING, this::setLowWatermark); clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING, this::setHighWatermark); - clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING, this::setFloodStage); - clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_SETTING, this::setFrozenFloodStage); + clusterSettings.addSettingsUpdateConsumer( + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING, + this::setFloodStageWatermark + ); + clusterSettings.addSettingsUpdateConsumer( + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING, + this::setFrozenFloodStageWatermark + ); clusterSettings.addSettingsUpdateConsumer( CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING, this::setFrozenFloodStageMaxHeadroom @@ -130,49 +125,81 @@ public DiskThresholdSettings(Settings settings, ClusterSettings clusterSettings) clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING, this::setEnabled); } - static final class LowDiskWatermarkValidator implements Setting.Validator { + /** + * Validates that low, high and flood stage watermarks are all either percentages or byte values, + * and that their values adhere to the comparison: low < high < flood. Else, throws an exception. + */ + static class WatermarkValidator implements Setting.Validator { @Override - public void validate(String value) { + public void validate(RelativeByteSizeValue value) { } @Override - public void validate(final String value, final Map, Object> settings) { - final String highWatermarkRaw = (String) settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING); - final String floodStageRaw = (String) settings.get(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING); - doValidate(value, highWatermarkRaw, floodStageRaw); - } - - @Override - public Iterator> settings() { - final List> settings = List.of( - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING, + public void validate(final RelativeByteSizeValue value, final Map, Object> settings) { + final RelativeByteSizeValue low = (RelativeByteSizeValue) settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING); + final RelativeByteSizeValue high = (RelativeByteSizeValue) settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING); + final RelativeByteSizeValue flood = (RelativeByteSizeValue) settings.get( CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING ); - return settings.iterator(); - } - - } - - static final class HighDiskWatermarkValidator implements Setting.Validator { - - @Override - public void validate(final String value) { - - } - @Override - public void validate(final String value, final Map, Object> settings) { - final String lowWatermarkRaw = (String) settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING); - final String floodStageRaw = (String) settings.get(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING); - doValidate(lowWatermarkRaw, value, floodStageRaw); + if (low.isAbsolute() == false && high.isAbsolute() == false && flood.isAbsolute() == false) { // Validate as percentages + final double lowWatermarkThreshold = low.getRatio().getAsPercent(); + final double highWatermarkThreshold = high.getRatio().getAsPercent(); + final double floodThreshold = flood.getRatio().getAsPercent(); + if (lowWatermarkThreshold > highWatermarkThreshold) { + throw new IllegalArgumentException( + "low disk watermark [" + low.getStringRep() + "] more than high disk watermark [" + high.getStringRep() + "]" + ); + } + if (highWatermarkThreshold > floodThreshold) { + throw new IllegalArgumentException( + "high disk watermark [" + + high.getStringRep() + + "] more than flood stage disk watermark [" + + flood.getStringRep() + + "]" + ); + } + } else if (low.isAbsolute() && high.isAbsolute() && flood.isAbsolute()) { // Validate as absolute values + final ByteSizeValue lowWatermarkBytes = low.getAbsolute(); + final ByteSizeValue highWatermarkBytes = high.getAbsolute(); + final ByteSizeValue floodStageBytes = flood.getAbsolute(); + + if (lowWatermarkBytes.getBytes() < highWatermarkBytes.getBytes()) { + throw new IllegalArgumentException( + "low disk watermark [" + low.getStringRep() + "] less than high disk watermark [" + high.getStringRep() + "]" + ); + } + if (highWatermarkBytes.getBytes() < floodStageBytes.getBytes()) { + throw new IllegalArgumentException( + "high disk watermark [" + + high.getStringRep() + + "] less than flood stage disk watermark [" + + flood.getStringRep() + + "]" + ); + } + } else { + final String message = Strings.format( + "unable to consistently parse [%s=%s], [%s=%s], and [%s=%s] as percentage or bytes", + CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), + low.getStringRep(), + CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), + high.getStringRep(), + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), + flood.getStringRep() + ); + throw new IllegalArgumentException(message); + } } @Override public Iterator> settings() { final List> settings = List.of( CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING, + CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING, CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING ); return settings.iterator(); @@ -180,93 +207,6 @@ public Iterator> settings() { } - static final class FloodStageValidator implements Setting.Validator { - - @Override - public void validate(final String value) { - - } - - @Override - public void validate(final String value, final Map, Object> settings) { - final String lowWatermarkRaw = (String) settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING); - final String highWatermarkRaw = (String) settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING); - doValidate(lowWatermarkRaw, highWatermarkRaw, value); - } - - @Override - public Iterator> settings() { - final List> settings = List.of( - CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING, - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING - ); - return settings.iterator(); - } - - } - - private static void doValidate(String low, String high, String flood) { - if (definitelyNotPercentage(low) == false) { // only try to validate as percentage if it isn't obviously a byte size value - try { - doValidateAsPercentage(low, high, flood); - return; // early return so that we do not try to parse as bytes - } catch (final ElasticsearchParseException e) { - // swallow as we are now going to try to parse as bytes - } - } - try { - doValidateAsBytes(low, high, flood); - } catch (final ElasticsearchParseException e) { - final String message = String.format( - Locale.ROOT, - "unable to consistently parse [%s=%s], [%s=%s], and [%s=%s] as percentage or bytes", - CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), - low, - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), - high, - CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), - flood - ); - throw new IllegalArgumentException(message, e); - } - } - - private static void doValidateAsPercentage(final String low, final String high, final String flood) { - final double lowWatermarkThreshold = thresholdPercentageFromWatermark(low, false); - final double highWatermarkThreshold = thresholdPercentageFromWatermark(high, false); - final double floodThreshold = thresholdPercentageFromWatermark(flood, false); - if (lowWatermarkThreshold > highWatermarkThreshold) { - throw new IllegalArgumentException("low disk watermark [" + low + "] more than high disk watermark [" + high + "]"); - } - if (highWatermarkThreshold > floodThreshold) { - throw new IllegalArgumentException("high disk watermark [" + high + "] more than flood stage disk watermark [" + flood + "]"); - } - } - - private static void doValidateAsBytes(final String low, final String high, final String flood) { - final ByteSizeValue lowWatermarkBytes = thresholdBytesFromWatermark( - low, - CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), - false - ); - final ByteSizeValue highWatermarkBytes = thresholdBytesFromWatermark( - high, - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), - false - ); - final ByteSizeValue floodStageBytes = thresholdBytesFromWatermark( - flood, - CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), - false - ); - if (lowWatermarkBytes.getBytes() < highWatermarkBytes.getBytes()) { - throw new IllegalArgumentException("low disk watermark [" + low + "] less than high disk watermark [" + high + "]"); - } - if (highWatermarkBytes.getBytes() < floodStageBytes.getBytes()) { - throw new IllegalArgumentException("high disk watermark [" + high + "] less than flood stage disk watermark [" + flood + "]"); - } - } - private void setRerouteInterval(TimeValue rerouteInterval) { this.rerouteInterval = rerouteInterval; } @@ -275,88 +215,64 @@ private void setEnabled(boolean enabled) { this.enabled = enabled; } - private void setLowWatermark(String lowWatermark) { - // Watermark is expressed in terms of used data, but we need "free" data watermark - this.lowWatermarkRaw = lowWatermark; - this.freeDiskThresholdLow = 100.0 - thresholdPercentageFromWatermark(lowWatermark); - this.freeBytesThresholdLow = thresholdBytesFromWatermark( - lowWatermark, - CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey() - ); + private void setLowWatermark(RelativeByteSizeValue lowWatermark) { + this.lowStageWatermark = lowWatermark; } - private void setHighWatermark(String highWatermark) { - // Watermark is expressed in terms of used data, but we need "free" data watermark - this.highWatermarkRaw = highWatermark; - this.freeDiskThresholdHigh = 100.0 - thresholdPercentageFromWatermark(highWatermark); - this.freeBytesThresholdHigh = thresholdBytesFromWatermark( - highWatermark, - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey() - ); + private void setHighWatermark(RelativeByteSizeValue highWatermark) { + this.highStageWatermark = highWatermark; } - private void setFloodStage(String floodStageRaw) { - // Watermark is expressed in terms of used data, but we need "free" data watermark - this.freeDiskThresholdFloodStage = 100.0 - thresholdPercentageFromWatermark(floodStageRaw); - this.freeBytesThresholdFloodStage = thresholdBytesFromWatermark( - floodStageRaw, - CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey() - ); + private void setFloodStageWatermark(RelativeByteSizeValue floodStage) { + this.floodStageWatermark = floodStage; } - private void setFrozenFloodStage(RelativeByteSizeValue floodStage) { - this.frozenFloodStage = floodStage; + private void setFrozenFloodStageWatermark(RelativeByteSizeValue floodStage) { + this.frozenFloodStageWatermark = floodStage; } private void setFrozenFloodStageMaxHeadroom(ByteSizeValue maxHeadroom) { this.frozenFloodStageMaxHeadroom = maxHeadroom; } - /** - * Gets the raw (uninterpreted) low watermark value as found in the settings. - */ - public String getLowWatermarkRaw() { - return lowWatermarkRaw; - } - - /** - * Gets the raw (uninterpreted) high watermark value as found in the settings. - */ - public String getHighWatermarkRaw() { - return highWatermarkRaw; - } - - public Double getFreeDiskThresholdLow() { - return freeDiskThresholdLow; + private ByteSizeValue getFreeBytesThreshold(ByteSizeValue total, RelativeByteSizeValue watermark, ByteSizeValue maxHeadroom) { + // If bytes are given, they can be readily returned as free bytes. If percentages are given, we need to calculate the free bytes. + if (watermark.isAbsolute()) { + return watermark.getAbsolute(); + } + return ByteSizeValue.ofBytes(total.getBytes() - watermark.calculateValue(total, maxHeadroom).getBytes()); } - public Double getFreeDiskThresholdHigh() { - return freeDiskThresholdHigh; + public ByteSizeValue getFreeBytesThresholdLowStage(ByteSizeValue total) { + return getFreeBytesThreshold(total, lowStageWatermark, ByteSizeValue.MINUS_ONE); } - public ByteSizeValue getFreeBytesThresholdLow() { - return freeBytesThresholdLow; + public ByteSizeValue getFreeBytesThresholdHighStage(ByteSizeValue total) { + return getFreeBytesThreshold(total, highStageWatermark, ByteSizeValue.MINUS_ONE); } - public ByteSizeValue getFreeBytesThresholdHigh() { - return freeBytesThresholdHigh; + public ByteSizeValue getFreeBytesThresholdFloodStage(ByteSizeValue total) { + return getFreeBytesThreshold(total, floodStageWatermark, ByteSizeValue.MINUS_ONE); } - public Double getFreeDiskThresholdFloodStage() { - return freeDiskThresholdFloodStage; + public ByteSizeValue getFreeBytesThresholdFrozenFloodStage(ByteSizeValue total) { + return getFreeBytesThreshold(total, frozenFloodStageWatermark, frozenFloodStageMaxHeadroom); } - public ByteSizeValue getFreeBytesThresholdFloodStage() { - return freeBytesThresholdFloodStage; - } + public ByteSizeValue getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue used) { + // If watermark is absolute, simply return total disk = used disk + free disk, where free disk bytes is the watermark value. + if (lowStageWatermark.isAbsolute()) { + return ByteSizeValue.ofBytes(lowStageWatermark.getAbsolute().getBytes() + used.getBytes()); + } - public ByteSizeValue getFreeBytesThresholdFrozenFloodStage(ByteSizeValue total) { - // flood stage bytes are reversed compared to percentage, so we special handle it. - RelativeByteSizeValue frozenFloodStage = this.frozenFloodStage; - if (frozenFloodStage.isAbsolute()) { - return frozenFloodStage.getAbsolute(); + // If watermark is percentage/ratio, calculate the total needed disk space. + double ratioThreshold = lowStageWatermark.getRatio().getAsRatio(); + if (ratioThreshold >= 0.0 && ratioThreshold < 1.0) { + ByteSizeValue totalBytes = ByteSizeValue.ofBytes((long) Math.ceil(used.getBytes() / ratioThreshold)); + return totalBytes; + } else { + return used; } - return ByteSizeValue.ofBytes(total.getBytes() - frozenFloodStage.calculateValue(total, frozenFloodStageMaxHeadroom).getBytes()); } public boolean isEnabled() { @@ -367,128 +283,68 @@ public TimeValue getRerouteInterval() { return rerouteInterval; } - String describeLowThreshold() { - return freeBytesThresholdLow.equals(ByteSizeValue.ZERO) - ? Strings.format1Decimals(100.0 - freeDiskThresholdLow, "%") - : freeBytesThresholdLow.toString(); - } - - String describeHighThreshold() { - return freeBytesThresholdHigh.equals(ByteSizeValue.ZERO) - ? Strings.format1Decimals(100.0 - freeDiskThresholdHigh, "%") - : freeBytesThresholdHigh.toString(); - } - - String describeFloodStageThreshold() { - return freeBytesThresholdFloodStage.equals(ByteSizeValue.ZERO) - ? Strings.format1Decimals(100.0 - freeDiskThresholdFloodStage, "%") - : freeBytesThresholdFloodStage.toString(); - } - - String describeFrozenFloodStageThreshold(ByteSizeValue total) { - ByteSizeValue maxHeadroom = this.frozenFloodStageMaxHeadroom; - RelativeByteSizeValue floodStage = this.frozenFloodStage; - if (floodStage.isAbsolute()) { - return floodStage.getStringRep(); - } else if (floodStage.calculateValue(total, maxHeadroom).equals(floodStage.calculateValue(total, null))) { - return Strings.format1Decimals(floodStage.getRatio().getAsPercent(), "%"); + private String describeThreshold( + ByteSizeValue total, + RelativeByteSizeValue watermark, + ByteSizeValue maxHeadroom, + boolean includeSettingKey, + String watermarkSettingKey, + String maxHeadroomSettingKey + ) { + if (watermark.isAbsolute()) { + return includeSettingKey ? watermarkSettingKey + "=" + watermark.getStringRep() : watermark.getStringRep(); + } else if (watermark.calculateValue(total, maxHeadroom).equals(watermark.calculateValue(total, null))) { + String value = watermark.getStringRep(); + return includeSettingKey ? watermarkSettingKey + "=" + value : value; } else { - return "max_headroom=" + maxHeadroom; + return includeSettingKey + ? maxHeadroomSettingKey + "=" + maxHeadroom.getStringRep() + : "max_headroom=" + maxHeadroom.getStringRep(); } } - /** - * Attempts to parse the watermark into a percentage, returning 100.0% if - * it cannot be parsed. - */ - private static double thresholdPercentageFromWatermark(String watermark) { - return thresholdPercentageFromWatermark(watermark, true); - } - - /** - * Attempts to parse the watermark into a percentage, returning 100.0% if it can not be parsed and the specified lenient parameter is - * true, otherwise throwing an {@link ElasticsearchParseException}. - * - * @param watermark the watermark to parse as a percentage - * @param lenient true if lenient parsing should be applied - * @return the parsed percentage - */ - private static double thresholdPercentageFromWatermark(String watermark, boolean lenient) { - if (lenient && definitelyNotPercentage(watermark)) { - // obviously not a percentage so return lenient fallback value like we would below on a parse failure - return 100.0; - } - try { - return RatioValue.parseRatioValue(watermark).getAsPercent(); - } catch (ElasticsearchParseException ex) { - // NOTE: this is not end-user leniency, since up above we check that it's a valid byte or percentage, and then store the two - // cases separately - if (lenient) { - return 100.0; - } - throw ex; - } + public String describeLowThreshold(ByteSizeValue total, boolean includeSettingKey) { + return describeThreshold( + total, + lowStageWatermark, + ByteSizeValue.MINUS_ONE, + includeSettingKey, + CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), + null + ); } - /** - * Attempts to parse the watermark into a {@link ByteSizeValue}, returning - * a ByteSizeValue of 0 bytes if the value cannot be parsed. - */ - private static ByteSizeValue thresholdBytesFromWatermark(String watermark, String settingName) { - return thresholdBytesFromWatermark(watermark, settingName, true); + public String describeHighThreshold(ByteSizeValue total, boolean includeSettingKey) { + return describeThreshold( + total, + highStageWatermark, + ByteSizeValue.MINUS_ONE, + includeSettingKey, + CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), + null + ); } - /** - * Attempts to parse the watermark into a {@link ByteSizeValue}, returning zero bytes if it can not be parsed and the specified lenient - * parameter is true, otherwise throwing an {@link ElasticsearchParseException}. - * - * @param watermark the watermark to parse as a byte size - * @param settingName the name of the setting - * @param lenient true if lenient parsing should be applied - * @return the parsed byte size value - */ - private static ByteSizeValue thresholdBytesFromWatermark(String watermark, String settingName, boolean lenient) { - try { - return ByteSizeValue.parseBytesSizeValue(watermark, settingName); - } catch (ElasticsearchParseException ex) { - // NOTE: this is not end-user leniency, since up above we check that it's a valid byte or percentage, and then store the two - // cases separately - if (lenient) { - return ByteSizeValue.ZERO; - } - throw ex; - } + public String describeFloodStageThreshold(ByteSizeValue total, boolean includeSettingKey) { + return describeThreshold( + total, + floodStageWatermark, + ByteSizeValue.MINUS_ONE, + includeSettingKey, + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), + null + ); } - /** - * Checks if a watermark string is a valid percentage or byte size value, - * @return the watermark value given - */ - private static String validWatermarkSetting(String watermark, String settingName) { - if (definitelyNotPercentage(watermark)) { - // short circuit to save expensive exception on obvious byte size value below - ByteSizeValue.parseBytesSizeValue(watermark, settingName); - return watermark; - } - try { - RatioValue.parseRatioValue(watermark); - } catch (ElasticsearchParseException e) { - try { - ByteSizeValue.parseBytesSizeValue(watermark, settingName); - } catch (ElasticsearchParseException ex) { - ex.addSuppressed(e); - throw ex; - } - } - return watermark; + public String describeFrozenFloodStageThreshold(ByteSizeValue total, boolean includeSettingKey) { + return describeThreshold( + total, + frozenFloodStageWatermark, + frozenFloodStageMaxHeadroom, + includeSettingKey, + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.getKey(), + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING.getKey() + ); } - // Checks that a value is definitely not a percentage by testing if it ends on `b` which implies that it is probably a byte size value - // instead. This is used to make setting validation skip attempting to parse a value as a percentage/ration for the settings in this - // class that accept either a byte size value. The main motivation of this method is to make tests faster. Some tests call this method - // frequently when starting up internal cluster nodes and using exception throwing and catching when trying to parse as a ratio as a - // means of identifying that a string is not a ratio is quite slow. - private static boolean definitelyNotPercentage(String value) { - return value.endsWith("b"); - } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index c2fb3a34780a8..076636f869485 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -36,9 +36,6 @@ import java.util.Map; import java.util.Set; -import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING; -import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING; - /** * The {@link DiskThresholdDecider} checks that the node a shard is potentially * being allocated to has enough disk space. @@ -56,8 +53,8 @@ * node to pass this watermark, it will not be allowed. It defaults to * 0.90 (90.0%). * - * Both watermark settings are expressed in terms of used disk percentage, or - * exact byte values for free space (like "500mb") + * The watermark settings are expressed in terms of used disk percentage/ratio, or + * exact byte values for free space (like "500mb"). * * cluster.routing.allocation.disk.threshold_enabled is used to * enable or disable this decider. It defaults to true (enabled). @@ -183,17 +180,13 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing return YES_DISK_WATERMARKS_IGNORED; } - final double usedDiskThresholdLow = 100.0 - diskThresholdSettings.getFreeDiskThresholdLow(); - final double usedDiskThresholdHigh = 100.0 - diskThresholdSettings.getFreeDiskThresholdHigh(); - // subtractLeavingShards is passed as false here, because they still use disk space, and therefore we should be extra careful // and take the size into account final DiskUsageWithRelocations usage = getDiskUsage(node, allocation, usages, false); - // First, check that the node currently over the low watermark - double freeDiskPercentage = usage.getFreeDiskAsPercentage(); // Cache the used disk percentage for displaying disk percentages consistent with documentation double usedDiskPercentage = usage.getUsedDiskAsPercentage(); long freeBytes = usage.getFreeBytes(); + final ByteSizeValue total = ByteSizeValue.ofBytes(usage.getTotalBytes()); if (freeBytes < 0L) { final long sizeOfRelocatingShards = sizeOfRelocatingShards( node, @@ -232,13 +225,12 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing && shardRouting.active() == false && shardRouting.recoverySource().getType() == RecoverySource.Type.EMPTY_STORE; - // checks for exact byte comparisons - if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLow().getBytes()) { + if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLowStage(total).getBytes()) { if (skipLowThresholdChecks == false) { if (logger.isDebugEnabled()) { logger.debug( "less than the required {} free bytes threshold ({} free) on node {}, preventing allocation", - diskThresholdSettings.getFreeBytesThresholdLow(), + diskThresholdSettings.getFreeBytesThresholdLowStage(total).getBytes(), freeBytesValue, node.nodeId() ); @@ -246,21 +238,21 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing return allocation.decision( Decision.NO, NAME, - "the node is above the low watermark cluster setting [%s=%s], having less than the minimum required [%s] free " - + "space, actual free: [%s]", - CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), - diskThresholdSettings.getLowWatermarkRaw(), - diskThresholdSettings.getFreeBytesThresholdLow(), - freeBytesValue + "the node is above the low watermark cluster setting [%s], having less than the minimum required [%s] free " + + "space, actual free: [%s], actual used: [%s]", + diskThresholdSettings.describeLowThreshold(total, true), + diskThresholdSettings.getFreeBytesThresholdLowStage(total), + freeBytesValue, + Strings.format1Decimals(usedDiskPercentage, "%") ); - } else if (freeBytes > diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()) { + } else if (freeBytes > diskThresholdSettings.getFreeBytesThresholdHighStage(total).getBytes()) { // Allow the shard to be allocated because it is primary that // has never been allocated if it's under the high watermark if (logger.isDebugEnabled()) { logger.debug( "less than the required {} free bytes threshold ({} free) on node {}, " + "but allowing allocation because primary has never been allocated", - diskThresholdSettings.getFreeBytesThresholdLow(), + diskThresholdSettings.getFreeBytesThresholdLowStage(total), freeBytesValue, node.nodeId() ); @@ -273,7 +265,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing logger.debug( "less than the required {} free bytes threshold ({} free) on node {}, " + "preventing allocation even though primary has never been allocated", - diskThresholdSettings.getFreeBytesThresholdHigh(), + diskThresholdSettings.getFreeBytesThresholdHighStage(total).getBytes(), freeBytesValue, node.nodeId() ); @@ -281,72 +273,12 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing return allocation.decision( Decision.NO, NAME, - "the node is above the high watermark cluster setting [%s=%s], having less than the minimum required [%s] free " - + "space, actual free: [%s]", - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), - diskThresholdSettings.getHighWatermarkRaw(), - diskThresholdSettings.getFreeBytesThresholdHigh(), - freeBytesValue - ); - } - } - - // checks for percentage comparisons - if (freeDiskPercentage < diskThresholdSettings.getFreeDiskThresholdLow()) { - // If the shard is a replica or is a non-empty primary, check the low threshold - if (skipLowThresholdChecks == false) { - if (logger.isDebugEnabled()) { - logger.debug( - "more than the allowed {} used disk threshold ({} used) on node [{}], preventing allocation", - Strings.format1Decimals(usedDiskThresholdLow, "%"), - Strings.format1Decimals(usedDiskPercentage, "%"), - node.nodeId() - ); - } - return allocation.decision( - Decision.NO, - NAME, - "the node is above the low watermark cluster setting [%s=%s], using more disk space than the maximum allowed " - + "[%s%%], actual free: [%s%%]", - CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), - diskThresholdSettings.getLowWatermarkRaw(), - usedDiskThresholdLow, - freeDiskPercentage - ); - } else if (freeDiskPercentage > diskThresholdSettings.getFreeDiskThresholdHigh()) { - // Allow the shard to be allocated because it is primary that - // has never been allocated if it's under the high watermark - if (logger.isDebugEnabled()) { - logger.debug( - "more than the allowed {} used disk threshold ({} used) on node [{}], " - + "but allowing allocation because primary has never been allocated", - Strings.format1Decimals(usedDiskThresholdLow, "%"), - Strings.format1Decimals(usedDiskPercentage, "%"), - node.nodeId() - ); - } - return YES_UNALLOCATED_PRIMARY_BETWEEN_WATERMARKS; - } else { - // Even though the primary has never been allocated, the node is - // above the high watermark, so don't allow allocating the shard - if (logger.isDebugEnabled()) { - logger.debug( - "less than the required {} free bytes threshold ({} bytes free) on node {}, " - + "preventing allocation even though primary has never been allocated", - Strings.format1Decimals(diskThresholdSettings.getFreeDiskThresholdHigh(), "%"), - Strings.format1Decimals(freeDiskPercentage, "%"), - node.nodeId() - ); - } - return allocation.decision( - Decision.NO, - NAME, - "the node is above the high watermark cluster setting [%s=%s], using more disk space than the maximum allowed " - + "[%s%%], actual free: [%s%%]", - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), - diskThresholdSettings.getHighWatermarkRaw(), - usedDiskThresholdHigh, - freeDiskPercentage + "the node is above the high watermark cluster setting [%s], having less than the minimum required [%s] free " + + "space, actual free: [%s], actual used: [%s]", + diskThresholdSettings.describeHighThreshold(total, true), + diskThresholdSettings.getFreeBytesThresholdHighStage(total), + freeBytesValue, + Strings.format1Decimals(usedDiskPercentage, "%") ); } } @@ -361,57 +293,40 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing allocation.routingTable() ); assert shardSize >= 0 : shardSize; - double freeSpaceAfterShard = freeDiskPercentageAfterShardAssigned(usage, shardSize); long freeBytesAfterShard = freeBytes - shardSize; - if (freeBytesAfterShard < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()) { + if (freeBytesAfterShard < diskThresholdSettings.getFreeBytesThresholdHighStage(total).getBytes()) { logger.warn( - "after allocating [{}] node [{}] would have less than the required threshold of " - + "{} free (currently {} free, estimated shard size is {}), preventing allocation", + "after allocating [{}] node [{}] would be above the high watermark setting [{}], having less than the minimum " + + "required {} of free space (actual free: {}, actual used: {}, estimated shard size: {}), preventing allocation", shardRouting, node.nodeId(), - diskThresholdSettings.getFreeBytesThresholdHigh(), + diskThresholdSettings.describeHighThreshold(total, false), + diskThresholdSettings.getFreeBytesThresholdHighStage(total), freeBytesValue, + Strings.format1Decimals(usedDiskPercentage, "%"), new ByteSizeValue(shardSize) ); return allocation.decision( Decision.NO, NAME, - "allocating the shard to this node will bring the node above the high watermark cluster setting [%s=%s] " - + "and cause it to have less than the minimum required [%s] of free space (free: [%s], estimated shard size: [%s])", - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), - diskThresholdSettings.getHighWatermarkRaw(), - diskThresholdSettings.getFreeBytesThresholdHigh(), + "allocating the shard to this node will bring the node above the high watermark cluster setting [%s] " + + "and cause it to have less than the minimum required [%s] of free space (free: [%s], used: [%s], estimated " + + "shard size: [%s])", + diskThresholdSettings.describeHighThreshold(total, true), + diskThresholdSettings.getFreeBytesThresholdHighStage(total), freeBytesValue, + Strings.format1Decimals(usedDiskPercentage, "%"), new ByteSizeValue(shardSize) ); } - if (freeSpaceAfterShard < diskThresholdSettings.getFreeDiskThresholdHigh()) { - logger.warn( - "after allocating [{}] node [{}] would have more than the allowed " - + "{} free disk threshold ({} free), preventing allocation", - shardRouting, - node.nodeId(), - Strings.format1Decimals(diskThresholdSettings.getFreeDiskThresholdHigh(), "%"), - Strings.format1Decimals(freeSpaceAfterShard, "%") - ); - return allocation.decision( - Decision.NO, - NAME, - "allocating the shard to this node will bring the node above the high watermark cluster setting [%s=%s] " - + "and cause it to use more disk space than the maximum allowed [%s%%] (free space after shard added: [%s%%])", - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), - diskThresholdSettings.getHighWatermarkRaw(), - usedDiskThresholdHigh, - freeSpaceAfterShard - ); - } assert freeBytesAfterShard >= 0 : freeBytesAfterShard; return allocation.decision( Decision.YES, NAME, - "enough disk for shard on node, free: [%s], shard size: [%s], free after allocating shard: [%s]", + "enough disk for shard on node, free: [%s], used: [%s], shard size: [%s], free after allocating shard: [%s]", freeBytesValue, + Strings.format1Decimals(usedDiskPercentage, "%"), new ByteSizeValue(shardSize), new ByteSizeValue(freeBytesAfterShard) ); @@ -483,6 +398,8 @@ public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting // If this node is already above the high threshold, the shard cannot remain (get it off!) final double freeDiskPercentage = usage.getFreeDiskAsPercentage(); final long freeBytes = usage.getFreeBytes(); + double usedDiskPercentage = usage.getUsedDiskAsPercentage(); + final ByteSizeValue total = ByteSizeValue.ofBytes(usage.getTotalBytes()); if (logger.isTraceEnabled()) { logger.trace("node [{}] has {}% free disk ({} bytes)", node.nodeId(), freeDiskPercentage, freeBytes); } @@ -514,44 +431,27 @@ public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting sizeOfRelocatingShards ); } - if (freeBytes < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()) { + if (freeBytes < diskThresholdSettings.getFreeBytesThresholdHighStage(total).getBytes()) { if (logger.isDebugEnabled()) { logger.debug( - "less than the required {} free bytes threshold ({} bytes free) on node {}, shard cannot remain", - diskThresholdSettings.getFreeBytesThresholdHigh(), + "node {} is over the high watermark setting [{}], having less than the required {} free space " + + "(actual free: {}, actual used: {}), shard cannot remain", + node.nodeId(), + diskThresholdSettings.describeHighThreshold(total, false), + diskThresholdSettings.getFreeBytesThresholdHighStage(total), freeBytes, - node.nodeId() + Strings.format1Decimals(usedDiskPercentage, "%") ); } return allocation.decision( Decision.NO, NAME, - "the shard cannot remain on this node because it is above the high watermark cluster setting [%s=%s] " - + "and there is less than the required [%s] free space on node, actual free: [%s]", - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), - diskThresholdSettings.getHighWatermarkRaw(), - diskThresholdSettings.getFreeBytesThresholdHigh(), - new ByteSizeValue(freeBytes) - ); - } - if (freeDiskPercentage < diskThresholdSettings.getFreeDiskThresholdHigh()) { - if (logger.isDebugEnabled()) { - logger.debug( - "less than the required {}% free disk threshold ({}% free) on node {}, shard cannot remain", - diskThresholdSettings.getFreeDiskThresholdHigh(), - freeDiskPercentage, - node.nodeId() - ); - } - return allocation.decision( - Decision.NO, - NAME, - "the shard cannot remain on this node because it is above the high watermark cluster setting [%s=%s] " - + "and there is less than the required [%s%%] free disk on node, actual free: [%s%%]", - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), - diskThresholdSettings.getHighWatermarkRaw(), - diskThresholdSettings.getFreeDiskThresholdHigh(), - freeDiskPercentage + "the shard cannot remain on this node because it is above the high watermark cluster setting [%s] " + + "and there is less than the required [%s] free space on node, actual free: [%s], actual used: [%s]", + diskThresholdSettings.describeHighThreshold(total, true), + diskThresholdSettings.getFreeBytesThresholdHighStage(total), + new ByteSizeValue(freeBytes), + Strings.format1Decimals(usedDiskPercentage, "%") ); } @@ -618,25 +518,6 @@ static DiskUsage averageUsage(RoutingNode node, Map usages) { return new DiskUsage(node.nodeId(), node.node().getName(), "_na_", totalBytes / usages.size(), freeBytes / usages.size()); } - /** - * Given the DiskUsage for a node and the size of the shard, return the - * percentage of free disk if the shard were to be allocated to the node. - * @param usage A DiskUsage for the node to have space computed for - * @param shardSize Size in bytes of the shard - * @return Percentage of free space after the shard is assigned to the node - */ - static double freeDiskPercentageAfterShardAssigned(DiskUsageWithRelocations usage, Long shardSize) { - shardSize = (shardSize == null) ? 0 : shardSize; - DiskUsage newUsage = new DiskUsage( - usage.getNodeId(), - usage.getNodeName(), - usage.getPath(), - usage.getTotalBytes(), - usage.getFreeBytes() - shardSize - ); - return newUsage.getFreeDiskAsPercentage(); - } - private static final Decision YES_DISABLED = Decision.single(Decision.Type.YES, NAME, "the disk threshold decider is disabled"); private static final Decision YES_USAGES_UNAVAILABLE = Decision.single(Decision.Type.YES, NAME, "disk usages are unavailable"); diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 1adf0d6afd0ee..d5c2f8e7d5d63 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -86,7 +86,7 @@ import java.util.Map; public class Lucene { - public static final String LATEST_CODEC = "Lucene93"; + public static final String LATEST_CODEC = "Lucene92"; public static final String SOFT_DELETES_FIELD = "__soft_deletes"; diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index c4024e0ba543d..19bbaeef72279 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -244,7 +244,7 @@ public void apply(Settings value, Settings current, Settings previous) { DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING, DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING, DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING, - DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_SETTING, + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING, DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING, DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING, DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING, diff --git a/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java b/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java index 83669ae94cb39..74f88d9aaa2cd 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java @@ -9,6 +9,7 @@ package org.elasticsearch.common.settings; import org.elasticsearch.common.util.ArrayUtils; +import org.elasticsearch.core.Booleans; import java.io.InputStream; import java.security.GeneralSecurityException; @@ -22,6 +23,9 @@ */ public abstract class SecureSetting extends Setting { + /** Determines whether legacy settings with sensitive values should be allowed. */ + private static final boolean ALLOW_INSECURE_SETTINGS = Booleans.parseBoolean(System.getProperty("es.allow_insecure_settings", "false")); + private static final Set ALLOWED_PROPERTIES = EnumSet.of( Property.Deprecated, Property.DeprecatedWarning, @@ -128,6 +132,16 @@ public static Setting secureString(String name, Setting insecureString(String name) { + return new InsecureStringSetting(name); + } + /** * A setting which contains a file. Reading the setting opens an input stream to the file. * @@ -159,6 +173,25 @@ SecureString getFallback(Settings settings) { } } + private static class InsecureStringSetting extends Setting { + private final String name; + + private InsecureStringSetting(String name) { + super(name, "", SecureString::new, Property.Deprecated, Property.Filtered, Property.NodeScope); + this.name = name; + } + + @Override + public SecureString get(Settings settings) { + if (ALLOW_INSECURE_SETTINGS == false && exists(settings)) { + throw new IllegalArgumentException( + "Setting [" + name + "] is insecure, " + "but property [allow_insecure_settings] is not set" + ); + } + return super.get(settings); + } + } + private static class SecureFileSetting extends SecureSetting { private final Setting fallback; diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java index a8f235df674c9..2fed8786ed344 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java @@ -166,7 +166,7 @@ public void configure(Binder binder) { * the setting during startup. */ private void registerSetting(Setting setting) { - if (setting.getKey().contains(".") == false) { + if (setting.getKey().contains(".") == false && isS3InsecureCredentials(setting) == false) { throw new IllegalArgumentException("setting [" + setting.getKey() + "] is not in any namespace, its name must contain a dot"); } if (setting.isFiltered()) { @@ -210,6 +210,13 @@ private void registerSetting(Setting setting) { } } + // TODO: remove this hack once we remove the deprecated ability to use repository settings in the cluster state in the S3 snapshot + // module + private boolean isS3InsecureCredentials(Setting setting) { + final String settingKey = setting.getKey(); + return settingKey.equals("access_key") || settingKey.equals("secret_key"); + } + /** * Registers a settings filter pattern that allows to filter out certain settings that for instance contain sensitive information * or if a setting is for internal purposes only. The given pattern must either be a valid settings key or a simple regexp pattern. diff --git a/server/src/main/java/org/elasticsearch/common/unit/RatioValue.java b/server/src/main/java/org/elasticsearch/common/unit/RatioValue.java index 755246b67ba4d..490515572f42c 100644 --- a/server/src/main/java/org/elasticsearch/common/unit/RatioValue.java +++ b/server/src/main/java/org/elasticsearch/common/unit/RatioValue.java @@ -65,12 +65,10 @@ public static RatioValue parseRatioValue(String sValue) { } /** - * Formats the input to a string with no trailing zeros and the '%' suffix. - * Note: this is not converting a ratio to a percentage. The input 0.75 will - * be formatted as 0.75%. + * Returns the percent as a string with no trailing zeros and the '%' suffix. */ - public static String formatNoTrailingZerosPercent(double percent) { - String value = String.valueOf(percent); + public String formatNoTrailingZerosPercent() { + String value = String.valueOf(getAsPercent()); int i = value.length() - 1; while (i >= 0 && value.charAt(i) == '0') { i--; diff --git a/server/src/main/java/org/elasticsearch/common/unit/RelativeByteSizeValue.java b/server/src/main/java/org/elasticsearch/common/unit/RelativeByteSizeValue.java index 7475f89f5910c..01449f4608c3d 100644 --- a/server/src/main/java/org/elasticsearch/common/unit/RelativeByteSizeValue.java +++ b/server/src/main/java/org/elasticsearch/common/unit/RelativeByteSizeValue.java @@ -71,15 +71,20 @@ public boolean isNonZeroSize() { } public static RelativeByteSizeValue parseRelativeByteSizeValue(String value, String settingName) { - try { - RatioValue ratio = RatioValue.parseRatioValue(value); - if (ratio.getAsPercent() != 0.0d || value.endsWith("%")) { - return new RelativeByteSizeValue(ratio); - } else { - return new RelativeByteSizeValue(ByteSizeValue.ZERO); + // If the value ends with a `b`, it implies it is probably a byte size value, so do not try to parse as a ratio/percentage at all. + // The main motivation is to make parsing faster. Using exception throwing and catching when trying to parse as a ratio as a + // means of identifying that a string is not a ratio can be quite slow. + if (value.endsWith("b") == false) { + try { + RatioValue ratio = RatioValue.parseRatioValue(value); + if (ratio.getAsPercent() != 0.0d || value.endsWith("%")) { + return new RelativeByteSizeValue(ratio); + } else { + return new RelativeByteSizeValue(ByteSizeValue.ZERO); + } + } catch (ElasticsearchParseException e) { + // ignore, see if it parses as bytes } - } catch (ElasticsearchParseException e) { - // ignore, see if it parses as bytes } try { return new RelativeByteSizeValue(ByteSizeValue.parseBytesSizeValue(value, settingName)); @@ -90,7 +95,7 @@ public static RelativeByteSizeValue parseRelativeByteSizeValue(String value, Str public String getStringRep() { if (ratio != null) { - return ratio.toString(); + return ratio.formatNoTrailingZerosPercent(); } else { return absolute.getStringRep(); } diff --git a/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadata.java b/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadata.java index 4cbe5988e3660..6666f67f53a10 100644 --- a/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadata.java +++ b/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadata.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.RatioValue; import org.elasticsearch.common.unit.RelativeByteSizeValue; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -195,7 +194,7 @@ private String getThresholdStringRep(RelativeByteSizeValue relativeByteSizeValue if (relativeByteSizeValue.isAbsolute()) { return relativeByteSizeValue.getAbsolute().getStringRep(); } else { - return RatioValue.formatNoTrailingZerosPercent(relativeByteSizeValue.getRatio().getAsPercent()); + return relativeByteSizeValue.getRatio().formatNoTrailingZerosPercent(); } } diff --git a/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadataService.java b/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadataService.java index 2293ca31c8b7d..7a8416de186b0 100644 --- a/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadataService.java +++ b/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadataService.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.RelativeByteSizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; @@ -32,7 +31,7 @@ import java.util.List; import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING; -import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_SETTING; +import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING; import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING; import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING; import static org.elasticsearch.health.node.selection.HealthNodeTaskExecutor.ENABLED_SETTING; @@ -69,15 +68,18 @@ public HealthMetadataService(ClusterService clusterService, Settings settings) { ClusterSettings clusterSettings = clusterService.getClusterSettings(); clusterSettings.addSettingsUpdateConsumer( CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING, - value -> updateOnSettingsUpdated(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), value) + value -> updateOnSettingsUpdated(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), value.getStringRep()) ); clusterSettings.addSettingsUpdateConsumer( CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING, - value -> updateOnSettingsUpdated(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), value) + value -> updateOnSettingsUpdated(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), value.getStringRep()) ); clusterSettings.addSettingsUpdateConsumer( - CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_SETTING, - value -> updateOnSettingsUpdated(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_SETTING.getKey(), value.getStringRep()) + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING, + value -> updateOnSettingsUpdated( + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.getKey(), + value.getStringRep() + ) ); clusterSettings.addSettingsUpdateConsumer( CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING, @@ -203,7 +205,7 @@ ClusterState execute(ClusterState clusterState) { if (CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey().equals(setting)) { builder.floodStageWatermark(value, setting); } - if (CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey().equals(setting)) { + if (CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.getKey().equals(setting)) { builder.frozenFloodStageWatermark(value, setting); } if (CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING.getKey().equals(setting)) { @@ -233,15 +235,9 @@ ClusterState execute(ClusterState clusterState) { HealthMetadata initialHealthMetadata = HealthMetadata.getHealthCustomMetadata(clusterState); final var finalHealthMetadata = new HealthMetadata( new HealthMetadata.Disk( - RelativeByteSizeValue.parseRelativeByteSizeValue( - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.get(settings), - CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey() - ), - RelativeByteSizeValue.parseRelativeByteSizeValue( - CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.get(settings), - CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey() - ), - CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_SETTING.get(settings), + CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.get(settings), + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.get(settings), + CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.get(settings), CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING.get(settings) ) ); diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java index f797c39a4ef29..bd9dfeca6d694 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -9,7 +9,7 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene93.Lucene93Codec; +import org.apache.lucene.codecs.lucene92.Lucene92Codec; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.MapperService; @@ -34,11 +34,11 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService) { final var codecs = new HashMap(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene93Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene93Codec(Lucene93Codec.Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene92Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Lucene92Codec.Mode.BEST_COMPRESSION)); } else { - codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene93Codec.Mode.BEST_SPEED, mapperService)); - codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene93Codec.Mode.BEST_COMPRESSION, mapperService)); + codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene92Codec.Mode.BEST_SPEED, mapperService)); + codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene92Codec.Mode.BEST_COMPRESSION, mapperService)); } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java index 9c2ffa77d51b2..517280dce73a2 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java @@ -13,7 +13,7 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; -import org.apache.lucene.codecs.lucene93.Lucene93Codec; +import org.apache.lucene.codecs.lucene92.Lucene92Codec; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; @@ -27,7 +27,7 @@ * per index in real time via the mapping API. If no specific postings format or vector format is * configured for a specific field the default postings or vector format is used. */ -public class PerFieldMapperCodec extends Lucene93Codec { +public class PerFieldMapperCodec extends Lucene92Codec { private final MapperService mapperService; private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 067061443d885..ac5c1f2706b04 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -294,17 +294,18 @@ private static void throwOnConcreteValue(ObjectMapper mapper, String currentFiel private static void innerParseObject(DocumentParserContext context, ObjectMapper mapper) throws IOException { - XContentParser.Token token = context.parser().currentToken(); - String currentFieldName = context.parser().currentName(); + final XContentParser parser = context.parser(); + XContentParser.Token token = parser.currentToken(); + String currentFieldName = null; assert token == XContentParser.Token.FIELD_NAME || token == XContentParser.Token.END_OBJECT; while (token != XContentParser.Token.END_OBJECT) { if (token == null) { - throwEOF(mapper, currentFieldName); + throwEOF(mapper, context); } switch (token) { case FIELD_NAME: - currentFieldName = context.parser().currentName(); + currentFieldName = parser.currentName(); if (currentFieldName.isBlank()) { throwFieldNameBlank(context, currentFieldName); } @@ -324,7 +325,7 @@ private static void innerParseObject(DocumentParserContext context, ObjectMapper } break; } - token = context.parser().nextToken(); + token = parser.nextToken(); } } @@ -334,12 +335,12 @@ private static void throwFieldNameBlank(DocumentParserContext context, String cu ); } - private static void throwEOF(ObjectMapper mapper, String currentFieldName) { + private static void throwEOF(ObjectMapper mapper, DocumentParserContext context) throws IOException { throw new MapperParsingException( "object mapping for [" + mapper.name() + "] tried to parse field [" - + currentFieldName + + context.parser().currentName() + "] as object, but got EOF, has a concrete value been provided to it?" ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index b66ae08386c68..13a85ac586b0b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -9,7 +9,7 @@ package org.elasticsearch.index.mapper.vectors; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat; +import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; import org.apache.lucene.document.KnnVectorField; @@ -523,7 +523,7 @@ public KnnVectorsFormat getKnnVectorsFormatForField() { return null; // use default format } else { HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions; - return new Lucene93HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction); + return new Lucene92HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction); } } } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index e25d72345ca8d..28750d6635821 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -8,17 +8,22 @@ package org.elasticsearch.indices; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.MinimizationOperations; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse; import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse.ResetFeatureStateStatus; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -105,6 +110,8 @@ public class SystemIndices { private static final Automaton EMPTY = Automata.makeEmpty(); + private static final Logger logger = LogManager.getLogger(SystemIndices.class); + /** * This is the source for non-plugin system features. */ @@ -328,6 +335,11 @@ public ExecutorSelector getExecutorSelector() { * @return The matching {@link SystemIndexDescriptor} or {@code null} if no descriptor is found */ public @Nullable SystemIndexDescriptor findMatchingDescriptor(String name) { + return findMatchingDescriptor(indexDescriptors, name); + } + + @Nullable + static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) { SystemIndexDescriptor matchingDescriptor = null; for (SystemIndexDescriptor systemIndexDescriptor : indexDescriptors) { if (systemIndexDescriptor.matchesIndexPattern(name)) { @@ -854,6 +866,27 @@ public MigrationCompletionHandler getPostMigrationFunction() { return postMigrationFunction; } + private static void cleanUpFeatureForIndices( + String name, + Client client, + String[] indexNames, + final ActionListener listener + ) { + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(); + deleteIndexRequest.indices(indexNames); + client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest, new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + listener.onResponse(ResetFeatureStateStatus.success(name)); + } + + @Override + public void onFailure(Exception e) { + listener.onResponse(ResetFeatureStateStatus.failure(name, e)); + } + }); + } + /** * Clean up the state of a feature * @param indexDescriptors List of descriptors of a feature's system indices @@ -864,39 +897,66 @@ public MigrationCompletionHandler getPostMigrationFunction() { * @param listener A listener to return success or failure of cleanup */ public static void cleanUpFeature( - Collection indexDescriptors, + Collection indexDescriptors, Collection associatedIndexDescriptors, String name, ClusterService clusterService, Client client, - ActionListener listener + final ActionListener listener ) { Metadata metadata = clusterService.state().getMetadata(); - List allIndices = Stream.concat(indexDescriptors.stream(), associatedIndexDescriptors.stream()) + List associatedIndices = associatedIndexDescriptors.stream() .map(descriptor -> descriptor.getMatchingIndices(metadata)) .flatMap(List::stream) .toList(); - if (allIndices.isEmpty()) { - // if no actual indices match the pattern, we can stop here + final int taskCount = ((associatedIndices.size() > 0) ? 1 : 0) + (int) indexDescriptors.stream() + .filter(id -> id.getMatchingIndices(metadata).isEmpty() == false) + .count(); + + // check if there's nothing to do and take an early out + if (taskCount == 0) { listener.onResponse(ResetFeatureStateStatus.success(name)); return; } - DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(); - deleteIndexRequest.indices(allIndices.toArray(Strings.EMPTY_ARRAY)); - client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest, new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - listener.onResponse(ResetFeatureStateStatus.success(name)); - } + GroupedActionListener groupedListener = new GroupedActionListener<>( + ActionListener.wrap(listenerResults -> { + List errors = listenerResults.stream() + .filter(status -> status.getStatus() == ResetFeatureStateResponse.ResetFeatureStateStatus.Status.FAILURE) + .collect(Collectors.toList()); - @Override - public void onFailure(Exception e) { - listener.onResponse(ResetFeatureStateStatus.failure(name, e)); + if (errors.isEmpty()) { + listener.onResponse(ResetFeatureStateStatus.success(name)); + } else { + StringBuilder exceptions = new StringBuilder("["); + exceptions.append(errors.stream().map(e -> e.getException().getMessage()).collect(Collectors.joining(", "))); + exceptions.append(']'); + errors.forEach(e -> logger.warn(() -> "error while resetting feature [" + name + "]", e.getException())); + listener.onResponse(ResetFeatureStateStatus.failure(name, new Exception(exceptions.toString()))); + } + }, listener::onFailure), + taskCount + ); + + // Send cleanup for the associated indices, they don't need special origin since they are not protected + if (associatedIndices.isEmpty() == false) { + cleanUpFeatureForIndices(name, client, associatedIndices.toArray(Strings.EMPTY_ARRAY), groupedListener); + } + + // One descriptor at a time, create an originating client and clean up the feature + for (var indexDescriptor : indexDescriptors) { + List matchingIndices = indexDescriptor.getMatchingIndices(metadata); + + if (matchingIndices.isEmpty() == false) { + final Client clientWithOrigin = (indexDescriptor.getOrigin() == null) + ? client + : new OriginSettingClient(client, indexDescriptor.getOrigin()); + + cleanUpFeatureForIndices(name, clientWithOrigin, matchingIndices.toArray(Strings.EMPTY_ARRAY), groupedListener); } - }); + } } // No-op pre-migration function to be used as the default in case none are provided. diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 8117bf7f10a23..3b6f61aad09ee 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -3517,26 +3517,16 @@ public int getReadBufferSizeInBytes() { /** * The result of removing a snapshot from a shard folder in the repository. + * + * @param indexId Index that the snapshot was removed from + * @param shardId Shard id that the snapshot was removed from + * @param newGeneration Id of the new index-${uuid} blob that does not include the snapshot any more + * @param blobsToDelete Blob names in the shard directory that have become unreferenced in the new shard generation */ - private static final class ShardSnapshotMetaDeleteResult { - - // Index that the snapshot was removed from - private final IndexId indexId; - - // Shard id that the snapshot was removed from - private final int shardId; - - // Id of the new index-${uuid} blob that does not include the snapshot any more - private final ShardGeneration newGeneration; - - // Blob names in the shard directory that have become unreferenced in the new shard generation - private final Collection blobsToDelete; - - ShardSnapshotMetaDeleteResult(IndexId indexId, int shardId, ShardGeneration newGeneration, Collection blobsToDelete) { - this.indexId = indexId; - this.shardId = shardId; - this.newGeneration = newGeneration; - this.blobsToDelete = blobsToDelete; - } - } + private record ShardSnapshotMetaDeleteResult( + IndexId indexId, + int shardId, + ShardGeneration newGeneration, + Collection blobsToDelete + ) {} } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestKnnSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestKnnSearchAction.java index 03af0f1e5ab46..30b25c8d4cfef 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestKnnSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestKnnSearchAction.java @@ -14,7 +14,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestStatusToXContentListener; -import org.elasticsearch.search.vectors.KnnSearchRequestBuilder; +import org.elasticsearch.search.vectors.KnnSearchRequestParser; import java.io.IOException; import java.util.List; @@ -44,10 +44,10 @@ public String getName() { protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { // This will allow to cancel the search request if the http channel is closed RestCancellableNodeClient cancellableNodeClient = new RestCancellableNodeClient(client, restRequest.getHttpChannel()); - KnnSearchRequestBuilder request = KnnSearchRequestBuilder.parseRestRequest(restRequest); + KnnSearchRequestParser parser = KnnSearchRequestParser.parseRestRequest(restRequest); SearchRequestBuilder searchRequestBuilder = cancellableNodeClient.prepareSearch(); - request.build(searchRequestBuilder); + parser.toSearchRequest(searchRequestBuilder); return channel -> searchRequestBuilder.execute(new RestStatusToXContentListener<>(channel)); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index bde5f2ae55e08..63660ed1a290f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -135,7 +135,7 @@ public static MultiSearchRequest parseRequest( parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, (searchRequest, parser) -> { searchRequest.source(SearchSourceBuilder.fromXContent(parser, false)); - RestSearchAction.checkRestTotalHits(restRequest, searchRequest); + RestSearchAction.validateSearchRequest(restRequest, searchRequest); if (searchRequest.pointInTimeBuilder() != null) { RestSearchAction.preparePointInTime(searchRequest, restRequest, namedWriteableRegistry); } else { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 34864ffdf3279..22de38bed4506 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -200,7 +200,7 @@ public static void parseSearchRequest( searchRequest.preference(request.param("preference")); searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); - checkRestTotalHits(request, searchRequest); + validateSearchRequest(request, searchRequest); if (searchRequest.pointInTimeBuilder() != null) { preparePointInTime(searchRequest, request, namedWriteableRegistry); @@ -398,6 +398,15 @@ static void preparePointInTime(SearchRequest request, RestRequest restRequest, N request.indices(searchContextId.getActualIndices()); } + /** + * Validates that no search request parameters conflict. This method + * might modify the search request to align certain parameters. + */ + public static void validateSearchRequest(RestRequest restRequest, SearchRequest searchRequest) { + checkRestTotalHits(restRequest, searchRequest); + checkSearchType(restRequest, searchRequest); + } + /** * Modify the search request to accurately count the total hits that match the query * if {@link #TOTAL_HITS_AS_INT_PARAM} is set. @@ -406,7 +415,7 @@ static void preparePointInTime(SearchRequest request, RestRequest restRequest, N * is used in conjunction with a lower bound value (other than {@link SearchContext#DEFAULT_TRACK_TOTAL_HITS_UP_TO}) * for the track_total_hits option. */ - public static void checkRestTotalHits(RestRequest restRequest, SearchRequest searchRequest) { + private static void checkRestTotalHits(RestRequest restRequest, SearchRequest searchRequest) { boolean totalHitsAsInt = restRequest.paramAsBoolean(TOTAL_HITS_AS_INT_PARAM, false); if (totalHitsAsInt == false) { return; @@ -429,6 +438,14 @@ public static void checkRestTotalHits(RestRequest restRequest, SearchRequest sea } } + private static void checkSearchType(RestRequest restRequest, SearchRequest searchRequest) { + if (restRequest.hasParam("search_type") && searchRequest.hasKnnSearch()) { + throw new IllegalArgumentException( + "cannot set [search_type] when using [knn] search, since the search type " + "is determined automatically" + ); + } + } + @Override protected Set responseParams() { return RESPONSE_PARAMS; diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index a5d4ddd950797..caa44ced40a63 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -252,6 +252,7 @@ import org.elasticsearch.search.suggest.phrase.StupidBackoff; import org.elasticsearch.search.suggest.term.TermSuggestion; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; +import org.elasticsearch.search.vectors.KnnScoreDocQueryBuilder; import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; @@ -1170,11 +1171,19 @@ private void registerQueryParsers(List plugins) { KnnVectorQueryBuilder.NAME, KnnVectorQueryBuilder::new, parser -> { - throw new IllegalArgumentException("[knn] queries cannot be provided directly, use the [_knn_search] endpoint instead"); + throw new IllegalArgumentException("[knn] queries cannot be provided directly, use the [knn] body parameter instead"); } ) ); + registerQuery( + new QuerySpec<>( + KnnScoreDocQueryBuilder.NAME, + KnnScoreDocQueryBuilder::new, + parser -> { throw new IllegalArgumentException("[score_doc] queries cannot be provided directly"); } + ) + ); + registerFromPlugin(plugins, SearchPlugin::getQueries, this::registerQuery); if (RestApiVersion.minimumSupported() == RestApiVersion.V_7) { diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index a38984911cb61..a42e77b7ee12d 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -43,6 +43,7 @@ import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.vectors.KnnSearchBuilder; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.ToXContentObject; @@ -78,6 +79,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R public static final ParseField TERMINATE_AFTER_FIELD = new ParseField("terminate_after"); public static final ParseField QUERY_FIELD = new ParseField("query"); public static final ParseField POST_FILTER_FIELD = new ParseField("post_filter"); + public static final ParseField KNN_FIELD = new ParseField("knn"); public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score"); public static final ParseField VERSION_FIELD = new ParseField("version"); public static final ParseField SEQ_NO_PRIMARY_TERM_FIELD = new ParseField("seq_no_primary_term"); @@ -135,6 +137,8 @@ public static HighlightBuilder highlight() { private QueryBuilder postQueryBuilder; + private KnnSearchBuilder knnSearch; + private int from = -1; private int size = -1; @@ -251,6 +255,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_7_11_0)) { runtimeMappings = in.readMap(); } + if (in.getVersion().onOrAfter(Version.V_8_4_0)) { + knnSearch = in.readOptionalWriteable(KnnSearchBuilder::new); + } } @Override @@ -318,6 +325,9 @@ public void writeTo(StreamOutput out) throws IOException { ); } } + if (out.getVersion().onOrAfter(Version.V_8_4_0)) { + out.writeOptionalWriteable(knnSearch); + } } /** @@ -354,6 +364,22 @@ public QueryBuilder postFilter() { return postQueryBuilder; } + /** + * Defines a kNN search. If a query is also provided, the kNN hits + * are combined with the query hits. + */ + public SearchSourceBuilder knnSearch(KnnSearchBuilder knnSearch) { + this.knnSearch = knnSearch; + return this; + } + + /** + * An optional kNN search definition. + */ + public KnnSearchBuilder knnSearch() { + return knnSearch; + } + /** * From index to start the search from. Defaults to {@code 0}. * Must be a positive value or 0. @@ -967,7 +993,7 @@ public List ext() { * @return true if the source only has suggest */ public boolean isSuggestOnly() { - return suggestBuilder != null && queryBuilder == null && aggregations == null; + return suggestBuilder != null && queryBuilder == null && knnSearch == null && aggregations == null; } /** @@ -1010,7 +1036,7 @@ public SearchSourceBuilder runtimeMappings(Map runtimeMappings) @SuppressWarnings({ "unchecked", "rawtypes" }) public SearchSourceBuilder rewrite(QueryRewriteContext context) throws IOException { assert (this.equals( - shallowCopy(queryBuilder, postQueryBuilder, aggregations, sliceBuilder, sorts, rescoreBuilders, highlightBuilder) + shallowCopy(queryBuilder, postQueryBuilder, knnSearch, aggregations, sliceBuilder, sorts, rescoreBuilders, highlightBuilder) )); QueryBuilder queryBuilder = null; if (this.queryBuilder != null) { @@ -1020,6 +1046,10 @@ public SearchSourceBuilder rewrite(QueryRewriteContext context) throws IOExcepti if (this.postQueryBuilder != null) { postQueryBuilder = this.postQueryBuilder.rewrite(context); } + KnnSearchBuilder knnSearch = null; + if (this.knnSearch != null) { + knnSearch = this.knnSearch.rewrite(context); + } AggregatorFactories.Builder aggregations = null; if (this.aggregations != null) { aggregations = this.aggregations.rewrite(context); @@ -1034,12 +1064,22 @@ public SearchSourceBuilder rewrite(QueryRewriteContext context) throws IOExcepti boolean rewritten = queryBuilder != this.queryBuilder || postQueryBuilder != this.postQueryBuilder + || knnSearch != this.knnSearch || aggregations != this.aggregations || rescoreBuilders != this.rescoreBuilders || sorts != this.sorts || this.highlightBuilder != highlightBuilder; if (rewritten) { - return shallowCopy(queryBuilder, postQueryBuilder, aggregations, this.sliceBuilder, sorts, rescoreBuilders, highlightBuilder); + return shallowCopy( + queryBuilder, + postQueryBuilder, + knnSearch, + aggregations, + this.sliceBuilder, + sorts, + rescoreBuilders, + highlightBuilder + ); } return this; } @@ -1048,7 +1088,7 @@ public SearchSourceBuilder rewrite(QueryRewriteContext context) throws IOExcepti * Create a shallow copy of this builder with a new slice configuration. */ public SearchSourceBuilder shallowCopy() { - return shallowCopy(queryBuilder, postQueryBuilder, aggregations, sliceBuilder, sorts, rescoreBuilders, highlightBuilder); + return shallowCopy(queryBuilder, postQueryBuilder, knnSearch, aggregations, sliceBuilder, sorts, rescoreBuilders, highlightBuilder); } /** @@ -1059,6 +1099,7 @@ public SearchSourceBuilder shallowCopy() { private SearchSourceBuilder shallowCopy( QueryBuilder queryBuilder, QueryBuilder postQueryBuilder, + KnnSearchBuilder knnSearch, AggregatorFactories.Builder aggregations, SliceBuilder slice, List> sorts, @@ -1078,6 +1119,7 @@ private SearchSourceBuilder shallowCopy( rewrittenBuilder.indexBoosts = indexBoosts; rewrittenBuilder.minScore = minScore; rewrittenBuilder.postQueryBuilder = postQueryBuilder; + rewrittenBuilder.knnSearch = knnSearch; rewrittenBuilder.profile = profile; rewrittenBuilder.queryBuilder = queryBuilder; rewrittenBuilder.rescoreBuilders = rescoreBuilders; @@ -1184,6 +1226,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th queryBuilder = parseInnerQueryBuilder(parser); } else if (POST_FILTER_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { postQueryBuilder = parseInnerQueryBuilder(parser); + } else if (KNN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + knnSearch = KnnSearchBuilder.fromXContent(parser); } else if (_SOURCE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { fetchSourceContext = FetchSourceContext.fromXContent(parser); } else if (SCRIPT_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { @@ -1354,6 +1398,12 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t builder.field(POST_FILTER_FIELD.getPreferredName(), postQueryBuilder); } + if (knnSearch != null) { + builder.startObject(KNN_FIELD.getPreferredName()); + knnSearch.toXContent(builder, params); + builder.endObject(); + } + if (minScore != null) { builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore); } @@ -1729,6 +1779,7 @@ public int hashCode() { minScore, postQueryBuilder, queryBuilder, + knnSearch, rescoreBuilders, scriptFields, size, @@ -1772,6 +1823,7 @@ public boolean equals(Object obj) { && Objects.equals(minScore, other.minScore) && Objects.equals(postQueryBuilder, other.postQueryBuilder) && Objects.equals(queryBuilder, other.queryBuilder) + && Objects.equals(knnSearch, other.knnSearch) && Objects.equals(rescoreBuilders, other.rescoreBuilders) && Objects.equals(scriptFields, other.scriptFields) && Objects.equals(size, other.size) diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsKnnResults.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsKnnResults.java new file mode 100644 index 0000000000000..5d68e520278bf --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsKnnResults.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.dfs; + +import org.apache.lucene.search.ScoreDoc; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.lucene.Lucene; + +import java.io.IOException; + +public class DfsKnnResults implements Writeable { + private final ScoreDoc[] scoreDocs; + + public DfsKnnResults(ScoreDoc[] scoreDocs) { + this.scoreDocs = scoreDocs; + } + + public DfsKnnResults(StreamInput in) throws IOException { + scoreDocs = in.readArray(Lucene::readScoreDoc, ScoreDoc[]::new); + } + + public ScoreDoc[] scoreDocs() { + return scoreDocs; + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeArray(Lucene::writeScoreDoc, scoreDocs); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java index 06602ff0b087a..5e35ae17e2199 100644 --- a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java @@ -14,8 +14,14 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.TopDocs; +import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.rescore.RescoreContext; +import org.elasticsearch.search.vectors.KnnSearchBuilder; import org.elasticsearch.tasks.TaskCancelledException; import java.io.IOException; @@ -23,8 +29,11 @@ import java.util.Map; /** - * Dfs phase of a search request, used to make scoring 100% accurate by collecting additional info from each shard before the query phase. - * The additional information is used to better compare the scores coming from all the shards, which depend on local factors (e.g. idf) + * DFS phase of a search request, used to make scoring 100% accurate by collecting additional info from each shard before the query phase. + * The additional information is used to better compare the scores coming from all the shards, which depend on local factors (e.g. idf). + * + * When a kNN search is provided alongside the query, the DFS phase is also used to gather the top k candidates from each shard. Then the + * global top k hits are passed on to the query phase. */ public class DfsPhase { @@ -75,6 +84,20 @@ public CollectionStatistics collectionStatistics(String field) throws IOExceptio .termsStatistics(terms, termStatistics) .fieldStatistics(fieldStatistics) .maxDoc(context.searcher().getIndexReader().maxDoc()); + + // If kNN search is requested, perform kNN query and gather top docs + SearchSourceBuilder source = context.request().source(); + if (source != null && source.knnSearch() != null) { + SearchExecutionContext searchExecutionContext = context.getSearchExecutionContext(); + KnnSearchBuilder knnSearch = source.knnSearch(); + + QueryBuilder queryBuilder = knnSearch.toQueryBuilder(); + ParsedQuery query = searchExecutionContext.toQuery(queryBuilder); + + TopDocs topDocs = searcher.search(query.query(), knnSearch.k()); + DfsKnnResults knnResults = new DfsKnnResults(topDocs.scoreDocs); + context.dfsResult().knnResults(knnResults); + } } catch (Exception e) { throw new DfsPhaseExecutionException(context.shardTarget(), "Exception during dfs phase", e); } diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java index 2094ec5fc5e75..ffa5f91f5b634 100644 --- a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java @@ -31,6 +31,7 @@ public class DfsSearchResult extends SearchPhaseResult { private Term[] terms; private TermStatistics[] termStatistics; private Map fieldStatistics = new HashMap<>(); + private DfsKnnResults knnResults; private int maxDoc; public DfsSearchResult(StreamInput in) throws IOException { @@ -52,6 +53,9 @@ public DfsSearchResult(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_7_10_0)) { setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new)); } + if (in.getVersion().onOrAfter(Version.V_8_4_0)) { + knnResults = in.readOptionalWriteable(DfsKnnResults::new); + } } public DfsSearchResult(ShardSearchContextId contextId, SearchShardTarget shardTarget, ShardSearchRequest shardSearchRequest) { @@ -80,6 +84,11 @@ public DfsSearchResult fieldStatistics(Map fieldSt return this; } + public DfsSearchResult knnResults(DfsKnnResults knnResults) { + this.knnResults = knnResults; + return this; + } + public Term[] terms() { return terms; } @@ -92,6 +101,10 @@ public Map fieldStatistics() { return fieldStatistics; } + public DfsKnnResults knnResults() { + return knnResults; + } + @Override public void writeTo(StreamOutput out) throws IOException { contextId.writeTo(out); @@ -105,6 +118,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_10_0)) { out.writeOptionalWriteable(getShardSearchRequest()); } + if (out.getVersion().onOrAfter(Version.V_8_4_0)) { + out.writeOptionalWriteable(knnResults); + } } public static void writeFieldStats(StreamOutput out, Map fieldStatistics) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java new file mode 100644 index 0000000000000..405c3c061a418 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java @@ -0,0 +1,210 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.vectors; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +/** + * A query that matches the provided docs with their scores. + * + * Note: this query was adapted from Lucene's DocAndScoreQuery from the class + * {@link org.apache.lucene.search.KnnVectorQuery}, which is package-private. + * There are no changes to the behavior, just some renames. + */ +class KnnScoreDocQuery extends Query { + private final int[] docs; + private final float[] scores; + private final int[] segmentStarts; + private final Object contextIdentity; + + /** + * Creates a query. + * + * @param docs the global doc IDs of documents that match, in ascending order + * @param scores the scores of the matching documents + * @param segmentStarts the indexes in docs and scores corresponding to the first matching + * document in each segment. If a segment has no matching documents, it should be assigned + * the index of the next segment that does. There should be a final entry that is always + * docs.length-1. + * @param contextIdentity an object identifying the reader context that was used to build this + * query + */ + KnnScoreDocQuery(int[] docs, float[] scores, int[] segmentStarts, Object contextIdentity) { + this.docs = docs; + this.scores = scores; + this.segmentStarts = segmentStarts; + this.contextIdentity = contextIdentity; + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + if (docs.length == 0) { + return new MatchNoDocsQuery(); + } + return this; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + if (searcher.getIndexReader().getContext().id() != contextIdentity) { + throw new IllegalStateException("This KnnScoreDocQuery was created by a different reader"); + } + return new Weight(this) { + @Override + public Explanation explain(LeafReaderContext context, int doc) { + int found = Arrays.binarySearch(docs, doc); + if (found < 0) { + return Explanation.noMatch("not in top k documents"); + } + return Explanation.match(scores[found] * boost, "within top k documents"); + } + + @Override + public Scorer scorer(LeafReaderContext context) { + + return new Scorer(this) { + final int lower = segmentStarts[context.ord]; + final int upper = segmentStarts[context.ord + 1]; + int upTo = -1; + + @Override + public DocIdSetIterator iterator() { + return new DocIdSetIterator() { + @Override + public int docID() { + return currentDocId(); + } + + @Override + public int nextDoc() { + if (upTo == -1) { + upTo = lower; + } else { + ++upTo; + } + return currentDocId(); + } + + @Override + public int advance(int target) throws IOException { + return slowAdvance(target); + } + + @Override + public long cost() { + return upper - lower; + } + }; + } + + @Override + public float getMaxScore(int docId) { + docId += context.docBase; + float maxScore = 0; + for (int idx = Math.max(0, upTo); idx < upper && docs[idx] <= docId; idx++) { + maxScore = Math.max(maxScore, scores[idx] * boost); + } + return maxScore; + } + + @Override + public float score() { + return scores[upTo] * boost; + } + + @Override + public int advanceShallow(int docId) { + int start = Math.max(upTo, lower); + int docIdIndex = Arrays.binarySearch(docs, start, upper, docId + context.docBase); + if (docIdIndex < 0) { + docIdIndex = -1 - docIdIndex; + } + if (docIdIndex >= upper) { + return NO_MORE_DOCS; + } + return docs[docIdIndex]; + } + + @Override + public int docID() { + return currentDocId(); + } + + private int currentDocId() { + if (upTo == -1) { + return -1; + } + if (upTo >= upper) { + return NO_MORE_DOCS; + } + return docs[upTo] - context.docBase; + } + + }; + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } + }; + } + + // visible for testing + int[] docs() { + return docs; + } + + // visible for testing + float[] scores() { + return scores; + } + + @Override + public String toString(String field) { + return "ScoreAndDocQuery"; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + + @Override + public boolean equals(Object obj) { + if (sameClassAs(obj) == false) { + return false; + } + return Arrays.equals(docs, ((KnnScoreDocQuery) obj).docs) + && Arrays.equals(scores, ((KnnScoreDocQuery) obj).scores) + && Arrays.equals(segmentStarts, ((KnnScoreDocQuery) obj).segmentStarts) + && contextIdentity == ((KnnScoreDocQuery) obj).contextIdentity; + } + + @Override + public int hashCode() { + return Objects.hash(classHash(), Arrays.hashCode(docs), Arrays.hashCode(scores), Arrays.hashCode(segmentStarts), contextIdentity); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java new file mode 100644 index 0000000000000..20dc8e7ce82de --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.vectors; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.MatchNoneQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +/** + * A query that matches the provided docs with their scores. This query is used + * when executing a kNN search during the search query phase, to include the documents + * that matched the initial kNN query during the DFS phase. + */ +public class KnnScoreDocQueryBuilder extends AbstractQueryBuilder { + public static final String NAME = "knn_score_doc"; + private final ScoreDoc[] scoreDocs; + + /** + * Creates a query builder. + * + * @param scoreDocs the docs and scores this query should match. The array must be + * sorted in order of ascending doc IDs. + */ + public KnnScoreDocQueryBuilder(ScoreDoc[] scoreDocs) { + this.scoreDocs = scoreDocs; + } + + public KnnScoreDocQueryBuilder(StreamInput in) throws IOException { + super(in); + this.scoreDocs = in.readArray(Lucene::readScoreDoc, ScoreDoc[]::new); + } + + @Override + public String getWriteableName() { + return NAME; + } + + public ScoreDoc[] scoreDocs() { + return scoreDocs; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeArray(Lucene::writeScoreDoc, scoreDocs); + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.startArray("values"); + for (ScoreDoc scoreDoc : scoreDocs) { + builder.startObject().field("doc", scoreDoc.doc).field("score", scoreDoc.score).endObject(); + } + builder.endArray(); + boostAndQueryNameToXContent(builder); + builder.endObject(); + } + + @Override + protected Query doToQuery(SearchExecutionContext context) throws IOException { + int numDocs = scoreDocs.length; + int[] docs = new int[numDocs]; + float[] scores = new float[numDocs]; + for (int i = 0; i < numDocs; i++) { + docs[i] = scoreDocs[i].doc; + scores[i] = scoreDocs[i].score; + } + + IndexReader reader = context.getIndexReader(); + int[] segmentStarts = findSegmentStarts(reader, docs); + return new KnnScoreDocQuery(docs, scores, segmentStarts, reader.getContext().id()); + } + + @Override + protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + if (scoreDocs.length == 0) { + return new MatchNoneQueryBuilder(); + } + return super.doRewrite(queryRewriteContext); + } + + private int[] findSegmentStarts(IndexReader reader, int[] docs) { + int[] starts = new int[reader.leaves().size() + 1]; + starts[starts.length - 1] = docs.length; + if (starts.length == 2) { + return starts; + } + int resultIndex = 0; + for (int i = 1; i < starts.length - 1; i++) { + int upper = reader.leaves().get(i).docBase; + resultIndex = Arrays.binarySearch(docs, resultIndex, docs.length, upper); + if (resultIndex < 0) { + resultIndex = -1 - resultIndex; + } + starts[i] = resultIndex; + } + return starts; + } + + @Override + protected boolean doEquals(KnnScoreDocQueryBuilder other) { + if (scoreDocs.length != other.scoreDocs.length) { + return false; + } + + for (int i = 0; i < scoreDocs.length; i++) { + ScoreDoc scoreDoc = scoreDocs[i]; + ScoreDoc otherScoreDoc = other.scoreDocs[i]; + + if ((scoreDoc.doc == otherScoreDoc.doc + && scoreDoc.score == otherScoreDoc.score + && scoreDoc.shardIndex == otherScoreDoc.shardIndex) == false) { + return false; + } + } + return true; + } + + @Override + protected int doHashCode() { + int result = 1; + for (ScoreDoc scoreDoc : scoreDocs) { + int hashCode = Objects.hash(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); + result = 31 * result + hashCode; + } + return result; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_8_4_0; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java new file mode 100644 index 0000000000000..f738b6108101c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -0,0 +1,210 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.vectors; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.Rewriteable; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; + +/** + * Defines a kNN search to run in the search request. + */ +public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewriteable { + private static final int NUM_CANDS_LIMIT = 10000; + static final ParseField FIELD_FIELD = new ParseField("field"); + static final ParseField K_FIELD = new ParseField("k"); + static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates"); + static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector"); + static final ParseField FILTER_FIELD = new ParseField("filter"); + static final ParseField BOOST_FIELD = AbstractQueryBuilder.BOOST_FIELD; + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("knn", args -> { + @SuppressWarnings("unchecked") + List vector = (List) args[1]; + float[] vectorArray = new float[vector.size()]; + for (int i = 0; i < vector.size(); i++) { + vectorArray[i] = vector.get(i); + } + return new KnnSearchBuilder((String) args[0], vectorArray, (int) args[2], (int) args[3]); + }); + + static { + PARSER.declareString(constructorArg(), FIELD_FIELD); + PARSER.declareFloatArray(constructorArg(), QUERY_VECTOR_FIELD); + PARSER.declareInt(constructorArg(), K_FIELD); + PARSER.declareInt(constructorArg(), NUM_CANDS_FIELD); + PARSER.declareFieldArray( + KnnSearchBuilder::addFilterQueries, + (p, c) -> AbstractQueryBuilder.parseInnerQueryBuilder(p), + FILTER_FIELD, + ObjectParser.ValueType.OBJECT_ARRAY + ); + PARSER.declareFloat(KnnSearchBuilder::boost, BOOST_FIELD); + } + + public static KnnSearchBuilder fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + final String field; + final float[] queryVector; + final int k; + final int numCands; + final List filterQueries; + float boost = AbstractQueryBuilder.DEFAULT_BOOST; + + /** + * Defines a kNN search. + * + * @param field the name of the vector field to search against + * @param queryVector the query vector + * @param k the final number of nearest neighbors to return as top hits + * @param numCands the number of nearest neighbor candidates to consider per shard + */ + public KnnSearchBuilder(String field, float[] queryVector, int k, int numCands) { + if (k < 1) { + throw new IllegalArgumentException("[" + K_FIELD.getPreferredName() + "] must be greater than 0"); + } + if (numCands < k) { + throw new IllegalArgumentException( + "[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot be less than " + "[" + K_FIELD.getPreferredName() + "]" + ); + } + if (numCands > NUM_CANDS_LIMIT) { + throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); + } + this.field = field; + this.queryVector = queryVector; + this.k = k; + this.numCands = numCands; + this.filterQueries = new ArrayList<>(); + } + + public KnnSearchBuilder(StreamInput in) throws IOException { + this.field = in.readString(); + this.k = in.readVInt(); + this.numCands = in.readVInt(); + this.queryVector = in.readFloatArray(); + this.filterQueries = in.readNamedWriteableList(QueryBuilder.class); + this.boost = in.readFloat(); + } + + public int k() { + return k; + } + + public KnnSearchBuilder addFilterQuery(QueryBuilder filterQuery) { + Objects.requireNonNull(filterQuery); + this.filterQueries.add(filterQuery); + return this; + } + + public KnnSearchBuilder addFilterQueries(List filterQueries) { + Objects.requireNonNull(filterQueries); + this.filterQueries.addAll(filterQueries); + return this; + } + + /** + * Set a boost to apply to the kNN search scores. + */ + public KnnSearchBuilder boost(float boost) { + this.boost = boost; + return this; + } + + @Override + public KnnSearchBuilder rewrite(QueryRewriteContext ctx) throws IOException { + boolean changed = false; + List rewrittenQueries = new ArrayList<>(filterQueries.size()); + for (QueryBuilder query : filterQueries) { + QueryBuilder rewrittenQuery = query.rewrite(ctx); + if (rewrittenQuery != query) { + changed = true; + } + rewrittenQueries.add(rewrittenQuery); + } + if (changed) { + return new KnnSearchBuilder(field, queryVector, k, numCands).boost(boost).addFilterQueries(rewrittenQueries); + } + return this; + } + + public KnnVectorQueryBuilder toQueryBuilder() { + return new KnnVectorQueryBuilder(field, queryVector, numCands).boost(boost).addFilterQueries(filterQueries); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KnnSearchBuilder that = (KnnSearchBuilder) o; + return k == that.k + && numCands == that.numCands + && Objects.equals(field, that.field) + && Arrays.equals(queryVector, that.queryVector) + && Objects.equals(filterQueries, that.filterQueries) + && boost == that.boost; + } + + @Override + public int hashCode() { + return Objects.hash(field, k, numCands, Arrays.hashCode(queryVector), Objects.hashCode(filterQueries), boost); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(FIELD_FIELD.getPreferredName(), field) + .field(K_FIELD.getPreferredName(), k) + .field(NUM_CANDS_FIELD.getPreferredName(), numCands) + .array(QUERY_VECTOR_FIELD.getPreferredName(), queryVector); + + if (filterQueries.isEmpty() == false) { + builder.startArray(FILTER_FIELD.getPreferredName()); + for (QueryBuilder filterQuery : filterQueries) { + filterQuery.toXContent(builder, params); + } + builder.endArray(); + } + + if (boost != AbstractQueryBuilder.DEFAULT_BOOST) { + builder.field(BOOST_FIELD.getPreferredName(), boost); + } + + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(field); + out.writeVInt(k); + out.writeVInt(numCands); + out.writeFloatArray(queryVector); + out.writeNamedWriteableList(filterQueries); + out.writeFloat(boost); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java similarity index 91% rename from server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestBuilder.java rename to server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java index 282ff0aebe56c..4c4ddb62b4bb7 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchRequestParser.java @@ -35,19 +35,19 @@ * A builder used in {@link RestKnnSearchAction} to convert the kNN REST request * into a {@link SearchRequestBuilder}. */ -public class KnnSearchRequestBuilder { +public class KnnSearchRequestParser { static final String INDEX_PARAM = "index"; static final String ROUTING_PARAM = "routing"; static final ParseField KNN_SECTION_FIELD = new ParseField("knn"); static final ParseField FILTER_FIELD = new ParseField("filter"); - private static final ObjectParser PARSER; + private static final ObjectParser PARSER; static { PARSER = new ObjectParser<>("knn-search"); - PARSER.declareField(KnnSearchRequestBuilder::knnSearch, KnnSearch::parse, KNN_SECTION_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareField(KnnSearchRequestParser::knnSearch, KnnSearch::parse, KNN_SECTION_FIELD, ObjectParser.ValueType.OBJECT); PARSER.declareFieldArray( - KnnSearchRequestBuilder::filter, + KnnSearchRequestParser::filter, (p, c) -> AbstractQueryBuilder.parseInnerQueryBuilder(p), FILTER_FIELD, ObjectParser.ValueType.OBJECT_ARRAY @@ -58,13 +58,13 @@ public class KnnSearchRequestBuilder { ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING ); PARSER.declareFieldArray( - KnnSearchRequestBuilder::fields, + KnnSearchRequestParser::fields, (p, c) -> FieldAndFormat.fromXContent(p), SearchSourceBuilder.FETCH_FIELDS_FIELD, ObjectParser.ValueType.OBJECT_ARRAY ); PARSER.declareFieldArray( - KnnSearchRequestBuilder::docValueFields, + KnnSearchRequestParser::docValueFields, (p, c) -> FieldAndFormat.fromXContent(p), SearchSourceBuilder.DOCVALUE_FIELDS_FIELD, ObjectParser.ValueType.OBJECT_ARRAY @@ -81,8 +81,8 @@ public class KnnSearchRequestBuilder { /** * Parses a {@link RestRequest} representing a kNN search into a request builder. */ - public static KnnSearchRequestBuilder parseRestRequest(RestRequest restRequest) throws IOException { - KnnSearchRequestBuilder builder = new KnnSearchRequestBuilder(Strings.splitStringByCommaToArray(restRequest.param("index"))); + public static KnnSearchRequestParser parseRestRequest(RestRequest restRequest) throws IOException { + KnnSearchRequestParser builder = new KnnSearchRequestParser(Strings.splitStringByCommaToArray(restRequest.param("index"))); builder.routing(restRequest.param("routing")); if (restRequest.hasContentOrSourceParam()) { @@ -103,7 +103,7 @@ public static KnnSearchRequestBuilder parseRestRequest(RestRequest restRequest) private List docValueFields; private StoredFieldsContext storedFields; - private KnnSearchRequestBuilder(String[] indices) { + private KnnSearchRequestParser(String[] indices) { this.indices = indices; } @@ -157,7 +157,7 @@ private void storedFields(StoredFieldsContext storedFields) { /** * Adds all the request components to the given {@link SearchRequestBuilder}. */ - public void build(SearchRequestBuilder builder) { + public void toSearchRequest(SearchRequestBuilder builder) { builder.setIndices(indices); builder.setRouting(routing); @@ -168,7 +168,7 @@ public void build(SearchRequestBuilder builder) { throw new IllegalArgumentException("missing required [" + KNN_SECTION_FIELD.getPreferredName() + "] section in search body"); } - KnnVectorQueryBuilder queryBuilder = knnSearch.buildQuery(); + KnnVectorQueryBuilder queryBuilder = knnSearch.toQueryBuilder(); if (filters != null) { queryBuilder.addFilterQueries(this.filters); } @@ -241,7 +241,7 @@ public static KnnSearch parse(XContentParser parser) throws IOException { this.numCands = numCands; } - public KnnVectorQueryBuilder buildQuery() { + public KnnVectorQueryBuilder toQueryBuilder() { // We perform validation here instead of the constructor because it makes the errors // much clearer. Otherwise, the error message is deeply nested under parsing exceptions. if (k < 1) { @@ -255,7 +255,6 @@ public KnnVectorQueryBuilder buildQuery() { if (numCands > NUM_CANDS_LIMIT) { throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } - return new KnnVectorQueryBuilder(field, queryVector, numCands); } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index dd977a10225c0..116ef025c15ef 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -104,6 +104,14 @@ protected void doWriteTo(StreamOutput out) throws IOException { @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME).field("field", fieldName).field("vector", queryVector).field("num_candidates", numCands); + if (filterQueries.isEmpty() == false) { + builder.startArray("filters"); + for (QueryBuilder filterQuery : filterQueries) { + filterQuery.toXContent(builder, params); + } + builder.endArray(); + } + builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 2850dbacf73aa..09bce137ab8af 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -52,6 +52,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -225,7 +226,24 @@ private void startNewSnapshots(List snapshotsInProgre } if (startedShards != null && startedShards.isEmpty() == false) { shardSnapshots.computeIfAbsent(snapshot, s -> new HashMap<>()).putAll(startedShards); - startNewShards(entry, startedShards); + + final List shardSnapshotTasks = new ArrayList<>(startedShards.size()); + for (final Map.Entry shardEntry : startedShards.entrySet()) { + final ShardId shardId = shardEntry.getKey(); + final IndexShardSnapshotStatus snapshotStatus = shardEntry.getValue(); + final IndexId indexId = entry.indices().get(shardId.getIndexName()); + assert indexId != null; + assert SnapshotsService.useShardGenerations(entry.version()) + || ShardGenerations.fixShardGeneration(snapshotStatus.generation()) == null + : "Found non-null, non-numeric shard generation [" + + snapshotStatus.generation() + + "] for snapshot with old-format compatibility"; + shardSnapshotTasks.add( + newShardSnapshotTask(shardId, snapshot, indexId, entry.userMetadata(), snapshotStatus, entry.version()) + ); + } + + threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> shardSnapshotTasks.forEach(Runnable::run)); } } else if (entryState == State.ABORTED) { // Abort all running shards for this snapshot @@ -248,53 +266,47 @@ private void startNewSnapshots(List snapshotsInProgre } } - private void startNewShards(SnapshotsInProgress.Entry entry, Map startedShards) { - threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> { - final Snapshot snapshot = entry.snapshot(); - for (final Map.Entry shardEntry : startedShards.entrySet()) { - final ShardId shardId = shardEntry.getKey(); - final IndexShardSnapshotStatus snapshotStatus = shardEntry.getValue(); - final IndexId indexId = entry.indices().get(shardId.getIndexName()); - assert indexId != null; - assert SnapshotsService.useShardGenerations(entry.version()) - || ShardGenerations.fixShardGeneration(snapshotStatus.generation()) == null - : "Found non-null, non-numeric shard generation [" - + snapshotStatus.generation() - + "] for snapshot with old-format compatibility"; - snapshot(shardId, snapshot, indexId, entry.userMetadata(), snapshotStatus, entry.version(), new ActionListener<>() { - @Override - public void onResponse(ShardSnapshotResult shardSnapshotResult) { - final ShardGeneration newGeneration = shardSnapshotResult.getGeneration(); - assert newGeneration != null; - assert newGeneration.equals(snapshotStatus.generation()); - if (logger.isDebugEnabled()) { - final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy(); - logger.debug( - "[{}][{}] completed snapshot to [{}] with status [{}] at generation [{}]", - shardId, - snapshot, - snapshot.getRepository(), - lastSnapshotStatus, - snapshotStatus.generation() - ); - } - notifySuccessfulSnapshotShard(snapshot, shardId, shardSnapshotResult); - } + private Runnable newShardSnapshotTask( + final ShardId shardId, + final Snapshot snapshot, + final IndexId indexId, + final Map userMetadata, + final IndexShardSnapshotStatus snapshotStatus, + final Version entryVersion + ) { + // separate method to make sure this lambda doesn't capture any heavy local objects like a SnapshotsInProgress.Entry + return () -> snapshot(shardId, snapshot, indexId, userMetadata, snapshotStatus, entryVersion, new ActionListener<>() { + @Override + public void onResponse(ShardSnapshotResult shardSnapshotResult) { + final ShardGeneration newGeneration = shardSnapshotResult.getGeneration(); + assert newGeneration != null; + assert newGeneration.equals(snapshotStatus.generation()); + if (logger.isDebugEnabled()) { + final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy(); + logger.debug( + "[{}][{}] completed snapshot to [{}] with status [{}] at generation [{}]", + shardId, + snapshot, + snapshot.getRepository(), + lastSnapshotStatus, + snapshotStatus.generation() + ); + } + notifySuccessfulSnapshotShard(snapshot, shardId, shardSnapshotResult); + } - @Override - public void onFailure(Exception e) { - final String failure; - if (e instanceof AbortedSnapshotException) { - failure = "aborted"; - logger.debug(() -> format("[%s][%s] aborted shard snapshot", shardId, snapshot), e); - } else { - failure = summarizeFailure(e); - logger.warn(() -> format("[%s][%s] failed to snapshot shard", shardId, snapshot), e); - } - snapshotStatus.moveToFailed(threadPool.absoluteTimeInMillis(), failure); - notifyFailedSnapshotShard(snapshot, shardId, failure, snapshotStatus.generation()); - } - }); + @Override + public void onFailure(Exception e) { + final String failure; + if (e instanceof AbortedSnapshotException) { + failure = "aborted"; + logger.debug(() -> format("[%s][%s] aborted shard snapshot", shardId, snapshot), e); + } else { + failure = summarizeFailure(e); + logger.warn(() -> format("[%s][%s] failed to snapshot shard", shardId, snapshot), e); + } + snapshotStatus.moveToFailed(threadPool.absoluteTimeInMillis(), failure); + notifyFailedSnapshotShard(snapshot, shardId, failure, snapshotStatus.generation()); } }); } diff --git a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java index 8eeb7b2ce6e04..d9f9bad00522d 100644 --- a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java +++ b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.util.Constants; import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.RoutingMissingException; +import org.elasticsearch.action.search.ReduceSearchPhaseException; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; @@ -202,6 +203,104 @@ public void testGuessRootCause() { } } + public void testReduceSearchPhaseExceptionWithNoShardFailuresAndNoCause() throws IOException { + final ReduceSearchPhaseException ex = new ReduceSearchPhaseException( + "search", + "no shard failure", + null, + ShardSearchFailure.EMPTY_ARRAY + ); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + ex.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + String expected = """ + { + "type": "reduce_search_phase_exception", + "reason": "[reduce] no shard failure", + "phase": "search", + "grouped": true, + "failed_shards": [] + }"""; + assertEquals(XContentHelper.stripWhitespace(expected), Strings.toString(builder)); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), ex.status().getStatus()); + } + + public void testReduceSearchPhaseExceptionWithNoShardFailuresAndCause() throws IOException { + final ReduceSearchPhaseException ex = new ReduceSearchPhaseException( + "search", + "no shard failure", + new IllegalArgumentException("illegal argument"), + ShardSearchFailure.EMPTY_ARRAY + ); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + ex.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + String expected = """ + { + "type": "reduce_search_phase_exception", + "reason": "[reduce] no shard failure", + "phase": "search", + "grouped": true, + "failed_shards": [], + "caused_by":{"type":"illegal_argument_exception","reason":"illegal argument"} + }"""; + assertEquals(XContentHelper.stripWhitespace(expected), Strings.toString(builder)); + assertEquals(RestStatus.BAD_REQUEST.getStatus(), ex.status().getStatus()); + } + + public void testSearchPhaseExecutionExceptionWithNoShardFailuresAndNoCause() throws IOException { + final SearchPhaseExecutionException ex = new SearchPhaseExecutionException( + "search", + "no shard failure", + null, + ShardSearchFailure.EMPTY_ARRAY + ); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + ex.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + String expected = """ + { + "type": "search_phase_execution_exception", + "reason": "no shard failure", + "phase": "search", + "grouped": true, + "failed_shards": [] + }"""; + assertEquals(XContentHelper.stripWhitespace(expected), Strings.toString(builder)); + assertEquals(RestStatus.SERVICE_UNAVAILABLE.getStatus(), ex.status().getStatus()); + } + + public void testReduceSearchPhaseExecutionExceptionWithNoShardFailuresAndCause() throws IOException { + final SearchPhaseExecutionException ex = new SearchPhaseExecutionException( + "search", + "no shard failure", + new IllegalArgumentException("illegal argument"), + ShardSearchFailure.EMPTY_ARRAY + ); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + ex.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + String expected = """ + { + "type": "search_phase_execution_exception", + "reason": "no shard failure", + "phase": "search", + "grouped": true, + "failed_shards": [], + "caused_by":{"type":"illegal_argument_exception","reason":"illegal argument"} + }"""; + assertEquals(XContentHelper.stripWhitespace(expected), Strings.toString(builder)); + assertEquals(RestStatus.BAD_REQUEST.getStatus(), ex.status().getStatus()); + } + public void testDeduplicate() throws IOException { { ShardSearchFailure failure = new ShardSearchFailure( diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java index 57f51457aa403..84022bc234436 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java @@ -12,7 +12,7 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; -import org.apache.lucene.codecs.lucene93.Lucene93Codec; +import org.apache.lucene.codecs.lucene92.Lucene92Codec; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.document.BinaryDocValuesField; @@ -297,7 +297,7 @@ public void testTriangle() throws Exception { public void testCompletionField() throws Exception { IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true) .setUseCompoundFile(false) - .setCodec(new Lucene93Codec(Lucene93Codec.Mode.BEST_SPEED) { + .setCodec(new Lucene92Codec(Lucene92Codec.Mode.BEST_SPEED) { @Override public PostingsFormat getPostingsFormatForField(String field) { if (field.startsWith("suggest_")) { @@ -384,25 +384,25 @@ private static void addFieldsToDoc(Document doc, IndexableField[] fields) { enum CodecMode { BEST_SPEED { @Override - Lucene93Codec.Mode mode() { - return Lucene93Codec.Mode.BEST_SPEED; + Lucene92Codec.Mode mode() { + return Lucene92Codec.Mode.BEST_SPEED; } }, BEST_COMPRESSION { @Override - Lucene93Codec.Mode mode() { - return Lucene93Codec.Mode.BEST_COMPRESSION; + Lucene92Codec.Mode mode() { + return Lucene92Codec.Mode.BEST_COMPRESSION; } }; - abstract Lucene93Codec.Mode mode(); + abstract Lucene92Codec.Mode mode(); } static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer addFields) throws IOException { IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true) .setUseCompoundFile(randomBoolean()) - .setCodec(new Lucene93Codec(codecMode.mode())); + .setCodec(new Lucene92Codec(codecMode.mode())); try (IndexWriter writer = new IndexWriter(directory, config)) { for (int i = 0; i < numDocs; i++) { final Document doc = new Document(); @@ -580,7 +580,7 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire try (DirectoryReader reader = DirectoryReader.open(source)) { IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) .setUseCompoundFile(randomBoolean()) - .setCodec(new Lucene93Codec(mode.mode()) { + .setCodec(new Lucene92Codec(mode.mode()) { @Override public PostingsFormat getPostingsFormatForField(String field) { return new Lucene90PostingsFormat(); diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java index f9132d7a4ae8d..a04916438b943 100644 --- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java @@ -23,7 +23,9 @@ import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.dfs.DfsSearchResult; +import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ShardSearchContextId; +import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.query.QuerySearchRequest; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.test.ESTestCase; @@ -39,6 +41,7 @@ public class DfsQueryPhaseTests extends ESTestCase { private static DfsSearchResult newSearchResult(int shardIndex, ShardSearchContextId contextId, SearchShardTarget target) { DfsSearchResult result = new DfsSearchResult(contextId, target, null); result.setShardIndex(shardIndex); + result.setShardSearchRequest(new ShardSearchRequest(target.getShardId(), System.currentTimeMillis(), AliasFilter.EMPTY)); return result; } @@ -111,7 +114,7 @@ public void sendExecuteQuery( results.length(), exc -> {} ); - DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, consumer, (response) -> new SearchPhase("test") { + DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, null, consumer, (response) -> new SearchPhase("test") { @Override public void run() throws IOException { responseRef.set(response.results); @@ -189,7 +192,7 @@ public void sendExecuteQuery( results.length(), exc -> {} ); - DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, consumer, (response) -> new SearchPhase("test") { + DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, null, consumer, (response) -> new SearchPhase("test") { @Override public void run() throws IOException { responseRef.set(response.results); @@ -269,7 +272,7 @@ public void sendExecuteQuery( results.length(), exc -> {} ); - DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, consumer, (response) -> new SearchPhase("test") { + DfsQueryPhase phase = new DfsQueryPhase(results.asList(), null, null, consumer, (response) -> new SearchPhase("test") { @Override public void run() throws IOException { responseRef.set(response.results); diff --git a/server/src/test/java/org/elasticsearch/action/search/KnnSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/KnnSearchActionTests.java deleted file mode 100644 index ac77ddb1fdce1..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/search/KnnSearchActionTests.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.search; - -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; - -import java.io.IOException; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; - -public class KnnSearchActionTests extends ESSingleNodeTestCase { - private static final int VECTOR_DIMENSION = 10; - - public void testTotalHits() throws IOException { - Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build(); - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject("vector") - .field("type", "dense_vector") - .field("dims", VECTOR_DIMENSION) - .field("index", true) - .field("similarity", "l2_norm") - .endObject() - .endObject() - .endObject(); - createIndex("index1", indexSettings, builder); - createIndex("index2", indexSettings, builder); - - for (int doc = 0; doc < 10; doc++) { - client().prepareIndex("index1").setId(String.valueOf(doc)).setSource("vector", randomVector()).get(); - client().prepareIndex("index2").setId(String.valueOf(doc)).setSource("vector", randomVector()).get(); - } - - client().admin().indices().prepareForceMerge("index1", "index2").setMaxNumSegments(1).get(); - client().admin().indices().prepareRefresh("index1", "index2").get(); - - // Since there's no kNN search action at the transport layer, we just emulate - // how the action works (it builds a kNN query under the hood) - float[] queryVector = randomVector(); - SearchResponse response = client().prepareSearch("index1", "index2") - .setQuery(new KnnVectorQueryBuilder("vector", queryVector, 5)) - .setSize(2) - .get(); - - // The total hits is num_cands * num_shards, since the query gathers num_cands hits from each shard - assertHitCount(response, 5 * 2); - assertEquals(2, response.getHits().getHits().length); - } - - private float[] randomVector() { - float[] vector = new float[VECTOR_DIMENSION]; - for (int i = 0; i < vector.length; i++) { - vector[i] = randomFloat(); - } - return vector; - } -} diff --git a/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java b/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java new file mode 100644 index 0000000000000..b71acb2a2531c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.indices.TermsLookup; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.vectors.KnnSearchBuilder; +import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; + +import java.io.IOException; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; + +public class KnnSearchSingleNodeTests extends ESSingleNodeTestCase { + private static final int VECTOR_DIMENSION = 10; + + public void testKnnWithQuery() throws IOException { + int numShards = 1 + randomInt(3); + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build(); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("vector") + .field("type", "dense_vector") + .field("dims", VECTOR_DIMENSION) + .field("index", true) + .field("similarity", "l2_norm") + .endObject() + .startObject("text") + .field("type", "text") + .endObject() + .endObject() + .endObject(); + createIndex("index", indexSettings, builder); + + for (int doc = 0; doc < 10; doc++) { + client().prepareIndex("index").setSource("vector", randomVector(), "text", "hello world").get(); + client().prepareIndex("index").setSource("text", "goodnight world").get(); + } + + client().admin().indices().prepareRefresh("index").get(); + + float[] queryVector = randomVector(); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50).boost(5.0f); + SearchResponse response = client().prepareSearch("index") + .setKnnSearch(knnSearch) + .setQuery(QueryBuilders.matchQuery("text", "goodnight")) + .addFetchField("*") + .setSize(10) + .get(); + + // The total hits is k plus the number of text matches + assertHitCount(response, 15); + assertEquals(10, response.getHits().getHits().length); + + // Because of the boost, vector results should appear first + assertNotNull(response.getHits().getAt(0).field("vector")); + } + + public void testKnnFilter() throws IOException { + int numShards = 1 + randomInt(3); + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build(); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("vector") + .field("type", "dense_vector") + .field("dims", VECTOR_DIMENSION) + .field("index", true) + .field("similarity", "l2_norm") + .endObject() + .startObject("field") + .field("type", "keyword") + .endObject() + .endObject() + .endObject(); + createIndex("index", indexSettings, builder); + + for (int doc = 0; doc < 10; doc++) { + String value = doc % 2 == 0 ? "first" : "second"; + client().prepareIndex("index").setId(String.valueOf(doc)).setSource("vector", randomVector(), "field", value).get(); + } + + client().admin().indices().prepareRefresh("index").get(); + + float[] queryVector = randomVector(); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50).addFilterQuery( + QueryBuilders.termsQuery("field", "second") + ); + SearchResponse response = client().prepareSearch("index").setKnnSearch(knnSearch).addFetchField("*").setSize(10).get(); + + assertHitCount(response, 5); + assertEquals(5, response.getHits().getHits().length); + for (SearchHit hit : response.getHits().getHits()) { + assertEquals("second", hit.field("field").getValue()); + } + } + + public void testKnnFilterWithRewrite() throws IOException { + int numShards = 1 + randomInt(3); + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build(); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("vector") + .field("type", "dense_vector") + .field("dims", VECTOR_DIMENSION) + .field("index", true) + .field("similarity", "l2_norm") + .endObject() + .startObject("field") + .field("type", "keyword") + .endObject() + .startObject("other-field") + .field("type", "keyword") + .endObject() + .endObject() + .endObject(); + createIndex("index", indexSettings, builder); + + for (int doc = 0; doc < 10; doc++) { + client().prepareIndex("index").setId(String.valueOf(doc)).setSource("vector", randomVector(), "field", "value").get(); + } + client().prepareIndex("index").setId("lookup-doc").setSource("other-field", "value").get(); + + client().admin().indices().prepareRefresh("index").get(); + + float[] queryVector = randomVector(); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50).addFilterQuery( + QueryBuilders.termsLookupQuery("field", new TermsLookup("index", "lookup-doc", "other-field")) + ); + SearchResponse response = client().prepareSearch("index").setKnnSearch(knnSearch).setSize(10).get(); + + assertHitCount(response, 5); + assertEquals(5, response.getHits().getHits().length); + } + + public void testKnnSearchAction() throws IOException { + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build(); + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("vector") + .field("type", "dense_vector") + .field("dims", VECTOR_DIMENSION) + .field("index", true) + .field("similarity", "l2_norm") + .endObject() + .endObject() + .endObject(); + createIndex("index1", indexSettings, builder); + createIndex("index2", indexSettings, builder); + + for (int doc = 0; doc < 10; doc++) { + client().prepareIndex("index1").setId(String.valueOf(doc)).setSource("vector", randomVector()).get(); + client().prepareIndex("index2").setId(String.valueOf(doc)).setSource("vector", randomVector()).get(); + } + + client().admin().indices().prepareForceMerge("index1", "index2").setMaxNumSegments(1).get(); + client().admin().indices().prepareRefresh("index1", "index2").get(); + + // Since there's no kNN search action at the transport layer, we just emulate + // how the action works (it builds a kNN query under the hood) + float[] queryVector = randomVector(); + SearchResponse response = client().prepareSearch("index1", "index2") + .setQuery(new KnnVectorQueryBuilder("vector", queryVector, 5)) + .setSize(2) + .get(); + + // The total hits is num_cands * num_shards, since the query gathers num_cands hits from each shard + assertHitCount(response, 5 * 2); + assertEquals(2, response.getHits().getHits().length); + } + + private float[] randomVector() { + float[] vector = new float[VECTOR_DIMENSION]; + for (int i = 0; i < vector.length; i++) { + vector[i] = randomFloat(); + } + return vector; + } +} diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 6fe576e1b3247..6b0706b811b31 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -8,6 +8,8 @@ package org.elasticsearch.action.search; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.apache.lucene.search.TotalHits; import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; @@ -68,6 +70,9 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchContextId; import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; +import org.elasticsearch.search.vectors.KnnSearchBuilder; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; @@ -1049,6 +1054,7 @@ public void testShouldMinimizeRoundtrips() throws Exception { SearchSourceBuilder source = searchRequest.source(); if (source != null) { source.pointInTimeBuilder(null); + source.knnSearch(null); CollapseBuilder collapse = source.collapse(); if (collapse != null) { collapse.setInnerHits(Collections.emptyList()); @@ -1059,6 +1065,48 @@ public void testShouldMinimizeRoundtrips() throws Exception { searchRequest.setCcsMinimizeRoundtrips(false); assertFalse(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest)); } + { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder source = new SearchSourceBuilder(); + source.knnSearch(new KnnSearchBuilder("field", new float[] { 1, 2, 3 }, 10, 50)); + searchRequest.source(source); + + searchRequest.setCcsMinimizeRoundtrips(true); + assertFalse(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest)); + searchRequest.setCcsMinimizeRoundtrips(false); + assertFalse(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest)); + } + } + + public void testAdjustSearchType() { + { + // If the search includes kNN, we should always use DFS_QUERY_THEN_FETCH + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder source = new SearchSourceBuilder(); + source.knnSearch(new KnnSearchBuilder("field", new float[] { 1, 2, 3 }, 10, 50)); + searchRequest.source(source); + + TransportSearchAction.adjustSearchType(searchRequest, randomBoolean()); + assertEquals(SearchType.DFS_QUERY_THEN_FETCH, searchRequest.searchType()); + } + { + // Suggest-only searches should always use QUERY_THEN_FETCH + SearchRequest searchRequest = new SearchRequest().searchType(RandomPicks.randomFrom(random(), SearchType.values())); + SearchSourceBuilder source = new SearchSourceBuilder(); + source.suggest(new SuggestBuilder().addSuggestion("field", new TermSuggestionBuilder("value"))); + searchRequest.source(source); + + TransportSearchAction.adjustSearchType(searchRequest, randomBoolean()); + assertFalse(searchRequest.requestCache()); + assertEquals(SearchType.QUERY_THEN_FETCH, searchRequest.searchType()); + } + { + // Single-shard searches should always use QUERY_THEN_FETCH in absence of kNN search + SearchRequest searchRequest = new SearchRequest().searchType(RandomPicks.randomFrom(random(), SearchType.values())); + + TransportSearchAction.adjustSearchType(searchRequest, true); + assertEquals(SearchType.QUERY_THEN_FETCH, searchRequest.searchType()); + } } public void testShouldPreFilterSearchShards() { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java index f0a2aa507b185..dbe88e3ae2442 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; import java.util.Locale; @@ -26,36 +27,195 @@ public void testDefaults() { ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); DiskThresholdSettings diskThresholdSettings = new DiskThresholdSettings(Settings.EMPTY, nss); - ByteSizeValue zeroBytes = ByteSizeValue.parseBytesSizeValue("0b", "test"); - assertEquals(zeroBytes, diskThresholdSettings.getFreeBytesThresholdHigh()); - assertEquals(10.0D, diskThresholdSettings.getFreeDiskThresholdHigh(), 0.0D); - assertEquals(zeroBytes, diskThresholdSettings.getFreeBytesThresholdLow()); - assertEquals(15.0D, diskThresholdSettings.getFreeDiskThresholdLow(), 0.0D); assertEquals(60L, diskThresholdSettings.getRerouteInterval().seconds()); assertTrue(diskThresholdSettings.isEnabled()); - assertEquals(zeroBytes, diskThresholdSettings.getFreeBytesThresholdFloodStage()); - assertEquals(5.0D, diskThresholdSettings.getFreeDiskThresholdFloodStage(), 0.0D); + + // Test default watermark percentages + ByteSizeValue hundredBytes = ByteSizeValue.parseBytesSizeValue("100b", "test"); + assertEquals(ByteSizeValue.ofBytes(15), diskThresholdSettings.getFreeBytesThresholdLowStage(hundredBytes)); + assertEquals(ByteSizeValue.ofBytes(10), diskThresholdSettings.getFreeBytesThresholdHighStage(hundredBytes)); + assertEquals(ByteSizeValue.ofBytes(5), diskThresholdSettings.getFreeBytesThresholdFloodStage(hundredBytes)); + assertEquals(ByteSizeValue.ofBytes(5), diskThresholdSettings.getFreeBytesThresholdFrozenFloodStage(hundredBytes)); + assertEquals("85%", diskThresholdSettings.describeLowThreshold(hundredBytes, false)); + assertEquals("90%", diskThresholdSettings.describeHighThreshold(hundredBytes, false)); + assertEquals("95%", diskThresholdSettings.describeFloodStageThreshold(hundredBytes, false)); + assertEquals("95%", diskThresholdSettings.describeFrozenFloodStageThreshold(hundredBytes, false)); + assertEquals( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey() + "=" + "85%", + diskThresholdSettings.describeLowThreshold(hundredBytes, true) + ); + assertEquals( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey() + "=" + "90%", + diskThresholdSettings.describeHighThreshold(hundredBytes, true) + ); + assertEquals( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey() + "=" + "95%", + diskThresholdSettings.describeFloodStageThreshold(hundredBytes, true) + ); + assertEquals( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.getKey() + "=" + "95%", + diskThresholdSettings.describeFrozenFloodStageThreshold(hundredBytes, true) + ); + + // Test default watermark max headroom values + ByteSizeValue thousandTb = ByteSizeValue.parseBytesSizeValue("1000tb", "test"); + ByteSizeValue frozenFloodHeadroom = ByteSizeValue.parseBytesSizeValue("20gb", "test"); + assertEquals(frozenFloodHeadroom, diskThresholdSettings.getFreeBytesThresholdFrozenFloodStage(thousandTb)); + assertEquals("max_headroom=20gb", diskThresholdSettings.describeFrozenFloodStageThreshold(thousandTb, false)); + assertEquals( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING.getKey() + "=" + "20gb", + diskThresholdSettings.describeFrozenFloodStageThreshold(thousandTb, true) + ); } - public void testUpdate() { + public void testMinimumTotalSizeForBelowLowWatermark() { + ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + DiskThresholdSettings diskThresholdSettings = new DiskThresholdSettings(Settings.EMPTY, nss); + + // Test default values + + // For 850 used bytes, we need 850 / 0.85 = 1000 total bytes. + assertEquals( + ByteSizeValue.ofBytes(1000), + diskThresholdSettings.getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue.ofBytes(850)) + ); + + // Test random factor. + final long factor = between(1, 1000); + assertThat( + diskThresholdSettings.getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue.ofBytes(85 * factor)), + Matchers.equalTo(ByteSizeValue.ofBytes(100L * factor)) + ); + + // Test absolute values + + Settings newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1gb") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "100mb") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "10mb") + .build(); + nss.applySettings(newSettings); + + // For 850 used bytes, we need 850b + 1GB total bytes. + assertEquals( + ByteSizeValue.ofBytes(ByteSizeValue.ofGb(1).getBytes() + ByteSizeValue.ofBytes(850).getBytes()), + diskThresholdSettings.getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue.ofBytes(850)) + ); + // For 100TB used bytes, we need 100TB+1GB total bytes. + assertEquals( + ByteSizeValue.ofBytes(ByteSizeValue.ofTb(100).getBytes() + ByteSizeValue.ofGb(1).getBytes()), + diskThresholdSettings.getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue.ofTb(100)) + ); + + newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "0.50") + .build(); + nss.applySettings(newSettings); + + // For 850 used bytes, we need 850 / 0.5 = 1700 total bytes + assertEquals( + ByteSizeValue.ofBytes(1700), + diskThresholdSettings.getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue.ofBytes(850)) + ); + // For 100TB used bytes, we need 100TB / 0.5 total bytes. + assertEquals( + ByteSizeValue.ofBytes((long) (ByteSizeValue.ofTb(100).getBytes() / 0.5)), + diskThresholdSettings.getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue.ofTb(100)) + ); + + newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "0.50") + .build(); + nss.applySettings(newSettings); + + // For 850 used bytes, we need 850 / 0.5 = 1700 total bytes + assertEquals( + ByteSizeValue.ofBytes(1700), + diskThresholdSettings.getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue.ofBytes(850)) + ); + + // Test random percentage + + // to make it easy, stay below high watermark. + final long percentage = between(1, 89); + newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), percentage + "%") + .build(); + nss.applySettings(newSettings); + + assertThat( + diskThresholdSettings.getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue.ofBytes(percentage * factor)), + Matchers.equalTo(ByteSizeValue.ofBytes(100L * factor)) + ); + + // Test random absolute values + + final long absolute = between(1, 1000); + newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), absolute + "b") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), absolute + "b") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), absolute + "b") + .build(); + nss.applySettings(newSettings); + + long needed = between(0, 1000); + assertThat( + diskThresholdSettings.getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue.ofBytes(needed)), + Matchers.equalTo(ByteSizeValue.ofBytes(needed + absolute)) + ); + } + + public void testUpdateWatermarkByteValues() { ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); DiskThresholdSettings diskThresholdSettings = new DiskThresholdSettings(Settings.EMPTY, nss); Settings newSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), false) - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "500mb") .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1000mb") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "500mb") .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "250mb") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.getKey(), "150mb") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "30s") + .build(); + nss.applySettings(newSettings); + + ByteSizeValue total = ByteSizeValue.parseBytesSizeValue("1000tb", "test"); + assertEquals(ByteSizeValue.parseBytesSizeValue("1000mb", "test"), diskThresholdSettings.getFreeBytesThresholdLowStage(total)); + assertEquals(ByteSizeValue.parseBytesSizeValue("500mb", "test"), diskThresholdSettings.getFreeBytesThresholdHighStage(total)); + assertEquals(ByteSizeValue.parseBytesSizeValue("250mb", "test"), diskThresholdSettings.getFreeBytesThresholdFloodStage(total)); + assertEquals( + ByteSizeValue.parseBytesSizeValue("150mb", "test"), + diskThresholdSettings.getFreeBytesThresholdFrozenFloodStage(total) + ); + assertEquals(30L, diskThresholdSettings.getRerouteInterval().seconds()); + assertFalse(diskThresholdSettings.isEnabled()); + } + + public void testUpdateWatermarkPercentageValues() { + ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + DiskThresholdSettings diskThresholdSettings = new DiskThresholdSettings(Settings.EMPTY, nss); + + Settings newSettings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), false) + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), randomBoolean() ? "50%" : "0.50") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), randomBoolean() ? "60%" : "0.60") + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), + randomBoolean() ? "75%" : "0.75" + ) + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.getKey(), + randomBoolean() ? "80%" : "0.80" + ) .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "30s") .build(); nss.applySettings(newSettings); - assertEquals(ByteSizeValue.parseBytesSizeValue("500mb", "test"), diskThresholdSettings.getFreeBytesThresholdHigh()); - assertEquals(0.0D, diskThresholdSettings.getFreeDiskThresholdHigh(), 0.0D); - assertEquals(ByteSizeValue.parseBytesSizeValue("1000mb", "test"), diskThresholdSettings.getFreeBytesThresholdLow()); - assertEquals(0.0D, diskThresholdSettings.getFreeDiskThresholdLow(), 0.0D); - assertEquals(ByteSizeValue.parseBytesSizeValue("250mb", "test"), diskThresholdSettings.getFreeBytesThresholdFloodStage()); - assertEquals(0.0D, diskThresholdSettings.getFreeDiskThresholdFloodStage(), 0.0D); + ByteSizeValue total = ByteSizeValue.parseBytesSizeValue("100b", "test"); + assertEquals(ByteSizeValue.parseBytesSizeValue("50b", "test"), diskThresholdSettings.getFreeBytesThresholdLowStage(total)); + assertEquals(ByteSizeValue.parseBytesSizeValue("40b", "test"), diskThresholdSettings.getFreeBytesThresholdHighStage(total)); + assertEquals(ByteSizeValue.parseBytesSizeValue("25b", "test"), diskThresholdSettings.getFreeBytesThresholdFloodStage(total)); + assertEquals(ByteSizeValue.parseBytesSizeValue("20b", "test"), diskThresholdSettings.getFreeBytesThresholdFrozenFloodStage(total)); assertEquals(30L, diskThresholdSettings.getRerouteInterval().seconds()); assertFalse(diskThresholdSettings.isEnabled()); } @@ -79,7 +239,7 @@ public void testInvalidLowHighPercentageUpdate() { final Settings newSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "80%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "83.45%") .build(); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings)); @@ -88,7 +248,7 @@ public void testInvalidLowHighPercentageUpdate() { assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); - assertThat(cause, hasToString(containsString("low disk watermark [90%] more than high disk watermark [80%]"))); + assertThat(cause, hasToString(containsString("low disk watermark [90%] more than high disk watermark [83.45%]"))); } public void testInvalidHighFloodPercentageUpdate() { @@ -96,13 +256,13 @@ public void testInvalidHighFloodPercentageUpdate() { new DiskThresholdSettings(Settings.EMPTY, clusterSettings); // this has the effect of registering the settings updater final Settings newSettings = Settings.builder() - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "50%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "50.1%") .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "60%") .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "55%") .build(); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings)); - final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [50%]"; + final String expected = "illegal value can't update [cluster.routing.allocation.disk.watermark.low] from [85%] to [50.1%]"; assertThat(e, hasToString(containsString(expected))); assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); @@ -126,7 +286,7 @@ public void testInvalidLowHighBytesUpdate() { assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); - assertThat(cause, hasToString(containsString("low disk watermark [500m] less than high disk watermark [1000m]"))); + assertThat(cause, hasToString(containsString("low disk watermark [500mb] less than high disk watermark [1000mb]"))); } public void testInvalidHighFloodBytesUpdate() { @@ -145,7 +305,7 @@ public void testInvalidHighFloodBytesUpdate() { assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); - assertThat(cause, hasToString(containsString("low disk watermark [500m] less than high disk watermark [1000m]"))); + assertThat(cause, hasToString(containsString("low disk watermark [500mb] less than high disk watermark [1000mb]"))); } public void testIncompatibleThresholdUpdate() { @@ -155,7 +315,7 @@ public void testIncompatibleThresholdUpdate() { final Settings newSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%") .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1000m") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "95%") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "95.2%") .build(); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(newSettings)); @@ -170,9 +330,9 @@ public void testIncompatibleThresholdUpdate() { DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%", DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), - "1000m", + "1000mb", DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), - "95%" + "95.2%" ); assertThat(cause, hasToString(containsString(incompatibleExpected))); } @@ -244,39 +404,155 @@ public void testSequenceOfUpdates() { } } - public void testThresholdDescriptions() { + private void doTestDescriptions(boolean includeKey) { final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + ByteSizeValue hundredBytes = ByteSizeValue.parseBytesSizeValue("100b", "test"); + ByteSizeValue thousandTb = ByteSizeValue.parseBytesSizeValue("1000tb", "test"); + String lowWatermarkPrefix = includeKey + ? DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey() + "=" + : ""; + String highWatermarkPrefix = includeKey + ? DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey() + "=" + : ""; + String floodWatermarkPrefix = includeKey + ? DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey() + "=" + : ""; + String frozenFloodWatermarkPrefix = includeKey + ? DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.getKey() + "=" + : ""; + String frozenFloodMaxHeadroomPrefix = includeKey + ? DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING.getKey() + "=" + : "max_headroom="; DiskThresholdSettings diskThresholdSettings = new DiskThresholdSettings(Settings.EMPTY, clusterSettings); - assertThat(diskThresholdSettings.describeLowThreshold(), equalTo("85%")); - assertThat(diskThresholdSettings.describeHighThreshold(), equalTo("90%")); - assertThat(diskThresholdSettings.describeFloodStageThreshold(), equalTo("95%")); + assertThat(diskThresholdSettings.describeLowThreshold(hundredBytes, includeKey), equalTo(lowWatermarkPrefix + "85%")); + assertThat(diskThresholdSettings.describeHighThreshold(hundredBytes, includeKey), equalTo(highWatermarkPrefix + "90%")); + assertThat(diskThresholdSettings.describeFloodStageThreshold(hundredBytes, includeKey), equalTo(floodWatermarkPrefix + "95%")); + assertThat( + diskThresholdSettings.describeFrozenFloodStageThreshold(hundredBytes, includeKey), + equalTo(frozenFloodWatermarkPrefix + "95%") + ); + + assertThat( + diskThresholdSettings.describeFrozenFloodStageThreshold(thousandTb, includeKey), + equalTo(frozenFloodMaxHeadroomPrefix + "20gb") + ); diskThresholdSettings = new DiskThresholdSettings( Settings.builder() - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "91.2%") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "91.3%") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "91.4%") + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), + randomBoolean() ? "91.2%" : "0.912" + ) + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), + randomBoolean() ? "91.3%" : "0.913" + ) + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), + randomBoolean() ? "91.4%" : "0.914" + ) + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.getKey(), + randomBoolean() ? "91.5%" : "0.915" + ) .build(), clusterSettings ); - assertThat(diskThresholdSettings.describeLowThreshold(), equalTo("91.2%")); - assertThat(diskThresholdSettings.describeHighThreshold(), equalTo("91.3%")); - assertThat(diskThresholdSettings.describeFloodStageThreshold(), equalTo("91.4%")); + assertThat(diskThresholdSettings.describeLowThreshold(hundredBytes, includeKey), equalTo(lowWatermarkPrefix + "91.2%")); + assertThat(diskThresholdSettings.describeHighThreshold(hundredBytes, includeKey), equalTo(highWatermarkPrefix + "91.3%")); + assertThat(diskThresholdSettings.describeFloodStageThreshold(hundredBytes, includeKey), equalTo(floodWatermarkPrefix + "91.4%")); + assertThat( + diskThresholdSettings.describeFrozenFloodStageThreshold(hundredBytes, includeKey), + equalTo(frozenFloodWatermarkPrefix + "91.5%") + ); + + // Even for 1000TB, the watermarks apply since they are set (any max headroom does not apply) + assertThat(diskThresholdSettings.describeLowThreshold(thousandTb, includeKey), equalTo(lowWatermarkPrefix + "91.2%")); + assertThat(diskThresholdSettings.describeHighThreshold(thousandTb, includeKey), equalTo(highWatermarkPrefix + "91.3%")); + assertThat(diskThresholdSettings.describeFloodStageThreshold(thousandTb, includeKey), equalTo(floodWatermarkPrefix + "91.4%")); + assertThat( + diskThresholdSettings.describeFrozenFloodStageThreshold(thousandTb, includeKey), + equalTo(frozenFloodWatermarkPrefix + "91.5%") + ); diskThresholdSettings = new DiskThresholdSettings( Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1GB") .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "10MB") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1B") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "2B") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.getKey(), "1B") + // Max headroom values should be ignored since the watermark values are set to absolute values + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING.getKey(), "10mb") .build(), clusterSettings ); - assertThat(diskThresholdSettings.describeLowThreshold(), equalTo("1gb")); - assertThat(diskThresholdSettings.describeHighThreshold(), equalTo("10mb")); - assertThat(diskThresholdSettings.describeFloodStageThreshold(), equalTo("1b")); + assertThat(diskThresholdSettings.describeLowThreshold(hundredBytes, includeKey), equalTo(lowWatermarkPrefix + "1gb")); + assertThat(diskThresholdSettings.describeHighThreshold(hundredBytes, includeKey), equalTo(highWatermarkPrefix + "10mb")); + assertThat(diskThresholdSettings.describeFloodStageThreshold(hundredBytes, includeKey), equalTo(floodWatermarkPrefix + "2b")); + assertThat( + diskThresholdSettings.describeFrozenFloodStageThreshold(hundredBytes, includeKey), + equalTo(frozenFloodWatermarkPrefix + "1b") + ); + + // Even for 1000TB, the watermarks apply since they are set to absolute values (max headroom values should be ignored) + assertThat(diskThresholdSettings.describeLowThreshold(thousandTb, includeKey), equalTo(lowWatermarkPrefix + "1gb")); + assertThat(diskThresholdSettings.describeHighThreshold(thousandTb, includeKey), equalTo(highWatermarkPrefix + "10mb")); + assertThat(diskThresholdSettings.describeFloodStageThreshold(thousandTb, includeKey), equalTo(floodWatermarkPrefix + "2b")); + assertThat( + diskThresholdSettings.describeFrozenFloodStageThreshold(thousandTb, includeKey), + equalTo(frozenFloodWatermarkPrefix + "1b") + ); + + // Test a mixture of percentages and max headroom values + diskThresholdSettings = new DiskThresholdSettings( + Settings.builder() + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), + randomBoolean() ? "31.2%" : "0.312" + ) + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), + randomBoolean() ? "31.3%" : "0.313" + ) + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), + randomBoolean() ? "31.4%" : "0.314" + ) + .put( + DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_WATERMARK_SETTING.getKey(), + randomBoolean() ? "31.5%" : "0.315" + ) + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_FROZEN_MAX_HEADROOM_SETTING.getKey(), "10gb") + .build(), + clusterSettings + ); + + assertThat(diskThresholdSettings.describeLowThreshold(hundredBytes, includeKey), equalTo(lowWatermarkPrefix + "31.2%")); + assertThat(diskThresholdSettings.describeHighThreshold(hundredBytes, includeKey), equalTo(highWatermarkPrefix + "31.3%")); + assertThat(diskThresholdSettings.describeFloodStageThreshold(hundredBytes, includeKey), equalTo(floodWatermarkPrefix + "31.4%")); + assertThat( + diskThresholdSettings.describeFrozenFloodStageThreshold(hundredBytes, includeKey), + equalTo(frozenFloodWatermarkPrefix + "31.5%") + ); + + assertThat(diskThresholdSettings.describeLowThreshold(thousandTb, includeKey), equalTo(lowWatermarkPrefix + "31.2%")); + assertThat(diskThresholdSettings.describeHighThreshold(thousandTb, includeKey), equalTo(highWatermarkPrefix + "31.3%")); + assertThat(diskThresholdSettings.describeFloodStageThreshold(thousandTb, includeKey), equalTo(floodWatermarkPrefix + "31.4%")); + assertThat( + diskThresholdSettings.describeFrozenFloodStageThreshold(thousandTb, includeKey), + equalTo(frozenFloodMaxHeadroomPrefix + "10gb") + ); + } + + public void testDescriptionsWithKeys() { + doTestDescriptions(true); + } + + public void testDescriptionsWithoutKeys() { + doTestDescriptions(false); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 8793f6f9c63e5..445a9d4853aef 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -663,20 +663,6 @@ public void testAverageUsage() { assertThat(node1Usage.getFreeBytes(), equalTo(25L)); } - public void testFreeDiskPercentageAfterShardAssigned() { - DiskThresholdDecider decider = makeDecider(Settings.EMPTY); - - Map usages = new HashMap<>(); - usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 50)); // 50% used - usages.put("node3", new DiskUsage("node3", "n3", "/dev/null", 100, 0)); // 100% used - - Double after = DiskThresholdDecider.freeDiskPercentageAfterShardAssigned( - new DiskThresholdDecider.DiskUsageWithRelocations(new DiskUsage("node2", "n2", "/dev/null", 100, 30), 0L), - 11L - ); - assertThat(after, equalTo(19.0)); - } - public void testShardRelocationsTakenIntoAccount() { Settings diskSettings = Settings.builder() .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true) @@ -788,9 +774,8 @@ public void testShardRelocationsTakenIntoAccount() { expectThrows(IllegalArgumentException.class, () -> strategy.reroute(clusterStateThatRejectsCommands, cmds, false, false)) .getMessage(), containsString( - "the node is above the low watermark cluster setting " - + "[cluster.routing.allocation.disk.watermark.low=0.7], using more disk space than the maximum " - + "allowed [70.0%], actual free: [26.0%]" + "the node is above the low watermark cluster setting [cluster.routing.allocation.disk.watermark.low=70%], " + + "having less than the minimum required [30b] free space, actual free: [26b], actual used: [74%]" ) ); @@ -930,8 +915,8 @@ public void testCanRemainWithShardRelocatingAway() { decision.getExplanation(), containsString( "the shard cannot remain on this node because it is above the high watermark cluster setting " - + "[cluster.routing.allocation.disk.watermark.high=70%] and there is less than the required [30.0%] free disk on node, " - + "actual free: [20.0%]" + + "[cluster.routing.allocation.disk.watermark.high=70%] and there is less than the required [30b] free space " + + "on node, actual free: [20b], actual used: [80%]" ) ); @@ -973,16 +958,16 @@ public void testCanRemainWithShardRelocatingAway() { assertThat( ((Decision.Single) decision).getExplanation(), containsString( - "the node is above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=70%], using " - + "more disk space than the maximum allowed [70.0%], actual free: [20.0%]" + "the node is above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=70%], " + + "having less than the minimum required [30b] free space, actual free: [20b], actual used: [80%]" ) ); } else { assertThat( ((Decision.Single) decision).getExplanation(), containsString( - "the node is above the low watermark cluster setting [cluster.routing.allocation.disk.watermark.low=60%], using more " - + "disk space than the maximum allowed [60.0%], actual free: [20.0%]" + "the node is above the low watermark cluster setting [cluster.routing.allocation.disk.watermark.low=60%], " + + "having less than the minimum required [40b] free space, actual free: [20b], actual used: [80%]" ) ); } @@ -1011,7 +996,7 @@ public void testCanRemainWithShardRelocatingAway() { EmptySnapshotsInfoService.INSTANCE ); // Ensure that the reroute call doesn't alter the routing table, since the first primary is relocating away - // and therefor we will have sufficient disk space on node1. + // and therefore we will have sufficient disk space on node1. ClusterState result = strategy.reroute(clusterState, "reroute"); assertThat(result, equalTo(clusterState)); assertThat(result.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED)); @@ -1130,8 +1115,8 @@ public void testWatermarksEnabledForSingleDataNode() { decision.getExplanation(), containsString( "the shard cannot remain on this node because it is above the high watermark cluster setting" - + " [cluster.routing.allocation.disk.watermark.high=70%] and there is less than the required [30.0%] free disk on node," - + " actual free: [20.0%]" + + " [cluster.routing.allocation.disk.watermark.high=70%] and there is less than the required [30b] free space " + + "on node, actual free: [20b], actual used: [80%]" ) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index dd380fb98e725..41a8b5794c10d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDeciderTests.DevNullClusterInfo; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; @@ -127,8 +128,8 @@ public void testCanAllocateUsesMaxAvailableSpace() { assertThat( ((Decision.Single) decision).getExplanation(), containsString( - "the node is above the high watermark cluster " - + "setting [cluster.routing.allocation.disk.watermark.high=90%], using more disk space than the maximum allowed [90.0%]" + "the node is above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=90%], " + + "having less than the minimum required" ) ); } @@ -200,15 +201,19 @@ public void testCannotAllocateDueToLackOfDiskResources() { Decision decision = decider.canAllocate(test_0, RoutingNodesHelper.routingNode("node_0", node_0), allocation); assertEquals(Decision.Type.NO, decision.type()); + double usedPercentage = 100.0 * (100 - freeBytes) / 100; + assertThat( decision.getExplanation(), containsString( "allocating the shard to this node will bring the node above the high watermark cluster setting " + "[cluster.routing.allocation.disk.watermark.high=90%] " - + "and cause it to have less than the minimum required [0b] of free space " + + "and cause it to have less than the minimum required [10b] of free space " + "(free: [" + freeBytes - + "b], estimated shard size: [" + + "b], used: [" + + Strings.format1Decimals(usedPercentage, "%") + + "], estimated shard size: [" + shardSize + "b])" ) @@ -331,8 +336,8 @@ public void testCanRemainUsesLeastAvailableSpace() { ((Decision.Single) decision).getExplanation(), containsString( "the shard cannot remain on this node because it is above the high watermark cluster setting " - + "[cluster.routing.allocation.disk.watermark.high=90%] and there is less than the required [10.0%] " - + "free disk on node, actual free: [9.0%]" + + "[cluster.routing.allocation.disk.watermark.high=90%] and there is less than the required [10b] " + + "free space on node, actual free: [9b], actual used: [91%]" ) ); try { diff --git a/server/src/test/java/org/elasticsearch/common/unit/RatioValueTests.java b/server/src/test/java/org/elasticsearch/common/unit/RatioValueTests.java index 830b222fb148e..6cf8431f19650 100644 --- a/server/src/test/java/org/elasticsearch/common/unit/RatioValueTests.java +++ b/server/src/test/java/org/elasticsearch/common/unit/RatioValueTests.java @@ -44,14 +44,14 @@ public void testNegativeCase() { } public void testToStringNoTrailingZeros() { - assertThat(RatioValue.formatNoTrailingZerosPercent(100.0), is("100%")); - assertThat(RatioValue.formatNoTrailingZerosPercent(.000000), is("0%")); - assertThat(RatioValue.formatNoTrailingZerosPercent(0.000000), is("0%")); - assertThat(RatioValue.formatNoTrailingZerosPercent(-0), is("0%")); - assertThat(RatioValue.formatNoTrailingZerosPercent(0), is("0%")); - assertThat(RatioValue.formatNoTrailingZerosPercent(15.1), is("15.1%")); - assertThat(RatioValue.formatNoTrailingZerosPercent(0.1000000), is("0.1%")); - assertThat(RatioValue.formatNoTrailingZerosPercent(1.1234567890), is("1.123456789%")); + assertThat((new RatioValue(100.0)).formatNoTrailingZerosPercent(), is("100%")); + assertThat((new RatioValue(.000000)).formatNoTrailingZerosPercent(), is("0%")); + assertThat((new RatioValue(0.000000)).formatNoTrailingZerosPercent(), is("0%")); + assertThat((new RatioValue(-0)).formatNoTrailingZerosPercent(), is("0%")); + assertThat((new RatioValue(0)).formatNoTrailingZerosPercent(), is("0%")); + assertThat((new RatioValue(15.1)).formatNoTrailingZerosPercent(), is("15.1%")); + assertThat((new RatioValue(0.1000000)).formatNoTrailingZerosPercent(), is("0.1%")); + assertThat((new RatioValue(1.1234567890)).formatNoTrailingZerosPercent(), is("1.123456789%")); } public void testInvalidRatio(String r) { diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 2a9f8dccf73d6..1b7e80c869496 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -10,7 +10,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene93.Lucene93Codec; +import org.apache.lucene.codecs.lucene92.Lucene92Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -41,21 +41,21 @@ public class CodecTests extends ESTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene93Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class)); } public void testDefault() throws Exception { Codec codec = createCodecService().codec("default"); - assertStoredFieldsCompressionEquals(Lucene93Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec); } public void testBestCompression() throws Exception { Codec codec = createCodecService().codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene93Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec); } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene93Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(null); iwc.setCodec(actual); @@ -67,7 +67,7 @@ private void assertStoredFieldsCompressionEquals(Lucene93Codec.Mode expected, Co SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene93Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene92Codec.Mode.valueOf(v)); ir.close(); dir.close(); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java index 83bc488ac0641..1e03c7d3571ad 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.index.engine; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene93.Lucene93Codec; +import org.apache.lucene.codecs.lucene92.Lucene92Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -45,7 +45,7 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); final PostingsFormat postingsFormat = new Completion90PostingsFormat(); - indexWriterConfig.setCodec(new Lucene93Codec() { + indexWriterConfig.setCodec(new Lucene92Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return postingsFormat; // all fields are suggest fields diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index fcdbbd3a7d268..d7c8fb8314b04 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -571,7 +571,7 @@ public void testScriptAndPrecludedParameters() { @Override protected Object generateRandomInputValue(MappedFieldType ft) { - return switch (between(0, 3)) { + return switch (between(0, 4)) { case 0 -> randomAlphaOfLengthBetween(1, 100); case 1 -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100); case 2 -> randomLong(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 069474cca314c..4fa3071890070 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -12,7 +12,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat; +import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.KnnVectorField; import org.apache.lucene.index.IndexableField; @@ -39,8 +39,8 @@ import java.util.Arrays; import java.util.List; -import static org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat.DEFAULT_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene93.Lucene93HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -454,8 +454,8 @@ public void testKnnVectorsFormat() throws IOException { Codec codec = codecService.codec("default"); assertThat(codec, instanceOf(PerFieldMapperCodec.class)); KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); - assertThat(knnVectorsFormat, instanceOf(Lucene93HnswVectorsFormat.class)); - String expectedString = "Lucene93HnswVectorsFormat(name=Lucene93HnswVectorsFormat, maxConn=" + assertThat(knnVectorsFormat, instanceOf(Lucene92HnswVectorsFormat.class)); + String expectedString = "lucene92HnswVectorsFormat(name = lucene92HnswVectorsFormat, maxConn = " + m + ", beamWidth=" + efConstruction diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderGeoPointTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderGeoPointTests.java new file mode 100644 index 0000000000000..aecd630b3d9ee --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderGeoPointTests.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.document.LatLonDocValuesField; +import org.apache.lucene.document.LatLonPoint; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.geo.GeoBoundingBoxQueryBuilderTestCase; + +import java.io.IOException; + +@SuppressWarnings("checkstyle:MissingJavadocMethod") +public class GeoBoundingBoxQueryBuilderGeoPointTests extends GeoBoundingBoxQueryBuilderTestCase { + + @Override + protected String getFieldName() { + return randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME); + } + + @Override + protected void doAssertLuceneQuery(GeoBoundingBoxQueryBuilder queryBuilder, Query query, SearchExecutionContext context) + throws IOException { + final MappedFieldType fieldType = context.getFieldType(queryBuilder.fieldName()); + if (fieldType == null) { + assertTrue("Found no indexed geo query.", query instanceof MatchNoDocsQuery); + } + assertEquals(GeoPointFieldMapper.GeoPointFieldType.class, fieldType.getClass()); + assertEquals(IndexOrDocValuesQuery.class, query.getClass()); + Query indexQuery = ((IndexOrDocValuesQuery) query).getIndexQuery(); + String expectedFieldName = expectedFieldName(queryBuilder.fieldName()); + double qMinLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(queryBuilder.bottomRight().lat())); + double qMaxLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(queryBuilder.topLeft().lat())); + double qMinLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(queryBuilder.topLeft().lon())); + double qMaxLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(queryBuilder.bottomRight().lon())); + assertEquals(LatLonPoint.newBoxQuery(expectedFieldName, qMinLat, qMaxLat, qMinLon, qMaxLon), indexQuery); + Query dvQuery = ((IndexOrDocValuesQuery) query).getRandomAccessQuery(); + assertEquals(LatLonDocValuesField.newSlowBoxQuery(expectedFieldName, qMinLat, qMaxLat, qMinLon, qMaxLon), dvQuery); + } + + public void testValidation() { + PointTester[] testers = { new TopTester(), new LeftTester(), new BottomTester(), new RightTester() }; + + for (PointTester tester : testers) { + GeoBoundingBoxQueryBuilder builder = createTestQueryBuilder(); + tester.invalidateCoordinate(builder.setValidationMethod(GeoValidationMethod.COERCE), false); + QueryValidationException except = builder.checkLatLon(); + assertNull( + "validation w/ coerce should ignore invalid " + + tester.getClass().getName() + + " coordinate: " + + tester.invalidCoordinate + + " ", + except + ); + + tester.invalidateCoordinate(builder.setValidationMethod(GeoValidationMethod.STRICT), false); + except = builder.checkLatLon(); + assertNotNull( + "validation w/o coerce should detect invalid coordinate: " + + tester.getClass().getName() + + " coordinate: " + + tester.invalidCoordinate, + except + ); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderGeoPointTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderGeoPointTests.java new file mode 100644 index 0000000000000..89b2d3ab853c2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderGeoPointTests.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.document.LatLonDocValuesField; +import org.apache.lucene.document.LatLonPoint; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.geo.GeoDistanceQueryBuilderTestCase; + +import java.io.IOException; + +@SuppressWarnings("checkstyle:MissingJavadocMethod") +public class GeoDistanceQueryBuilderGeoPointTests extends GeoDistanceQueryBuilderTestCase { + + @Override + protected String getFieldName() { + return randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME); + } + + @Override + protected void doAssertLuceneQuery(GeoDistanceQueryBuilder queryBuilder, Query query, SearchExecutionContext context) + throws IOException { + final MappedFieldType fieldType = context.getFieldType(queryBuilder.fieldName()); + if (fieldType == null) { + assertTrue("Found no indexed geo query.", query instanceof MatchNoDocsQuery); + } + assertEquals(GeoPointFieldMapper.GeoPointFieldType.class, fieldType.getClass()); + assertEquals(IndexOrDocValuesQuery.class, query.getClass()); + Query indexQuery = ((IndexOrDocValuesQuery) query).getIndexQuery(); + String expectedFieldName = expectedFieldName(queryBuilder.fieldName()); + double qLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(queryBuilder.point().lat())); + double qLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(queryBuilder.point().lon())); + assertEquals(LatLonPoint.newDistanceQuery(expectedFieldName, qLat, qLon, queryBuilder.distance()), indexQuery); + Query dvQuery = ((IndexOrDocValuesQuery) query).getRandomAccessQuery(); + assertEquals(LatLonDocValuesField.newSlowDistanceQuery(expectedFieldName, qLat, qLon, queryBuilder.distance()), dvQuery); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoPointTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoPointTests.java index 373e5cd619a49..cc220438fb448 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoPointTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoPointTests.java @@ -10,23 +10,25 @@ import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.search.geo.GeoShapeQueryBuilderTestCase; -public class GeoShapeQueryBuilderGeoPointTests extends GeoShapeQueryBuilderTests { +public class GeoShapeQueryBuilderGeoPointTests extends GeoShapeQueryBuilderTestCase { - protected String fieldName() { - return GEO_POINT_FIELD_NAME; + protected String getFieldName() { + return randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME); } + @Override protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { Geometry geometry = GeometryTestUtils.randomPolygon(false); GeoShapeQueryBuilder builder; clearShapeFields(); if (indexedShape == false) { - builder = new GeoShapeQueryBuilder(fieldName(), geometry); + builder = new GeoShapeQueryBuilder(getFieldName(), geometry); } else { indexedShapeToReturn = geometry; indexedShapeId = randomAlphaOfLengthBetween(3, 20); - builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId); + builder = new GeoShapeQueryBuilder(getFieldName(), indexedShapeId); if (randomBoolean()) { indexedShapeIndex = randomAlphaOfLengthBetween(3, 20); builder.indexedShapeIndex(indexedShapeIndex); @@ -49,5 +51,4 @@ protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { } return builder; } - } diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java deleted file mode 100644 index f359f4600faaf..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.index.query; - -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.geo.GeometryTestUtils; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.ShapeType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; - -public class GeoShapeQueryBuilderGeoShapeTests extends GeoShapeQueryBuilderTests { - - private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; - - @Override - protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { - XContentBuilder builder = PutMappingRequest.simpleMapping(GEO_SHAPE_FIELD_NAME, "type=geo_shape"); - mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); - } - - @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled - protected Collection> getPlugins() { - return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); - } - - protected String fieldName() { - return GEO_SHAPE_FIELD_NAME; - } - - protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { - ShapeType shapeType = randomFrom( - ShapeType.POINT, - ShapeType.MULTIPOINT, - ShapeType.LINESTRING, - ShapeType.MULTILINESTRING, - ShapeType.POLYGON - ); - Geometry geometry = GeometryTestUtils.randomGeometry(shapeType, false); - GeoShapeQueryBuilder builder; - clearShapeFields(); - if (indexedShape == false) { - builder = new GeoShapeQueryBuilder(fieldName(), geometry); - } else { - indexedShapeToReturn = geometry; - indexedShapeId = randomAlphaOfLengthBetween(3, 20); - builder = new GeoShapeQueryBuilder(fieldName(), indexedShapeId); - if (randomBoolean()) { - indexedShapeIndex = randomAlphaOfLengthBetween(3, 20); - builder.indexedShapeIndex(indexedShapeIndex); - } - if (randomBoolean()) { - indexedShapePath = randomAlphaOfLengthBetween(3, 20); - builder.indexedShapePath(indexedShapePath); - } - if (randomBoolean()) { - indexedShapeRouting = randomAlphaOfLengthBetween(3, 20); - builder.indexedShapeRouting(indexedShapeRouting); - } - } - if (randomBoolean()) { - SearchExecutionContext context = createSearchExecutionContext(); - if (context.indexVersionCreated().onOrAfter(Version.V_7_5_0)) { // CONTAINS is only supported from version 7.5 - if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) { - builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.CONTAINS)); - } else { - builder.relation( - randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN, ShapeRelation.CONTAINS) - ); - } - } else { - if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) { - builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS)); - } else { - builder.relation(randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN)); - } - } - } - - if (randomBoolean()) { - builder.ignoreUnmapped(randomBoolean()); - } - return builder; - } -} diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index ac0e967face4f..33196e4214460 100644 --- a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -16,6 +16,8 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.plugins.spi.BarPlugin; +import org.elasticsearch.plugins.spi.BarTestService; import org.elasticsearch.plugins.spi.TestService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.PrivilegedOperations; @@ -41,8 +43,11 @@ import java.util.Map; import java.util.Set; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.instanceOf; @@ -695,6 +700,34 @@ public void testLoadServiceProviders() throws Exception { } } + // The mock node loads plugins in the same class loader, make sure we can find the appropriate + // plugin to use in the constructor in that case too + public void testLoadServiceProvidersInSameClassLoader() { + PluginsService service = newMockPluginsService(List.of(BarPlugin.class, PluginOther.class)); + + // There's only one TestService implementation, FooTestService which uses FooPlugin in the constructor. + // We should find only one instance of this service when we load with two plugins in the same class loader. + @SuppressWarnings("unchecked") + List testServices = (List) service.loadServiceProviders(TestService.class); + assertEquals(1, testServices.size()); + + var fooPlugin = (BarPlugin) service.plugins().stream().filter(p -> p.instance() instanceof BarPlugin).findAny().get().instance(); + var othPlugin = (PluginOther) service.plugins() + .stream() + .filter(p -> p.instance() instanceof PluginOther) + .findAny() + .get() + .instance(); + + // We shouldn't find the FooTestService implementation with PluginOther + assertThat(MockPluginsService.createExtensions(TestService.class, othPlugin), empty()); + + // We should find the FooTestService implementation when we use FooPlugin, because it matches the constructor arg. + var providers = MockPluginsService.createExtensions(TestService.class, fooPlugin); + + assertThat(providers, allOf(hasSize(1), everyItem(instanceOf(BarTestService.class)))); + } + private static class TestExtensiblePlugin extends Plugin implements ExtensiblePlugin { private List extensions; @@ -734,4 +767,8 @@ public ThrowingConstructorExtension() { throw new IllegalArgumentException("test constructor failure"); } } + + public static class PluginOther extends Plugin { + public PluginOther() {} + } } diff --git a/server/src/test/java/org/elasticsearch/plugins/spi/BarPlugin.java b/server/src/test/java/org/elasticsearch/plugins/spi/BarPlugin.java new file mode 100644 index 0000000000000..5e7a13f02ef2a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/plugins/spi/BarPlugin.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.spi; + +import org.elasticsearch.plugins.Plugin; + +public class BarPlugin extends Plugin { + public BarPlugin() {} +} diff --git a/server/src/test/java/org/elasticsearch/plugins/spi/BarTestService.java b/server/src/test/java/org/elasticsearch/plugins/spi/BarTestService.java new file mode 100644 index 0000000000000..84d2c46ea6b66 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/plugins/spi/BarTestService.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.spi; + +public class BarTestService implements TestService { + BarPlugin plugin; + + @SuppressWarnings("unused") + public BarTestService() {} + + @SuppressWarnings("unused") + public BarTestService(final BarPlugin plugin) { + this.plugin = plugin; + } + + @Override + public String name() { + return "aaa"; + } +} diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java index ed1f380aa5331..adb7e4c437213 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java @@ -7,11 +7,15 @@ */ package org.elasticsearch.rest.action.search; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; +import org.elasticsearch.search.vectors.KnnSearchBuilder; import org.elasticsearch.test.rest.FakeRestChannel; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.test.rest.RestActionTestCase; @@ -71,6 +75,47 @@ public void testEnableFieldsEmulationNoErrors() throws Exception { action.handleRequest(request, new FakeRestChannel(request, false, 1), verifyingClient); } + public void testValidateSearchRequest() { + { + Map params = new HashMap<>(); + params.put("rest_total_hits_as_int", "true"); + + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( + Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) + ).withMethod(RestRequest.Method.GET).withPath("/some_index/_search").withParams(params).build(); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(new SearchSourceBuilder().trackTotalHitsUpTo(100)); + + Exception ex = expectThrows( + IllegalArgumentException.class, + () -> RestSearchAction.validateSearchRequest(request, searchRequest) + ); + assertEquals("[rest_total_hits_as_int] cannot be used if the tracking of total hits is not accurate, got 100", ex.getMessage()); + } + { + Map params = new HashMap<>(); + params.put("search_type", randomFrom(SearchType.values()).name()); + + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( + Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) + ).withMethod(RestRequest.Method.GET).withPath("/some_index/_search").withParams(params).build(); + + SearchRequest searchRequest = new SearchRequest(); + KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", new float[] { 1, 1, 1 }, 10, 100); + searchRequest.source(new SearchSourceBuilder().knnSearch(knnSearch)); + + Exception ex = expectThrows( + IllegalArgumentException.class, + () -> RestSearchAction.validateSearchRequest(request, searchRequest) + ); + assertEquals( + "cannot set [search_type] when using [knn] search, since the search type is determined automatically", + ex.getMessage() + ); + } + } + /** * Using an illegal search type on the request should throw an error */ diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java index 812ce12337fd4..9fe6829861ba1 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -454,6 +454,7 @@ public CheckedBiConsumer getReque "query_string", "range", "regexp", + "knn_score_doc", "script", "script_score", "simple_query_string", diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java new file mode 100644 index 0000000000000..5e571d669ee09 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.vectors; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.index.query.MatchNoneQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; + +public class KnnScoreDocQueryBuilderTests extends AbstractQueryTestCase { + + @Override + protected Collection> getPlugins() { + return Arrays.asList(TestGeoShapeFieldMapperPlugin.class); + } + + @Override + protected KnnScoreDocQueryBuilder doCreateTestQueryBuilder() { + List scoreDocs = new ArrayList<>(); + int numDocs = randomInt(10); + for (int doc = 0; doc < numDocs; doc++) { + scoreDocs.add(new ScoreDoc(doc, randomFloat())); + } + return new KnnScoreDocQueryBuilder(scoreDocs.toArray(new ScoreDoc[0])); + } + + @Override + public void testValidOutput() { + KnnScoreDocQueryBuilder query = new KnnScoreDocQueryBuilder(new ScoreDoc[] { new ScoreDoc(0, 4.25f), new ScoreDoc(5, 1.6f) }); + String expected = """ + { + "knn_score_doc" : { + "values" : [ + { + "doc" : 0, + "score" : 4.25 + }, + { + "doc" : 5, + "score" : 1.6 + } + ] + } + }"""; + assertEquals(expected, query.toString()); + } + + @Override + protected void doAssertLuceneQuery(KnnScoreDocQueryBuilder queryBuilder, Query query, SearchExecutionContext context) + throws IOException { + // Not needed since we override testToQuery + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testToQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(new Document()); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(new IndexSearcher(reader)); + KnnScoreDocQueryBuilder queryBuilder = createTestQueryBuilder(); + Query query = queryBuilder.doToQuery(context); + + assertTrue(query instanceof KnnScoreDocQuery); + KnnScoreDocQuery scoreDocQuery = (KnnScoreDocQuery) query; + + int expectedDocs = queryBuilder.scoreDocs().length; + assertEquals(expectedDocs, scoreDocQuery.docs().length); + assertEquals(expectedDocs, scoreDocQuery.scores().length); + } + } + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testCacheability() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(new Document()); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(new IndexSearcher(reader)); + KnnScoreDocQueryBuilder queryBuilder = createTestQueryBuilder(); + QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new SearchExecutionContext(context)); + assertNotNull(rewriteQuery.toQuery(context)); + assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable()); + } + } + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testMustRewrite() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(new Document()); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(new IndexSearcher(reader)); + context.setAllowUnmappedFields(true); + KnnScoreDocQueryBuilder queryBuilder = createTestQueryBuilder(); + queryBuilder.toQuery(context); + } + } + } + + public void testRewriteToMatchNone() throws IOException { + KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder(new ScoreDoc[0]); + SearchExecutionContext context = createSearchExecutionContext(); + assertEquals(new MatchNoneQueryBuilder(), queryBuilder.rewrite(context)); + } + + @Override + public void testUnknownObjectException() { + // Test isn't relevant, since query is never parsed from xContent + } + + @Override + public void testFromXContent() throws IOException { + // Test isn't relevant, since query is never parsed from xContent + } + + @Override + public void testUnknownField() { + // Test isn't relevant, since query is never parsed from xContent + } + + public void testScoreDocQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + for (int i = 0; i < 50; i++) { + Document doc = new Document(); + doc.add(new StringField("field", "value" + i, Field.Store.NO)); + iw.addDocument(doc); + if (i % 10 == 0) { + iw.flush(); + } + } + + try (IndexReader reader = iw.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + SearchExecutionContext context = createSearchExecutionContext(searcher); + + List scoreDocsList = new ArrayList<>(); + for (int doc = 0; doc < 50; doc += 1 + random().nextInt(5)) { + ScoreDoc scoreDoc = new ScoreDoc(doc, randomFloat()); + scoreDocsList.add(scoreDoc); + } + ScoreDoc[] scoreDocs = scoreDocsList.toArray(new ScoreDoc[0]); + + KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder(scoreDocs); + Query query = queryBuilder.doToQuery(context); + + TopDocs topDocs = searcher.search(query, 100); + assertEquals(scoreDocs.length, topDocs.totalHits.value); + assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); + + Arrays.sort(topDocs.scoreDocs, Comparator.comparingInt(scoreDoc -> scoreDoc.doc)); + assertEquals(scoreDocs.length, topDocs.scoreDocs.length); + for (int i = 0; i < scoreDocs.length; i++) { + assertEquals(scoreDocs[i].doc, topDocs.scoreDocs[i].doc); + assertEquals(scoreDocs[i].score, topDocs.scoreDocs[i].score, 0.0001f); + + } + } + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java new file mode 100644 index 0000000000000..e019b00fa3b49 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.vectors; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParser; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.hamcrest.Matchers.containsString; + +public class KnnSearchBuilderTests extends AbstractSerializingTestCase { + private NamedWriteableRegistry namedWriteableRegistry; + private NamedXContentRegistry namedXContentRegistry; + + @Before + public void registerNamedXContents() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + namedXContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents()); + namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return namedXContentRegistry; + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return namedWriteableRegistry; + } + + @Override + protected KnnSearchBuilder doParseInstance(XContentParser parser) throws IOException { + return KnnSearchBuilder.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return KnnSearchBuilder::new; + } + + @Override + protected KnnSearchBuilder createTestInstance() { + String field = randomAlphaOfLength(6); + int dim = randomIntBetween(2, 30); + float[] vector = randomVector(dim); + int k = randomIntBetween(1, 100); + int numCands = randomIntBetween(k + 20, 1000); + + KnnSearchBuilder builder = new KnnSearchBuilder(field, vector, k, numCands); + if (randomBoolean()) { + builder.boost(randomFloat()); + } + + int numFilters = randomIntBetween(0, 3); + for (int i = 0; i < numFilters; i++) { + builder.addFilterQuery(QueryBuilders.termQuery(randomAlphaOfLength(5), randomAlphaOfLength(10))); + } + + return builder; + } + + @Override + protected KnnSearchBuilder mutateInstance(KnnSearchBuilder instance) throws IOException { + switch (random().nextInt(6)) { + + case 0: + String newField = randomValueOtherThan(instance.field, () -> randomAlphaOfLength(5)); + return new KnnSearchBuilder(newField, instance.queryVector, instance.k, instance.numCands + 3).boost(instance.boost); + case 1: + float[] newVector = randomValueOtherThan(instance.queryVector, () -> randomVector(5)); + return new KnnSearchBuilder(instance.field, newVector, instance.k + 3, instance.numCands).boost(instance.boost); + case 2: + return new KnnSearchBuilder(instance.field, instance.queryVector, instance.k + 3, instance.numCands).boost(instance.boost); + case 3: + return new KnnSearchBuilder(instance.field, instance.queryVector, instance.k, instance.numCands + 3).boost(instance.boost); + case 4: + return new KnnSearchBuilder(instance.field, instance.queryVector, instance.k, instance.numCands).addFilterQueries( + instance.filterQueries + ).addFilterQuery(QueryBuilders.termQuery("new_field", "new-value")).boost(instance.boost); + case 5: + float newBoost = randomValueOtherThan(instance.boost, ESTestCase::randomFloat); + return new KnnSearchBuilder(instance.field, instance.queryVector, instance.k, instance.numCands).addFilterQueries( + instance.filterQueries + ).boost(newBoost); + default: + throw new IllegalStateException(); + } + } + + public void testToQueryBuilder() { + String field = randomAlphaOfLength(6); + float[] vector = randomVector(randomIntBetween(2, 30)); + int k = randomIntBetween(1, 100); + int numCands = randomIntBetween(k, 1000); + KnnSearchBuilder builder = new KnnSearchBuilder(field, vector, k, numCands); + + float boost = AbstractQueryBuilder.DEFAULT_BOOST; + if (randomBoolean()) { + boost = randomFloat(); + builder.boost(boost); + } + + int numFilters = random().nextInt(3); + List filterQueries = new ArrayList<>(); + for (int i = 0; i < numFilters; i++) { + QueryBuilder filter = QueryBuilders.termQuery(randomAlphaOfLength(5), randomAlphaOfLength(5)); + filterQueries.add(filter); + builder.addFilterQuery(filter); + } + + QueryBuilder expected = new KnnVectorQueryBuilder(field, vector, numCands).addFilterQueries(filterQueries).boost(boost); + assertEquals(expected, builder.toQueryBuilder()); + } + + public void testNumCandsLessThanK() { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new KnnSearchBuilder("field", randomVector(3), 50, 10) + ); + assertThat(e.getMessage(), containsString("[num_candidates] cannot be less than [k]")); + } + + public void testNumCandsExceedsLimit() { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new KnnSearchBuilder("field", randomVector(3), 100, 10002) + ); + assertThat(e.getMessage(), containsString("[num_candidates] cannot exceed [10000]")); + } + + public void testInvalidK() { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new KnnSearchBuilder("field", randomVector(3), 0, 100) + ); + assertThat(e.getMessage(), containsString("[k] must be greater than 0")); + } + + private static float[] randomVector(int dim) { + float[] vector = new float[dim]; + for (int i = 0; i < vector.length; i++) { + vector[i] = randomFloat(); + } + return vector; + } +} diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java similarity index 87% rename from server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestBuilderTests.java rename to server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java index d851e7ea89be6..2937fb2dfd081 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; -import org.elasticsearch.search.vectors.KnnSearchRequestBuilder.KnnSearch; +import org.elasticsearch.search.vectors.KnnSearchRequestParser.KnnSearch; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -40,7 +40,7 @@ import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchSourceBuilder; import static org.hamcrest.Matchers.containsString; -public class KnnSearchRequestBuilderTests extends ESTestCase { +public class KnnSearchRequestParserTests extends ESTestCase { private NamedXContentRegistry namedXContentRegistry; @Before @@ -55,16 +55,16 @@ protected NamedXContentRegistry xContentRegistry() { return namedXContentRegistry; } - public void testBuildSearchRequest() throws IOException { + public void testParseSearchRequest() throws IOException { // Choose random REST parameters Map params = new HashMap<>(); String[] indices = generateRandomStringArray(5, 10, false, true); - params.put(KnnSearchRequestBuilder.INDEX_PARAM, String.join(",", indices)); + params.put(KnnSearchRequestParser.INDEX_PARAM, String.join(",", indices)); String routing = null; if (randomBoolean()) { routing = randomAlphaOfLengthBetween(3, 10); - params.put(KnnSearchRequestBuilder.ROUTING_PARAM, routing); + params.put(KnnSearchRequestParser.ROUTING_PARAM, routing); } // Create random request body @@ -81,13 +81,13 @@ public void testBuildSearchRequest() throws IOException { XContentBuilder builder = createRequestBody(knnSearch, filterQueries, searchSource); // Convert the REST request to a search request and check the components - SearchRequestBuilder searchRequestBuilder = buildSearchRequest(builder, params); + SearchRequestBuilder searchRequestBuilder = parseSearchRequest(builder, params); SearchRequest searchRequest = searchRequestBuilder.request(); assertArrayEquals(indices, searchRequest.indices()); assertEquals(routing, searchRequest.routing()); - KnnVectorQueryBuilder query = knnSearch.buildQuery(); + KnnVectorQueryBuilder query = knnSearch.toQueryBuilder(); if (filterQueries.isEmpty() == false) { query.addFilterQueries(filterQueries); } @@ -107,7 +107,7 @@ public void testParseSourceString() throws IOException { KnnSearch knnSearch = randomKnnSearch(); builder.startObject() - .startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .startObject(KnnSearchRequestParser.KNN_SECTION_FIELD.getPreferredName()) .field(KnnSearch.FIELD_FIELD.getPreferredName(), knnSearch.field) .field(KnnSearch.K_FIELD.getPreferredName(), knnSearch.k) .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands) @@ -118,7 +118,7 @@ public void testParseSourceString() throws IOException { builder.endObject(); // Convert the REST request to a search request and check the components - SearchRequestBuilder searchRequestBuilder = buildSearchRequest(builder); + SearchRequestBuilder searchRequestBuilder = parseSearchRequest(builder); SearchRequest searchRequest = searchRequestBuilder.request(); FetchSourceContext fetchSource = searchRequest.source().fetchSource(); @@ -133,7 +133,7 @@ public void testParseSourceArray() throws IOException { KnnSearch knnSearch = randomKnnSearch(); builder.startObject() - .startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .startObject(KnnSearchRequestParser.KNN_SECTION_FIELD.getPreferredName()) .field(KnnSearch.FIELD_FIELD.getPreferredName(), knnSearch.field) .field(KnnSearch.K_FIELD.getPreferredName(), knnSearch.k) .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands) @@ -144,7 +144,7 @@ public void testParseSourceArray() throws IOException { builder.endObject(); // Convert the REST request to a search request and check the components - SearchRequestBuilder searchRequestBuilder = buildSearchRequest(builder); + SearchRequestBuilder searchRequestBuilder = parseSearchRequest(builder); SearchRequest searchRequest = searchRequestBuilder.request(); FetchSourceContext fetchSource = searchRequest.source().fetchSource(); @@ -159,7 +159,7 @@ public void testMissingKnnSection() throws IOException { .array(SearchSourceBuilder.FETCH_FIELDS_FIELD.getPreferredName(), "field1", "field2") .endObject(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> buildSearchRequest(builder)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseSearchRequest(builder)); assertThat(e.getMessage(), containsString("missing required [knn] section in search body")); } @@ -167,7 +167,7 @@ public void testNumCandsLessThanK() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()) .startObject() - .startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .startObject(KnnSearchRequestParser.KNN_SECTION_FIELD.getPreferredName()) .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") .field(KnnSearch.K_FIELD.getPreferredName(), 100) .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 80) @@ -175,7 +175,7 @@ public void testNumCandsLessThanK() throws IOException { .endObject() .endObject(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> buildSearchRequest(builder)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseSearchRequest(builder)); assertThat(e.getMessage(), containsString("[num_candidates] cannot be less than [k]")); } @@ -183,7 +183,7 @@ public void testNumCandsExceedsLimit() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()) .startObject() - .startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .startObject(KnnSearchRequestParser.KNN_SECTION_FIELD.getPreferredName()) .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") .field(KnnSearch.K_FIELD.getPreferredName(), 100) .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 10002) @@ -191,7 +191,7 @@ public void testNumCandsExceedsLimit() throws IOException { .endObject() .endObject(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> buildSearchRequest(builder)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseSearchRequest(builder)); assertThat(e.getMessage(), containsString("[num_candidates] cannot exceed [10000]")); } @@ -199,7 +199,7 @@ public void testInvalidK() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()) .startObject() - .startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .startObject(KnnSearchRequestParser.KNN_SECTION_FIELD.getPreferredName()) .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") .field(KnnSearch.K_FIELD.getPreferredName(), 0) .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 10) @@ -207,24 +207,24 @@ public void testInvalidK() throws IOException { .endObject() .endObject(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> buildSearchRequest(builder)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseSearchRequest(builder)); assertThat(e.getMessage(), containsString("[k] must be greater than 0")); } - private SearchRequestBuilder buildSearchRequest(XContentBuilder builder) throws IOException { - Map params = Map.of(KnnSearchRequestBuilder.INDEX_PARAM, "index"); - return buildSearchRequest(builder, params); + private SearchRequestBuilder parseSearchRequest(XContentBuilder builder) throws IOException { + Map params = Map.of(KnnSearchRequestParser.INDEX_PARAM, "index"); + return parseSearchRequest(builder, params); } - private SearchRequestBuilder buildSearchRequest(XContentBuilder builder, Map params) throws IOException { - KnnSearchRequestBuilder knnRequestBuilder = KnnSearchRequestBuilder.parseRestRequest( + private SearchRequestBuilder parseSearchRequest(XContentBuilder builder, Map params) throws IOException { + KnnSearchRequestParser knnRequestBuilder = KnnSearchRequestParser.parseRestRequest( new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST) .withParams(params) .withContent(BytesReference.bytes(builder), builder.contentType()) .build() ); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(null, SearchAction.INSTANCE); - knnRequestBuilder.build(searchRequestBuilder); + knnRequestBuilder.toSearchRequest(searchRequestBuilder); return searchRequestBuilder; } @@ -257,7 +257,7 @@ private XContentBuilder createRequestBody(KnnSearch knnSearch, List 1) { builder.startArray(); } diff --git a/server/src/test/resources/META-INF/services/org.elasticsearch.plugins.spi.TestService b/server/src/test/resources/META-INF/services/org.elasticsearch.plugins.spi.TestService new file mode 100644 index 0000000000000..ffeecb79691ed --- /dev/null +++ b/server/src/test/resources/META-INF/services/org.elasticsearch.plugins.spi.TestService @@ -0,0 +1,9 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +org.elasticsearch.plugins.spi.BarTestService diff --git a/test/framework/src/integTest/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/integTest/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java index e5f6c059b94f9..835ab2e5475a6 100644 --- a/test/framework/src/integTest/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java +++ b/test/framework/src/integTest/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java @@ -17,7 +17,6 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportException; @@ -75,7 +74,7 @@ private Tuple, Set> prepareDisruptedCluster() { public void testClearDisruptionSchemeWhenNodeIsDown() throws IOException { Tuple, Set> sides = prepareDisruptedCluster(); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(randomFrom(sides.v2()))); + internalCluster().stopNode(randomFrom(sides.v2())); internalCluster().clearDisruptionScheme(); } diff --git a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java index 60c10f430c911..fda64f9e654a3 100644 --- a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java +++ b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java @@ -14,12 +14,16 @@ import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.spi.SPIClassIterator; +import java.lang.reflect.Constructor; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; public class MockPluginsService extends PluginsService { @@ -76,4 +80,51 @@ public PluginsAndModules info() { List.of() ); } + + @Override + public List loadServiceProviders(Class service) { + // We use a set here to avoid duplicates because SPIClassIterator will match + // all plugins in MockNode, because all plugins are loaded by the same class loader. + Set result = new HashSet<>(); + + for (LoadedPlugin pluginTuple : plugins()) { + result.addAll(createExtensions(service, pluginTuple.instance())); + } + + return List.copyOf(result); + } + + /** + * When we load tests with MockNode, all plugins are loaded with the same class loader, + * which breaks loading service providers with our SPIClassIterator. Since all plugins are + * loaded in the same class loader, we find all plugins for any class found by the SPIClassIterator + * causing us to pass wrong plugin type to createExtension. This modified createExtensions, checks for + * the type and returns an empty list if the plugin class type is incompatible. + */ + static List createExtensions(Class extensionPointType, Plugin plugin) { + SPIClassIterator classIterator = SPIClassIterator.get(extensionPointType, plugin.getClass().getClassLoader()); + List extensions = new ArrayList<>(); + while (classIterator.hasNext()) { + Class extensionClass = classIterator.next(); + + @SuppressWarnings("unchecked") + Constructor[] constructors = (Constructor[]) extensionClass.getConstructors(); + boolean compatible = true; + + // We only check if we have incompatible one argument constructor, otherwise we let the code + // fall-through to the PluginsService method that will check if we have valid service provider. + // For one argument constructors we cannot validate from which plugin they should be loaded, which + // is why we de-dup the instances by using a Set in loadServiceProviders. + for (var constructor : constructors) { + if (constructor.getParameterCount() == 1 && constructor.getParameterTypes()[0] != plugin.getClass()) { + compatible = false; + break; + } + } + if (compatible) { + extensions.add(createExtension(extensionClass, extensionPointType, plugin)); + } + } + return extensions; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index eef6c2a87cca2..a0bb992745354 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -32,6 +32,7 @@ import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.vectors.KnnSearchBuilder; import org.elasticsearch.test.AbstractQueryTestCase; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -50,6 +51,7 @@ import static org.elasticsearch.test.ESTestCase.between; import static org.elasticsearch.test.ESTestCase.generateRandomStringArray; import static org.elasticsearch.test.ESTestCase.mockScript; +import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween; import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomByte; @@ -245,6 +247,19 @@ public static SearchSourceBuilder randomSearchSourceBuilder( if (randomBoolean()) { builder.postFilter(QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20))); } + + if (randomBoolean()) { + String field = randomAlphaOfLength(6); + int dim = randomIntBetween(2, 30); + float[] vector = new float[dim]; + for (int i = 0; i < vector.length; i++) { + vector[i] = randomFloat(); + } + int k = randomIntBetween(1, 100); + int numCands = randomIntBetween(k, 1000); + builder.knnSearch(new KnnSearchBuilder(field, vector, k, numCands)); + } + if (randomBoolean()) { int numSorts = randomIntBetween(1, 5); for (int i = 0; i < numSorts; i++) { diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryBuilderTestCase.java similarity index 78% rename from server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java rename to test/framework/src/main/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryBuilderTestCase.java index c0875b88207da..b5de27adbf677 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java +++ b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryBuilderTestCase.java @@ -6,66 +6,38 @@ * Side Public License, v 1. */ -package org.elasticsearch.index.query; +package org.elasticsearch.search.geo; -import org.apache.lucene.document.LatLonDocValuesField; -import org.apache.lucene.document.LatLonPoint; -import org.apache.lucene.geo.GeoEncodingUtils; -import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.Geohash; -import org.elasticsearch.index.mapper.GeoPointFieldMapper; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; +import org.elasticsearch.index.query.GeoValidationMethod; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; -import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.Collection; -import java.util.Collections; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.notNullValue; @SuppressWarnings("checkstyle:MissingJavadocMethod") -public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase { +public abstract class GeoBoundingBoxQueryBuilderTestCase extends AbstractQueryTestCase { /** Randomly generate either NaN or one of the two infinity values. */ private static final Double[] brokenDoubles = { Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY }; - private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; - protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; - @Override - protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { - final XContentBuilder builder = PutMappingRequest.simpleMapping( - GEO_SHAPE_FIELD_NAME, - "type=geo_shape", - GEO_SHAPE_ALIAS_FIELD_NAME, - "type=alias,path=" + GEO_SHAPE_FIELD_NAME - ); - mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); - } - - @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled - protected Collection> getPlugins() { - return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); - } + protected abstract String getFieldName(); @Override protected GeoBoundingBoxQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); + String fieldName = getFieldName(); GeoBoundingBoxQueryBuilder builder = new GeoBoundingBoxQueryBuilder(fieldName); // make sure that minX != maxX and minY != maxY after geohash encoding Rectangle box = randomValueOtherThanMany( @@ -141,34 +113,6 @@ public void testBrokenCoordinateCanBeSetWithIgnoreMalformed() { } } - public void testValidation() { - PointTester[] testers = { new TopTester(), new LeftTester(), new BottomTester(), new RightTester() }; - - for (PointTester tester : testers) { - GeoBoundingBoxQueryBuilder builder = createTestQueryBuilder(); - tester.invalidateCoordinate(builder.setValidationMethod(GeoValidationMethod.COERCE), false); - QueryValidationException except = builder.checkLatLon(); - assertNull( - "validation w/ coerce should ignore invalid " - + tester.getClass().getName() - + " coordinate: " - + tester.invalidCoordinate - + " ", - except - ); - - tester.invalidateCoordinate(builder.setValidationMethod(GeoValidationMethod.STRICT), false); - except = builder.checkLatLon(); - assertNotNull( - "validation w/o coerce should detect invalid coordinate: " - + tester.getClass().getName() - + " coordinate: " - + tester.invalidCoordinate, - except - ); - } - } - public void testTopBottomCannotBeFlipped() { GeoBoundingBoxQueryBuilder builder = createTestQueryBuilder(); double top = builder.topLeft().getLat(); @@ -241,37 +185,15 @@ public void testStrictnessDefault() { ); } - @Override - protected void doAssertLuceneQuery(GeoBoundingBoxQueryBuilder queryBuilder, Query query, SearchExecutionContext context) - throws IOException { - final MappedFieldType fieldType = context.getFieldType(queryBuilder.fieldName()); - if (fieldType == null) { - assertTrue("Found no indexed geo query.", query instanceof MatchNoDocsQuery); - } else if (fieldType instanceof GeoPointFieldMapper.GeoPointFieldType) { - assertEquals(IndexOrDocValuesQuery.class, query.getClass()); - Query indexQuery = ((IndexOrDocValuesQuery) query).getIndexQuery(); - String expectedFieldName = expectedFieldName(queryBuilder.fieldName()); - double qMinLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(queryBuilder.bottomRight().lat())); - double qMaxLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(queryBuilder.topLeft().lat())); - double qMinLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(queryBuilder.topLeft().lon())); - double qMaxLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(queryBuilder.bottomRight().lon())); - assertEquals(LatLonPoint.newBoxQuery(expectedFieldName, qMinLat, qMaxLat, qMinLon, qMaxLon), indexQuery); - Query dvQuery = ((IndexOrDocValuesQuery) query).getRandomAccessQuery(); - assertEquals(LatLonDocValuesField.newSlowBoxQuery(expectedFieldName, qMinLat, qMaxLat, qMinLon, qMaxLon), dvQuery); - } else { - assertEquals(GeoShapeFieldMapper.GeoShapeFieldType.class, fieldType.getClass()); - } - } - - private abstract static class PointTester { + protected abstract static class PointTester { private final double brokenCoordinate = randomFrom(brokenDoubles); - private final double invalidCoordinate; + public final double invalidCoordinate; - private PointTester(double invalidCoodinate) { + protected PointTester(double invalidCoodinate) { this.invalidCoordinate = invalidCoodinate; } - private void invalidateCoordinate(GeoBoundingBoxQueryBuilder qb, boolean useBrokenDouble) { + public void invalidateCoordinate(GeoBoundingBoxQueryBuilder qb, boolean useBrokenDouble) { if (useBrokenDouble) { fillIn(brokenCoordinate, qb); } else { @@ -282,8 +204,8 @@ private void invalidateCoordinate(GeoBoundingBoxQueryBuilder qb, boolean useBrok protected abstract void fillIn(double fillIn, GeoBoundingBoxQueryBuilder qb); } - private static class TopTester extends PointTester { - private TopTester() { + protected static class TopTester extends PointTester { + public TopTester() { super(randomDoubleBetween(GeoUtils.MAX_LAT, Double.MAX_VALUE, false)); } @@ -293,8 +215,8 @@ public void fillIn(double coordinate, GeoBoundingBoxQueryBuilder qb) { } } - private static class LeftTester extends PointTester { - private LeftTester() { + protected static class LeftTester extends PointTester { + public LeftTester() { super(randomDoubleBetween(-Double.MAX_VALUE, GeoUtils.MIN_LON, true)); } @@ -304,8 +226,8 @@ public void fillIn(double coordinate, GeoBoundingBoxQueryBuilder qb) { } } - private static class BottomTester extends PointTester { - private BottomTester() { + protected static class BottomTester extends PointTester { + public BottomTester() { super(randomDoubleBetween(-Double.MAX_VALUE, GeoUtils.MIN_LAT, false)); } @@ -315,8 +237,8 @@ public void fillIn(double coordinate, GeoBoundingBoxQueryBuilder qb) { } } - private static class RightTester extends PointTester { - private RightTester() { + protected static class RightTester extends PointTester { + public RightTester() { super(randomDoubleBetween(GeoUtils.MAX_LON, Double.MAX_VALUE, true)); } diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoDistanceQueryBuilderTestCase.java similarity index 81% rename from server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java rename to test/framework/src/main/java/org/elasticsearch/search/geo/GeoDistanceQueryBuilderTestCase.java index a564b04122537..5293d88e86121 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java +++ b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoDistanceQueryBuilderTestCase.java @@ -6,64 +6,36 @@ * Side Public License, v 1. */ -package org.elasticsearch.index.query; +package org.elasticsearch.search.geo; -import org.apache.lucene.document.LatLonDocValuesField; -import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.geo.GeoEncodingUtils; -import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.geo.GeometryTestUtils; -import org.elasticsearch.index.mapper.GeoPointFieldMapper; -import org.elasticsearch.index.mapper.GeoShapeFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.index.query.GeoDistanceQueryBuilder; +import org.elasticsearch.index.query.GeoValidationMethod; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; -import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.Collection; -import java.util.Collections; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.notNullValue; @SuppressWarnings("checkstyle:MissingJavadocMethod") -public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase { +public abstract class GeoDistanceQueryBuilderTestCase extends AbstractQueryTestCase { - private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; - protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; - - @Override - protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { - final XContentBuilder builder = PutMappingRequest.simpleMapping( - GEO_SHAPE_FIELD_NAME, - "type=geo_shape", - GEO_SHAPE_ALIAS_FIELD_NAME, - "type=alias,path=" + GEO_SHAPE_FIELD_NAME - ); - mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); - } - - @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled - protected Collection> getPlugins() { - return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); - } + protected abstract String getFieldName(); @Override protected GeoDistanceQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); + String fieldName = getFieldName(); GeoDistanceQueryBuilder qb = new GeoDistanceQueryBuilder(fieldName); String distance = "" + randomDouble(); if (randomBoolean()) { @@ -140,26 +112,6 @@ public void testToQuery() throws IOException { super.testToQuery(); } - @Override - protected void doAssertLuceneQuery(GeoDistanceQueryBuilder queryBuilder, Query query, SearchExecutionContext context) - throws IOException { - final MappedFieldType fieldType = context.getFieldType(queryBuilder.fieldName()); - if (fieldType == null) { - assertTrue("Found no indexed geo query.", query instanceof MatchNoDocsQuery); - } else if (fieldType instanceof GeoPointFieldMapper.GeoPointFieldType) { - Query indexQuery = ((IndexOrDocValuesQuery) query).getIndexQuery(); - - String expectedFieldName = expectedFieldName(queryBuilder.fieldName()); - double qLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(queryBuilder.point().lat())); - double qLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(queryBuilder.point().lon())); - assertEquals(LatLonPoint.newDistanceQuery(expectedFieldName, qLat, qLon, queryBuilder.distance()), indexQuery); - Query dvQuery = ((IndexOrDocValuesQuery) query).getRandomAccessQuery(); - assertEquals(LatLonDocValuesField.newSlowDistanceQuery(expectedFieldName, qLat, qLon, queryBuilder.distance()), dvQuery); - } else { - assertEquals(GeoShapeFieldMapper.GeoShapeFieldType.class, fieldType.getClass()); - } - } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/86834") public void testParsingAndToQueryGeoJSON() throws IOException { // TODO: GeoJSON support missing for geo_distance query, although all other point formats work diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoShapeQueryBuilderTestCase.java similarity index 90% rename from server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java rename to test/framework/src/main/java/org/elasticsearch/search/geo/GeoShapeQueryBuilderTestCase.java index 308324acc43c6..507ea6a89aa04 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java +++ b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoShapeQueryBuilderTestCase.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.index.query; +package org.elasticsearch.search.geo; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; @@ -22,6 +22,13 @@ import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.index.get.GetResult; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.GeoShapeQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.index.query.Rewriteable; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.test.AbstractQueryTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -39,7 +46,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -public abstract class GeoShapeQueryBuilderTests extends AbstractQueryTestCase { +public abstract class GeoShapeQueryBuilderTestCase extends AbstractQueryTestCase { protected static String indexedShapeId; protected static String indexedShapePath; @@ -47,14 +54,14 @@ public abstract class GeoShapeQueryBuilderTests extends AbstractQueryTestCase new GeoShapeQueryBuilder(fieldName(), (Geometry) null)); + expectThrows(IllegalArgumentException.class, () -> new GeoShapeQueryBuilder(getFieldName(), (Geometry) null)); } public void testNoIndexedShape() { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new GeoShapeQueryBuilder(fieldName(), null, null)); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new GeoShapeQueryBuilder(getFieldName(), null, null) + ); assertEquals("either shape or indexedShapeId is required", e.getMessage()); } public void testNoRelation() { Geometry shape = GeometryTestUtils.randomGeometry(false); - GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(fieldName(), shape); + GeoShapeQueryBuilder builder = new GeoShapeQueryBuilder(getFieldName(), shape); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.relation(null)); assertEquals("No Shape Relation defined", e.getMessage()); } @@ -158,7 +168,7 @@ public void testMustRewrite() throws IOException { ); assertEquals("query must be rewritten first", e.getMessage()); QueryBuilder rewrite = rewriteAndFetch(query, createSearchExecutionContext()); - GeoShapeQueryBuilder geoShapeQueryBuilder = new GeoShapeQueryBuilder(fieldName(), indexedShapeToReturn); + GeoShapeQueryBuilder geoShapeQueryBuilder = new GeoShapeQueryBuilder(query.fieldName(), indexedShapeToReturn); geoShapeQueryBuilder.strategy(query.strategy()); geoShapeQueryBuilder.relation(query.relation()); assertEquals(geoShapeQueryBuilder, rewrite); @@ -169,7 +179,7 @@ public void testMultipleRewrite() { QueryBuilder builder = new BoolQueryBuilder().should(shape).should(shape); builder = rewriteAndFetch(builder, createSearchExecutionContext()); - GeoShapeQueryBuilder expectedShape = new GeoShapeQueryBuilder(fieldName(), indexedShapeToReturn); + GeoShapeQueryBuilder expectedShape = new GeoShapeQueryBuilder(shape.fieldName(), indexedShapeToReturn); expectedShape.strategy(shape.strategy()); expectedShape.relation(shape.relation()); QueryBuilder expected = new BoolQueryBuilder().should(expectedShape).should(expectedShape); diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 83f81b57dc3d0..81757fd24e2ed 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -197,7 +197,7 @@ public static int numberOfFiles(Path dir) throws IOException { protected void stopNode(final String node) throws IOException { logger.info("--> stopping node {}", node); - internalCluster().stopRandomNode(settings -> settings.get("node.name").equals(node)); + internalCluster().stopNode(node); } protected static String startDataNodeWithLargeSnapshotPool() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index a21455da86902..b29af2adc8d4c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -59,7 +59,6 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasables; @@ -1675,6 +1674,7 @@ public synchronized boolean stopNode(String nodeName) throws IOException { ensureOpen(); Optional nodeToStop = nodes.values().stream().filter(n -> n.getName().equals(nodeName)).findFirst(); if (nodeToStop.isPresent()) { + ensureNotTheLastMasterEligibleNode(nodeToStop.get()); logger.info("Closing node [{}]", nodeToStop.get().name); stopNodesAndClient(nodeToStop.get()); return true; @@ -1690,22 +1690,26 @@ public synchronized void stopRandomNode(final Predicate filter) throws ensureOpen(); NodeAndClient nodeAndClient = getRandomNodeAndClient(nc -> filter.test(nc.node.settings())); if (nodeAndClient != null) { - if (nodePrefix.equals(ESIntegTestCase.SUITE_CLUSTER_NODE_PREFIX) - && nodeAndClient.nodeAndClientId() < sharedNodesSeeds.length - && nodeAndClient.isMasterEligible() - && autoManageMasterNodes - && nodes.values() - .stream() - .filter(NodeAndClient::isMasterEligible) - .filter(n -> n.nodeAndClientId() < sharedNodesSeeds.length) - .count() == 1) { - throw new AssertionError("Tried to stop the only master eligible shared node"); - } + ensureNotTheLastMasterEligibleNode(nodeAndClient); logger.info("Closing filtered random node [{}] ", nodeAndClient.name); stopNodesAndClient(nodeAndClient); } } + private void ensureNotTheLastMasterEligibleNode(NodeAndClient nodeAndClient) { + if (nodePrefix.equals(ESIntegTestCase.SUITE_CLUSTER_NODE_PREFIX) + && nodeAndClient.nodeAndClientId() < sharedNodesSeeds.length + && nodeAndClient.isMasterEligible() + && autoManageMasterNodes + && nodes.values() + .stream() + .filter(NodeAndClient::isMasterEligible) + .filter(n -> n.nodeAndClientId() < sharedNodesSeeds.length) + .count() == 1) { + throw new AssertionError("Tried to stop the only master eligible shared node"); + } + } + /** * Stops the current master node forcefully */ @@ -2384,14 +2388,6 @@ public NamedWriteableRegistry getNamedWriteableRegistry() { return getInstance(NamedWriteableRegistry.class); } - /** - * Returns a predicate that only accepts settings of nodes with one of the given names. - */ - public static Predicate nameFilter(String... nodeNames) { - final Set nodes = Sets.newHashSet(nodeNames); - return settings -> nodes.contains(settings.get("node.name")); - } - /** * An abstract class that is called during {@link #rollingRestart(InternalTestCluster.RestartCallback)} * and / or {@link #fullRestart(InternalTestCluster.RestartCallback)} to execute actions at certain diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index a37fd7c0814e2..ed4ef0c201abc 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -307,7 +307,7 @@ public Path nodeConfigPath(int nodeOrdinal) { final Settings poorNodeDataPathSettings = cluster.dataPathSettings(poorNode); final Path testMarker = dataPath.resolve("testMarker"); Files.createDirectories(testMarker); - cluster.stopRandomNode(InternalTestCluster.nameFilter(poorNode)); + cluster.stopNode(poorNode); assertFileExists(testMarker); // stopping a node half way shouldn't clean data final String stableNode = randomFrom(cluster.getNodeNames()); diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index b7e6fb0d9ea8c..20b8a7013bdd6 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -60,7 +60,7 @@ /** * Runs a suite of yaml tests shared with all the official Elasticsearch - * clients against against an elasticsearch cluster. + * clients against an elasticsearch cluster. * * The suite timeout is extended to account for projects with a large number of tests. */ diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index db4209ec60e9d..b65c0fb3da31b 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -228,7 +228,9 @@ event action. ["read","maintenance"]},{"names":["in*","alias*"],"privileges":["read"], "field_security":{"grant":["field1*","@timestamp"],"except":["field11"]}}], "applications":[],"run_as":[]},{"cluster":["all"],"indices":[{"names": -["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}]}}} +["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}], +"metadata":{"application":"my-application","environment":{"level": 1, +"tags":["dev","staging"]}}}}} ==== [[event-change-apikey]] @@ -251,7 +253,9 @@ event action. ["read","maintenance"]},{"names":["in*","alias*"],"privileges":["read"], "field_security":{"grant":["field1*","@timestamp"],"except":["field11"]}}], "applications":[],"run_as":[]},{"cluster":["all"],"indices":[{"names": -["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}]}}} +["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}], +"metadata":{"application":"my-application","environment":{"level": 1, +"tags":["dev","staging"]}}}}} ==== [[event-delete-privileges]] @@ -757,7 +761,8 @@ the <>. + [source,js] ---- -`{"id": , "name": , "expiration": , "role_descriptors" []}` +`{"id": , "name": , "expiration": , "role_descriptors" [], +"metadata" []}` ---- // NOTCONSOLE + diff --git a/x-pack/docs/en/security/fips-140-compliance.asciidoc b/x-pack/docs/en/security/fips-140-compliance.asciidoc index 13583400545c5..c34e9e8e92cfd 100644 --- a/x-pack/docs/en/security/fips-140-compliance.asciidoc +++ b/x-pack/docs/en/security/fips-140-compliance.asciidoc @@ -113,11 +113,12 @@ keys must have corresponding length according to the following table: |======================= [discrete] -==== Password Hashing +==== Stored password hashing +[[fips-stored-password-hashing]] -While {es} offers a number of algorithms for securely hashing credentials in memory and +While {es} offers a number of algorithms for securely hashing credentials on disk, only the `PBKDF2` based family of algorithms is compliant with FIPS -140-2 for password hashing. However, since `PBKDF2` is essentially a key derivation +140-2 for stored password hashing. However, since `PBKDF2` is essentially a key derivation function, your JVM security provider may enforce a <>. Although FIPS 140-2 does not mandate user password standards, this requirement may affect password @@ -131,8 +132,7 @@ NOTE: You can still use one of the plain `pbkdf2` options instead of `pbkdf2_str you have external policies and tools that can ensure all user passwords for the reserved, native, and file realms are longer than 14 bytes. -You must set the `cache.hash_algo` realm settings -and the `xpack.security.authc.password_hashing.algorithm` setting to one of the +You must set the `xpack.security.authc.password_hashing.algorithm` setting to one of the available `pbkdf_stretch_*` values. When FIPS-140 mode is enabled, the default value for `xpack.security.authc.password_hashing.algorithm` is `pbkdf2_stretch`. @@ -147,9 +147,27 @@ for the file realm and the <> and <> APIs for the native and reserved realms. Other types of realms are not affected and do not require any changes. -The user cache will be emptied upon node restart, so any existing hashes using -non-compliant algorithms will be discarded and the new ones will be created -using the compliant `PBKDF2` based algorithm you have selected. +[discrete] +==== Cached password hashing +[[fips-cached-password-hashing]] + +`ssha256` (salted `sha256`) is recommended for cache hashing. Though +`PBKDF2` is compliant with FIPS-140-2, it is -- by design -- slow, and +thus not generally suitable as a cache hashing algorithm. Cached +credentials are never stored on disk, and salted `sha256` provides an +adequate level of security for in-memory credential hashing, without +imposing prohibitive performance overhead. You _may_ use `PBKDF2`, +however you should carefully assess performance impact first. Depending +on your deployment, the overhead of `PBKDF2` could undo most of the +performance gain of using a cache. + +Either set all `cache.hash_algo` settings to `ssha256` or leave +them undefined, since `ssha256` is the default value for all +`cache.hash_algo` settings. See <>. + +The user cache will be emptied upon node restart, so any existing +hashes using non-compliant algorithms will be discarded and the new +ones will be created using the algorithm you have selected. [discrete] === Limitations diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java index 476c6b21df034..c9e24032bf37f 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java @@ -218,18 +218,8 @@ static Optional singleNoDecision(Decision decision, Predicate } } - static long nodeSizeForDataBelowLowWatermark(long bytes, DiskThresholdSettings thresholdSettings) { - ByteSizeValue bytesThreshold = thresholdSettings.getFreeBytesThresholdLow(); - if (bytesThreshold.getBytes() != 0) { - return bytesThreshold.getBytes() + bytes; - } else { - double percentThreshold = thresholdSettings.getFreeDiskThresholdLow(); - if (percentThreshold >= 0.0 && percentThreshold < 100.0) { - return (long) (100 * bytes / (100 - percentThreshold)); - } else { - return bytes; - } - } + static long nodeSizeForDataBelowLowWatermark(long neededBytes, DiskThresholdSettings thresholdSettings) { + return thresholdSettings.getMinimumTotalSizeForBelowLowWatermark(ByteSizeValue.ofBytes(neededBytes)).getBytes(); } // todo: move this to top level class. @@ -547,22 +537,12 @@ long unmovableSize(String nodeId, Collection shards) { return 0; } - long threshold = Math.max( - diskThresholdSettings.getFreeBytesThresholdHigh().getBytes(), - thresholdFromPercentage(diskThresholdSettings.getFreeDiskThresholdHigh(), diskUsage) - ); + long threshold = diskThresholdSettings.getFreeBytesThresholdHighStage(ByteSizeValue.ofBytes(diskUsage.getTotalBytes())) + .getBytes(); long missing = threshold - diskUsage.getFreeBytes(); return Math.max(missing, shards.stream().mapToLong(this::sizeOf).min().orElseThrow()); } - private long thresholdFromPercentage(Double percentage, DiskUsage diskUsage) { - if (percentage == null) { - return 0L; - } - - return (long) Math.ceil(diskUsage.getTotalBytes() * percentage / 100); - } - Stream nodesInTier(RoutingNodes routingNodes) { return nodeIds.stream().map(routingNodes::node); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java index 353fa8d789c84..ccfaa5131fb0d 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java @@ -355,39 +355,6 @@ public void testNodeLockSplitClone() { assertThat(createAllocationState(shardSizes, clusterState).maxNodeLockedSize(), equalTo(sourceSize * 2)); } - public void testNodeSizeForDataBelowLowWatermark() { - final ClusterSettings emptyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - final DiskThresholdSettings defaultSettings = new DiskThresholdSettings(Settings.EMPTY, emptyClusterSettings); - final long factor = between(1, 1000); - assertThat(ReactiveStorageDeciderService.nodeSizeForDataBelowLowWatermark(85 * factor, defaultSettings), equalTo(100L * factor)); - - // to make it easy, stay below high watermark. - final long percentage = between(1, 89); - final DiskThresholdSettings relativeSettings = new DiskThresholdSettings( - Settings.builder() - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), percentage + "%") - .build(), - emptyClusterSettings - ); - assertThat( - ReactiveStorageDeciderService.nodeSizeForDataBelowLowWatermark(percentage * factor, relativeSettings), - equalTo(100L * factor) - ); - - final long absolute = between(1, 1000); - final DiskThresholdSettings absoluteSettings = new DiskThresholdSettings( - Settings.builder() - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), absolute + "b") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), absolute + "b") - .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), absolute + "b") - .build(), - emptyClusterSettings - ); - - long needed = between(0, 1000); - assertThat(ReactiveStorageDeciderService.nodeSizeForDataBelowLowWatermark(needed, absoluteSettings), equalTo(needed + absolute)); - } - private Settings.Builder addRandomNodeLockUsingAttributes(Settings.Builder settings) { String setting = randomFrom( IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleAction.java index bee317085815a..62aeba41d81b8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; @@ -62,8 +63,12 @@ public LifecyclePolicy getPolicy() { @Override public ActionRequestValidationException validate() { - this.policy.validate(); ActionRequestValidationException err = null; + try { + this.policy.validate(); + } catch (IllegalArgumentException iae) { + err = ValidateActions.addValidationError(iae.getMessage(), null); + } String phaseTimingErr = TimeseriesLifecycleType.validateMonotonicallyIncreasingPhaseTimings(this.policy.getPhases().values()); if (Strings.hasText(phaseTimingErr)) { err = new ActionRequestValidationException(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index 40e6b3fe2bf6f..02f6b24d17db1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -719,10 +719,16 @@ public void cleanUpFeature( List systemPlugins = filterPlugins(SystemIndexPlugin.class); GroupedActionListener allListeners = new GroupedActionListener<>( - ActionListener.wrap( - listenerResults -> finalListener.onResponse(ResetFeatureStateStatus.success(getFeatureName())), - finalListener::onFailure - ), + ActionListener.wrap(listenerResults -> { + // If the clean-up produced only one result, use that to pass along. In most + // cases it should be 1-1 mapping of feature to response. Passing back success + // prevents us from writing validation tests on this API. + if (listenerResults != null && listenerResults.size() == 1) { + finalListener.onResponse(listenerResults.stream().findFirst().get()); + } else { + finalListener.onResponse(ResetFeatureStateStatus.success(getFeatureName())); + } + }, finalListener::onFailure), systemPlugins.size() ); systemPlugins.forEach(plugin -> plugin.cleanUpFeature(clusterService, client, allListeners)); diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java index 22cb856cccec3..32e936346a589 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.tasks.Task; @@ -158,9 +159,9 @@ static DeprecationIssue checkDiskLowWatermark( DiskUsage usage = clusterInfo.getNodeMostAvailableDiskUsages().get(nodeId); if (usage != null) { long freeBytes = usage.getFreeBytes(); - double freeDiskPercentage = usage.getFreeDiskAsPercentage(); - if (exceedsLowWatermark(nodeSettings, clusterSettings, freeBytes, freeDiskPercentage) - || exceedsLowWatermark(dynamicSettings, clusterSettings, freeBytes, freeDiskPercentage)) { + long totalBytes = usage.getTotalBytes(); + if (exceedsLowWatermark(nodeSettings, clusterSettings, freeBytes, totalBytes) + || exceedsLowWatermark(dynamicSettings, clusterSettings, freeBytes, totalBytes)) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, "Disk usage exceeds low watermark", @@ -179,15 +180,9 @@ static DeprecationIssue checkDiskLowWatermark( return null; } - private static boolean exceedsLowWatermark( - Settings settingsToCheck, - ClusterSettings clusterSettings, - long freeBytes, - double freeDiskPercentage - ) { + private static boolean exceedsLowWatermark(Settings settingsToCheck, ClusterSettings clusterSettings, long freeBytes, long totalBytes) { DiskThresholdSettings diskThresholdSettings = new DiskThresholdSettings(settingsToCheck, clusterSettings); - if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLow().getBytes() - || freeDiskPercentage < diskThresholdSettings.getFreeDiskThresholdLow()) { + if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLowStage(ByteSizeValue.ofBytes(totalBytes)).getBytes()) { return true; } return false; diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java index 9d2887b69b3ed..a5368cb38d5dd 100644 --- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java +++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java @@ -35,7 +35,6 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; import org.elasticsearch.xpack.frozen.FrozenIndices; @@ -104,7 +103,7 @@ public void testTimestampRangeRecalculatedOnStalePrimaryAllocation() throws IOEx sameInstance(IndexLongFieldRange.EMPTY) ); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeNames.get(1))); + internalCluster().stopNode(nodeNames.get(1)); assertThat(client().admin().cluster().prepareHealth("index").get().getUnassignedShards(), equalTo(2)); assertAcked(client().admin().indices().prepareUpdateSettings("index").setSettings(Settings.builder().putNull(excludeSetting))); assertThat(client().admin().cluster().prepareHealth("index").get().getUnassignedShards(), equalTo(2)); diff --git a/x-pack/plugin/ml/licenses/lucene-analysis-icu-9.3.0-snapshot-2d05f5c623e.jar.sha1 b/x-pack/plugin/ml/licenses/lucene-analysis-icu-9.3.0-snapshot-2d05f5c623e.jar.sha1 deleted file mode 100644 index 73d5dfdcae436..0000000000000 --- a/x-pack/plugin/ml/licenses/lucene-analysis-icu-9.3.0-snapshot-2d05f5c623e.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -87772b1d9907c63287d7bc79cf8fa3e9a46c9c77 \ No newline at end of file diff --git a/x-pack/plugin/ml/licenses/lucene-analysis-icu-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 b/x-pack/plugin/ml/licenses/lucene-analysis-icu-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 new file mode 100644 index 0000000000000..ceea8ba4f6855 --- /dev/null +++ b/x-pack/plugin/ml/licenses/lucene-analysis-icu-9.3.0-snapshot-b8d1fcfd0ec.jar.sha1 @@ -0,0 +1 @@ +501aa4f0028424a994b06627f30ffb36150ffbe2 \ No newline at end of file diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index a3a4235357c04..df1224cf760c2 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -22,7 +22,6 @@ import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; @@ -422,7 +421,7 @@ public void testMlStateAndResultsIndicesNotAvailable() throws Exception { }); logger.info("Stop non ml node"); Settings nonMLNodeDataPathSettings = internalCluster().dataPathSettings(nonMLNode); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nonMLNode)); + internalCluster().stopNode(nonMLNode); ensureStableCluster(1); Exception e = expectThrows( diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index 328677ff16d69..864a832ce03b5 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -108,7 +108,7 @@ public void testFailOver() throws Exception { GetJobsStatsAction.Request request = new GetJobsStatsAction.Request("fail-over-job"); GetJobsStatsAction.Response response = client().execute(GetJobsStatsAction.INSTANCE, request).actionGet(); DiscoveryNode discoveryNode = response.getResponse().results().get(0).getNode(); - internalCluster().stopRandomNode(settings -> discoveryNode.getName().equals(settings.get("node.name"))); + internalCluster().stopNode(discoveryNode.getName()); ensureStableCluster(); }); } @@ -331,7 +331,7 @@ public void testCloseUnassignedFailedJobAndStopUnassignedStoppingDatafeed() thro // Stop the node running the failed job/stopping datafeed ensureGreen(); // replicas must be assigned, otherwise we could lose a whole index - internalCluster().stopRandomNode(settings -> jobNode.getName().equals(settings.get("node.name"))); + internalCluster().stopNode(jobNode.getName()); ensureStableCluster(3); // We should be allowed to force stop the unassigned datafeed even though it is stopping and its job has failed diff --git a/x-pack/plugin/repository-encrypted/src/internalClusterTest/java/org/elasticsearch/repositories/encrypted/EncryptedRepositorySecretIntegTests.java b/x-pack/plugin/repository-encrypted/src/internalClusterTest/java/org/elasticsearch/repositories/encrypted/EncryptedRepositorySecretIntegTests.java index 579af5f4905cb..19473c3784b5d 100644 --- a/x-pack/plugin/repository-encrypted/src/internalClusterTest/java/org/elasticsearch/repositories/encrypted/EncryptedRepositorySecretIntegTests.java +++ b/x-pack/plugin/repository-encrypted/src/internalClusterTest/java/org/elasticsearch/repositories/encrypted/EncryptedRepositorySecretIntegTests.java @@ -118,7 +118,7 @@ public void testRepositoryCreationFailsForMissingPassword() throws Exception { if (randomBoolean()) { // stop the node with the missing password - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodeName)); + internalCluster().stopNode(masterNodeName); ensureStableCluster(2); } else { // restart the node with the missing password @@ -183,7 +183,7 @@ public void testRepositoryVerificationFailsForMissingPassword() throws Exception expectThrows(RepositoryVerificationException.class, () -> client().admin().cluster().prepareVerifyRepository(repositoryName).get()); if (randomBoolean()) { // stop the node with the missing password - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(otherNodeName)); + internalCluster().stopNode(otherNodeName); ensureStableCluster(1); // repository verification now succeeds VerifyRepositoryResponse verifyRepositoryResponse = client().admin().cluster().prepareVerifyRepository(repositoryName).get(); @@ -686,7 +686,7 @@ public void testSnapshotFailsForMasterFailoverWithWrongPassword() throws Excepti client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName).setIndices(indexName).setWaitForCompletion(false).get(); // stop master - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNode)); + internalCluster().stopNode(masterNode); ensureStableCluster(3); otherNodeEncryptedRepo.unblockSnapshotShard(); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java index 4fe7e3636c62c..e888776cd1b4a 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java @@ -47,7 +47,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient size -> searchRequest.source().size(size) ) ); - RestSearchAction.checkRestTotalHits(restRequest, searchRequest); + RestSearchAction.validateSearchRequest(restRequest, searchRequest); return channel -> client.execute(RollupSearchAction.INSTANCE, searchRequest, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index a9498c785971f..17479e850bd5b 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -212,7 +212,6 @@ private class TimeSeriesBucketCollector extends BucketCollector { private final RollupBucketBuilder rollupBucketBuilder = new RollupBucketBuilder(); long lastTimestamp = Long.MAX_VALUE; long lastHistoTimestamp = Long.MAX_VALUE; - BytesRef lastTsid = null; TimeSeriesBucketCollector(BulkProcessor bulkProcessor) { this.bulkProcessor = bulkProcessor; @@ -256,6 +255,7 @@ public void collect(int docId, long owningBucketOrd) throws IOException { * - _tsid must be sorted in ascending order * - @timestamp must be sorted in descending order within the same _tsid */ + BytesRef lastTsid = rollupBucketBuilder.tsid(); assert lastTsid == null || lastTsid.compareTo(tsid) <= 0 : "_tsid is not sorted in ascending order: [" + DocValueFormat.TIME_SERIES_ID.format(lastTsid) @@ -268,7 +268,6 @@ public void collect(int docId, long owningBucketOrd) throws IOException { + "] -> [" + timestampFormat.format(timestamp) + "]"; - lastTsid = BytesRef.deepCopyOf(tsid); lastTimestamp = timestamp; if (tsidChanged || rollupBucketBuilder.timestamp() != lastHistoTimestamp) { @@ -279,7 +278,12 @@ public void collect(int docId, long owningBucketOrd) throws IOException { } // Create new rollup bucket - rollupBucketBuilder.init(tsid, lastHistoTimestamp); + if (tsidChanged) { + rollupBucketBuilder.resetTsid(tsid, lastHistoTimestamp); + } else { + rollupBucketBuilder.resetTimestamp(lastHistoTimestamp); + } + bucketsCreated++; } @@ -348,8 +352,18 @@ private class RollupBucketBuilder { this.metricFieldProducers = MetricFieldProducer.buildMetricFieldProducers(searchExecutionContext, metricFields); } - public RollupBucketBuilder init(BytesRef tsid, long timestamp) { + /** + * tsid changed, reset tsid and timestamp + */ + public RollupBucketBuilder resetTsid(BytesRef tsid, long timestamp) { this.tsid = BytesRef.deepCopyOf(tsid); + return resetTimestamp(timestamp); + } + + /** + * timestamp change, reset builder + */ + public RollupBucketBuilder resetTimestamp(long timestamp) { this.timestamp = timestamp; this.docCount = 0; this.metricFieldProducers.values().stream().forEach(p -> p.reset()); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java index 56ba6093551c4..e49cfa10f364a 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java @@ -210,7 +210,9 @@ public void testDecay() throws IOException { } public void testCacheSizeRejectedOnNonFrozenNodes() { - String cacheSize = randomBoolean() ? new ByteSizeValue(size(500)).getStringRep() : new RatioValue(between(1, 100)).toString(); + String cacheSize = randomBoolean() + ? new ByteSizeValue(size(500)).getStringRep() + : (new RatioValue(between(1, 100))).formatNoTrailingZerosPercent(); final Settings settings = Settings.builder() .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), cacheSize) .put(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), new ByteSizeValue(size(100)).getStringRep()) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureResetTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureResetTests.java new file mode 100644 index 0000000000000..3e75e2d95e8b2 --- /dev/null +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureResetTests.java @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.integration; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateAction; +import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateRequest; +import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.test.SecurityIntegTestCase; +import org.elasticsearch.test.SecuritySettingsSource; +import org.elasticsearch.test.SecuritySettingsSourceField; +import org.elasticsearch.test.TestSecurityClient; +import org.elasticsearch.xpack.core.security.user.User; +import org.junit.Before; + +import java.util.Collections; + +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER; +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.hamcrest.Matchers.containsString; + +/** + * These tests ensure that the Feature Reset API works for users with default superuser and manage roles. + * This can be complex due to restrictions on system indices and the need to use the correct origin for + * each index. See also https://github.com/elastic/elasticsearch/issues/88617 + */ +public class SecurityFeatureResetTests extends SecurityIntegTestCase { + private static final SecureString SUPER_USER_PASSWD = SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; + + @Override + protected boolean addMockHttpTransport() { + return false; // enable http + } + + @Before + public void setupForTests() throws Exception { + // adds a dummy user to the native realm to force .security index creation + new TestSecurityClient(getRestClient(), SecuritySettingsSource.SECURITY_REQUEST_OPTIONS).putUser( + new User("dummy_user", "missing_role"), + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING + ); + assertSecurityIndexActive(); + } + + @Override + protected String configUsers() { + final String usersPasswHashed = new String(getFastStoredHashAlgoForTests().hash(SUPER_USER_PASSWD)); + return super.configUsers() + + "su:" + + usersPasswHashed + + "\n" + + "manager:" + + usersPasswHashed + + "\n" + + "usr:" + + usersPasswHashed + + "\n"; + } + + @Override + protected String configUsersRoles() { + return super.configUsersRoles() + """ + superuser:su + role1:manager + role2:usr"""; + } + + @Override + protected String configRoles() { + return super.configRoles() + """ + %s + role1: + cluster: [ manage ] + indices: + - names: '*' + privileges: [ manage ] + role2: + cluster: [ monitor ] + indices: + - names: '*' + privileges: [ read ] + """; + } + + public void testFeatureResetSuperuser() { + assertResetSuccessful("su", SUPER_USER_PASSWD); + } + + public void testFeatureResetManageRole() { + assertResetSuccessful("manager", SUPER_USER_PASSWD); + } + + public void testFeatureResetNoManageRole() { + final ResetFeatureStateRequest req = new ResetFeatureStateRequest(); + + client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("usr", SUPER_USER_PASSWD))) + .admin() + .cluster() + .execute(ResetFeatureStateAction.INSTANCE, req, new ActionListener<>() { + @Override + public void onResponse(ResetFeatureStateResponse response) { + fail("Shouldn't reach here"); + } + + @Override + public void onFailure(Exception e) { + assertThat( + e.getMessage(), + containsString("action [cluster:admin/features/reset] is unauthorized for user [usr] with roles [role2]") + ); + } + }); + + // Manually delete the security index, reset shouldn't work + deleteSecurityIndex(); + } + + private void assertResetSuccessful(String user, SecureString password) { + final ResetFeatureStateRequest req = new ResetFeatureStateRequest(); + + client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue(user, password))) + .admin() + .cluster() + .execute(ResetFeatureStateAction.INSTANCE, req, new ActionListener<>() { + @Override + public void onResponse(ResetFeatureStateResponse response) { + long failures = response.getFeatureStateResetStatuses() + .stream() + .filter(status -> status.getStatus() == ResetFeatureStateResponse.ResetFeatureStateStatus.Status.FAILURE) + .count(); + assertEquals(0, failures); + } + + @Override + public void onFailure(Exception e) { + fail("Shouldn't reach here"); + } + }); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 867dd495e82a0..503d5becab553 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -1241,8 +1241,11 @@ private void withRequestBody(XContentBuilder builder, CreateApiKeyRequest create for (RoleDescriptor roleDescriptor : createApiKeyRequest.getRoleDescriptors()) { withRoleDescriptor(builder, roleDescriptor); } - builder.endArray() // role_descriptors - .endObject(); // apikey + builder.endArray(); // role_descriptors + if (createApiKeyRequest.getMetadata() != null && createApiKeyRequest.getMetadata().isEmpty() == false) { + builder.field("metadata", createApiKeyRequest.getMetadata()); + } + builder.endObject(); // apikey } private void withRequestBody(final XContentBuilder builder, final UpdateApiKeyRequest updateApiKeyRequest) throws IOException { @@ -1254,6 +1257,11 @@ private void withRequestBody(final XContentBuilder builder, final UpdateApiKeyRe } builder.endArray(); } + if (updateApiKeyRequest.getMetadata() != null) { + // Include in entry even if metadata is empty. It's meaningful to track an empty metadata request parameter + // because it replaces any metadata previously associated with the API key + builder.field("metadata", updateApiKeyRequest.getMetadata()); + } builder.endObject(); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java index bcda68c08f39d..3486671bcaa58 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.security.authc.jwt; +import com.nimbusds.jose.JWSHeader; import com.nimbusds.jose.jwk.JWK; import com.nimbusds.jose.jwk.OctetSequenceKey; import com.nimbusds.jwt.JWTClaimsSet; @@ -15,12 +16,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.SettingsException; +import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.common.util.concurrent.ReleasableLock; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Releasable; @@ -37,18 +41,19 @@ import org.elasticsearch.xpack.core.security.support.CacheIteratorHelper; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.ssl.SSLService; -import org.elasticsearch.xpack.security.authc.BytesKey; import org.elasticsearch.xpack.security.authc.support.ClaimParser; import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport; import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; import static java.lang.String.join; import static org.elasticsearch.core.Strings.format; @@ -60,11 +65,34 @@ public class JwtRealm extends Realm implements CachingRealm, Releasable { private static final Logger LOGGER = LogManager.getLogger(JwtRealm.class); - record ExpiringUser(User user, Date exp) {} + // Cached authenticated users, and adjusted JWT expiration date (=exp+skew) for checking if the JWT expired before the cache entry + record ExpiringUser(User user, Date exp) { + ExpiringUser { + Objects.requireNonNull(user, "User must not be null"); + Objects.requireNonNull(exp, "Expiration date must not be null"); + } + } + // Original PKC/HMAC JWKSet or HMAC JWK content (for comparison during refresh), and filtered JWKs and Algs + record ContentAndJwksAlgs(byte[] sha256, JwksAlgs jwksAlgs) { + ContentAndJwksAlgs { + Objects.requireNonNull(jwksAlgs, "Filters JWKs and Algs must not be null"); + } + + boolean isEmpty() { + return ((this.sha256 == null) || this.sha256.length == 0) && this.jwksAlgs.isEmpty(); + } + } + + // Filtered JWKs and Algs record JwksAlgs(List jwks, List algs) { + JwksAlgs { + Objects.requireNonNull(jwks, "JWKs must not be null"); + Objects.requireNonNull(algs, "Algs must not be null"); + } + boolean isEmpty() { - return jwks.isEmpty() && algs.isEmpty(); + return this.jwks.isEmpty() && this.algs.isEmpty(); } } @@ -78,9 +106,11 @@ boolean isEmpty() { final String allowedIssuer; final List allowedAudiences; final String jwkSetPath; - final CloseableHttpAsyncClient httpClient; - final JwtRealm.JwksAlgs jwksAlgsHmac; - final JwtRealm.JwksAlgs jwksAlgsPkc; + final boolean isConfiguredJwkSetPkc; + final boolean isConfiguredJwkSetHmac; + final boolean isConfiguredJwkOidcHmac; + private final CloseableHttpAsyncClient httpClient; + final JwkSetLoader jwkSetLoader; final TimeValue allowedClockSkew; final Boolean populateUserMetadata; final ClaimParser claimParserPrincipal; @@ -90,9 +120,14 @@ boolean isEmpty() { final ClaimParser claimParserName; final JwtRealmSettings.ClientAuthenticationType clientAuthenticationType; final SecureString clientAuthenticationSharedSecret; - final Cache jwtCache; - final CacheIteratorHelper jwtCacheHelper; + final Cache jwtCache; + final CacheIteratorHelper jwtCacheHelper; + final List allowedJwksAlgsPkc; + final List allowedJwksAlgsHmac; DelegatedAuthorizationSupport delegatedAuthorizationSupport = null; + ContentAndJwksAlgs contentAndJwksAlgsPkc; + ContentAndJwksAlgs contentAndJwksAlgsHmac; + final URI jwkSetPathUri; JwtRealm( final RealmConfig realmConfig, @@ -127,9 +162,17 @@ boolean isEmpty() { this.clientAuthenticationSharedSecret ); - if (config.hasSetting(JwtRealmSettings.HMAC_KEY) == false - && config.hasSetting(JwtRealmSettings.HMAC_JWKSET) == false - && config.hasSetting(JwtRealmSettings.PKC_JWKSET_PATH) == false) { + // Split configured signature algorithms by PKC and HMAC. Useful during validation, error logging, and JWK vs Alg filtering. + final List algs = super.config.getSetting(JwtRealmSettings.ALLOWED_SIGNATURE_ALGORITHMS); + this.allowedJwksAlgsHmac = algs.stream().filter(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC::contains).toList(); + this.allowedJwksAlgsPkc = algs.stream().filter(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC::contains).toList(); + + // PKC JWKSet can be URL, file, or not set; only initialize HTTP client if PKC JWKSet is a URL. + this.jwkSetPath = super.config.getSetting(JwtRealmSettings.PKC_JWKSET_PATH); + this.isConfiguredJwkSetPkc = Strings.hasText(this.jwkSetPath); + this.isConfiguredJwkSetHmac = Strings.hasText(super.config.getSetting(JwtRealmSettings.HMAC_JWKSET)); + this.isConfiguredJwkOidcHmac = Strings.hasText(super.config.getSetting(JwtRealmSettings.HMAC_KEY)); + if (this.isConfiguredJwkSetPkc == false && this.isConfiguredJwkSetHmac == false && this.isConfiguredJwkOidcHmac == false) { throw new SettingsException( "At least one of [" + RealmSettings.getFullSettingKey(realmConfig, JwtRealmSettings.HMAC_KEY) @@ -141,44 +184,39 @@ boolean isEmpty() { ); } - // PKC JWKSet can be URL, file, or not set; only initialize HTTP client if PKC JWKSet is a URL. - this.jwkSetPath = super.config.getSetting(JwtRealmSettings.PKC_JWKSET_PATH); - if (Strings.hasText(this.jwkSetPath)) { - final URI jwkSetPathPkcUri = JwtUtil.parseHttpsUri(this.jwkSetPath); - if (jwkSetPathPkcUri == null) { - this.httpClient = null; // local file means no HTTP client + if (this.isConfiguredJwkSetPkc) { + final URI jwkSetPathUri = JwtUtil.parseHttpsUri(jwkSetPath); + if (jwkSetPathUri == null) { + this.jwkSetPathUri = null; // local file path + this.httpClient = null; } else { - this.httpClient = JwtUtil.createHttpClient(super.config, sslService); + this.jwkSetPathUri = jwkSetPathUri; // HTTPS URL + this.httpClient = JwtUtil.createHttpClient(this.config, sslService); } + this.jwkSetLoader = new JwkSetLoader(); // PKC JWKSet loader for HTTPS URL or local file path } else { - this.httpClient = null; // no setting means no HTTP client + this.jwkSetPathUri = null; // not configured + this.httpClient = null; + this.jwkSetLoader = null; } - // If HTTPS client was created in JWT realm, any exception after that point requires closing it to avoid a thread pool leak + // Any exception during loading requires closing JwkSetLoader's HTTP client to avoid a thread pool leak try { - this.jwksAlgsHmac = this.parseJwksAlgsHmac(); - this.jwksAlgsPkc = this.parseJwksAlgsPkc(); + this.contentAndJwksAlgsHmac = this.parseJwksAlgsHmac(); + this.contentAndJwksAlgsPkc = this.parseJwksAlgsPkc(); this.verifyAnyAvailableJwkAndAlgPair(); } catch (Throwable t) { + // ASSUME: Tests or startup only. Catch and rethrow Throwable here, in case some code throws an uncaught RuntimeException. this.close(); throw t; } } - private Cache buildJwtCache() { - final TimeValue jwtCacheTtl = super.config.getSetting(JwtRealmSettings.JWT_CACHE_TTL); - final int jwtCacheSize = super.config.getSetting(JwtRealmSettings.JWT_CACHE_SIZE); - if ((jwtCacheTtl.getNanos() > 0) && (jwtCacheSize > 0)) { - return CacheBuilder.builder().setExpireAfterWrite(jwtCacheTtl).setMaximumWeight(jwtCacheSize).build(); - } - return null; - } - - // must call parseAlgsAndJwksHmac() before parseAlgsAndJwksPkc() - private JwtRealm.JwksAlgs parseJwksAlgsHmac() { + private ContentAndJwksAlgs parseJwksAlgsHmac() { final JwtRealm.JwksAlgs jwksAlgsHmac; final SecureString hmacJwkSetContents = super.config.getSetting(JwtRealmSettings.HMAC_JWKSET); final SecureString hmacKeyContents = super.config.getSetting(JwtRealmSettings.HMAC_KEY); + byte[] hmacStringContentsSha256 = null; if (Strings.hasText(hmacJwkSetContents) && Strings.hasText(hmacKeyContents)) { // HMAC Key vs HMAC JWKSet settings must be mutually exclusive throw new SettingsException( @@ -195,68 +233,52 @@ private JwtRealm.JwksAlgs parseJwksAlgsHmac() { // At this point, one-and-only-one of the HMAC Key or HMAC JWKSet settings are set List jwksHmac; if (Strings.hasText(hmacJwkSetContents)) { + hmacStringContentsSha256 = JwtUtil.sha256(hmacJwkSetContents.toString()); jwksHmac = JwkValidateUtil.loadJwksFromJwkSetString( RealmSettings.getFullSettingKey(super.config, JwtRealmSettings.HMAC_JWKSET), hmacJwkSetContents.toString() ); } else { final OctetSequenceKey hmacKey = JwkValidateUtil.loadHmacJwkFromJwkString( - RealmSettings.getFullSettingKey(super.config, JwtRealmSettings.HMAC_JWKSET), + RealmSettings.getFullSettingKey(super.config, JwtRealmSettings.HMAC_KEY), hmacKeyContents ); + assert hmacKey != null : "Null HMAC key should not happen here"; jwksHmac = List.of(hmacKey); + hmacStringContentsSha256 = JwtUtil.sha256(hmacKeyContents.toString()); } + // Filter JWK(s) vs signature algorithms. Only keep JWKs with a matching alg. Only keep algs with a matching JWK. - final List algs = super.config.getSetting(JwtRealmSettings.ALLOWED_SIGNATURE_ALGORITHMS); - final List algsHmac = algs.stream().filter(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC::contains).toList(); - jwksAlgsHmac = JwkValidateUtil.filterJwksAndAlgorithms(jwksHmac, algsHmac); + jwksAlgsHmac = JwkValidateUtil.filterJwksAndAlgorithms(jwksHmac, this.allowedJwksAlgsHmac); } LOGGER.info("Usable HMAC: JWKs [{}]. Algorithms [{}].", jwksAlgsHmac.jwks.size(), String.join(",", jwksAlgsHmac.algs())); - return jwksAlgsHmac; + return new ContentAndJwksAlgs(hmacStringContentsSha256, jwksAlgsHmac); } - private JwtRealm.JwksAlgs parseJwksAlgsPkc() { - final JwtRealm.JwksAlgs jwksAlgsPkc; - if (Strings.hasText(this.jwkSetPath) == false) { - jwksAlgsPkc = new JwtRealm.JwksAlgs(Collections.emptyList(), Collections.emptyList()); + private ContentAndJwksAlgs parseJwksAlgsPkc() { + if (this.isConfiguredJwkSetPkc == false) { + return new ContentAndJwksAlgs(null, new JwksAlgs(Collections.emptyList(), Collections.emptyList())); } else { - // PKC JWKSet get contents from local file or remote HTTPS URL - final byte[] jwkSetContentBytesPkc; - if (this.httpClient == null) { - jwkSetContentBytesPkc = JwtUtil.readFileContents( - RealmSettings.getFullSettingKey(super.config, JwtRealmSettings.PKC_JWKSET_PATH), - this.jwkSetPath, - super.config.env() - ); - } else { - final URI jwkSetPathPkcUri = JwtUtil.parseHttpsUri(this.jwkSetPath); - jwkSetContentBytesPkc = JwtUtil.readUriContents( - RealmSettings.getFullSettingKey(super.config, JwtRealmSettings.PKC_JWKSET_PATH), - jwkSetPathPkcUri, - this.httpClient - ); - } - final String jwkSetContentsPkc = new String(jwkSetContentBytesPkc, StandardCharsets.UTF_8); - - // PKC JWKSet parse contents - final List jwksPkc = JwkValidateUtil.loadJwksFromJwkSetString( - RealmSettings.getFullSettingKey(super.config, JwtRealmSettings.PKC_JWKSET_PATH), - jwkSetContentsPkc - ); + // ASSUME: Blocking read operations are OK during startup + final PlainActionFuture future = new PlainActionFuture<>(); + this.jwkSetLoader.load(future); + return future.actionGet(); + } + } - // PKC JWKSet filter contents - final List algs = super.config.getSetting(JwtRealmSettings.ALLOWED_SIGNATURE_ALGORITHMS); - final List algsPkc = algs.stream().filter(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC::contains).toList(); - jwksAlgsPkc = JwkValidateUtil.filterJwksAndAlgorithms(jwksPkc, algsPkc); + private Cache buildJwtCache() { + final TimeValue jwtCacheTtl = super.config.getSetting(JwtRealmSettings.JWT_CACHE_TTL); + final int jwtCacheSize = super.config.getSetting(JwtRealmSettings.JWT_CACHE_SIZE); + if ((jwtCacheTtl.getNanos() > 0) && (jwtCacheSize > 0)) { + return CacheBuilder.builder().setExpireAfterWrite(jwtCacheTtl).setMaximumWeight(jwtCacheSize).build(); } - LOGGER.info("Usable PKC: JWKs [{}]. Algorithms [{}].", jwksAlgsPkc.jwks().size(), String.join(",", jwksAlgsPkc.algs())); - return jwksAlgsPkc; + return null; } private void verifyAnyAvailableJwkAndAlgPair() { - assert this.jwksAlgsHmac != null : "HMAC not initialized"; - assert this.jwksAlgsPkc != null : "PKC not initialized"; - if (this.jwksAlgsHmac.isEmpty() && this.jwksAlgsPkc.isEmpty()) { + assert this.contentAndJwksAlgsHmac != null : "HMAC not initialized"; + assert this.contentAndJwksAlgsPkc != null : "PKC not initialized"; + if (this.contentAndJwksAlgsHmac.jwksAlgs.isEmpty() && this.contentAndJwksAlgsPkc.jwksAlgs.isEmpty()) { final String msg = "No available JWK and algorithm for HMAC or PKC. Realm authentication expected to fail until this is fixed."; throw new SettingsException(msg); } @@ -289,13 +311,31 @@ public void initialize(final Iterable allRealms, final XPackLicenseState */ @Override public void close() { - if (this.jwtCache != null) { + this.invalidateJwtCache(); + this.closeHttpClient(); + } + + /** + * Clean up JWT cache (if enabled). + */ + private void invalidateJwtCache() { + if ((this.jwtCache != null) && (this.jwtCacheHelper != null)) { try { - this.jwtCache.invalidateAll(); + LOGGER.trace("Invalidating JWT cache for realm [{}]", super.name()); + try (ReleasableLock ignored = this.jwtCacheHelper.acquireUpdateLock()) { + this.jwtCache.invalidateAll(); + } + LOGGER.debug("Invalidated JWT cache for realm [{}]", super.name()); } catch (Exception e) { LOGGER.warn("Exception invalidating JWT cache for realm [" + super.name() + "]", e); } } + } + + /** + * Clean up HTTPS client cache (if enabled). + */ + private void closeHttpClient() { if (this.httpClient != null) { try { this.httpClient.close(); @@ -314,21 +354,17 @@ public void lookupUser(final String username, final ActionListener listene @Override public void expire(final String username) { this.ensureInitialized(); - LOGGER.trace("Expiring JWT cache entries for realm [" + super.name() + "] principal=[" + username + "]"); if (this.jwtCacheHelper != null) { + LOGGER.trace("Expiring JWT cache entries for realm [{}] principal=[{}]", super.name(), username); this.jwtCacheHelper.removeValuesIf(expiringUser -> expiringUser.user.principal().equals(username)); + LOGGER.trace("Expired JWT cache entries for realm [{}] principal=[{}]", super.name(), username); } } @Override public void expireAll() { this.ensureInitialized(); - if ((this.jwtCache != null) && (this.jwtCacheHelper != null)) { - LOGGER.trace("Invalidating JWT cache for realm [" + super.name() + "]"); - try (ReleasableLock ignored = this.jwtCacheHelper.acquireUpdateLock()) { - this.jwtCache.invalidateAll(); - } - } + this.invalidateJwtCache(); } @Override @@ -365,7 +401,7 @@ public void authenticate(final AuthenticationToken authenticationToken, final Ac // JWT cache final SecureString serializedJwt = jwtAuthenticationToken.getEndUserSignedJwt(); - final BytesKey jwtCacheKey = (this.jwtCache == null) ? null : computeBytesKey(serializedJwt); + final BytesArray jwtCacheKey = (this.jwtCache == null) ? null : new BytesArray(JwtUtil.sha256(serializedJwt)); if (jwtCacheKey != null) { final ExpiringUser expiringUser = this.jwtCache.get(jwtCacheKey); if (expiringUser == null) { @@ -417,99 +453,201 @@ public void authenticate(final AuthenticationToken authenticationToken, final Ac } // Validate JWT: Extract JWT and claims set, and validate JWT. - final SignedJWT jwt; - final JWTClaimsSet claimsSet; - try { - jwt = SignedJWT.parse(serializedJwt.toString()); - final String jwtAlg = jwt.getHeader().getAlgorithm().getName(); - final boolean isJwtAlgHmac = JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC.contains(jwtAlg); - final JwtRealm.JwksAlgs jwksAndAlgs = isJwtAlgHmac ? this.jwksAlgsHmac : this.jwksAlgsPkc; - JwtValidateUtil.validate( - jwt, - this.allowedIssuer, - this.allowedAudiences, - this.allowedClockSkew.seconds(), - jwksAndAlgs.algs, - jwksAndAlgs.jwks + validateJwt( + serializedJwt, + tokenPrincipal, + ActionListener.wrap(claimsSet -> processValidatedJwt(tokenPrincipal, jwtCacheKey, claimsSet, listener), ex -> { + final String msg = "Realm [" + super.name() + "] JWT validation failed for token=[" + tokenPrincipal + "]."; + LOGGER.debug(msg, ex); + listener.onResponse(AuthenticationResult.unsuccessful(msg, ex)); + }) + ); + } else { + final String className = (authenticationToken == null) ? "null" : authenticationToken.getClass().getCanonicalName(); + final String msg = "Realm [" + super.name() + "] does not support AuthenticationToken [" + className + "]."; + LOGGER.trace(msg); + listener.onResponse(AuthenticationResult.unsuccessful(msg, null)); + } + } + + private void validateJwt(SecureString serializedJwt, String tokenPrincipal, ActionListener listener) { + final SignedJWT jwt; + final JWSHeader header; + final JWTClaimsSet claimsSet; + final String alg; + try { + jwt = SignedJWT.parse(serializedJwt.toString()); + header = jwt.getHeader(); + alg = header.getAlgorithm().getName(); + claimsSet = jwt.getJWTClaimsSet(); + final Date now = new Date(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Realm [{}] JWT parse succeeded for token=[{}]." + + "Validating JWT, now [{}], alg [{}], issuer [{}], audiences [{}], kty [{}]," + + " auth_time [{}], iat [{}], nbf [{}], exp [{}], kid [{}], jti [{}]", + super.name(), + tokenPrincipal, + now, + alg, + claimsSet.getIssuer(), + claimsSet.getAudience(), + header.getType(), + claimsSet.getDateClaim("auth_time"), + claimsSet.getIssueTime(), + claimsSet.getNotBeforeTime(), + claimsSet.getExpirationTime(), + header.getKeyID(), + claimsSet.getJWTID() ); - claimsSet = jwt.getJWTClaimsSet(); - LOGGER.trace("Realm [{}] JWT validation succeeded for token=[{}].", super.name(), tokenPrincipal); - } catch (Exception e) { - final String msg = "Realm [" + super.name() + "] JWT validation failed for token=[" + tokenPrincipal + "]."; - final AuthenticationResult failure = AuthenticationResult.unsuccessful(msg, e); - LOGGER.debug(msg, e); - listener.onResponse(failure); - return; } + // Validate all else before signature, because these checks are more helpful diagnostics than rejected signatures. + final boolean isJwtAlgHmac = JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC.contains(alg); + JwtValidateUtil.validateType(jwt); + JwtValidateUtil.validateIssuer(jwt, allowedIssuer); + JwtValidateUtil.validateAudiences(jwt, allowedAudiences); + JwtValidateUtil.validateSignatureAlgorithm(jwt, isJwtAlgHmac ? this.allowedJwksAlgsHmac : this.allowedJwksAlgsPkc); + JwtValidateUtil.validateAuthTime(jwt, now, this.allowedClockSkew.seconds()); + JwtValidateUtil.validateIssuedAtTime(jwt, now, this.allowedClockSkew.seconds()); + JwtValidateUtil.validateNotBeforeTime(jwt, now, this.allowedClockSkew.seconds()); + JwtValidateUtil.validateExpiredTime(jwt, now, this.allowedClockSkew.seconds()); + + // At this point, client authc and JWT kty+alg+iss+aud+time filters passed. Do sig last, in case JWK reload is expensive. + validateSignature(jwt, isJwtAlgHmac, tokenPrincipal, listener.map(ignored -> claimsSet)); + + } catch (Exception e) { + listener.onFailure(e); + } + } - // At this point, JWT is validated. Parse the JWT claims using realm settings. - - final String principal = this.claimParserPrincipal.getClaimValue(claimsSet); - if (Strings.hasText(principal) == false) { - final String msg = "Realm [" - + super.name() - + "] no principal for token=[" - + tokenPrincipal - + "] parser=[" - + this.claimParserPrincipal - + "] claims=[" - + claimsSet - + "]."; - LOGGER.debug(msg); - listener.onResponse(AuthenticationResult.unsuccessful(msg, null)); - return; - } + private void processValidatedJwt( + String tokenPrincipal, + BytesArray jwtCacheKey, + JWTClaimsSet claimsSet, + ActionListener> listener + ) { + // At this point, JWT is validated. Parse the JWT claims using realm settings. + final String principal = this.claimParserPrincipal.getClaimValue(claimsSet); + if (Strings.hasText(principal) == false) { + final String msg = "Realm [" + + super.name() + + "] no principal for token=[" + + tokenPrincipal + + "] parser=[" + + this.claimParserPrincipal + + "] claims=[" + + claimsSet + + "]."; + LOGGER.debug(msg); + listener.onResponse(AuthenticationResult.unsuccessful(msg, null)); + return; + } - // Roles listener: Log roles from delegated authz lookup or role mapping, and cache User if JWT cache is enabled. - final ActionListener> logAndCacheListener = ActionListener.wrap(result -> { - if (result.isAuthenticated()) { - final User user = result.getValue(); - LOGGER.debug( - () -> format("Realm [%s] roles [%s] for principal=[%s].", super.name(), join(",", user.roles()), principal) - ); - if ((this.jwtCache != null) && (this.jwtCacheHelper != null)) { - try (ReleasableLock ignored = this.jwtCacheHelper.acquireUpdateLock()) { - final long expWallClockMillis = claimsSet.getExpirationTime().getTime() + this.allowedClockSkew.getMillis(); - this.jwtCache.put(jwtCacheKey, new ExpiringUser(result.getValue(), new Date(expWallClockMillis))); - } + // Roles listener: Log roles from delegated authz lookup or role mapping, and cache User if JWT cache is enabled. + final ActionListener> logAndCacheListener = ActionListener.wrap(result -> { + if (result.isAuthenticated()) { + final User user = result.getValue(); + LOGGER.debug(() -> format("Realm [%s] roles [%s] for principal=[%s].", super.name(), join(",", user.roles()), principal)); + if ((this.jwtCache != null) && (this.jwtCacheHelper != null)) { + try (ReleasableLock ignored = this.jwtCacheHelper.acquireUpdateLock()) { + final long expWallClockMillis = claimsSet.getExpirationTime().getTime() + this.allowedClockSkew.getMillis(); + this.jwtCache.put(jwtCacheKey, new ExpiringUser(result.getValue(), new Date(expWallClockMillis))); } } - listener.onResponse(result); - }, listener::onFailure); - - // Delegated role lookup or Role mapping: Use the above listener to log roles and cache User. - if (this.delegatedAuthorizationSupport.hasDelegation()) { - this.delegatedAuthorizationSupport.resolve(principal, logAndCacheListener); - return; } + listener.onResponse(result); + }, listener::onFailure); - // User metadata: If enabled, extract metadata from JWT claims set. Use it in UserRoleMapper.UserData and User constructors. - final Map userMetadata; - try { - userMetadata = this.populateUserMetadata ? JwtUtil.toUserMetadata(jwt) : Map.of(); - } catch (Exception e) { - final String msg = "Realm [" + super.name() + "] parse metadata failed for principal=[" + principal + "]."; - final AuthenticationResult unsuccessful = AuthenticationResult.unsuccessful(msg, e); - LOGGER.debug(msg, e); - listener.onResponse(unsuccessful); + // Delegated role lookup or Role mapping: Use the above listener to log roles and cache User. + if (this.delegatedAuthorizationSupport.hasDelegation()) { + this.delegatedAuthorizationSupport.resolve(principal, logAndCacheListener); + return; + } + + // User metadata: If enabled, extract metadata from JWT claims set. Use it in UserRoleMapper.UserData and User constructors. + final Map userMetadata; + try { + userMetadata = this.populateUserMetadata ? JwtUtil.toUserMetadata(claimsSet) : Map.of(); + } catch (Exception e) { + final String msg = "Realm [" + super.name() + "] parse metadata failed for principal=[" + principal + "]."; + LOGGER.debug(msg, e); + listener.onResponse(AuthenticationResult.unsuccessful(msg, e)); + return; + } + + // Role resolution: Handle role mapping in JWT Realm. + final List groups = this.claimParserGroups.getClaimValues(claimsSet); + final String dn = this.claimParserDn.getClaimValue(claimsSet); + final String mail = this.claimParserMail.getClaimValue(claimsSet); + final String name = this.claimParserName.getClaimValue(claimsSet); + final UserRoleMapper.UserData userData = new UserRoleMapper.UserData(principal, dn, groups, userMetadata, super.config); + this.userRoleMapper.resolveRoles(userData, ActionListener.wrap(rolesSet -> { + final User user = new User(principal, rolesSet.toArray(Strings.EMPTY_ARRAY), name, mail, userData.getMetadata(), true); + logAndCacheListener.onResponse(AuthenticationResult.success(user)); + }, logAndCacheListener::onFailure)); + } + + private void validateSignature( + final SignedJWT jwt, + final boolean isJwtAlgHmac, + final String tokenPrincipal, + final ActionListener listener + ) throws Exception { + try { + JwtValidateUtil.validateSignature( + jwt, + isJwtAlgHmac ? this.contentAndJwksAlgsHmac.jwksAlgs.jwks : this.contentAndJwksAlgsPkc.jwksAlgs.jwks + ); + listener.onResponse(null); + } catch (Exception primaryException) { + if (isJwtAlgHmac || this.jwkSetLoader == null) { + listener.onFailure(primaryException); // HMAC reload not supported at this time return; } - // Role resolution: Handle role mapping in JWT Realm. - final List groups = this.claimParserGroups.getClaimValues(claimsSet); - final String dn = this.claimParserDn.getClaimValue(claimsSet); - final String mail = this.claimParserMail.getClaimValue(claimsSet); - final String name = this.claimParserName.getClaimValue(claimsSet); - final UserRoleMapper.UserData userData = new UserRoleMapper.UserData(principal, dn, groups, userMetadata, super.config); - this.userRoleMapper.resolveRoles(userData, ActionListener.wrap(rolesSet -> { - final User user = new User(principal, rolesSet.toArray(Strings.EMPTY_ARRAY), name, mail, userData.getMetadata(), true); - logAndCacheListener.onResponse(AuthenticationResult.success(user)); - }, logAndCacheListener::onFailure)); - } else { - final String className = (authenticationToken == null) ? "null" : authenticationToken.getClass().getCanonicalName(); - final String msg = "Realm [" + super.name() + "] does not support AuthenticationToken [" + className + "]."; - LOGGER.trace(msg); - listener.onResponse(AuthenticationResult.unsuccessful(msg, null)); + LOGGER.debug( + () -> org.elasticsearch.core.Strings.format( + "Signature verification failed for [%s] reloading JWKSet (was: #[%s] JWKs, #[%s] algs, sha256=[%s])", + tokenPrincipal, + this.contentAndJwksAlgsPkc.jwksAlgs.jwks().size(), + this.contentAndJwksAlgsPkc.jwksAlgs.algs().size(), + MessageDigests.toHexString(this.contentAndJwksAlgsPkc.sha256()) + ), + primaryException + ); + + this.jwkSetLoader.load(ActionListener.wrap(newContentAndJwksAlgs -> { + if (Arrays.equals(this.contentAndJwksAlgsPkc.sha256, newContentAndJwksAlgs.sha256)) { + // No change in JWKSet + logger.debug("Reloaded same PKC JWKs, can't retry verify JWT token=[{}]", tokenPrincipal); + listener.onFailure(primaryException); + return; + } + this.contentAndJwksAlgsPkc = newContentAndJwksAlgs; + // If all PKC JWKs were replaced, all PKC JWT cache entries need to be invalidated. + // Enhancement idea: Use separate caches for PKC vs HMAC JWKs, so only PKC entries get invalidated. + // Enhancement idea: When some JWKs are retained (ex: rotation), only invalidate for removed JWKs. + this.invalidateJwtCache(); + + if (this.contentAndJwksAlgsPkc.jwksAlgs.isEmpty()) { + logger.debug("Reloaded empty PKC JWKs, verification of JWT token will fail [{}]", tokenPrincipal); + // fall through and let try/catch below handle empty JWKs failure log and response + } + + try { + JwtValidateUtil.validateSignature(jwt, this.contentAndJwksAlgsPkc.jwksAlgs.jwks); + listener.onResponse(null); + } catch (Exception secondaryException) { + logger.debug( + "Verification of JWT token for [{}] failed - original failure=[{}], failure after reload=[{}]", + tokenPrincipal, + primaryException.getMessage(), + secondaryException.getMessage() + ); + secondaryException.addSuppressed(primaryException); + listener.onFailure(secondaryException); + } + }, listener::onFailure)); } } @@ -522,9 +660,75 @@ public void usageStats(final ActionListener> listener) { }, listener::onFailure)); } - static BytesKey computeBytesKey(final CharSequence charSequence) { - final MessageDigest messageDigest = MessageDigests.sha256(); - messageDigest.update(charSequence.toString().getBytes(StandardCharsets.UTF_8)); - return new BytesKey(messageDigest.digest()); + private class JwkSetLoader { + private final AtomicReference> reloadFutureRef = new AtomicReference<>(); + + void load(final ActionListener listener) { + final ListenableFuture future = this.getFuture(); + future.addListener(listener); + } + + private ListenableFuture getFuture() { + for (;;) { + final ListenableFuture existingFuture = this.reloadFutureRef.get(); + if (existingFuture != null) { + return existingFuture; + } + + final ListenableFuture newFuture = new ListenableFuture<>(); + if (this.reloadFutureRef.compareAndSet(null, newFuture)) { + loadInternal(ActionListener.runAfter(newFuture, () -> this.reloadFutureRef.compareAndSet(newFuture, null))); + return newFuture; + } + // else, Another thread set the future-ref before us, just try it all again + } + } + + private void loadInternal(final ActionListener listener) { + // PKC JWKSet get contents from local file or remote HTTPS URL + if (JwtRealm.this.httpClient == null) { + LOGGER.trace("Loading PKC JWKs from path [{}]", JwtRealm.this.jwkSetPath); + listener.onResponse( + this.parseContent( + JwtUtil.readFileContents( + RealmSettings.getFullSettingKey(JwtRealm.this.config, JwtRealmSettings.PKC_JWKSET_PATH), + JwtRealm.this.jwkSetPath, + JwtRealm.this.config.env() + ) + ) + ); + } else { + LOGGER.trace("Loading PKC JWKs from https URI [{}]", JwtRealm.this.jwkSetPathUri); + JwtUtil.readUriContents( + RealmSettings.getFullSettingKey(JwtRealm.this.config, JwtRealmSettings.PKC_JWKSET_PATH), + JwtRealm.this.jwkSetPathUri, + JwtRealm.this.httpClient, + listener.map(bytes -> { + LOGGER.trace("Loaded bytes [{}] from [{}]", bytes.length, JwtRealm.this.jwkSetPathUri); + return this.parseContent(bytes); + }) + ); + } + } + + private ContentAndJwksAlgs parseContent(final byte[] jwkSetContentBytesPkc) { + final String jwkSetContentsPkc = new String(jwkSetContentBytesPkc, StandardCharsets.UTF_8); + final byte[] jwkSetContentsPkcSha256 = JwtUtil.sha256(jwkSetContentsPkc); + + // PKC JWKSet parse contents + final List jwksPkc = JwkValidateUtil.loadJwksFromJwkSetString( + RealmSettings.getFullSettingKey(JwtRealm.this.config, JwtRealmSettings.PKC_JWKSET_PATH), + jwkSetContentsPkc + ); + // Filter JWK(s) vs signature algorithms. Only keep JWKs with a matching alg. Only keep algs with a matching JWK. + final JwksAlgs jwksAlgsPkc = JwkValidateUtil.filterJwksAndAlgorithms(jwksPkc, JwtRealm.this.allowedJwksAlgsPkc); + LOGGER.info( + "Usable PKC: JWKs=[{}] algorithms=[{}] sha256=[{}]", + jwksAlgsPkc.jwks().size(), + String.join(",", jwksAlgsPkc.algs()), + MessageDigests.toHexString(jwkSetContentsPkcSha256) + ); + return new ContentAndJwksAlgs(jwkSetContentsPkcSha256, jwksAlgsPkc); + } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java index 9bef31abf5d51..b7cf1c515b9bd 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java @@ -11,7 +11,6 @@ import com.nimbusds.jose.jwk.JWKSet; import com.nimbusds.jose.util.JSONObjectUtils; import com.nimbusds.jwt.JWTClaimsSet; -import com.nimbusds.jwt.SignedJWT; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -33,8 +32,9 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.SpecialPermission; -import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.ssl.SslConfiguration; @@ -51,6 +51,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.security.AccessController; +import java.security.MessageDigest; import java.security.PrivilegedAction; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; @@ -185,16 +186,25 @@ public static URI parseHttpsUri(final String uriString) { return null; } - public static byte[] readUriContents( + public static void readUriContents( final String jwkSetConfigKeyPkc, final URI jwkSetPathPkcUri, - final CloseableHttpAsyncClient httpClient - ) throws SettingsException { - try { - return JwtUtil.readBytes(httpClient, jwkSetPathPkcUri); - } catch (Exception e) { - throw new SettingsException("Can't get contents for setting [" + jwkSetConfigKeyPkc + "] value [" + jwkSetPathPkcUri + "].", e); - } + final CloseableHttpAsyncClient httpClient, + final ActionListener listener + ) { + JwtUtil.readBytes( + httpClient, + jwkSetPathPkcUri, + ActionListener.wrap( + listener::onResponse, + ex -> listener.onFailure( + new SettingsException( + "Can't get contents for setting [" + jwkSetConfigKeyPkc + "] value [" + jwkSetPathPkcUri + "].", + ex + ) + ) + ) + ); } public static byte[] readFileContents(final String jwkSetConfigKeyPkc, final String jwkSetPathPkc, final Environment environment) @@ -211,7 +221,7 @@ public static byte[] readFileContents(final String jwkSetConfigKeyPkc, final Str } public static String serializeJwkSet(final JWKSet jwkSet, final boolean publicKeysOnly) { - if ((jwkSet == null) || (jwkSet.getKeys().isEmpty())) { + if (jwkSet == null) { return null; } return JSONObjectUtils.toJSONString(jwkSet.toJSONObject(publicKeysOnly)); @@ -262,13 +272,11 @@ public static CloseableHttpAsyncClient createHttpClient(final RealmConfig realmC } /** - * Use the HTTP Client to get URL content bytes up to N max bytes. + * Use the HTTP Client to get URL content bytes. * @param httpClient Configured HTTP/HTTPS client. * @param uri URI to download. - * @return Byte array of the URI contents up to N max bytes. */ - public static byte[] readBytes(final CloseableHttpAsyncClient httpClient, final URI uri) { - final PlainActionFuture plainActionFuture = PlainActionFuture.newFuture(); + public static void readBytes(final CloseableHttpAsyncClient httpClient, final URI uri, ActionListener listener) { AccessController.doPrivileged((PrivilegedAction) () -> { httpClient.execute(new HttpGet(uri), new FutureCallback<>() { @Override @@ -278,12 +286,12 @@ public void completed(final HttpResponse result) { if (statusCode == 200) { final HttpEntity entity = result.getEntity(); try (InputStream inputStream = entity.getContent()) { - plainActionFuture.onResponse(inputStream.readAllBytes()); + listener.onResponse(inputStream.readAllBytes()); } catch (Exception e) { - plainActionFuture.onFailure(e); + listener.onFailure(e); } } else { - plainActionFuture.onFailure( + listener.onFailure( new ElasticsearchSecurityException( "Get [" + uri + "] failed, status [" + statusCode + "], reason [" + statusLine.getReasonPhrase() + "]." ) @@ -293,17 +301,16 @@ public void completed(final HttpResponse result) { @Override public void failed(Exception e) { - plainActionFuture.onFailure(new ElasticsearchSecurityException("Get [" + uri + "] failed.", e)); + listener.onFailure(new ElasticsearchSecurityException("Get [" + uri + "] failed.", e)); } @Override public void cancelled() { - plainActionFuture.onFailure(new ElasticsearchSecurityException("Get [" + uri + "] was cancelled.")); + listener.onFailure(new ElasticsearchSecurityException("Get [" + uri + "] was cancelled.")); } }); return null; }); - return plainActionFuture.actionGet(); } public static Path resolvePath(final Environment environment, final String jwkSetPath) { @@ -335,14 +342,10 @@ public static SecureString join(final CharSequence delimiter, final CharSequence * JWSHeader: Header are not support. * JWTClaimsSet: Claims are supported. Claim keys are prefixed by "jwt_claim_". * Base64URL: Signature is not supported. - * @param jwt SignedJWT object. * @return Map of formatted and filtered values to be used as user metadata. - * @throws Exception Parse error. */ - // // Values will be filtered by type using isAllowedTypeForClaim(). - public static Map toUserMetadata(final SignedJWT jwt) throws Exception { - final JWTClaimsSet claimsSet = jwt.getJWTClaimsSet(); + public static Map toUserMetadata(JWTClaimsSet claimsSet) { return claimsSet.getClaims() .entrySet() .stream() @@ -366,4 +369,10 @@ static boolean isAllowedTypeForClaim(final Object value) { || (value instanceof Collection && ((Collection) value).stream().allMatch(e -> e instanceof String || e instanceof Boolean || e instanceof Number))); } + + public static byte[] sha256(final CharSequence charSequence) { + final MessageDigest messageDigest = MessageDigests.sha256(); + messageDigest.update(charSequence.toString().getBytes(StandardCharsets.UTF_8)); + return messageDigest.digest(); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtil.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtil.java index b7b7a90b78fd5..3e87be0c9be50 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtil.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtil.java @@ -32,6 +32,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.settings.SecureString; import java.util.Date; @@ -48,55 +49,6 @@ public class JwtValidateUtil { null ); - /** - * Validate a SignedJWT. Use iss/aud/alg filters for those claims, JWKSet for signature, and skew seconds for time claims. - * @param jwt Signed JWT to be validated. - * @param allowedIssuer Filter for the "iss" claim. - * @param allowedAudiences Filter for the "aud" claim. - * @param allowedClockSkewSeconds Skew tolerance for the "auth_time", "iat", "nbf", and "exp" claims. - * @param allowedSignatureAlgorithms Filter for the "aud" header. - * @param jwks JWKs of HMAC secret keys or RSA/EC public keys. - * @throws Exception Error for the first validation to fail. - */ - public static void validate( - final SignedJWT jwt, - final String allowedIssuer, - final List allowedAudiences, - final long allowedClockSkewSeconds, - final List allowedSignatureAlgorithms, - final List jwks - ) throws Exception { - final Date now = new Date(); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Validating JWT, now [{}], alg [{}], issuer [{}], audiences [{}], typ [{}]," - + " auth_time [{}], iat [{}], nbf [{}], exp [{}], kid [{}], jti [{}]", - now, - jwt.getHeader().getAlgorithm(), - jwt.getJWTClaimsSet().getIssuer(), - jwt.getJWTClaimsSet().getAudience(), - jwt.getHeader().getType(), - jwt.getJWTClaimsSet().getDateClaim("auth_time"), - jwt.getJWTClaimsSet().getIssueTime(), - jwt.getJWTClaimsSet().getNotBeforeTime(), - jwt.getJWTClaimsSet().getExpirationTime(), - jwt.getHeader().getKeyID(), - jwt.getJWTClaimsSet().getJWTID() - ); - } - // validate claims before signature, because log messages about rejected claims can be more helpful than rejected signatures - JwtValidateUtil.validateType(jwt); - JwtValidateUtil.validateIssuer(jwt, allowedIssuer); - JwtValidateUtil.validateAudiences(jwt, allowedAudiences); - JwtValidateUtil.validateSignatureAlgorithm(jwt, allowedSignatureAlgorithms); - JwtValidateUtil.validateAuthTime(jwt, now, allowedClockSkewSeconds); - JwtValidateUtil.validateIssuedAtTime(jwt, now, allowedClockSkewSeconds); - JwtValidateUtil.validateNotBeforeTime(jwt, now, allowedClockSkewSeconds); - JwtValidateUtil.validateExpiredTime(jwt, now, allowedClockSkewSeconds); - JwtValidateUtil.validateSignature(jwt, jwks); - } - public static void validateType(final SignedJWT jwt) throws Exception { final JOSEObjectType jwtHeaderType = jwt.getHeader().getType(); try { @@ -277,7 +229,10 @@ static void validateExpiredTime(final Date exp, final Date now, final long allow * @throws Exception Error if JWKs fail to validate the Signed JWT. */ public static void validateSignature(final SignedJWT jwt, final List jwks) throws Exception { - assert jwks != null && jwks.isEmpty() == false : "Caller must provide a non-empty JWK list"; + assert jwks != null : "Verify requires a non-null JWK list"; + if (jwks.isEmpty()) { + throw new ElasticsearchException("Verify requires a non-empty JWK list"); + } final String id = jwt.getHeader().getKeyID(); final JWSAlgorithm alg = jwt.getHeader().getAlgorithm(); LOGGER.trace("JWKs [{}], JWT KID [{}], and JWT Algorithm [{}] before filters.", jwks.size(), id, alg.getName()); @@ -305,12 +260,35 @@ public static void validateSignature(final SignedJWT jwt, final List jwks) final List jwksStrength = jwksAlg.stream().filter(j -> JwkValidateUtil.isMatch(j, alg.getName())).toList(); LOGGER.debug("JWKs [{}] after Algorithm [{}] match filter.", jwksStrength.size(), alg); + // No JWKs passed the kid, alg, and strength checks, so nothing left to use in verifying the JWT signature + if (jwksStrength.isEmpty()) { + throw new ElasticsearchException("Verify failed because all " + jwks.size() + " provided JWKs were filtered."); + } + for (final JWK jwk : jwksStrength) { if (jwt.verify(JwtValidateUtil.createJwsVerifier(jwk))) { - return; // VERIFY SUCCEEDED + LOGGER.trace( + "JWT signature validation succeeded with JWK kty=[{}], jwtAlg=[{}], jwtKid=[{}], use=[{}], ops=[{}]", + jwk.getKeyType(), + jwk.getAlgorithm(), + jwk.getKeyID(), + jwk.getKeyUse(), + jwk.getKeyOperations() + ); + return; + } else { + LOGGER.trace( + "JWT signature validation failed with JWK kty=[{}], jwtAlg=[{}], jwtKid=[{}], use=[{}], ops={}", + jwk.getKeyType(), + jwk.getAlgorithm(), + jwk.getKeyID(), + jwk.getKeyUse(), + jwk.getKeyOperations() == null ? "[null]" : jwk.getKeyOperations() + ); } } - throw new Exception("Verify failed using " + jwksStrength.size() + " of " + jwks.size() + " provided JWKs."); + + throw new ElasticsearchException("Verify failed using " + jwksStrength.size() + " of " + jwks.size() + " provided JWKs."); } public static JWSVerifier createJwsVerifier(final JWK jwk) throws JOSEException { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index f85b28bcbba98..77345333629a1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -46,7 +46,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.security.action.apikey.ApiKeyTests; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyAction; @@ -585,17 +584,21 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles); final Authentication authentication = createAuthentication(); - + final ApiKeyMetadataWithSerialization metadataWithSerialization = randomApiKeyMetadataWithSerialization(); CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(keyName, keyRoleDescriptors, expiration); createApiKeyRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); + createApiKeyRequest.setMetadata(metadataWithSerialization.metadata()); auditTrail.accessGranted(requestId, authentication, CreateApiKeyAction.NAME, createApiKeyRequest, authorizationInfo); String expectedCreateKeyAuditEventString = """ - "create":{"apikey":{"id":"%s","name":"%s","expiration":%s,%s}}\ + "create":{"apikey":{"id":"%s","name":"%s","expiration":%s,%s%s}}\ """.formatted( createApiKeyRequest.getId(), keyName, expiration != null ? "\"" + expiration + "\"" : "null", - roleDescriptorsStringBuilder + roleDescriptorsStringBuilder, + createApiKeyRequest.getMetadata() == null || createApiKeyRequest.getMetadata().isEmpty() + ? "" + : ",\"metadata\":%s".formatted(metadataWithSerialization.serialization()) ); List output = CapturingLogger.output(logger.getName(), Level.INFO); assertThat(output.size(), is(2)); @@ -617,12 +620,16 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException final var updateApiKeyRequest = new UpdateApiKeyRequest( keyId, randomBoolean() ? null : keyRoleDescriptors, - ApiKeyTests.randomMetadata() + metadataWithSerialization.metadata() ); auditTrail.accessGranted(requestId, authentication, UpdateApiKeyAction.NAME, updateApiKeyRequest, authorizationInfo); final var expectedUpdateKeyAuditEventString = """ - "change":{"apikey":{"id":"%s"%s}}\ - """.formatted(keyId, updateApiKeyRequest.getRoleDescriptors() == null ? "" : "," + roleDescriptorsStringBuilder); + "change":{"apikey":{"id":"%s"%s%s}}\ + """.formatted( + keyId, + updateApiKeyRequest.getRoleDescriptors() == null ? "" : "," + roleDescriptorsStringBuilder, + updateApiKeyRequest.getMetadata() == null ? "" : ",\"metadata\":%s".formatted(metadataWithSerialization.serialization()) + ); output = CapturingLogger.output(logger.getName(), Level.INFO); assertThat(output.size(), is(2)); String generatedUpdateKeyAuditEventString = output.get(1); @@ -658,8 +665,12 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException .append("\",\"expiration\":") .append(expiration != null ? "\"" + expiration + "\"" : "null") .append(",") - .append(roleDescriptorsStringBuilder) - .append("},\"grant\":{\"type\":"); + .append(roleDescriptorsStringBuilder); + if (grantApiKeyRequest.getApiKeyRequest().getMetadata() != null + && grantApiKeyRequest.getApiKeyRequest().getMetadata().isEmpty() == false) { + grantKeyAuditEventStringBuilder.append(",\"metadata\":").append(metadataWithSerialization.serialization()); + } + grantKeyAuditEventStringBuilder.append("},\"grant\":{\"type\":"); if (grantApiKeyRequest.getGrant().getType() != null) { grantKeyAuditEventStringBuilder.append("\"").append(grantApiKeyRequest.getGrant().getType()).append("\""); } else { @@ -2901,4 +2912,27 @@ private static void indicesRequest( checkedFields.put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, MockRequest.class.getSimpleName()); } } + + private record ApiKeyMetadataWithSerialization(Map metadata, String serialization) {}; + + private ApiKeyMetadataWithSerialization randomApiKeyMetadataWithSerialization() { + final int metadataCase = randomInt(3); + return switch (metadataCase) { + case 0 -> new ApiKeyMetadataWithSerialization(null, null); + case 1 -> new ApiKeyMetadataWithSerialization(Map.of(), "{}"); + case 2 -> { + final Map metadata = new TreeMap<>(); + metadata.put("test", true); + metadata.put("ans", 42); + yield new ApiKeyMetadataWithSerialization(metadata, "{\"ans\":42,\"test\":true}"); + } + case 3 -> { + final Map metadata = new TreeMap<>(); + metadata.put("ans", List.of(42, true)); + metadata.put("other", Map.of("42", true)); + yield new ApiKeyMetadataWithSerialization(metadata, "{\"ans\":[42,true],\"other\":{\"42\":true}}"); + } + default -> throw new IllegalStateException("Unexpected case number: " + metadataCase); + }; + } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkValidateUtilTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkValidateUtilTests.java index 3d1cdeef77872..8957105d57ca7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkValidateUtilTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkValidateUtilTests.java @@ -10,13 +10,14 @@ import com.nimbusds.jose.JWSAlgorithm; import com.nimbusds.jose.jwk.JWK; import com.nimbusds.jose.jwk.OctetSequenceKey; -import com.nimbusds.jose.util.Base64URL; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.hamcrest.Matchers.anyOf; @@ -27,46 +28,27 @@ public class JwkValidateUtilTests extends JwtTestCase { private static final Logger LOGGER = LogManager.getLogger(JwkValidateUtilTests.class); - // HMAC JWKSet setting can use keys from randomJwkHmac() - // HMAC key setting cannot use randomJwkHmac(), it must use randomJwkHmacString() - public void testConvertHmacJwkToStringToJwk() throws Exception { - final JWSAlgorithm jwsAlgorithm = JWSAlgorithm.parse(randomFrom(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC)); - - // Use HMAC random bytes for OIDC JWKSet setting only. Demonstrate encode/decode fails if used in OIDC HMAC key setting. - final OctetSequenceKey hmacKeyRandomBytes = JwtTestCase.randomJwkHmac(jwsAlgorithm); - assertThat(this.hmacEncodeDecodeAsPasswordTestHelper(hmacKeyRandomBytes), is(false)); - - // Convert HMAC random bytes to UTF8 bytes. This makes it usable as an OIDC HMAC key setting. - final OctetSequenceKey hmacKeyString1 = JwtTestCase.conditionJwkHmacForOidc(hmacKeyRandomBytes); - assertThat(this.hmacEncodeDecodeAsPasswordTestHelper(hmacKeyString1), is(true)); - - // Generate HMAC UTF8 bytes. This is usable as an OIDC HMAC key setting. - final OctetSequenceKey hmacKeyString2 = JwtTestCase.randomJwkHmacOidc(jwsAlgorithm); - assertThat(this.hmacEncodeDecodeAsPasswordTestHelper(hmacKeyString2), is(true)); + // Test decode bytes as UTF8 to String, encode back to UTF8, and compare to original bytes. If same, it is safe for OIDC JWK encode. + static boolean isJwkHmacOidcSafe(final JWK jwk) { + if (jwk instanceof OctetSequenceKey jwkHmac) { + final byte[] rawKeyBytes = jwkHmac.getKeyValue().decode(); + return Arrays.equals(rawKeyBytes, new String(rawKeyBytes, StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8)); + } + return true; } - private boolean hmacEncodeDecodeAsPasswordTestHelper(final OctetSequenceKey hmacKey) { - final OctetSequenceKey hmacKeyNoAttributes = JwtTestCase.jwkHmacRemoveAttributes(hmacKey); - // Encode input key as Base64(keyBytes) and Utf8String(keyBytes) - final String keyBytesToBase64 = hmacKey.getKeyValue().toString(); - final String keyBytesAsUtf8 = hmacKey.getKeyValue().decodeToString(); - - // Decode Base64(keyBytes) into new key and compare to original. This always works. - final OctetSequenceKey decodeFromBase64 = new OctetSequenceKey.Builder(new Base64URL(keyBytesToBase64)).build(); - LOGGER.info("Base64 enc/dec test:\ngen: [" + hmacKey + "]\nenc: [" + keyBytesToBase64 + "]\ndec: [" + decodeFromBase64 + "]\n"); - if (decodeFromBase64.equals(hmacKeyNoAttributes) == false) { - return false; + static boolean areJwkHmacOidcSafe(final Collection jwks) { + for (final JWK jwk : jwks) { + if (JwkValidateUtilTests.isJwkHmacOidcSafe(jwk) == false) { + return false; + } } - - // Decode Utf8String(keyBytes) into new key and compare to original. Only works for randomJwkHmacString, fails for randomJwkHmac. - final OctetSequenceKey decodeFromUtf8 = new OctetSequenceKey.Builder(keyBytesAsUtf8.getBytes(StandardCharsets.UTF_8)).build(); - LOGGER.info("UTF8 enc/dec test:\ngen: [" + hmacKey + "]\nenc: [" + keyBytesAsUtf8 + "]\ndec: [" + decodeFromUtf8 + "]\n"); - return decodeFromUtf8.equals(hmacKeyNoAttributes); + return true; } public void testComputeBitLengthRsa() throws Exception { for (final String signatureAlgorithmRsa : JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_RSA) { - final JWK jwk = JwtTestCase.randomJwk(signatureAlgorithmRsa); + final JWK jwk = JwtTestCase.randomJwkRsa(JWSAlgorithm.parse(signatureAlgorithmRsa)); final int minLength = JwkValidateUtil.computeBitLengthRsa(jwk.toRSAKey().toPublicKey()); assertThat(minLength, is(anyOf(equalTo(2048), equalTo(3072)))); } @@ -86,7 +68,7 @@ public void testAlgsJwksAllPkcNotFiltered() throws Exception { private void filterJwksAndAlgorithmsTestHelper(final List candidateAlgs) throws JOSEException { final List algsRandom = randomOfMinUnique(2, candidateAlgs); // duplicates allowed - final List algJwkPairsAll = JwtTestCase.randomJwks(algsRandom); + final List algJwkPairsAll = JwtTestCase.randomJwks(algsRandom, randomBoolean()); final List jwks = algJwkPairsAll.stream().map(JwtIssuer.AlgJwkPair::jwk).toList(); final List algsAll = algJwkPairsAll.stream().map(JwtIssuer.AlgJwkPair::alg).toList(); final List jwksAll = algJwkPairsAll.stream().map(JwtIssuer.AlgJwkPair::jwk).toList(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationTokenTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationTokenTests.java index bb47cf7465676..120040b30444c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationTokenTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationTokenTests.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.security.authc.jwt; +import com.nimbusds.jose.JOSEObjectType; import com.nimbusds.jose.jwk.JWK; import com.nimbusds.jwt.SignedJWT; @@ -27,7 +28,7 @@ public class JwtAuthenticationTokenTests extends JwtTestCase { public void testJwtAuthenticationTokenParse() throws Exception { final String signatureAlgorithm = randomFrom(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS); - final JWK jwk = JwtTestCase.randomJwk(signatureAlgorithm); + final JWK jwk = JwtTestCase.randomJwk(signatureAlgorithm, randomBoolean()); final SecureString jwt = JwtTestCase.randomBespokeJwt(jwk, signatureAlgorithm); // bespoke JWT, not tied to any JWT realm final SecureString clientSharedSecret = randomBoolean() ? null : new SecureString(randomAlphaOfLengthBetween(10, 20).toCharArray()); @@ -65,24 +66,25 @@ public void testPrincipalForJwtWithoutSub() throws Exception { final String principalClaimValue = randomAlphaOfLengthBetween(8, 32); final String signatureAlgorithm = randomFrom(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS); - final JWK jwk = JwtTestCase.randomJwk(signatureAlgorithm); + final JWK jwk = JwtTestCase.randomJwk(signatureAlgorithm, randomBoolean()); final Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt( - null, // type - signatureAlgorithm, + randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty + randomBoolean() ? null : jwk.getKeyID(), // kid + signatureAlgorithm, // alg null, // jwtID - issuer, - List.of(audience), + issuer, // iss + List.of(audience), // aud null, // sub claim value principalClaimName, // principal claim name principalClaimValue, // principal claim value null, // groups claim List.of(), // groups - Date.from(now.minusSeconds(randomLongBetween(10, 20))), // auth_time - Date.from(now), // iat - Date.from(now.minusSeconds(randomLongBetween(5, 10))), // nbf - Date.from(now.plusSeconds(randomLongBetween(3600, 7200))), // exp + Date.from(now.minusSeconds(60 * randomLongBetween(10, 20))), // auth_time + Date.from(now.minusSeconds(randomBoolean() ? 0 : 60 * randomLongBetween(5, 10))), // iat + Date.from(now), // nbf + Date.from(now.plusSeconds(60 * randomLongBetween(3600, 7200))), // exp null, // nonce Map.of() // other claims ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuer.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuer.java index dde44555e5b85..008f48681855e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuer.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuer.java @@ -7,22 +7,23 @@ package org.elasticsearch.xpack.security.authc.jwt; +import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.jwk.JWK; import com.nimbusds.jose.jwk.JWKSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings; import org.elasticsearch.xpack.core.security.user.User; import java.io.Closeable; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; + +import static org.elasticsearch.test.ESTestCase.randomBoolean; /** * Test class with settings for a JWT issuer to sign JWTs for users. @@ -38,59 +39,66 @@ record AlgJwkPair(String alg, JWK jwk) {} final List audiencesClaimValue; // claim name is hard-coded to `aud` for OIDC ID Token compatibility final String principalClaimName; // claim name is configurable, EX: Users (sub, oid, email, dn, uid), Clients (azp, appid, client_id) final Map principals; // principals with roles, for sending encoded JWTs into JWT realms for authc/authz verification - final List algAndJwksPkc; - final List algAndJwksHmac; - final AlgJwkPair algAndJwkHmacOidc; + final JwtIssuerHttpsServer httpsServer; + + List algorithmsAll; // Computed values - final List algAndJwksAll; - final Set algorithmsAll; - final String encodedJwkSetPkcPrivate; - final String encodedJwkSetPkcPublic; - final String encodedJwkSetHmac; - final String encodedKeyHmacOidc; - final JwtIssuerHttpsServer httpsServer; + List algAndJwksPkc; + List algAndJwksHmac; + AlgJwkPair algAndJwkHmacOidc; + List algAndJwksAll; + String encodedJwkSetPkcPublicPrivate; + String encodedJwkSetPkcPublic; + String encodedJwkSetHmac; + String encodedKeyHmacOidc; JwtIssuer( final String issuerClaimValue, final List audiencesClaimValue, final String principalClaimName, final Map principals, - final List algAndJwksPkc, - final List algAndJwksHmac, - final AlgJwkPair algAndJwkHmacOidc, final boolean createHttpsServer ) throws Exception { this.issuerClaimValue = issuerClaimValue; this.audiencesClaimValue = audiencesClaimValue; this.principalClaimName = principalClaimName; this.principals = principals; - this.algAndJwksPkc = algAndJwksPkc; - this.algAndJwksHmac = algAndJwksHmac; - this.algAndJwkHmacOidc = algAndJwkHmacOidc; - - this.algAndJwksAll = new ArrayList<>(this.algAndJwksPkc.size() + this.algAndJwksHmac.size() + 1); - this.algAndJwksAll.addAll(this.algAndJwksPkc); - this.algAndJwksAll.addAll(this.algAndJwksHmac); - if (this.algAndJwkHmacOidc != null) { - this.algAndJwksAll.add(this.algAndJwkHmacOidc); - } + this.httpsServer = createHttpsServer ? new JwtIssuerHttpsServer(null) : null; + } - this.algorithmsAll = this.algAndJwksAll.stream().map(p -> p.alg).collect(Collectors.toSet()); + // The flag areHmacJwksOidcSafe indicates if all provided HMAC JWKs are UTF8, for HMAC OIDC JWK encoding compatibility. + void setJwks(final List algAndJwks, final boolean areHmacJwksOidcSafe) throws JOSEException { + this.algorithmsAll = algAndJwks.stream().map(e -> e.alg).toList(); + LOGGER.info("Setting JWKs: algorithms=[{}], areHmacJwksOidcSafe=[{}]", String.join(",", this.algorithmsAll), areHmacJwksOidcSafe); + this.algAndJwksAll = algAndJwks; + this.algAndJwksPkc = this.algAndJwksAll.stream() + .filter(e -> JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC.contains(e.alg)) + .toList(); + this.algAndJwksHmac = this.algAndJwksAll.stream() + .filter(e -> JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC.contains(e.alg)) + .toList(); + if ((this.algAndJwksHmac.size() == 1) && (areHmacJwksOidcSafe) && (randomBoolean())) { + this.algAndJwkHmacOidc = this.algAndJwksHmac.get(0); + this.algAndJwksHmac = Collections.emptyList(); + } else { + this.algAndJwkHmacOidc = null; + } + // Encode PKC JWKSet (key material bytes are wrapped in Base64URL, and then wraps in JSON) final JWKSet jwkSetPkc = new JWKSet(this.algAndJwksPkc.stream().map(p -> p.jwk).toList()); + this.encodedJwkSetPkcPublicPrivate = JwtUtil.serializeJwkSet(jwkSetPkc, false); + this.encodedJwkSetPkcPublic = JwtUtil.serializeJwkSet(jwkSetPkc, true); + + // Encode HMAC JWKSet (key material bytes are wrapped in Base64URL, and then wraps in JSON) final JWKSet jwkSetHmac = new JWKSet(this.algAndJwksHmac.stream().map(p -> p.jwk).toList()); + this.encodedJwkSetHmac = JwtUtil.serializeJwkSet(jwkSetHmac, false); - this.encodedJwkSetPkcPrivate = jwkSetPkc.getKeys().isEmpty() ? null : JwtUtil.serializeJwkSet(jwkSetPkc, false); - this.encodedJwkSetPkcPublic = jwkSetPkc.getKeys().isEmpty() ? null : JwtUtil.serializeJwkSet(jwkSetPkc, true); - this.encodedJwkSetHmac = jwkSetHmac.getKeys().isEmpty() ? null : JwtUtil.serializeJwkSet(jwkSetHmac, false); + // Encode HMAC OIDC JWK (key material bytes are decoded from UTF8 to UNICODE String) this.encodedKeyHmacOidc = (algAndJwkHmacOidc == null) ? null : JwtUtil.serializeJwkHmacOidc(this.algAndJwkHmacOidc.jwk); - if ((Strings.hasText(this.encodedJwkSetPkcPublic) == false) || (createHttpsServer == false)) { - this.httpsServer = null; // no PKC JWKSet, or skip HTTPS server because caller will use local file instead - } else { - final byte[] encodedJwkSetPkcPublicBytes = this.encodedJwkSetPkcPublic.getBytes(StandardCharsets.UTF_8); - this.httpsServer = new JwtIssuerHttpsServer(encodedJwkSetPkcPublicBytes); + if (this.httpsServer != null) { + this.httpsServer.updateJwkSetPkcContents(this.encodedJwkSetPkcPublic.getBytes(StandardCharsets.UTF_8)); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuerHttpsServer.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuerHttpsServer.java index 24eedcabc622f..96be8c784c6b0 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuerHttpsServer.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuerHttpsServer.java @@ -68,6 +68,11 @@ public JwtIssuerHttpsServer(final byte[] encodedJwkSetPkcPublicBytes) throws Exc LOGGER.debug("Started [{}]", this.url); } + public void updateJwkSetPkcContents(final byte[] encodedJwkSetPkcPublicBytes) { + this.httpsServer.removeContext(PATH); + this.httpsServer.createContext(PATH, new JwtIssuerHttpHandler(encodedJwkSetPkcPublicBytes)); + } + @Override public void close() throws IOException { if (this.httpsServer != null) { @@ -92,8 +97,12 @@ public void handle(final HttpExchange httpExchange) throws IOException { final String path = httpExchange.getRequestURI().getPath(); // EX: "/", "/valid/", "/valid/pkc_jwkset.json" LOGGER.trace("Request: [{}]", path); try (OutputStream os = httpExchange.getResponseBody()) { - httpExchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, this.encodedJwkSetPkcPublicBytes.length); - os.write(this.encodedJwkSetPkcPublicBytes); + if (encodedJwkSetPkcPublicBytes == null) { + httpExchange.sendResponseHeaders(HttpURLConnection.HTTP_NOT_FOUND, 0); + } else { + httpExchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, this.encodedJwkSetPkcPublicBytes.length); + os.write(this.encodedJwkSetPkcPublicBytes); + } } LOGGER.trace("Response: [{}]", path); // Confirm client didn't disconnect before flush } catch (Throwable t) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java index 693fdda71882e..8a8b955a81385 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java @@ -13,6 +13,8 @@ import com.nimbusds.jwt.PlainJWT; import com.nimbusds.jwt.SignedJWT; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.MockSecureSettings; @@ -26,9 +28,9 @@ import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings; import org.elasticsearch.xpack.core.security.user.User; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; @@ -39,6 +41,7 @@ import static org.hamcrest.Matchers.nullValue; public class JwtRealmAuthenticateTests extends JwtRealmTestCase { + private static final Logger LOGGER = LogManager.getLogger(JwtRealmAuthenticateTests.class); /** * Test with empty roles. @@ -90,6 +93,162 @@ public void testJwtAuthcRealmAuthcAuthzWithoutAuthzRealms() throws Exception { this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcRange); } + /** + * Test with updated/removed/restored JWKs. + * @throws Exception Unexpected test failure + */ + public void testJwkSetUpdates() throws Exception { + this.jwtIssuerAndRealms = this.generateJwtIssuerRealmPairs( + this.createJwtRealmsSettingsBuilder(), + new MinMax(1, 3), // realmsRange + new MinMax(0, 0), // authzRange + new MinMax(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange + new MinMax(1, 3), // audiencesRange + new MinMax(1, 3), // usersRange + new MinMax(0, 3), // rolesRange + new MinMax(0, 1), // jwtCacheSizeRange + randomBoolean() // createHttpsServer + ); + final JwtIssuerAndRealm jwtIssuerAndRealm = this.randomJwtIssuerRealmPair(); + assertThat(jwtIssuerAndRealm.realm().delegatedAuthorizationSupport.hasDelegation(), is(false)); + + final User user = this.randomUser(jwtIssuerAndRealm.issuer()); + final SecureString jwtJwks1 = this.randomJwt(jwtIssuerAndRealm, user); + final SecureString clientSecret = jwtIssuerAndRealm.realm().clientAuthenticationSharedSecret; + final MinMax jwtAuthcRange = new MinMax(2, 3); + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange); + + // Details about first JWT using the JWT issuer original JWKs + final String jwt1JwksAlg = SignedJWT.parse(jwtJwks1.toString()).getHeader().getAlgorithm().getName(); + final boolean isPkcJwtJwks1 = JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC.contains(jwt1JwksAlg); + LOGGER.debug("JWT alg=[{}]", jwt1JwksAlg); + + // Backup JWKs 1 + final List jwtIssuerJwks1Backup = jwtIssuerAndRealm.issuer().algAndJwksAll; + final boolean jwtIssuerJwks1OidcSafe = JwkValidateUtilTests.areJwkHmacOidcSafe( + jwtIssuerJwks1Backup.stream().map(e -> e.jwk()).toList() + ); + LOGGER.debug("JWKs 1, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll)); + + // Empty all JWT issuer JWKs. + LOGGER.debug("JWKs 1 backed up, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll)); + jwtIssuerAndRealm.issuer().setJwks(Collections.emptyList(), jwtIssuerJwks1OidcSafe); + super.printJwtIssuer(jwtIssuerAndRealm.issuer()); + super.copyIssuerJwksToRealmConfig(jwtIssuerAndRealm); + LOGGER.debug("JWKs 1 emptied, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll)); + + // Original JWT continues working, because JWT realm cached old JWKs in memory. + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange); + LOGGER.debug("JWT 1 still worked, because JWT realm has old JWKs cached in memory"); + + // Restore original JWKs 1 into the JWT issuer. + jwtIssuerAndRealm.issuer().setJwks(jwtIssuerJwks1Backup, jwtIssuerJwks1OidcSafe); + super.printJwtIssuer(jwtIssuerAndRealm.issuer()); + super.copyIssuerJwksToRealmConfig(jwtIssuerAndRealm); + LOGGER.debug("JWKs 1 restored, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll)); + + // Original JWT continues working, because JWT realm cached old JWKs in memory. + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange); + LOGGER.debug("JWT 1 still worked, because JWT realm has old JWKs cached in memory"); + + // Generate a replacement set of JWKs 2 for the JWT issuer. + final List jwtIssuerJwks2Backup = JwtRealmTestCase.randomJwks( + jwtIssuerJwks1Backup.stream().map(e -> e.alg()).toList(), + jwtIssuerJwks1OidcSafe + ); + jwtIssuerAndRealm.issuer().setJwks(jwtIssuerJwks2Backup, jwtIssuerJwks1OidcSafe); + super.printJwtIssuer(jwtIssuerAndRealm.issuer()); + super.copyIssuerJwksToRealmConfig(jwtIssuerAndRealm); + LOGGER.debug("JWKs 2 created, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll)); + + // Original JWT continues working, because JWT realm still has original JWKs cached in memory. + // - jwtJwks1(PKC): Pass (Original PKC JWKs are still in the realm) + // - jwtJwks1(HMAC): Pass (Original HMAC JWKs are still in the realm) + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange); + LOGGER.debug("JWT 1 still worked, because JWT realm has old JWKs cached in memory"); + + // Create a JWT using the new JWKs. + final SecureString jwtJwks2 = this.randomJwt(jwtIssuerAndRealm, user); + final String jwtJwks2Alg = SignedJWT.parse(jwtJwks2.toString()).getHeader().getAlgorithm().getName(); + final boolean isPkcJwtJwks2 = JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC.contains(jwtJwks2Alg); + LOGGER.debug("Created JWT 2: oidcSafe=[{}], algs=[{}, {}]", jwtIssuerJwks1OidcSafe, jwt1JwksAlg, jwtJwks2Alg); + + // Try new JWT. + // - jwtJwks2(PKC): PKC reload triggered and loaded new JWKs, so PASS + // - jwtJwks2(HMAC): HMAC reload triggered but it is a no-op, so FAIL + if (isPkcJwtJwks2) { + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcRange); + LOGGER.debug("PKC JWT 2 worked with JWKs 2"); + } else { + this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret); + LOGGER.debug("HMAC JWT 2 failed with JWKs 1"); + } + + // Try old JWT. + // - jwtJwks2(PKC): PKC reload triggered and loaded new JWKs, jwtJwks1(PKC): PKC reload triggered and loaded new JWKs, so FAIL + // - jwtJwks2(PKC): PKC reload triggered and loaded new JWKs, jwtJwks1(HMAC): HMAC reload not triggered, so PASS + // - jwtJwks2(HMAC): HMAC reload triggered but it is a no-op, jwtJwks1(PKC): PKC reload not triggered, so PASS + // - jwtJwks2(HMAC): HMAC reload triggered but it is a no-op, jwtJwks1(HMAC): HMAC reload not triggered, so PASS + if (isPkcJwtJwks1 == false || isPkcJwtJwks2 == false) { + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange); + } else { + this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks1, clientSecret); + } + + // Empty all JWT issuer JWKs. + jwtIssuerAndRealm.issuer().setJwks(Collections.emptyList(), jwtIssuerJwks1OidcSafe); + super.printJwtIssuer(jwtIssuerAndRealm.issuer()); + super.copyIssuerJwksToRealmConfig(jwtIssuerAndRealm); + + // New JWT continues working because JWT realm will end up with PKC JWKs 2 and HMAC JWKs 1 in memory + if (isPkcJwtJwks2) { + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcRange); + } else { + this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret); + } + + // Trigger JWT realm to reload JWKs and go into a degraded state + // - jwtJwks1(HMAC): HMAC reload not triggered, so PASS + // - jwtJwks1(PKC): PKC reload triggered and loaded new JWKs, so FAIL + if (isPkcJwtJwks1 == false || isPkcJwtJwks2 == false) { + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange); + } else { + this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks1, clientSecret); + } + + // Try new JWT and verify degraded state caused by empty PKC JWKs + // - jwtJwks1(PKC) + jwtJwks2(PKC): If second JWT is PKC, and first JWT is PKC, degraded state can be tested. + // - jwtJwks1(HMAC) + jwtJwks2(PKC): If second JWT is PKC, but first JWT is HMAC, HMAC JWT 1 above didn't trigger PKC reload. + // - jwtJwks1(PKC) + jwtJwks2(HMAC): If second JWT is HMAC, it always fails because HMAC reload not supported. + // - jwtJwks1(HMAC) + jwtJwks2(HMAC): If second JWT is HMAC, it always fails because HMAC reload not supported. + if (isPkcJwtJwks1 == false && isPkcJwtJwks2) { + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcRange); + } else { + this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret); + } + + // Restore JWKs 2 to the realm + jwtIssuerAndRealm.issuer().setJwks(jwtIssuerJwks2Backup, jwtIssuerJwks1OidcSafe); + super.copyIssuerJwksToRealmConfig(jwtIssuerAndRealm); + super.printJwtIssuer(jwtIssuerAndRealm.issuer()); + + // Trigger JWT realm to reload JWKs and go into a recovered state + // - jwtJwks2(PKC): Pass (Triggers PKC reload, gets newer PKC JWKs), jwtJwks1(PKC): Fail (Triggers PKC reload, gets new PKC JWKs) + // - jwtJwks2(PKC): Pass (Triggers PKC reload, gets newer PKC JWKs), jwtJwks1(HMAC): Pass (HMAC reload was a no-op) + // - jwtJwks2(HMAC): Fail (Triggers HMAC reload, but it is a no-op), jwtJwks1(PKC): Fail (Triggers PKC reload, gets new PKC JWKs) + // - jwtJwks2(HMAC): Fail (Triggers HMAC reload, but it is a no-op), jwtJwks1(HMAC): Pass (HMAC reload was a no-op) + if (isPkcJwtJwks2) { + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcRange); + } else { + this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret); + } + if (isPkcJwtJwks1 == false || isPkcJwtJwks2 == false) { + this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange); + } else { + this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks1, clientSecret); + } + } + /** * Test with authz realms. * @throws Exception Unexpected test failure @@ -253,16 +412,16 @@ public void testJwtValidationFailures() throws Exception { { // Verify rejection of a tampered header (flip HMAC=>RSA or RSA/EC=>HMAC) final String mixupAlg; // Check if there are any algorithms available in the realm for attempting a flip test if (JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC.contains(validHeader.getAlgorithm().getName())) { - if (jwtIssuerAndRealm.realm().jwksAlgsPkc.algs().isEmpty()) { + if (jwtIssuerAndRealm.realm().contentAndJwksAlgsPkc.jwksAlgs().algs().isEmpty()) { mixupAlg = null; // cannot flip HMAC to PKC (no PKC algs available) } else { - mixupAlg = randomFrom(jwtIssuerAndRealm.realm().jwksAlgsPkc.algs()); // flip HMAC to PKC + mixupAlg = randomFrom(jwtIssuerAndRealm.realm().contentAndJwksAlgsPkc.jwksAlgs().algs()); // flip HMAC to PKC } } else { - if (jwtIssuerAndRealm.realm().jwksAlgsHmac.algs().isEmpty()) { + if (jwtIssuerAndRealm.realm().contentAndJwksAlgsHmac.jwksAlgs().algs().isEmpty()) { mixupAlg = null; // cannot flip PKC to HMAC (no HMAC algs available) } else { - mixupAlg = randomFrom(jwtIssuerAndRealm.realm().jwksAlgsHmac.algs()); // flip HMAC to PKC + mixupAlg = randomFrom(jwtIssuerAndRealm.realm().contentAndJwksAlgsHmac.jwksAlgs().algs()); // flip HMAC to PKC } } // This check can only be executed if there is a flip algorithm available in the realm @@ -328,6 +487,7 @@ public void testSameIssuerTwoRealmsDifferentClientSecrets() throws Exception { final int realmsCount = 2; final List allRealms = new ArrayList<>(realmsCount); // two identical realms for same issuer, except different client secret final JwtIssuer jwtIssuer = this.createJwtIssuer(0, principalClaimName, 12, 1, 1, 1, false); + super.printJwtIssuer(jwtIssuer); this.jwtIssuerAndRealms = new ArrayList<>(realmsCount); for (int i = 0; i < realmsCount; i++) { final String realmName = "realm_" + jwtIssuer.issuerClaimValue + "_" + i; @@ -346,21 +506,21 @@ public void testSameIssuerTwoRealmsDifferentClientSecrets() throws Exception { RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.CLIENT_AUTHENTICATION_TYPE), JwtRealmSettings.ClientAuthenticationType.SHARED_SECRET.value() ); - if (Strings.hasText(jwtIssuer.encodedJwkSetPkcPublic)) { + if (jwtIssuer.encodedJwkSetPkcPublic.isEmpty() == false) { authcSettings.put( RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.PKC_JWKSET_PATH), - super.saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic.getBytes(StandardCharsets.UTF_8)) + super.saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic) ); } // JWT authc realm secure settings final MockSecureSettings secureSettings = new MockSecureSettings(); - if (Strings.hasText(jwtIssuer.encodedJwkSetHmac)) { + if (jwtIssuer.algAndJwksHmac.isEmpty() == false) { secureSettings.setString( RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.HMAC_JWKSET), jwtIssuer.encodedJwkSetHmac ); } - if (Strings.hasText(jwtIssuer.encodedKeyHmacOidc)) { + if (jwtIssuer.encodedKeyHmacOidc != null) { secureSettings.setString( RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.HMAC_KEY), jwtIssuer.encodedKeyHmacOidc @@ -376,6 +536,7 @@ public void testSameIssuerTwoRealmsDifferentClientSecrets() throws Exception { jwtRealm.initialize(allRealms, super.licenseState); final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, jwtRealmSettingsBuilder); this.jwtIssuerAndRealms.add(jwtIssuerAndRealm); // add them so the test will clean them up + super.printJwtRealm(jwtRealm); } // pick 2nd realm and use its secret, verify 2nd realm does authc, which implies 1st realm rejects the secret diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmGenerateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmGenerateTests.java index aa8c1e0786928..663b6d8c6d1cd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmGenerateTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmGenerateTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.security.authc.jwt; import com.nimbusds.jose.JOSEObjectType; +import com.nimbusds.jose.JWSAlgorithm; import com.nimbusds.jose.jwk.JWK; import com.nimbusds.jose.jwk.OctetSequenceKey; import com.nimbusds.jose.jwk.RSAKey; @@ -77,11 +78,9 @@ public void testCreateJwtSmokeTestRealm() throws Exception { List.of("aud8"), // aud principalClaimName, // sub Collections.singletonMap("security_test_user", new User("security_test_user", "security_test_role")), // users - Collections.emptyList(), // algJwkPairsPkc - Collections.emptyList(), // algJwkPairsHmac - algJwkPairHmac, // algJwkPairHmac false // createHttpsServer ); + jwtIssuer.setJwks(List.of(algJwkPairHmac), true); // Create realm settings final String realmName = "jwt8"; @@ -121,11 +120,13 @@ public void testCreateJwtSmokeTestRealm() throws Exception { final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = new JwtRealmSettingsBuilder(realmName, configBuilder); final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, jwtRealmSettingsBuilder); super.jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // super.shutdown() closes issuer+realm if necessary + super.printJwtRealmAndIssuer(jwtIssuerAndRealm); // Create JWT final User user = this.randomUser(jwtIssuerAndRealm.issuer()); final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt( - randomBoolean() ? null : JOSEObjectType.JWT.toString(), + randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty + randomBoolean() ? null : algJwkPairHmac.jwk().getKeyID(), // kid algJwkPairHmac.alg(), // alg null, // jwtID jwtIssuerAndRealm.realm().allowedIssuer, // iss @@ -156,7 +157,7 @@ public void testCreateJwtSmokeTestRealm() throws Exception { */ public void testCreateJwtIntegrationTestRealm1() throws Exception { // Create RSA key for algorithm RS256 - final JWK jwk = new RSAKey.Builder((RSAKey) JwtTestCase.randomJwk("RS256")).keyID("test-rsa-key").build(); + final JWK jwk = new RSAKey.Builder(JwtTestCase.randomJwkRsa(JWSAlgorithm.RS256)).keyID("test-rsa-key").build(); final JwtIssuer.AlgJwkPair algJwkPairPkc = new JwtIssuer.AlgJwkPair("RS256", jwk); final String principalClaimName = "sub"; @@ -171,11 +172,9 @@ public void testCreateJwtIntegrationTestRealm1() throws Exception { List.of("https://audience.example.com/"), // aud claim value principalClaimName, // principal claim name Collections.singletonMap("user1", new User("user1", "role1")), // users - List.of(algJwkPairPkc), // algJwkPairsPkc - Collections.emptyList(), // algJwkPairsHmac - null, // algJwkPairHmac false // createHttpsServer ); + jwtIssuer.setJwks(List.of(algJwkPairPkc), false); // Create realm settings (no secure settings) final String realmName = "jwt1"; @@ -201,7 +200,7 @@ public void testCreateJwtIntegrationTestRealm1() throws Exception { ) .put( RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.PKC_JWKSET_PATH), - super.saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic.getBytes(StandardCharsets.UTF_8)) + super.saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic) ); // Create realm @@ -213,11 +212,13 @@ public void testCreateJwtIntegrationTestRealm1() throws Exception { final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = new JwtRealmSettingsBuilder(realmName, configBuilder); final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, jwtRealmSettingsBuilder); super.jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // super.shutdown() closes issuer+realm if necessary + super.printJwtRealmAndIssuer(jwtIssuerAndRealm); // Create JWT final User user = this.randomUser(jwtIssuerAndRealm.issuer()); final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt( - randomBoolean() ? null : JOSEObjectType.JWT.toString(), + randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty + randomBoolean() ? null : algJwkPairPkc.jwk().getKeyID(), // kid algJwkPairPkc.alg(), // alg null, // jwtID jwtIssuerAndRealm.realm().allowedIssuer, // iss @@ -266,11 +267,9 @@ public void testCreateJwtIntegrationTestRealm2() throws Exception { List.of("es01", "es02", "es03"), // aud claim value principalClaimName, // principal claim name Collections.singletonMap("user2", new User("user2", "role2")), // users - Collections.emptyList(), // algJwkPairsPkc - Collections.emptyList(), // algJwkPairsHmac - algJwkPairHmac, // algJwkPairHmac false // createHttpsServer ); + jwtIssuer.setJwks(List.of(algJwkPairHmac), true); // Create realm settings final String realmName = "jwt2"; @@ -319,11 +318,13 @@ public void testCreateJwtIntegrationTestRealm2() throws Exception { final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = new JwtRealmSettingsBuilder(realmName, configBuilder); final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, jwtRealmSettingsBuilder); super.jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // super.shutdown() closes issuer+realm if necessary + super.printJwtRealmAndIssuer(jwtIssuerAndRealm); // Create JWT final User user = this.randomUser(jwtIssuerAndRealm.issuer()); final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt( - JOSEObjectType.JWT.toString(), + randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty + randomBoolean() ? null : algJwkPairHmac.jwk().getKeyID(), // kid algJwkPairHmac.alg(), // alg null, // jwtID jwtIssuerAndRealm.realm().allowedIssuer, // iss @@ -358,7 +359,6 @@ public void testCreateJwtIntegrationTestRealm3() throws Exception { new JwtIssuer.AlgJwkPair("HS384", new OctetSequenceKey.Builder(randomByteArrayOfLength(48)).keyID("test-hmac-384").build()), new JwtIssuer.AlgJwkPair("HS512", new OctetSequenceKey.Builder(randomByteArrayOfLength(64)).keyID("test-hmac-512").build()) ); - var selectedHmac = randomFrom(hmacKeys); final String principalClaimName = "sub"; final Settings.Builder jwtRealmsServiceSettings = Settings.builder() @@ -372,11 +372,9 @@ public void testCreateJwtIntegrationTestRealm3() throws Exception { List.of("jwt3-audience"), // aud claim value principalClaimName, // principal claim name Collections.singletonMap("user3", new User("user3", "role3")), // users - Collections.emptyList(), // algJwkPairsPkc - hmacKeys, // algJwkPairsHmac - null, // algJwkPairHmac false // createHttpsServer ); + jwtIssuer.setJwks(hmacKeys, false); // Create realm settings final String realmName = "jwt3"; @@ -416,11 +414,14 @@ public void testCreateJwtIntegrationTestRealm3() throws Exception { final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = new JwtRealmSettingsBuilder(realmName, configBuilder); final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, jwtRealmSettingsBuilder); super.jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // super.shutdown() closes issuer+realm if necessary + super.printJwtRealmAndIssuer(jwtIssuerAndRealm); // Create JWT + final JwtIssuer.AlgJwkPair selectedHmac = randomFrom(hmacKeys); final User user = this.randomUser(jwtIssuerAndRealm.issuer()); final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt( - JOSEObjectType.JWT.toString(), + randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty + randomBoolean() ? null : selectedHmac.jwk().getKeyID(), // kid selectedHmac.alg(), // alg null, // jwtID jwtIssuerAndRealm.realm().allowedIssuer, // iss @@ -483,7 +484,7 @@ private static String printIssuerSettings(final JwtIssuer jwtIssuer) { sb.append("Audiences: ").append(String.join(",", jwtIssuer.audiencesClaimValue)).append('\n'); sb.append("Algorithms: ").append(String.join(",", jwtIssuer.algorithmsAll)).append("\n"); if (jwtIssuer.algAndJwksPkc.isEmpty() == false) { - sb.append("PKC JWKSet (Private): ").append(jwtIssuer.encodedJwkSetPkcPrivate).append("\n"); + sb.append("PKC JWKSet (Private): ").append(jwtIssuer.encodedJwkSetPkcPublicPrivate).append("\n"); sb.append("PKC JWKSet (Public): ").append(jwtIssuer.encodedJwkSetPkcPublic).append("\n"); } if (jwtIssuer.algAndJwksHmac.isEmpty() == false) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmTestCase.java index 68de795f4ebe6..ec64938a6e34c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmTestCase.java @@ -7,18 +7,18 @@ package org.elasticsearch.xpack.security.authc.jwt; import com.nimbusds.jose.JOSEObjectType; -import com.nimbusds.jose.jwk.OctetSequenceKey; +import com.nimbusds.jose.jwk.JWK; import com.nimbusds.jwt.SignedJWT; import com.nimbusds.openid.connect.sdk.Nonce; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.PathUtils; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.threadpool.TestThreadPool; @@ -42,7 +42,9 @@ import org.junit.After; import org.junit.Before; -import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.ParseException; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -199,45 +201,25 @@ protected JwtIssuer createJwtIssuer( final boolean createHttpsServer ) throws Exception { final String issuer = "iss" + (i + 1) + "_" + randomIntBetween(0, 9999); - - // Allow algorithm repeats, to cover testing of multiple JWKs for same algorithm - final List algs = randomOfMinMaxNonUnique(algsCount, algsCount, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS); - final List algsPkc = algs.stream().filter(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC::contains).toList(); - final List algsHmac = algs.stream().filter(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC::contains).toList(); - final List algJwkPairsPkc = JwtTestCase.randomJwks(algsPkc); - // Key setting vs JWKSet setting are mutually exclusive, do not populate both - final List algJwkPairsHmac = new ArrayList<>(JwtTestCase.randomJwks(algsHmac)); // allow remove/add below - final JwtIssuer.AlgJwkPair algJwkPairHmacOidc; - if ((algJwkPairsHmac.size() == 0) || (randomBoolean())) { - algJwkPairHmacOidc = null; // list(0||1||N) => Key=null and JWKSet(N) - } else { - // Change one of the HMAC random bytes keys to an OIDC UTF8 key. Put it in either the Key setting or JWKSet setting. - final JwtIssuer.AlgJwkPair algJwkPairRandomBytes = algJwkPairsHmac.get(0); - final OctetSequenceKey jwkHmacRandomBytes = JwtTestCase.conditionJwkHmacForOidc((OctetSequenceKey) algJwkPairRandomBytes.jwk()); - final JwtIssuer.AlgJwkPair algJwkPairUtf8Bytes = new JwtIssuer.AlgJwkPair(algJwkPairRandomBytes.alg(), jwkHmacRandomBytes); - if ((algJwkPairsHmac.size() == 1) && (randomBoolean())) { - algJwkPairHmacOidc = algJwkPairUtf8Bytes; // list(1) => Key=OIDC and JWKSet(0) - algJwkPairsHmac.remove(0); - } else { - algJwkPairHmacOidc = null; // list(N) => Key=null and JWKSet(OIDC+N-1) - algJwkPairsHmac.set(0, algJwkPairUtf8Bytes); - } - } - final List audiences = IntStream.range(0, audiencesCount).mapToObj(j -> issuer + "_aud" + (j + 1)).toList(); final Map users = JwtTestCase.generateTestUsersWithRoles(userCount, roleCount); + // Allow algorithm repeats, to cover testing of multiple JWKs for same algorithm + final JwtIssuer jwtIssuer = new JwtIssuer(issuer, audiences, principalClaimName, users, createHttpsServer); + final List algorithms = randomOfMinMaxNonUnique(algsCount, algsCount, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS); + final boolean areHmacJwksOidcSafe = randomBoolean(); + final List algAndJwks = JwtRealmTestCase.randomJwks(algorithms, areHmacJwksOidcSafe); + jwtIssuer.setJwks(algAndJwks, areHmacJwksOidcSafe); + return jwtIssuer; + } - // Decide if public PKC JWKSet will be hosted in a local file or an HTTPS URL. If HTTPS URL, tell issuer to set up an HTTPS server. - return new JwtIssuer( - issuer, - audiences, - principalClaimName, - users, - algJwkPairsPkc, - algJwkPairsHmac, - algJwkPairHmacOidc, - createHttpsServer - ); + protected void copyIssuerJwksToRealmConfig(final JwtIssuerAndRealm jwtIssuerAndRealm) throws Exception { + if ((jwtIssuerAndRealm.realm.isConfiguredJwkSetPkc) && (jwtIssuerAndRealm.realm.jwkSetPathUri == null)) { + LOGGER.trace("Updating JwtRealm PKC public JWKSet local file"); + final Path path = PathUtils.get(jwtIssuerAndRealm.realm.jwkSetPath); + Files.writeString(path, jwtIssuerAndRealm.issuer.encodedJwkSetPkcPublic); + } + + // TODO If x-pack Security plug-in add supports for reloadable settings, update HMAC JWKSet and HMAC OIDC JWK in ES Keystore } protected JwtRealmsServiceSettingsBuilder createJwtRealmsSettingsBuilder() throws Exception { @@ -290,10 +272,10 @@ protected JwtRealmSettingsBuilder createJwtRealmSettingsBuilder(final JwtIssuer randomBoolean() ? "-1" : randomBoolean() ? "0" : randomIntBetween(1, 5) + randomFrom("s", "m", "h") ); } - if (Strings.hasText(jwtIssuer.encodedJwkSetPkcPublic)) { + if (jwtIssuer.encodedJwkSetPkcPublic.isEmpty() == false) { final String jwkSetPath; // file or HTTPS URL if (jwtIssuer.httpsServer == null) { - jwkSetPath = super.saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic.getBytes(StandardCharsets.UTF_8)); + jwkSetPath = super.saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic); } else { authcSettings.putList( RealmSettings.getFullSettingKey( @@ -377,13 +359,13 @@ protected JwtRealmSettingsBuilder createJwtRealmSettingsBuilder(final JwtIssuer // JWT authc realm secure settings final MockSecureSettings secureSettings = new MockSecureSettings(); - if (Strings.hasText(jwtIssuer.encodedJwkSetHmac)) { + if (jwtIssuer.algAndJwksHmac.isEmpty() == false) { secureSettings.setString( RealmSettings.getFullSettingKey(authcRealmName, JwtRealmSettings.HMAC_JWKSET), jwtIssuer.encodedJwkSetHmac ); } - if (Strings.hasText(jwtIssuer.encodedKeyHmacOidc)) { + if (jwtIssuer.encodedKeyHmacOidc != null) { secureSettings.setString( RealmSettings.getFullSettingKey(authcRealmName, JwtRealmSettings.HMAC_KEY), jwtIssuer.encodedKeyHmacOidc @@ -435,7 +417,7 @@ protected JwtRealm createJwtRealm( return jwtRealm; } - protected JwtIssuerAndRealm randomJwtIssuerRealmPair() { + protected JwtIssuerAndRealm randomJwtIssuerRealmPair() throws ParseException { // Select random JWT issuer and JWT realm pair, and log the realm settings assertThat(this.jwtIssuerAndRealms, is(notNullValue())); assertThat(this.jwtIssuerAndRealms, is(not(empty()))); @@ -444,41 +426,7 @@ protected JwtIssuerAndRealm randomJwtIssuerRealmPair() { assertThat(jwtRealm, is(notNullValue())); assertThat(jwtRealm.allowedIssuer, is(equalTo(jwtIssuerAndRealm.issuer.issuerClaimValue))); // assert equal, don't print both assertThat(jwtIssuerAndRealm.issuer.audiencesClaimValue.stream().anyMatch(jwtRealm.allowedAudiences::contains), is(true)); - LOGGER.info( - "REALM[" - + jwtRealm.name() - + "," - + jwtRealm.order() - + "/" - + this.jwtIssuerAndRealms.size() - + "], iss=[" - + jwtIssuerAndRealm.issuer - + "], iss.aud=" - + jwtIssuerAndRealm.issuer.audiencesClaimValue - + ", realm.aud=" - + jwtRealm.allowedAudiences - + ", HMAC alg=" - + jwtRealm.jwksAlgsHmac.algs() - + ", PKC alg=" - + jwtRealm.jwksAlgsPkc.algs() - + ", client=[" - + jwtRealm.clientAuthenticationType - + "], meta=[" - + jwtRealm.populateUserMetadata - + "], authz=[" - + jwtRealm.delegatedAuthorizationSupport.hasDelegation() - + "], jwkSetPath=[" - + jwtRealm.jwkSetPath - + "], claimPrincipal=[" - + jwtRealm.claimParserPrincipal.getClaimName() - + "], claimGroups=[" - + jwtRealm.claimParserGroups.getClaimName() - + "], clientAuthenticationSharedSecret=[" - + jwtRealm.clientAuthenticationSharedSecret - + "], authz=[" - + jwtRealm.delegatedAuthorizationSupport - + "]" - ); + this.printJwtRealmAndIssuer(jwtIssuerAndRealm); return jwtIssuerAndRealm; } @@ -498,9 +446,8 @@ protected void doMultipleAuthcAuthzAndVerifySuccess( final int jwtAuthcRepeats = randomIntBetween(jwtAuthcRange.min(), jwtAuthcRange.max()); for (int authcRun = 1; authcRun <= jwtAuthcRepeats; authcRun++) { // Create request with headers set - LOGGER.info("RUN[" + authcRun + "/" + jwtAuthcRepeats + "], jwt=[" + jwt + "], secret=[" + sharedSecret + "]."); final ThreadContext requestThreadContext = super.createThreadContext(jwt, sharedSecret); - LOGGER.info(requestThreadContext.getHeaders().toString()); // TODO Remove debug log + LOGGER.info("REQ[" + authcRun + "/" + jwtAuthcRepeats + "] HEADERS=" + requestThreadContext.getHeaders()); // Loop through all authc/authz realms. Confirm a JWT authc realm recognizes and extracts the request headers. JwtAuthenticationToken jwtAuthenticationToken = null; @@ -527,15 +474,16 @@ protected void doMultipleAuthcAuthzAndVerifySuccess( } assertThat(tokenSecret, is(equalTo(sharedSecret))); } - LOGGER.info("TOKEN[" + tokenPrincipal + "]: jwt=[" + tokenJwt + "], secret=[" + tokenSecret + "]."); + LOGGER.info("GOT TOKEN: principal=[" + tokenPrincipal + "], jwt=[" + tokenJwt + "], secret=[" + tokenSecret + "]."); - // Loop through all authc/authz realms. Confirm authenticatedUser is returned with expected principal and roles. + // Loop through all authc/authz realms. Confirm user is returned with expected principal and roles. User authenticatedUser = null; final List realmAuthenticationResults = new ArrayList<>(); final List realmUsageStats = new ArrayList<>(); final List realmFailureExceptions = new ArrayList<>(jwtRealmsList.size()); try { for (final JwtRealm candidateJwtRealm : jwtRealmsList) { + LOGGER.info("TRY AUTHC: expected=[" + jwtRealm.name() + "], candidate[" + candidateJwtRealm.name() + "]."); final PlainActionFuture> authenticateFuture = PlainActionFuture.newFuture(); try { candidateJwtRealm.authenticate(jwtAuthenticationToken, authenticateFuture); @@ -566,21 +514,21 @@ protected void doMultipleAuthcAuthzAndVerifySuccess( realmFailureExceptions.add(new Exception(realmResult, authenticationResultException)); switch (authenticationResult.getStatus()) { case SUCCESS: - assertThat(candidateJwtRealm.name(), is(equalTo(jwtRealm.name()))); - assertThat(authenticationResult.isAuthenticated(), is(equalTo(true))); - assertThat(authenticationResult.getException(), is(nullValue())); - assertThat(authenticationResult.getMessage(), is(nullValue())); - assertThat(authenticationResult.getMetadata(), is(anEmptyMap())); + assertThat("Unexpected realm SUCCESS status", candidateJwtRealm.name(), is(equalTo(jwtRealm.name()))); + assertThat("Expected realm authc false", authenticationResult.isAuthenticated(), is(equalTo(true))); + assertThat("Expected realm exception thrown", authenticationResult.getException(), is(nullValue())); + assertThat("Expected realm message null", authenticationResult.getMessage(), is(nullValue())); + assertThat("Expected realm metadata empty", authenticationResult.getMetadata(), is(anEmptyMap())); authenticatedUser = authenticationResult.getValue(); - assertThat(authenticatedUser, is(notNullValue())); + assertThat("Expected realm user null", authenticatedUser, is(notNullValue())); break; case CONTINUE: - assertThat(candidateJwtRealm.name(), is(not(equalTo(jwtRealm.name())))); - assertThat(authenticationResult.isAuthenticated(), is(equalTo(false))); + assertThat("Expected realm CONTINUE status", candidateJwtRealm.name(), is(not(equalTo(jwtRealm.name())))); + assertThat("Unexpected realm authc success", authenticationResult.isAuthenticated(), is(equalTo(false))); continue; case TERMINATE: - assertThat(candidateJwtRealm.name(), is(not(equalTo(jwtRealm.name())))); - assertThat(authenticationResult.isAuthenticated(), is(equalTo(false))); + assertThat("Expected realm TERMINATE status", candidateJwtRealm.name(), is(not(equalTo(jwtRealm.name())))); + assertThat("Unexpected realm authc success", authenticationResult.isAuthenticated(), is(equalTo(false))); break; default: fail("Unexpected AuthenticationResult.Status=[" + authenticationResult.getStatus() + "]"); @@ -588,7 +536,8 @@ protected void doMultipleAuthcAuthzAndVerifySuccess( } break; // Only SUCCESS falls through to here, break out of the loop } catch (Exception e) { - realmFailureExceptions.add(new Exception("Caught Exception.", e)); + realmFailureExceptions.add(e); + throw e; } finally { final PlainActionFuture> usageStatsFuture = PlainActionFuture.newFuture(); candidateJwtRealm.usageStats(usageStatsFuture); @@ -605,7 +554,7 @@ protected void doMultipleAuthcAuthzAndVerifySuccess( ); } } - // Loop ended. Confirm authenticatedUser is returned with expected principal and roles. + // Loop ended. Confirm user is returned with expected principal and roles. assertThat("Expected realm " + jwtRealm.name() + " to authenticate.", authenticatedUser, is(notNullValue())); assertThat(user.principal(), equalTo(authenticatedUser.principal())); assertThat(new TreeSet<>(Arrays.asList(user.roles())), equalTo(new TreeSet<>(Arrays.asList(authenticatedUser.roles())))); @@ -617,11 +566,9 @@ protected void doMultipleAuthcAuthzAndVerifySuccess( assertThat(authenticatedUser.metadata(), is(anEmptyMap())); // role mapping with flag false returns empty } } catch (Throwable t) { - final Exception authcFailed = new Exception("Authentication test failed."); - realmFailureExceptions.forEach(authcFailed::addSuppressed); // realm exceptions - authcFailed.addSuppressed(t); // final throwable (ex: assertThat) - LOGGER.error("Unexpected exception.", authcFailed); - throw authcFailed; + realmFailureExceptions.forEach(t::addSuppressed); // all previous realm exceptions + // LOGGER.error("Unexpected exception.", t); + throw t; } finally { LOGGER.info("STATS: expected=[" + jwtRealm.name() + "]\n" + String.join("\n", realmUsageStats)); if (authenticatedUser != null) { @@ -640,11 +587,29 @@ protected User randomUser(final JwtIssuer jwtIssuer) { protected SecureString randomJwt(final JwtIssuerAndRealm jwtIssuerAndRealm, User user) throws Exception { final JwtIssuer.AlgJwkPair algJwkPair = randomFrom(jwtIssuerAndRealm.issuer.algAndJwksAll); - LOGGER.info("JWK=[" + algJwkPair.jwk().getKeyType() + "/" + algJwkPair.jwk().size() + "], alg=[" + algJwkPair.alg() + "]."); + final JWK jwk = algJwkPair.jwk(); + LOGGER.info( + "ALG[" + + algJwkPair.alg() + + "]. JWK: kty=[" + + jwk.getKeyType() + + "], len=[" + + jwk.size() + + "], alg=[" + + jwk.getAlgorithm() + + "], use=[" + + jwk.getKeyUse() + + "], ops=[" + + jwk.getKeyOperations() + + "], kid=[" + + jwk.getKeyID() + + "]." + ); final Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt( - randomBoolean() ? null : JOSEObjectType.JWT.toString(), + randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty + randomBoolean() ? null : jwk.getKeyID(), // kid algJwkPair.alg(), // alg randomAlphaOfLengthBetween(10, 20), // jwtID jwtIssuerAndRealm.realm.allowedIssuer, // iss @@ -654,16 +619,89 @@ protected SecureString randomJwt(final JwtIssuerAndRealm jwtIssuerAndRealm, User user.principal(), // principal claim value jwtIssuerAndRealm.realm.claimParserGroups.getClaimName(), // group claim name List.of(user.roles()), // group claim value - Date.from(now.minusSeconds(randomLongBetween(10, 20))), // auth_time - Date.from(now), // iat - Date.from(now.minusSeconds(randomLongBetween(5, 10))), // nbf - Date.from(now.plusSeconds(randomLongBetween(3600, 7200))), // exp + Date.from(now.minusSeconds(60 * randomLongBetween(10, 20))), // auth_time + Date.from(now.minusSeconds(randomBoolean() ? 0 : 60 * randomLongBetween(5, 10))), // iat + Date.from(now), // nbf + Date.from(now.plusSeconds(60 * randomLongBetween(3600, 7200))), // exp randomBoolean() ? null : new Nonce(32).toString(), randomBoolean() ? null : Map.of("other1", randomAlphaOfLength(10), "other2", randomAlphaOfLength(10)) ); - final SecureString signedJWT = JwtValidateUtil.signJwt(algJwkPair.jwk(), unsignedJwt); - assertThat(JwtValidateUtil.verifyJwt(algJwkPair.jwk(), SignedJWT.parse(signedJWT.toString())), is(equalTo(true))); + final SecureString signedJWT = JwtValidateUtil.signJwt(jwk, unsignedJwt); + assertThat(JwtValidateUtil.verifyJwt(jwk, SignedJWT.parse(signedJWT.toString())), is(equalTo(true))); return signedJWT; } + protected void printJwtRealmAndIssuer(JwtIssuerAndRealm jwtIssuerAndRealm) throws ParseException { + this.printJwtIssuer(jwtIssuerAndRealm.issuer()); + this.printJwtRealm(jwtIssuerAndRealm.realm()); + } + + protected void printJwtRealm(final JwtRealm jwtRealm) { + LOGGER.info( + "REALM[" + + jwtRealm.name() + + "," + + jwtRealm.order() + + "/" + + this.jwtIssuerAndRealms.size() + + "]: clientType=[" + + jwtRealm.clientAuthenticationType + + "], clientSecret=[" + + jwtRealm.clientAuthenticationSharedSecret + + "], iss=[" + + jwtRealm.allowedIssuer + + "], aud=" + + jwtRealm.allowedAudiences + + ", algsHmac=" + + jwtRealm.allowedJwksAlgsHmac + + ", filteredHmac=" + + jwtRealm.contentAndJwksAlgsHmac.jwksAlgs().algs() + + ", algsPkc=" + + jwtRealm.allowedJwksAlgsPkc + + ", filteredPkc=" + + jwtRealm.contentAndJwksAlgsPkc.jwksAlgs().algs() + + ", claimPrincipal=[" + + jwtRealm.claimParserPrincipal.getClaimName() + + "], claimGroups=[" + + jwtRealm.claimParserGroups.getClaimName() + + "], authz=[" + + jwtRealm.delegatedAuthorizationSupport.hasDelegation() + + "], meta=[" + + jwtRealm.populateUserMetadata + + "], jwkSetPath=[" + + jwtRealm.jwkSetPath + + "]." + ); + for (final JWK jwk : jwtRealm.contentAndJwksAlgsHmac.jwksAlgs().jwks()) { + LOGGER.info("REALM HMAC: jwk=[{}]", jwk); + } + for (final JWK jwk : jwtRealm.contentAndJwksAlgsPkc.jwksAlgs().jwks()) { + LOGGER.info("REALM PKC: jwk=[{}]", jwk); + } + } + + protected void printJwtIssuer(final JwtIssuer jwtIssuer) { + LOGGER.info( + "ISSUER: iss=[" + + jwtIssuer.issuerClaimValue + + "], aud=[" + + String.join(",", jwtIssuer.audiencesClaimValue) + + "], principal=[" + + jwtIssuer.principalClaimName + + "], algorithms=[" + + String.join(",", jwtIssuer.algorithmsAll) + + "], httpServer=[" + + (jwtIssuer.httpsServer != null) + + "]." + ); + if (jwtIssuer.algAndJwkHmacOidc != null) { + LOGGER.info("ISSUER HMAC OIDC: alg=[{}] jwk=[{}]", jwtIssuer.algAndJwkHmacOidc.alg(), jwtIssuer.encodedKeyHmacOidc); + } + for (final JwtIssuer.AlgJwkPair pair : jwtIssuer.algAndJwksHmac) { + LOGGER.info("ISSUER HMAC: alg=[{}] jwk=[{}]", pair.alg(), pair.jwk()); + } + for (final JwtIssuer.AlgJwkPair pair : jwtIssuer.algAndJwksPkc) { + LOGGER.info("ISSUER PKC: alg=[{}] jwk=[{}]", pair.alg(), pair.jwk()); + } + } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTestCase.java index d08672456b9fa..b9821606de6eb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTestCase.java @@ -49,7 +49,6 @@ import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings; import org.junit.Before; import org.mockito.Mockito; -import org.mockito.stubbing.Answer; import java.io.IOException; import java.net.URL; @@ -59,17 +58,14 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; import java.util.stream.IntStream; import static org.elasticsearch.test.ActionListenerUtils.anyActionListener; @@ -241,16 +237,6 @@ protected RealmConfig buildRealmConfig( return new RealmConfig(realmIdentifier, settings, this.env, this.threadContext); } - protected Answer> getAnswer(AtomicReference userData) { - return invocation -> { - userData.set((UserRoleMapper.UserData) invocation.getArguments()[0]); - @SuppressWarnings("unchecked") - ActionListener> listener = (ActionListener>) invocation.getArguments()[1]; - listener.onResponse(new HashSet<>(Arrays.asList("kibana_user", "role1"))); - return null; - }; - } - protected UserRoleMapper buildRoleMapper(final Map registeredUsers) { final UserRoleMapper roleMapper = mock(UserRoleMapper.class); Mockito.doAnswer(invocation -> { @@ -268,18 +254,19 @@ protected UserRoleMapper buildRoleMapper(final Map registeredUsers return roleMapper; } - public static List randomJwks(final List signatureAlgorithms) throws JOSEException { + public static List randomJwks(final List signatureAlgorithms, final boolean requireOidcSafe) + throws JOSEException { final List algAndJwks = new ArrayList<>(); for (final String signatureAlgorithm : signatureAlgorithms) { - algAndJwks.add(new JwtIssuer.AlgJwkPair(signatureAlgorithm, JwtTestCase.randomJwk(signatureAlgorithm))); + algAndJwks.add(new JwtIssuer.AlgJwkPair(signatureAlgorithm, JwtTestCase.randomJwk(signatureAlgorithm, requireOidcSafe))); } return algAndJwks; } - public static JWK randomJwk(final String signatureAlgorithm) throws JOSEException { + public static JWK randomJwk(final String signatureAlgorithm, final boolean requireOidcSafe) throws JOSEException { final JWSAlgorithm jwsAlgorithm = JWSAlgorithm.parse(signatureAlgorithm); if (JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC.contains(signatureAlgorithm)) { - return JwtTestCase.randomJwkHmac(jwsAlgorithm); + return JwtTestCase.randomJwkHmac(jwsAlgorithm, requireOidcSafe); } else if (JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_RSA.contains(signatureAlgorithm)) { return JwtTestCase.randomJwkRsa(jwsAlgorithm); } else if (JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_EC.contains(signatureAlgorithm)) { @@ -294,14 +281,31 @@ public static JWK randomJwk(final String signatureAlgorithm) throws JOSEExceptio ); } - // Generate using random bytes - // - random byte => 2^8 => search space 8-bit per byte - public static OctetSequenceKey randomJwkHmac(final JWSAlgorithm jwsAlgorithm) throws JOSEException { + public static OctetSequenceKey randomJwkHmac(final JWSAlgorithm jwsAlgorithm, final boolean requireOidcSafe) throws JOSEException { final int minHmacLengthBytes = MACSigner.getMinRequiredSecretLength(jwsAlgorithm) / 8; - final int hmacLengthBits = scaledRandomIntBetween(minHmacLengthBytes, minHmacLengthBytes * 2) * 8; // Double it: Nice to have - final OctetSequenceKeyGenerator jwkGenerator = new OctetSequenceKeyGenerator(hmacLengthBits); - JwtTestCase.randomSettingsForJwkGenerator(jwkGenerator, jwsAlgorithm); // options: kid, alg, use, ops - return jwkGenerator.generate(); + final int hmacLengthBytes = scaledRandomIntBetween(minHmacLengthBytes, minHmacLengthBytes * 2); // Double it: Nice to have + if (requireOidcSafe == false && randomBoolean()) { + // random byte => 2^8 search space per 1 byte => 8 bits per byte + final OctetSequenceKeyGenerator jwkGenerator = new OctetSequenceKeyGenerator(hmacLengthBytes * 8); + return JwtTestCase.randomSettingsForJwkGenerator(jwkGenerator, jwsAlgorithm).generate().toOctetSequenceKey(); // kid,alg,use,ops + } + final String passwordKey; + if (randomBoolean()) { + // Base 64 byte => 2^6 search space per 1 byte => 6 bits per byte + passwordKey = Base64URL.encode(randomByteArrayOfLength(hmacLengthBytes)).toString(); + } else { + // UTF8 1 byte => 2^7 search space per 1 byte => 7 bits per byte + // UTF8 2 byte => 2^11 search space per 2 byte => 5.5 bits per byte + // UTF8 3 byte => 2^16 search space per 3 byte => 5.333 bits per byte + // UTF8 4 byte => 2^21 search space per 4 byte => 5.25 bits per byte (theoretical, UNICODE currently only allocates 1.1M of 2M) + passwordKey = randomAlphaOfLength(hmacLengthBytes); + } + final OctetSequenceKey.Builder hmacKeyBuilder = new OctetSequenceKey.Builder(passwordKey.getBytes(StandardCharsets.UTF_8)); + return JwtTestCase.randomSettingsForHmacJwkBuilder(hmacKeyBuilder, jwsAlgorithm).build(); // kid,alg,use,ops + } + + public static OctetSequenceKey randomJwkHmacOidcSafe(final JWSAlgorithm jwsAlgorithm) throws JOSEException { + return JwtTestCase.randomJwkHmac(jwsAlgorithm, true); } public static RSAKey randomJwkRsa(final JWSAlgorithm jwsAlgorithm) throws JOSEException { @@ -318,46 +322,6 @@ public static ECKey randomJwkEc(final JWSAlgorithm jwsAlgorithm) throws JOSEExce return jwkGenerator.generate(); } - public static OctetSequenceKey randomJwkHmacOidc(final JWSAlgorithm jwsAlgorithm) throws JOSEException { - return JwtTestCase.conditionJwkHmacForOidc(JwtTestCase.randomJwkHmac(jwsAlgorithm)); - } - - /** - * Input HMAC key is assumed random bytes. Generating random bytes is useful to guarantee min search space (aka strength, entropy). - * - * OIDC HMAC key must be UTF8 bytes (aka password). Encoding random bytes as UTF8 doesn't work, and UTF8 search space is smaller. - * - * To satisfy min search space and OIDC UTF8 encoding, Base64(randomBytes) is used as the bytes of a new HMAC OIDC key. - * - * Search space comparisons of random bytes, base 64, and UTF-8. - * - random byte => 2^8 search space per 1 byte => 8 bits per byte - * - Base 64 byte => 2^6 search space per 1 byte => 6 bits per byte - * - UTF8 1 byte => 2^7 search space per 1 byte => 7 bits per byte - * - UTF8 2 byte => 2^11 search space per 2 byte => 5.5 bits per byte - * - UTF8 3 byte => 2^16 search space per 3 byte => 5.333 bits per byte - * - UTF8 4 byte => 2^21 search space per 4 byte => 5.25 bits per byte (theoretical, UNICODE currently only allocates 1.1M of 2M) - * - * @param hmacKey HMAC key with random bytes. - * @return HMAC key with UTF-8 bytes, making the key bytes compatible with OIDC UTF-8 string encoding. - */ - public static OctetSequenceKey conditionJwkHmacForOidc(final OctetSequenceKey hmacKey) { - final String passwordKey; - if (randomBoolean()) { - final Base64URL hmacKeyBytesBase64 = hmacKey.getKeyValue(); // Random bytes => 8 bits/byte search space - passwordKey = hmacKeyBytesBase64.toString(); // Use Base64(randomBytes) as UTF8 bytes for a new password with same search space - } else { - final int numLetters = hmacKey.toByteArray().length * 8; // Random [A-Za-z] => 5.7 bits/byte - passwordKey = randomAlphaOfLength(numLetters); // Use length * ceil(2.3) to avoid reducing search space below 8 bits/byte - } - final OctetSequenceKey.Builder hmacKeyBuilder = new OctetSequenceKey.Builder(passwordKey.getBytes(StandardCharsets.UTF_8)); - hmacKeyBuilder.keyID(hmacKey.getKeyID()); // Copy null attribute is OK (no-op) - hmacKeyBuilder.algorithm(hmacKey.getAlgorithm()); - hmacKeyBuilder.keyUse(hmacKey.getKeyUse()); - hmacKeyBuilder.keyOperations(hmacKey.getKeyOperations()); - hmacKeyBuilder.keyStore(hmacKey.getKeyStore()); - return hmacKeyBuilder.build(); - } - public static OctetSequenceKey jwkHmacRemoveAttributes(final OctetSequenceKey hmacKey) { final String keyBytesAsUtf8 = hmacKey.getKeyValue().decodeToString(); return new OctetSequenceKey.Builder(keyBytesAsUtf8.getBytes(StandardCharsets.UTF_8)).build(); @@ -382,9 +346,29 @@ public static JWKGenerator randomSettingsForJwkGenerator( return jwkGenerator; } + public static OctetSequenceKey.Builder randomSettingsForHmacJwkBuilder( + final OctetSequenceKey.Builder jwkGenerator, + final JWSAlgorithm jwsAlgorithm + ) { + if (randomBoolean()) { + jwkGenerator.keyID(UUID.randomUUID().toString()); + } + if (randomBoolean()) { + jwkGenerator.algorithm(jwsAlgorithm); + } + if (randomBoolean()) { + jwkGenerator.keyUse(KeyUse.SIGNATURE); + } + if (randomBoolean()) { + jwkGenerator.keyOperations(Set.of(KeyOperation.SIGN, KeyOperation.VERIFY)); + } + return jwkGenerator; + } + public static SignedJWT buildUnsignedJwt( final String type, - final String signatureAlgorithm, + final String kid, + final String alg, final String jwtId, final String issuer, final List audiences, @@ -400,7 +384,10 @@ public static SignedJWT buildUnsignedJwt( final String nonce, final Map otherClaims ) { - final JWSHeader.Builder jwsHeaderBuilder = new JWSHeader.Builder(JWSAlgorithm.parse(signatureAlgorithm)); + final JWSHeader.Builder jwsHeaderBuilder = new JWSHeader.Builder(JWSAlgorithm.parse(alg)); + if (kid != null) { + jwsHeaderBuilder.keyID(kid); + } if (type != null) { jwsHeaderBuilder.type(new JOSEObjectType(type)); } @@ -447,20 +434,22 @@ public static SignedJWT buildUnsignedJwt( if (otherClaims != null) { for (final Map.Entry entry : otherClaims.entrySet()) { if (Strings.hasText(entry.getKey()) == false) { - throw new IllegalArgumentException("Null or blank other claim key allowed."); + throw new IllegalArgumentException("Null or blank other claim key not allowed."); } else if (entry.getValue() == null) { - throw new IllegalArgumentException("Null other claim value allowed."); + throw new IllegalArgumentException("Null other claim value not allowed."); } jwtClaimsSetBuilder.claim(entry.getKey(), entry.getValue()); } } final JWTClaimsSet jwtClaimsSet = jwtClaimsSetBuilder.build(); LOGGER.info( - "CLAIMS: alg=[" - + jwtHeader.getAlgorithm().getName() - + "], jwtId=[" - + jwtClaimsSet.getJWTID() - + "], iss=[" + "JWT: HEADER{alg=[" + + jwtHeader.getAlgorithm() + + "], kid=[" + + jwtHeader.getKeyID() + + "], kty=[" + + jwtHeader.getType() + + "]}. CLAIMS: {iss=[" + jwtClaimsSet.getIssuer() + "], aud=" + jwtClaimsSet.getAudience() @@ -474,19 +463,21 @@ public static SignedJWT buildUnsignedJwt( + groupsClaimName + "=" + jwtClaimsSet.getClaim(groupsClaimName) - + "], nbf=[" - + jwtClaimsSet.getNotBeforeTime() + "], auth_time=[" + jwtClaimsSet.getClaim("auth_time") + "], iat=[" + jwtClaimsSet.getIssueTime() + + "], nbf=[" + + jwtClaimsSet.getNotBeforeTime() + "], exp=[" + jwtClaimsSet.getExpirationTime() + "], nonce=[" + jwtClaimsSet.getClaim("nonce") + + "], jid=[" + + jwtClaimsSet.getJWTID() + "], other=[" + otherClaims - + "]" + + "]}." ); return JwtValidateUtil.buildUnsignedJwt(jwtHeader, jwtClaimsSet); } @@ -494,7 +485,8 @@ public static SignedJWT buildUnsignedJwt( public static SecureString randomBespokeJwt(final JWK jwk, final String signatureAlgorithm) throws Exception { final Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt( - randomBoolean() ? null : JOSEObjectType.JWT.toString(), + randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty + randomBoolean() ? null : jwk.getKeyID(), // kid signatureAlgorithm, // alg randomAlphaOfLengthBetween(10, 20), // jwtID randomFrom("https://www.example.com/", "") + "iss1" + randomIntBetween(0, 99), @@ -504,10 +496,10 @@ public static SecureString randomBespokeJwt(final JWK jwk, final String signatur "principal1", // principal claim value randomBoolean() ? null : randomFrom("groups", "roles", "other"), randomFrom(List.of(""), List.of("grp1"), List.of("rol1", "rol2", "rol3"), List.of("per1")), - Date.from(now.minusSeconds(randomLongBetween(10, 20))), // auth_time - Date.from(now), // iat - Date.from(now.minusSeconds(randomLongBetween(5, 10))), // nbf - Date.from(now.plusSeconds(randomLongBetween(3600, 7200))), // exp + Date.from(now.minusSeconds(60 * randomLongBetween(10, 20))), // auth_time + Date.from(now.minusSeconds(randomBoolean() ? 0 : 60 * randomLongBetween(5, 10))), // iat + Date.from(now), // nbf + Date.from(now.plusSeconds(60 * randomLongBetween(3600, 7200))), // exp randomBoolean() ? null : new Nonce(32).toString(), randomBoolean() ? null : Map.of("other1", randomAlphaOfLength(10), "other2", randomAlphaOfLength(10)) ); @@ -573,9 +565,9 @@ public static List randomOfMinMaxNonUnique(final int min, final int max, return IntStream.rangeClosed(1, minToMaxInclusive).mapToObj(i -> randomFrom(collection)).toList(); // 1..N inclusive } - public String saveToTempFile(final String prefix, final String suffix, final byte[] content) throws IOException { + public String saveToTempFile(final String prefix, final String suffix, final String content) throws IOException { final Path path = Files.createTempFile(PathUtils.get(this.pathHome), prefix, suffix); - Files.write(path, content); + Files.writeString(path, content); return path.toString(); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtilTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtilTests.java index bfdaca7c65c9b..4b4af611b146d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtilTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtilTests.java @@ -26,10 +26,9 @@ public class JwtValidateUtilTests extends JwtTestCase { private static final Logger LOGGER = LogManager.getLogger(JwtValidateUtilTests.class); - private boolean helpTestSignatureAlgorithm(final String signatureAlgorithm) throws Exception { - LOGGER.info("Testing signature algorithm " + signatureAlgorithm); - // randomSecretOrSecretKeyOrKeyPair() randomizes which JwtUtil methods to call, so it indirectly covers most JwtUtil code - final JWK jwk = JwtTestCase.randomJwk(signatureAlgorithm); + private boolean helpTestSignatureAlgorithm(final String signatureAlgorithm, final boolean requireOidcSafe) throws Exception { + LOGGER.trace("Testing signature algorithm " + signatureAlgorithm); + final JWK jwk = JwtTestCase.randomJwk(signatureAlgorithm, requireOidcSafe); final SecureString serializedJWTOriginal = JwtTestCase.randomBespokeJwt(jwk, signatureAlgorithm); final SignedJWT parsedSignedJWT = SignedJWT.parse(serializedJWTOriginal.toString()); return JwtValidateUtil.verifyJwt(jwk, parsedSignedJWT); @@ -38,10 +37,13 @@ private boolean helpTestSignatureAlgorithm(final String signatureAlgorithm) thro public void testJwtSignVerifyPassedForAllSupportedAlgorithms() throws Exception { // Pass: "ES256", "ES384", "ES512", RS256", "RS384", "RS512", "PS256", "PS384", "PS512, "HS256", "HS384", "HS512" for (final String signatureAlgorithm : JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS) { - assertThat(this.helpTestSignatureAlgorithm(signatureAlgorithm), is(true)); + assertThat(this.helpTestSignatureAlgorithm(signatureAlgorithm, false), is(true)); } // Fail: "ES256K" - final Exception exp1 = expectThrows(JOSEException.class, () -> this.helpTestSignatureAlgorithm(JWSAlgorithm.ES256K.getName())); + final Exception exp1 = expectThrows( + JOSEException.class, + () -> this.helpTestSignatureAlgorithm(JWSAlgorithm.ES256K.getName(), false) + ); final String msg1 = "Unsupported signature algorithm [" + JWSAlgorithm.ES256K + "]. Supported signature algorithms are " diff --git a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownDelayedAllocationIT.java b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownDelayedAllocationIT.java index a85c8f7340185..4231d4094f193 100644 --- a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownDelayedAllocationIT.java +++ b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownDelayedAllocationIT.java @@ -102,7 +102,7 @@ public void testShardAllocationWillProceedAfterTimeout() throws Exception { assertTrue(putShutdownResponse.isAcknowledged()); // Actually stop the node - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeToRestartName)); + internalCluster().stopNode(nodeToRestartName); // And the index should turn green again well within the 30-second timeout ensureGreen("test"); @@ -210,7 +210,7 @@ private String setupLongTimeoutTestCase() throws Exception { } // Actually stop the node - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeToRestartName)); + internalCluster().stopNode(nodeToRestartName); // Verify that the shard's allocation is delayed assertBusy(() -> { assertThat(client().admin().cluster().prepareHealth().get().getDelayedUnassignedShards(), equalTo(1)); }); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoBoundingBoxQueryBuilderGeoShapeTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoBoundingBoxQueryBuilderGeoShapeTests.java new file mode 100644 index 0000000000000..9f84b954f20ac --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoBoundingBoxQueryBuilderGeoShapeTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.index.query; + +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.geo.GeoBoundingBoxQueryBuilderTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; +import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +@SuppressWarnings("checkstyle:MissingJavadocMethod") +public class GeoBoundingBoxQueryBuilderGeoShapeTests extends GeoBoundingBoxQueryBuilderTestCase { + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + final XContentBuilder builder = PutMappingRequest.simpleMapping( + GEO_SHAPE_FIELD_NAME, + "type=geo_shape", + GEO_SHAPE_ALIAS_FIELD_NAME, + "type=alias,path=" + GEO_SHAPE_FIELD_NAME + ); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + + @Override + protected Collection> getPlugins() { + return Arrays.asList(LocalStateSpatialPlugin.class); + } + + @Override + protected String getFieldName() { + return randomFrom(GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); + } + + @Override + protected void doAssertLuceneQuery(GeoBoundingBoxQueryBuilder queryBuilder, Query query, SearchExecutionContext context) { + final MappedFieldType fieldType = context.getFieldType(queryBuilder.fieldName()); + if (fieldType == null) { + assertTrue("Found no indexed geo query.", query instanceof MatchNoDocsQuery); + } + assertEquals(GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType.class, fieldType.getClass()); + assertEquals(IndexOrDocValuesQuery.class, query.getClass()); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/88711") + public void testToQuery() throws IOException { + super.testToQuery(); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoDistanceQueryBuilderGeoShapeTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoDistanceQueryBuilderGeoShapeTests.java new file mode 100644 index 0000000000000..e04b26dc68a16 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoDistanceQueryBuilderGeoShapeTests.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.index.query; + +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.GeoDistanceQueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.geo.GeoDistanceQueryBuilderTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; +import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +@SuppressWarnings("checkstyle:MissingJavadocMethod") +public class GeoDistanceQueryBuilderGeoShapeTests extends GeoDistanceQueryBuilderTestCase { + + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + final XContentBuilder builder = PutMappingRequest.simpleMapping( + GEO_SHAPE_FIELD_NAME, + "type=geo_shape", + GEO_SHAPE_ALIAS_FIELD_NAME, + "type=alias,path=" + GEO_SHAPE_FIELD_NAME + ); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + + @Override + protected Collection> getPlugins() { + return Arrays.asList(LocalStateSpatialPlugin.class); + } + + @Override + protected String getFieldName() { + return randomFrom(GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); + } + + @Override + protected void doAssertLuceneQuery(GeoDistanceQueryBuilder queryBuilder, Query query, SearchExecutionContext context) { + final MappedFieldType fieldType = context.getFieldType(queryBuilder.fieldName()); + if (fieldType == null) { + assertTrue("Found no indexed geo query.", query instanceof MatchNoDocsQuery); + } + assertEquals(GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType.class, fieldType.getClass()); + assertEquals(IndexOrDocValuesQuery.class, query.getClass()); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/88712") + public void testToQuery() throws IOException { + super.testToQuery(); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoShapeQueryBuilderGeoShapeTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoShapeQueryBuilderGeoShapeTests.java new file mode 100644 index 0000000000000..596e0d9c730c8 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoShapeQueryBuilderGeoShapeTests.java @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.index.query; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.GeoShapeQueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.geo.GeoShapeQueryBuilderTestCase; +import org.elasticsearch.test.AbstractBuilderTestCase; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +public class GeoShapeQueryBuilderGeoShapeTests extends GeoShapeQueryBuilderTestCase { + + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + + protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + final XContentBuilder builder = PutMappingRequest.simpleMapping( + GEO_SHAPE_FIELD_NAME, + "type=geo_shape", + GEO_SHAPE_ALIAS_FIELD_NAME, + "type=alias,path=" + GEO_SHAPE_FIELD_NAME + ); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + + @Override + protected Collection> getPlugins() { + return Collections.singletonList(LocalStateSpatialPlugin.class); + } + + @Override + protected String getFieldName() { + return randomFrom(GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); + } + + @Override + protected GeoShapeQueryBuilder doCreateTestQueryBuilder(boolean indexedShape) { + ShapeType shapeType = ESTestCase.randomFrom( + ShapeType.POINT, + ShapeType.MULTIPOINT, + ShapeType.LINESTRING, + ShapeType.MULTILINESTRING, + ShapeType.POLYGON + ); + Geometry geometry = GeometryTestUtils.randomGeometry(shapeType, false); + GeoShapeQueryBuilder builder; + clearShapeFields(); + if (indexedShape == false) { + builder = new GeoShapeQueryBuilder(getFieldName(), geometry); + } else { + indexedShapeToReturn = geometry; + indexedShapeId = ESTestCase.randomAlphaOfLengthBetween(3, 20); + builder = new GeoShapeQueryBuilder(getFieldName(), indexedShapeId); + if (ESTestCase.randomBoolean()) { + indexedShapeIndex = ESTestCase.randomAlphaOfLengthBetween(3, 20); + builder.indexedShapeIndex(indexedShapeIndex); + } + if (ESTestCase.randomBoolean()) { + indexedShapePath = ESTestCase.randomAlphaOfLengthBetween(3, 20); + builder.indexedShapePath(indexedShapePath); + } + if (ESTestCase.randomBoolean()) { + indexedShapeRouting = ESTestCase.randomAlphaOfLengthBetween(3, 20); + builder.indexedShapeRouting(indexedShapeRouting); + } + } + if (ESTestCase.randomBoolean()) { + SearchExecutionContext context = AbstractBuilderTestCase.createSearchExecutionContext(); + if (context.indexVersionCreated().onOrAfter(Version.V_7_5_0)) { // CONTAINS is only supported from version 7.5 + if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) { + builder.relation(ESTestCase.randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.CONTAINS)); + } else { + builder.relation( + ESTestCase.randomFrom( + ShapeRelation.DISJOINT, + ShapeRelation.INTERSECTS, + ShapeRelation.WITHIN, + ShapeRelation.CONTAINS + ) + ); + } + } else { + if (shapeType == ShapeType.LINESTRING || shapeType == ShapeType.MULTILINESTRING) { + builder.relation(ESTestCase.randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS)); + } else { + builder.relation(ESTestCase.randomFrom(ShapeRelation.DISJOINT, ShapeRelation.INTERSECTS, ShapeRelation.WITHIN)); + } + } + } + + if (ESTestCase.randomBoolean()) { + builder.ignoreUnmapped(ESTestCase.randomBoolean()); + } + return builder; + } +} diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java index b3c128ef9efa9..94eefd4fb5e78 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java @@ -322,6 +322,12 @@ public void testStopWaitForCheckpoint() throws Exception { startTransform(config.getId(), RequestOptions.DEFAULT); + // wait until transform has been triggered and indexed at least 1 document + assertBusy(() -> { + var stateAndStats = getTransformStats(config.getId()); + assertThat((Integer) XContentMapValues.extractValue("stats.documents_indexed", stateAndStats), greaterThan(1)); + }); + // waitForCheckpoint: true should make the transform continue until we hit the first checkpoint, then it will stop stopTransform(transformId, false, null, true);