From 621984d82d1b4d6406fd458dd6edaa6c43653613 Mon Sep 17 00:00:00 2001 From: SolrBot Date: Thu, 23 Oct 2025 01:45:32 +0000 Subject: [PATCH 01/83] Update software.amazon.awssdk:* to v2.35.10 (branch_9x) --- solr/licenses/annotations-2.31.77.jar.sha1 | 1 - solr/licenses/annotations-2.35.10.jar.sha1 | 1 + solr/licenses/apache-client-2.31.77.jar.sha1 | 1 - solr/licenses/apache-client-2.35.10.jar.sha1 | 1 + solr/licenses/arns-2.31.77.jar.sha1 | 1 - solr/licenses/arns-2.35.10.jar.sha1 | 1 + solr/licenses/auth-2.31.77.jar.sha1 | 1 - solr/licenses/auth-2.35.10.jar.sha1 | 1 + solr/licenses/aws-core-2.31.77.jar.sha1 | 1 - solr/licenses/aws-core-2.35.10.jar.sha1 | 1 + .../aws-query-protocol-2.31.77.jar.sha1 | 1 - .../aws-query-protocol-2.35.10.jar.sha1 | 1 + .../aws-xml-protocol-2.31.77.jar.sha1 | 1 - .../aws-xml-protocol-2.35.10.jar.sha1 | 1 + solr/licenses/checksums-2.31.77.jar.sha1 | 1 - solr/licenses/checksums-2.35.10.jar.sha1 | 1 + solr/licenses/checksums-spi-2.31.77.jar.sha1 | 1 - solr/licenses/checksums-spi-2.35.10.jar.sha1 | 1 + solr/licenses/crt-core-2.31.77.jar.sha1 | 1 - solr/licenses/crt-core-2.35.10.jar.sha1 | 1 + solr/licenses/endpoints-spi-2.31.77.jar.sha1 | 1 - solr/licenses/endpoints-spi-2.35.10.jar.sha1 | 1 + solr/licenses/http-auth-2.31.77.jar.sha1 | 1 - solr/licenses/http-auth-2.35.10.jar.sha1 | 1 + solr/licenses/http-auth-aws-2.31.77.jar.sha1 | 1 - solr/licenses/http-auth-aws-2.35.10.jar.sha1 | 1 + ...http-auth-aws-eventstream-2.31.77.jar.sha1 | 1 - ...http-auth-aws-eventstream-2.35.10.jar.sha1 | 1 + solr/licenses/http-auth-spi-2.31.77.jar.sha1 | 1 - solr/licenses/http-auth-spi-2.35.10.jar.sha1 | 1 + .../licenses/http-client-spi-2.31.77.jar.sha1 | 1 - .../licenses/http-client-spi-2.35.10.jar.sha1 | 1 + solr/licenses/identity-spi-2.31.77.jar.sha1 | 1 - solr/licenses/identity-spi-2.35.10.jar.sha1 | 1 + solr/licenses/json-utils-2.31.77.jar.sha1 | 1 - solr/licenses/json-utils-2.35.10.jar.sha1 | 1 + solr/licenses/metrics-spi-2.31.77.jar.sha1 | 1 - solr/licenses/metrics-spi-2.35.10.jar.sha1 | 1 + solr/licenses/profiles-2.31.77.jar.sha1 | 1 - solr/licenses/profiles-2.35.10.jar.sha1 | 1 + solr/licenses/protocol-core-2.31.77.jar.sha1 | 1 - solr/licenses/protocol-core-2.35.10.jar.sha1 | 1 + solr/licenses/regions-2.31.77.jar.sha1 | 1 - solr/licenses/regions-2.35.10.jar.sha1 | 1 + solr/licenses/retries-2.31.77.jar.sha1 | 1 - solr/licenses/retries-2.35.10.jar.sha1 | 1 + solr/licenses/retries-spi-2.31.77.jar.sha1 | 1 - solr/licenses/retries-spi-2.35.10.jar.sha1 | 1 + solr/licenses/s3-2.31.77.jar.sha1 | 1 - solr/licenses/s3-2.35.10.jar.sha1 | 1 + solr/licenses/sdk-core-2.31.77.jar.sha1 | 1 - solr/licenses/sdk-core-2.35.10.jar.sha1 | 1 + solr/licenses/sts-2.31.77.jar.sha1 | 1 - solr/licenses/sts-2.35.10.jar.sha1 | 1 + .../third-party-jackson-core-2.31.77.jar.sha1 | 1 - .../third-party-jackson-core-2.35.10.jar.sha1 | 1 + .../url-connection-client-2.31.77.jar.sha1 | 1 - .../url-connection-client-2.35.10.jar.sha1 | 1 + solr/licenses/utils-2.31.77.jar.sha1 | 1 - solr/licenses/utils-2.35.10.jar.sha1 | 1 + solr/licenses/utils-lite-2.35.10.jar.sha1 | 1 + versions.lock | 63 ++++++++++--------- versions.props | 2 +- 63 files changed, 64 insertions(+), 62 deletions(-) delete mode 100644 solr/licenses/annotations-2.31.77.jar.sha1 create mode 100644 solr/licenses/annotations-2.35.10.jar.sha1 delete mode 100644 solr/licenses/apache-client-2.31.77.jar.sha1 create mode 100644 solr/licenses/apache-client-2.35.10.jar.sha1 delete mode 100644 solr/licenses/arns-2.31.77.jar.sha1 create mode 100644 solr/licenses/arns-2.35.10.jar.sha1 delete mode 100644 solr/licenses/auth-2.31.77.jar.sha1 create mode 100644 solr/licenses/auth-2.35.10.jar.sha1 delete mode 100644 solr/licenses/aws-core-2.31.77.jar.sha1 create mode 100644 solr/licenses/aws-core-2.35.10.jar.sha1 delete mode 100644 solr/licenses/aws-query-protocol-2.31.77.jar.sha1 create mode 100644 solr/licenses/aws-query-protocol-2.35.10.jar.sha1 delete mode 100644 solr/licenses/aws-xml-protocol-2.31.77.jar.sha1 create mode 100644 solr/licenses/aws-xml-protocol-2.35.10.jar.sha1 delete mode 100644 solr/licenses/checksums-2.31.77.jar.sha1 create mode 100644 solr/licenses/checksums-2.35.10.jar.sha1 delete mode 100644 solr/licenses/checksums-spi-2.31.77.jar.sha1 create mode 100644 solr/licenses/checksums-spi-2.35.10.jar.sha1 delete mode 100644 solr/licenses/crt-core-2.31.77.jar.sha1 create mode 100644 solr/licenses/crt-core-2.35.10.jar.sha1 delete mode 100644 solr/licenses/endpoints-spi-2.31.77.jar.sha1 create mode 100644 solr/licenses/endpoints-spi-2.35.10.jar.sha1 delete mode 100644 solr/licenses/http-auth-2.31.77.jar.sha1 create mode 100644 solr/licenses/http-auth-2.35.10.jar.sha1 delete mode 100644 solr/licenses/http-auth-aws-2.31.77.jar.sha1 create mode 100644 solr/licenses/http-auth-aws-2.35.10.jar.sha1 delete mode 100644 solr/licenses/http-auth-aws-eventstream-2.31.77.jar.sha1 create mode 100644 solr/licenses/http-auth-aws-eventstream-2.35.10.jar.sha1 delete mode 100644 solr/licenses/http-auth-spi-2.31.77.jar.sha1 create mode 100644 solr/licenses/http-auth-spi-2.35.10.jar.sha1 delete mode 100644 solr/licenses/http-client-spi-2.31.77.jar.sha1 create mode 100644 solr/licenses/http-client-spi-2.35.10.jar.sha1 delete mode 100644 solr/licenses/identity-spi-2.31.77.jar.sha1 create mode 100644 solr/licenses/identity-spi-2.35.10.jar.sha1 delete mode 100644 solr/licenses/json-utils-2.31.77.jar.sha1 create mode 100644 solr/licenses/json-utils-2.35.10.jar.sha1 delete mode 100644 solr/licenses/metrics-spi-2.31.77.jar.sha1 create mode 100644 solr/licenses/metrics-spi-2.35.10.jar.sha1 delete mode 100644 solr/licenses/profiles-2.31.77.jar.sha1 create mode 100644 solr/licenses/profiles-2.35.10.jar.sha1 delete mode 100644 solr/licenses/protocol-core-2.31.77.jar.sha1 create mode 100644 solr/licenses/protocol-core-2.35.10.jar.sha1 delete mode 100644 solr/licenses/regions-2.31.77.jar.sha1 create mode 100644 solr/licenses/regions-2.35.10.jar.sha1 delete mode 100644 solr/licenses/retries-2.31.77.jar.sha1 create mode 100644 solr/licenses/retries-2.35.10.jar.sha1 delete mode 100644 solr/licenses/retries-spi-2.31.77.jar.sha1 create mode 100644 solr/licenses/retries-spi-2.35.10.jar.sha1 delete mode 100644 solr/licenses/s3-2.31.77.jar.sha1 create mode 100644 solr/licenses/s3-2.35.10.jar.sha1 delete mode 100644 solr/licenses/sdk-core-2.31.77.jar.sha1 create mode 100644 solr/licenses/sdk-core-2.35.10.jar.sha1 delete mode 100644 solr/licenses/sts-2.31.77.jar.sha1 create mode 100644 solr/licenses/sts-2.35.10.jar.sha1 delete mode 100644 solr/licenses/third-party-jackson-core-2.31.77.jar.sha1 create mode 100644 solr/licenses/third-party-jackson-core-2.35.10.jar.sha1 delete mode 100644 solr/licenses/url-connection-client-2.31.77.jar.sha1 create mode 100644 solr/licenses/url-connection-client-2.35.10.jar.sha1 delete mode 100644 solr/licenses/utils-2.31.77.jar.sha1 create mode 100644 solr/licenses/utils-2.35.10.jar.sha1 create mode 100644 solr/licenses/utils-lite-2.35.10.jar.sha1 diff --git a/solr/licenses/annotations-2.31.77.jar.sha1 b/solr/licenses/annotations-2.31.77.jar.sha1 deleted file mode 100644 index 993c2c0f639d..000000000000 --- a/solr/licenses/annotations-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fd0536be5ab3299a6f35d24c3baaf63ffcb90c21 diff --git a/solr/licenses/annotations-2.35.10.jar.sha1 b/solr/licenses/annotations-2.35.10.jar.sha1 new file mode 100644 index 000000000000..cce2502558b5 --- /dev/null +++ b/solr/licenses/annotations-2.35.10.jar.sha1 @@ -0,0 +1 @@ +ba69d7c4e14e325acaa2b4b27fe3375412ab3d0f diff --git a/solr/licenses/apache-client-2.31.77.jar.sha1 b/solr/licenses/apache-client-2.31.77.jar.sha1 deleted file mode 100644 index 8f208c3a1332..000000000000 --- a/solr/licenses/apache-client-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -072af5b4b875856b4b9ea39f9744a0223b0f7388 diff --git a/solr/licenses/apache-client-2.35.10.jar.sha1 b/solr/licenses/apache-client-2.35.10.jar.sha1 new file mode 100644 index 000000000000..54ee99dbf1d1 --- /dev/null +++ b/solr/licenses/apache-client-2.35.10.jar.sha1 @@ -0,0 +1 @@ +022ee1cada92c23bc423921df33fd1abbd49137f diff --git a/solr/licenses/arns-2.31.77.jar.sha1 b/solr/licenses/arns-2.31.77.jar.sha1 deleted file mode 100644 index fbf9a6601dad..000000000000 --- a/solr/licenses/arns-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a86b2f68842758bfea4fdc93bd78eae6dbf17f33 diff --git a/solr/licenses/arns-2.35.10.jar.sha1 b/solr/licenses/arns-2.35.10.jar.sha1 new file mode 100644 index 000000000000..e021ae6c31c0 --- /dev/null +++ b/solr/licenses/arns-2.35.10.jar.sha1 @@ -0,0 +1 @@ +2e5266a36270e0bfad9bfc4157f455c5c39cc958 diff --git a/solr/licenses/auth-2.31.77.jar.sha1 b/solr/licenses/auth-2.31.77.jar.sha1 deleted file mode 100644 index d66d2a23e453..000000000000 --- a/solr/licenses/auth-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d5d128441995040d2a357b09e3b31b2bd2f4c45b diff --git a/solr/licenses/auth-2.35.10.jar.sha1 b/solr/licenses/auth-2.35.10.jar.sha1 new file mode 100644 index 000000000000..cb91273bcddf --- /dev/null +++ b/solr/licenses/auth-2.35.10.jar.sha1 @@ -0,0 +1 @@ +841e6bee4d79a8b9d420991ad54937fde6ef2843 diff --git a/solr/licenses/aws-core-2.31.77.jar.sha1 b/solr/licenses/aws-core-2.31.77.jar.sha1 deleted file mode 100644 index 7d4cebbf8325..000000000000 --- a/solr/licenses/aws-core-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0e1fc6b0c6af44adb6e5fceffe4e0f33571b694 diff --git a/solr/licenses/aws-core-2.35.10.jar.sha1 b/solr/licenses/aws-core-2.35.10.jar.sha1 new file mode 100644 index 000000000000..461edcd279a2 --- /dev/null +++ b/solr/licenses/aws-core-2.35.10.jar.sha1 @@ -0,0 +1 @@ +ef1bf21d99dc5510b737e6b15bb52c5a0a324a7a diff --git a/solr/licenses/aws-query-protocol-2.31.77.jar.sha1 b/solr/licenses/aws-query-protocol-2.31.77.jar.sha1 deleted file mode 100644 index 443572c371f1..000000000000 --- a/solr/licenses/aws-query-protocol-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -668f8dfed02d4ddac13d8ad232b0ed195cf9076f diff --git a/solr/licenses/aws-query-protocol-2.35.10.jar.sha1 b/solr/licenses/aws-query-protocol-2.35.10.jar.sha1 new file mode 100644 index 000000000000..f5c892016415 --- /dev/null +++ b/solr/licenses/aws-query-protocol-2.35.10.jar.sha1 @@ -0,0 +1 @@ +e2257d00f1220bc39a3612bc3df1f6765c2ccef4 diff --git a/solr/licenses/aws-xml-protocol-2.31.77.jar.sha1 b/solr/licenses/aws-xml-protocol-2.31.77.jar.sha1 deleted file mode 100644 index 0cf94fc66d49..000000000000 --- a/solr/licenses/aws-xml-protocol-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a35a6784bea0b5446e2771f26c5137f2335a4b90 diff --git a/solr/licenses/aws-xml-protocol-2.35.10.jar.sha1 b/solr/licenses/aws-xml-protocol-2.35.10.jar.sha1 new file mode 100644 index 000000000000..bda39358afcd --- /dev/null +++ b/solr/licenses/aws-xml-protocol-2.35.10.jar.sha1 @@ -0,0 +1 @@ +b2cd28d8b301bfe1ec9a57ee04dbc58dea0ae1a3 diff --git a/solr/licenses/checksums-2.31.77.jar.sha1 b/solr/licenses/checksums-2.31.77.jar.sha1 deleted file mode 100644 index c80b6c0ca7ae..000000000000 --- a/solr/licenses/checksums-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1e09242172b9ddcaaeefab31fc483914863bc0d6 diff --git a/solr/licenses/checksums-2.35.10.jar.sha1 b/solr/licenses/checksums-2.35.10.jar.sha1 new file mode 100644 index 000000000000..3bfb1b98cd9e --- /dev/null +++ b/solr/licenses/checksums-2.35.10.jar.sha1 @@ -0,0 +1 @@ +ac37b1500f329c12374f76bd32501ee48cdb7ab2 diff --git a/solr/licenses/checksums-spi-2.31.77.jar.sha1 b/solr/licenses/checksums-spi-2.31.77.jar.sha1 deleted file mode 100644 index b4f35559667f..000000000000 --- a/solr/licenses/checksums-spi-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -83ebf7c1b41195a498f76216f095bfa8ce2c9002 diff --git a/solr/licenses/checksums-spi-2.35.10.jar.sha1 b/solr/licenses/checksums-spi-2.35.10.jar.sha1 new file mode 100644 index 000000000000..2f3a2bc62321 --- /dev/null +++ b/solr/licenses/checksums-spi-2.35.10.jar.sha1 @@ -0,0 +1 @@ +685b825fd9132410dbc6c30bb02be43e59ba7920 diff --git a/solr/licenses/crt-core-2.31.77.jar.sha1 b/solr/licenses/crt-core-2.31.77.jar.sha1 deleted file mode 100644 index ad79020b61f1..000000000000 --- a/solr/licenses/crt-core-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -408b5feb87a706b8c1624b739760e9f17f7c7db5 diff --git a/solr/licenses/crt-core-2.35.10.jar.sha1 b/solr/licenses/crt-core-2.35.10.jar.sha1 new file mode 100644 index 000000000000..a1355d4ba6c8 --- /dev/null +++ b/solr/licenses/crt-core-2.35.10.jar.sha1 @@ -0,0 +1 @@ +0d303bb215277960a6f4edceddb8888bad966378 diff --git a/solr/licenses/endpoints-spi-2.31.77.jar.sha1 b/solr/licenses/endpoints-spi-2.31.77.jar.sha1 deleted file mode 100644 index f144b620dbca..000000000000 --- a/solr/licenses/endpoints-spi-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -384d751218565ae19217bb70e8866eb4093319c3 diff --git a/solr/licenses/endpoints-spi-2.35.10.jar.sha1 b/solr/licenses/endpoints-spi-2.35.10.jar.sha1 new file mode 100644 index 000000000000..95060bd0e8a0 --- /dev/null +++ b/solr/licenses/endpoints-spi-2.35.10.jar.sha1 @@ -0,0 +1 @@ +f04d68579874a977cf8da2cb27f8e452ff76e9d2 diff --git a/solr/licenses/http-auth-2.31.77.jar.sha1 b/solr/licenses/http-auth-2.31.77.jar.sha1 deleted file mode 100644 index 5da4d324200c..000000000000 --- a/solr/licenses/http-auth-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5f266baa4395bd8a96bcf66b2412731ab482c59b diff --git a/solr/licenses/http-auth-2.35.10.jar.sha1 b/solr/licenses/http-auth-2.35.10.jar.sha1 new file mode 100644 index 000000000000..08e732724919 --- /dev/null +++ b/solr/licenses/http-auth-2.35.10.jar.sha1 @@ -0,0 +1 @@ +b7322df88e01f97e1e6da2505c4efe2bb9cc51a5 diff --git a/solr/licenses/http-auth-aws-2.31.77.jar.sha1 b/solr/licenses/http-auth-aws-2.31.77.jar.sha1 deleted file mode 100644 index 1d96629ab912..000000000000 --- a/solr/licenses/http-auth-aws-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -60a26f2820ff87be082061cb22af7a8572e05388 diff --git a/solr/licenses/http-auth-aws-2.35.10.jar.sha1 b/solr/licenses/http-auth-aws-2.35.10.jar.sha1 new file mode 100644 index 000000000000..a614a54386fe --- /dev/null +++ b/solr/licenses/http-auth-aws-2.35.10.jar.sha1 @@ -0,0 +1 @@ +f42d21abf1e09d590e3ebb27c64496afefb45fbd diff --git a/solr/licenses/http-auth-aws-eventstream-2.31.77.jar.sha1 b/solr/licenses/http-auth-aws-eventstream-2.31.77.jar.sha1 deleted file mode 100644 index 8ae0e6f3173a..000000000000 --- a/solr/licenses/http-auth-aws-eventstream-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a126d4034cd431963a417ed21e426a1ae30a21c diff --git a/solr/licenses/http-auth-aws-eventstream-2.35.10.jar.sha1 b/solr/licenses/http-auth-aws-eventstream-2.35.10.jar.sha1 new file mode 100644 index 000000000000..dc7324caa926 --- /dev/null +++ b/solr/licenses/http-auth-aws-eventstream-2.35.10.jar.sha1 @@ -0,0 +1 @@ +53d2eed18c752487fbf6aa2369ca52b040fdf8b8 diff --git a/solr/licenses/http-auth-spi-2.31.77.jar.sha1 b/solr/licenses/http-auth-spi-2.31.77.jar.sha1 deleted file mode 100644 index 16908c7c1731..000000000000 --- a/solr/licenses/http-auth-spi-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -94845f78603272544442bec2f52852b59859451c diff --git a/solr/licenses/http-auth-spi-2.35.10.jar.sha1 b/solr/licenses/http-auth-spi-2.35.10.jar.sha1 new file mode 100644 index 000000000000..cd9c4eb28903 --- /dev/null +++ b/solr/licenses/http-auth-spi-2.35.10.jar.sha1 @@ -0,0 +1 @@ +a54798be45bff1e7c3cafc76a8a93d43e41c6f4a diff --git a/solr/licenses/http-client-spi-2.31.77.jar.sha1 b/solr/licenses/http-client-spi-2.31.77.jar.sha1 deleted file mode 100644 index 50407e98a4b0..000000000000 --- a/solr/licenses/http-client-spi-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -211c6fed3efe982f3470063a970189e71ee6a81e diff --git a/solr/licenses/http-client-spi-2.35.10.jar.sha1 b/solr/licenses/http-client-spi-2.35.10.jar.sha1 new file mode 100644 index 000000000000..9355d369a849 --- /dev/null +++ b/solr/licenses/http-client-spi-2.35.10.jar.sha1 @@ -0,0 +1 @@ +e9b34eadb312b699290da830f28b7e7393d3f5b8 diff --git a/solr/licenses/identity-spi-2.31.77.jar.sha1 b/solr/licenses/identity-spi-2.31.77.jar.sha1 deleted file mode 100644 index 3e53e08927dc..000000000000 --- a/solr/licenses/identity-spi-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ce3910c6db3986a089ab82b4196edbe69d1a016b diff --git a/solr/licenses/identity-spi-2.35.10.jar.sha1 b/solr/licenses/identity-spi-2.35.10.jar.sha1 new file mode 100644 index 000000000000..0ea540a86f16 --- /dev/null +++ b/solr/licenses/identity-spi-2.35.10.jar.sha1 @@ -0,0 +1 @@ +38c3e34170f93f07de740435828098f9aa8d7694 diff --git a/solr/licenses/json-utils-2.31.77.jar.sha1 b/solr/licenses/json-utils-2.31.77.jar.sha1 deleted file mode 100644 index 22026fe79219..000000000000 --- a/solr/licenses/json-utils-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -666b548875365b7d5160dab6f8b673b6bee8be33 diff --git a/solr/licenses/json-utils-2.35.10.jar.sha1 b/solr/licenses/json-utils-2.35.10.jar.sha1 new file mode 100644 index 000000000000..ef729bbe9e5b --- /dev/null +++ b/solr/licenses/json-utils-2.35.10.jar.sha1 @@ -0,0 +1 @@ +1f8cf8870a6eeadc59fc5b53c21148ad97b8ac11 diff --git a/solr/licenses/metrics-spi-2.31.77.jar.sha1 b/solr/licenses/metrics-spi-2.31.77.jar.sha1 deleted file mode 100644 index 43682eb4611d..000000000000 --- a/solr/licenses/metrics-spi-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fb83360704593dd3b2742ae3eae061a4146e695a diff --git a/solr/licenses/metrics-spi-2.35.10.jar.sha1 b/solr/licenses/metrics-spi-2.35.10.jar.sha1 new file mode 100644 index 000000000000..5285da58b4ce --- /dev/null +++ b/solr/licenses/metrics-spi-2.35.10.jar.sha1 @@ -0,0 +1 @@ +db2b5f8b8340176eb380e91008a502bfb0ac4b61 diff --git a/solr/licenses/profiles-2.31.77.jar.sha1 b/solr/licenses/profiles-2.31.77.jar.sha1 deleted file mode 100644 index e87a2c83e760..000000000000 --- a/solr/licenses/profiles-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbc4c3b6258f4deba8154c3eb4a9a6c2b5e53f51 diff --git a/solr/licenses/profiles-2.35.10.jar.sha1 b/solr/licenses/profiles-2.35.10.jar.sha1 new file mode 100644 index 000000000000..abe22c6c52db --- /dev/null +++ b/solr/licenses/profiles-2.35.10.jar.sha1 @@ -0,0 +1 @@ +625267d4ddef03df4e49be7a292e6f7430b698de diff --git a/solr/licenses/protocol-core-2.31.77.jar.sha1 b/solr/licenses/protocol-core-2.31.77.jar.sha1 deleted file mode 100644 index c59c62f21eb9..000000000000 --- a/solr/licenses/protocol-core-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10e1830e00cad105be1727db59d15fbe29ccb372 diff --git a/solr/licenses/protocol-core-2.35.10.jar.sha1 b/solr/licenses/protocol-core-2.35.10.jar.sha1 new file mode 100644 index 000000000000..8ad567dd0f0e --- /dev/null +++ b/solr/licenses/protocol-core-2.35.10.jar.sha1 @@ -0,0 +1 @@ +38960c68eb67b144bf077541f772fc42b3966e5f diff --git a/solr/licenses/regions-2.31.77.jar.sha1 b/solr/licenses/regions-2.31.77.jar.sha1 deleted file mode 100644 index 5473b1d0c3fe..000000000000 --- a/solr/licenses/regions-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -92da8b47fbf21df27a026a5e345f2309f1bbb6c7 diff --git a/solr/licenses/regions-2.35.10.jar.sha1 b/solr/licenses/regions-2.35.10.jar.sha1 new file mode 100644 index 000000000000..80a5addcf375 --- /dev/null +++ b/solr/licenses/regions-2.35.10.jar.sha1 @@ -0,0 +1 @@ +d4f3f2c92cd5351069d153ac7b3a346e7dbf00cb diff --git a/solr/licenses/retries-2.31.77.jar.sha1 b/solr/licenses/retries-2.31.77.jar.sha1 deleted file mode 100644 index 21fc273d340c..000000000000 --- a/solr/licenses/retries-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d909f3a7d0f55fc8d77e055830dfa24f343864dc diff --git a/solr/licenses/retries-2.35.10.jar.sha1 b/solr/licenses/retries-2.35.10.jar.sha1 new file mode 100644 index 000000000000..b362a84877ac --- /dev/null +++ b/solr/licenses/retries-2.35.10.jar.sha1 @@ -0,0 +1 @@ +bee496fc90334ba8c31be4909f8d68a070690b12 diff --git a/solr/licenses/retries-spi-2.31.77.jar.sha1 b/solr/licenses/retries-spi-2.31.77.jar.sha1 deleted file mode 100644 index 21f98fba5967..000000000000 --- a/solr/licenses/retries-spi-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -de182d8aa722f68ab074a765e59bd1e1784cdc2f diff --git a/solr/licenses/retries-spi-2.35.10.jar.sha1 b/solr/licenses/retries-spi-2.35.10.jar.sha1 new file mode 100644 index 000000000000..810b1f97b875 --- /dev/null +++ b/solr/licenses/retries-spi-2.35.10.jar.sha1 @@ -0,0 +1 @@ +609e15015e3961059e9fe67af38d0378305a45dc diff --git a/solr/licenses/s3-2.31.77.jar.sha1 b/solr/licenses/s3-2.31.77.jar.sha1 deleted file mode 100644 index 1447265a1a3b..000000000000 --- a/solr/licenses/s3-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ee870a078a441c9e06c2ae5abd268285d76b2759 diff --git a/solr/licenses/s3-2.35.10.jar.sha1 b/solr/licenses/s3-2.35.10.jar.sha1 new file mode 100644 index 000000000000..a5fc5f892f61 --- /dev/null +++ b/solr/licenses/s3-2.35.10.jar.sha1 @@ -0,0 +1 @@ +543274787737678ba31a9b82aa25a94d2341fa20 diff --git a/solr/licenses/sdk-core-2.31.77.jar.sha1 b/solr/licenses/sdk-core-2.31.77.jar.sha1 deleted file mode 100644 index efed247beff6..000000000000 --- a/solr/licenses/sdk-core-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4ddfde140a85ae83c8c82423b66785a7db77921b diff --git a/solr/licenses/sdk-core-2.35.10.jar.sha1 b/solr/licenses/sdk-core-2.35.10.jar.sha1 new file mode 100644 index 000000000000..4112cf09388d --- /dev/null +++ b/solr/licenses/sdk-core-2.35.10.jar.sha1 @@ -0,0 +1 @@ +003f7a2ca5b4f1289f298e9a3db3e3fe905f1b8d diff --git a/solr/licenses/sts-2.31.77.jar.sha1 b/solr/licenses/sts-2.31.77.jar.sha1 deleted file mode 100644 index fd29fb9821a9..000000000000 --- a/solr/licenses/sts-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aec46859f01f4ca616cc90a67e4bce7622511fe0 diff --git a/solr/licenses/sts-2.35.10.jar.sha1 b/solr/licenses/sts-2.35.10.jar.sha1 new file mode 100644 index 000000000000..cc6675492181 --- /dev/null +++ b/solr/licenses/sts-2.35.10.jar.sha1 @@ -0,0 +1 @@ +b485ac4f9fd9ae20cf0eb69fb09c99d577374108 diff --git a/solr/licenses/third-party-jackson-core-2.31.77.jar.sha1 b/solr/licenses/third-party-jackson-core-2.31.77.jar.sha1 deleted file mode 100644 index c89ee16b9502..000000000000 --- a/solr/licenses/third-party-jackson-core-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -64d134e5d930933ffcbd80976a10cbd9fd082969 diff --git a/solr/licenses/third-party-jackson-core-2.35.10.jar.sha1 b/solr/licenses/third-party-jackson-core-2.35.10.jar.sha1 new file mode 100644 index 000000000000..8b9514dcca67 --- /dev/null +++ b/solr/licenses/third-party-jackson-core-2.35.10.jar.sha1 @@ -0,0 +1 @@ +61af899f8ced215b1decca7efe15b2935078b213 diff --git a/solr/licenses/url-connection-client-2.31.77.jar.sha1 b/solr/licenses/url-connection-client-2.31.77.jar.sha1 deleted file mode 100644 index c1482b916131..000000000000 --- a/solr/licenses/url-connection-client-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5bacf67a99c78684bfbf06b457c2700ab604b095 diff --git a/solr/licenses/url-connection-client-2.35.10.jar.sha1 b/solr/licenses/url-connection-client-2.35.10.jar.sha1 new file mode 100644 index 000000000000..fdf8a5a67f1f --- /dev/null +++ b/solr/licenses/url-connection-client-2.35.10.jar.sha1 @@ -0,0 +1 @@ +f782233a89a60a611f5d377725e20809be542a19 diff --git a/solr/licenses/utils-2.31.77.jar.sha1 b/solr/licenses/utils-2.31.77.jar.sha1 deleted file mode 100644 index b16a6897988b..000000000000 --- a/solr/licenses/utils-2.31.77.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6936b4777c4cdeb7a25c1a4d43fc8c92c5c01cfb diff --git a/solr/licenses/utils-2.35.10.jar.sha1 b/solr/licenses/utils-2.35.10.jar.sha1 new file mode 100644 index 000000000000..d5d4647688ea --- /dev/null +++ b/solr/licenses/utils-2.35.10.jar.sha1 @@ -0,0 +1 @@ +95e82dcd0e69a8c359e60007226aba8792401c97 diff --git a/solr/licenses/utils-lite-2.35.10.jar.sha1 b/solr/licenses/utils-lite-2.35.10.jar.sha1 new file mode 100644 index 000000000000..be437e0ec975 --- /dev/null +++ b/solr/licenses/utils-lite-2.35.10.jar.sha1 @@ -0,0 +1 @@ +6eb0b1c525a2a2df355933a24bda4eee591b4d07 diff --git a/versions.lock b/versions.lock index b99b5dd0c59d..a36e34e9d567 100644 --- a/versions.lock +++ b/versions.lock @@ -344,36 +344,37 @@ org.slf4j:jul-to-slf4j:2.0.17 (2 constraints: c11bd27e) org.slf4j:slf4j-api:2.0.17 (68 constraints: 68d3e189) org.threeten:threetenbp:1.6.9 (4 constraints: 2833ea68) org.xerial.snappy:snappy-java:1.1.10.8 (5 constraints: c146fa38) -software.amazon.awssdk:annotations:2.31.77 (29 constraints: 59bcb697) -software.amazon.awssdk:apache-client:2.31.77 (4 constraints: 112ac6e3) -software.amazon.awssdk:arns:2.31.77 (2 constraints: 23186ec1) -software.amazon.awssdk:auth:2.31.77 (5 constraints: 51384741) -software.amazon.awssdk:aws-core:2.31.77 (6 constraints: 044e1521) -software.amazon.awssdk:aws-query-protocol:2.31.77 (3 constraints: 5e2a08de) -software.amazon.awssdk:aws-xml-protocol:2.31.77 (2 constraints: 23186ec1) -software.amazon.awssdk:bom:2.31.77 (1 constraints: 7605b740) -software.amazon.awssdk:checksums:2.31.77 (4 constraints: 90363cf6) -software.amazon.awssdk:checksums-spi:2.31.77 (5 constraints: 754526bd) -software.amazon.awssdk:crt-core:2.31.77 (1 constraints: c60b9cf9) -software.amazon.awssdk:endpoints-spi:2.31.77 (5 constraints: 13413975) -software.amazon.awssdk:http-auth:2.31.77 (5 constraints: ad3fdf79) -software.amazon.awssdk:http-auth-aws:2.31.77 (5 constraints: a43f5e5d) -software.amazon.awssdk:http-auth-aws-eventstream:2.31.77 (2 constraints: 2f1957ff) -software.amazon.awssdk:http-auth-spi:2.31.77 (8 constraints: d86ccfdd) -software.amazon.awssdk:http-client-spi:2.31.77 (15 constraints: 86da4c2d) -software.amazon.awssdk:identity-spi:2.31.77 (9 constraints: 0f7d16c5) -software.amazon.awssdk:json-utils:2.31.77 (5 constraints: 833f973f) -software.amazon.awssdk:metrics-spi:2.31.77 (7 constraints: 44620c16) -software.amazon.awssdk:profiles:2.31.77 (8 constraints: 6f61f249) -software.amazon.awssdk:protocol-core:2.31.77 (5 constraints: 8f482f5b) -software.amazon.awssdk:regions:2.31.77 (7 constraints: 4c50bfb1) -software.amazon.awssdk:retries:2.31.77 (3 constraints: d5283128) -software.amazon.awssdk:retries-spi:2.31.77 (7 constraints: a554369f) -software.amazon.awssdk:s3:2.31.77 (4 constraints: e72f0cc1) -software.amazon.awssdk:sdk-core:2.31.77 (10 constraints: 9287ebec) -software.amazon.awssdk:sts:2.31.77 (2 constraints: d3114f15) -software.amazon.awssdk:third-party-jackson-core:2.31.77 (2 constraints: 951b30a8) -software.amazon.awssdk:utils:2.31.77 (26 constraints: 58849766) +software.amazon.awssdk:annotations:2.35.10 (30 constraints: 76ca7ed2) +software.amazon.awssdk:apache-client:2.35.10 (4 constraints: ed29fedb) +software.amazon.awssdk:arns:2.35.10 (2 constraints: 111824c0) +software.amazon.awssdk:auth:2.35.10 (5 constraints: 24382434) +software.amazon.awssdk:aws-core:2.35.10 (6 constraints: ce4d330c) +software.amazon.awssdk:aws-query-protocol:2.35.10 (3 constraints: 432a18da) +software.amazon.awssdk:aws-xml-protocol:2.35.10 (2 constraints: 111824c0) +software.amazon.awssdk:bom:2.35.10 (1 constraints: 6d05b440) +software.amazon.awssdk:checksums:2.35.10 (4 constraints: 6c3693ed) +software.amazon.awssdk:checksums-spi:2.35.10 (7 constraints: 3e62d5f4) +software.amazon.awssdk:crt-core:2.35.10 (1 constraints: bd0b99f9) +software.amazon.awssdk:endpoints-spi:2.35.10 (5 constraints: e640b367) +software.amazon.awssdk:http-auth:2.35.10 (5 constraints: 803fbc6c) +software.amazon.awssdk:http-auth-aws:2.35.10 (5 constraints: 773fd84f) +software.amazon.awssdk:http-auth-aws-eventstream:2.35.10 (2 constraints: 1d1904fe) +software.amazon.awssdk:http-auth-spi:2.35.10 (8 constraints: 906c33b6) +software.amazon.awssdk:http-client-spi:2.35.10 (15 constraints: ffd99689) +software.amazon.awssdk:identity-spi:2.35.10 (9 constraints: be7c2991) +software.amazon.awssdk:json-utils:2.35.10 (5 constraints: 563f5932) +software.amazon.awssdk:metrics-spi:2.35.10 (7 constraints: 056269f8) +software.amazon.awssdk:profiles:2.35.10 (8 constraints: 27619a23) +software.amazon.awssdk:protocol-core:2.35.10 (5 constraints: 6248c44d) +software.amazon.awssdk:regions:2.35.10 (7 constraints: 1650b09c) +software.amazon.awssdk:retries:2.35.10 (3 constraints: ba28e723) +software.amazon.awssdk:retries-spi:2.35.10 (7 constraints: 66547a82) +software.amazon.awssdk:s3:2.35.10 (4 constraints: d52f98bc) +software.amazon.awssdk:sdk-core:2.35.10 (10 constraints: 3887a8ab) +software.amazon.awssdk:sts:2.35.10 (2 constraints: c111fc13) +software.amazon.awssdk:third-party-jackson-core:2.35.10 (2 constraints: 831b9ea6) +software.amazon.awssdk:utils:2.35.10 (26 constraints: 7783fb97) +software.amazon.awssdk:utils-lite:2.35.10 (2 constraints: 8c1a0271) software.amazon.eventstream:eventstream:1.0.1 (3 constraints: cd2e5385) ua.net.nlp:morfologik-ukrainian-search:4.9.1 (1 constraints: d5126e1e) @@ -463,5 +464,5 @@ org.springframework.boot:spring-boot-starter-logging:2.7.13 (1 constraints: 6e13 org.springframework.boot:spring-boot-starter-web:2.7.13 (1 constraints: f30a39d6) org.testcontainers:testcontainers:1.20.4 (1 constraints: 3905313b) org.yaml:snakeyaml:1.30 (1 constraints: 0713d91f) -software.amazon.awssdk:url-connection-client:2.31.77 (2 constraints: 481f03f7) +software.amazon.awssdk:url-connection-client:2.35.10 (2 constraints: 3f1f00f7) software.amazon.ion:ion-java:1.0.2 (1 constraints: 720db831) diff --git a/versions.props b/versions.props index b95645350ce2..f878c2d4dc1f 100644 --- a/versions.props +++ b/versions.props @@ -85,4 +85,4 @@ org.semver4j:semver4j=5.8.0 org.slf4j:*=2.0.17 org.testcontainers:testcontainers*=1.20.4 org.xerial.snappy:snappy-java=1.1.10.8 -software.amazon.awssdk:*=2.31.77 +software.amazon.awssdk:*=2.35.10 From 3dfdf1c348afb95a74f030573f04c0f92bb1f59b Mon Sep 17 00:00:00 2001 From: SolrBot Date: Fri, 2 Jan 2026 22:39:53 +0000 Subject: [PATCH 02/83] Add changelog entry --- ...93-update-software-amazon-awssdk-to-v2-35-10-branch.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-35-10-branch.yml diff --git a/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-35-10-branch.yml b/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-35-10-branch.yml new file mode 100644 index 000000000000..0b83b53c949d --- /dev/null +++ b/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-35-10-branch.yml @@ -0,0 +1,7 @@ +title: Update software.amazon.awssdk:* to v2.35.10 (branch_9x) - abandoned +type: dependency_update +authors: +- name: solrbot +links: +- name: PR#3593 + url: https://github.com/apache/solr/pull/3593 From 49f14fc6d3d700225935a4c74b30d48f1b27934f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 3 Jan 2026 01:20:19 +0100 Subject: [PATCH 03/83] v2.41.0 --- solr/licenses/annotations-2.35.10.jar.sha1 | 1 - solr/licenses/annotations-2.41.0.jar.sha1 | 1 + solr/licenses/apache-client-2.35.10.jar.sha1 | 1 - solr/licenses/apache-client-2.41.0.jar.sha1 | 1 + solr/licenses/arns-2.35.10.jar.sha1 | 1 - solr/licenses/arns-2.41.0.jar.sha1 | 1 + solr/licenses/auth-2.35.10.jar.sha1 | 1 - solr/licenses/auth-2.41.0.jar.sha1 | 1 + solr/licenses/aws-core-2.35.10.jar.sha1 | 1 - solr/licenses/aws-core-2.41.0.jar.sha1 | 1 + .../aws-query-protocol-2.35.10.jar.sha1 | 1 - .../aws-query-protocol-2.41.0.jar.sha1 | 1 + .../aws-xml-protocol-2.35.10.jar.sha1 | 1 - .../licenses/aws-xml-protocol-2.41.0.jar.sha1 | 1 + solr/licenses/checksums-2.35.10.jar.sha1 | 1 - solr/licenses/checksums-2.41.0.jar.sha1 | 1 + solr/licenses/checksums-spi-2.35.10.jar.sha1 | 1 - solr/licenses/checksums-spi-2.41.0.jar.sha1 | 1 + solr/licenses/crt-core-2.35.10.jar.sha1 | 1 - solr/licenses/crt-core-2.41.0.jar.sha1 | 1 + solr/licenses/endpoints-spi-2.35.10.jar.sha1 | 1 - solr/licenses/endpoints-spi-2.41.0.jar.sha1 | 1 + solr/licenses/http-auth-2.35.10.jar.sha1 | 1 - solr/licenses/http-auth-2.41.0.jar.sha1 | 1 + solr/licenses/http-auth-aws-2.35.10.jar.sha1 | 1 - solr/licenses/http-auth-aws-2.41.0.jar.sha1 | 1 + ...http-auth-aws-eventstream-2.35.10.jar.sha1 | 1 - .../http-auth-aws-eventstream-2.41.0.jar.sha1 | 1 + solr/licenses/http-auth-spi-2.35.10.jar.sha1 | 1 - solr/licenses/http-auth-spi-2.41.0.jar.sha1 | 1 + .../licenses/http-client-spi-2.35.10.jar.sha1 | 1 - solr/licenses/http-client-spi-2.41.0.jar.sha1 | 1 + solr/licenses/identity-spi-2.35.10.jar.sha1 | 1 - solr/licenses/identity-spi-2.41.0.jar.sha1 | 1 + solr/licenses/json-utils-2.35.10.jar.sha1 | 1 - solr/licenses/json-utils-2.41.0.jar.sha1 | 1 + solr/licenses/metrics-spi-2.35.10.jar.sha1 | 1 - solr/licenses/metrics-spi-2.41.0.jar.sha1 | 1 + solr/licenses/profiles-2.35.10.jar.sha1 | 1 - solr/licenses/profiles-2.41.0.jar.sha1 | 1 + solr/licenses/protocol-core-2.35.10.jar.sha1 | 1 - solr/licenses/protocol-core-2.41.0.jar.sha1 | 1 + solr/licenses/regions-2.35.10.jar.sha1 | 1 - solr/licenses/regions-2.41.0.jar.sha1 | 1 + solr/licenses/retries-2.35.10.jar.sha1 | 1 - solr/licenses/retries-2.41.0.jar.sha1 | 1 + solr/licenses/retries-spi-2.35.10.jar.sha1 | 1 - solr/licenses/retries-spi-2.41.0.jar.sha1 | 1 + solr/licenses/s3-2.35.10.jar.sha1 | 1 - solr/licenses/s3-2.41.0.jar.sha1 | 1 + solr/licenses/sdk-core-2.35.10.jar.sha1 | 1 - solr/licenses/sdk-core-2.41.0.jar.sha1 | 1 + solr/licenses/sts-2.35.10.jar.sha1 | 1 - solr/licenses/sts-2.41.0.jar.sha1 | 1 + .../third-party-jackson-core-2.35.10.jar.sha1 | 1 - .../third-party-jackson-core-2.41.0.jar.sha1 | 1 + .../url-connection-client-2.35.10.jar.sha1 | 1 - .../url-connection-client-2.41.0.jar.sha1 | 1 + solr/licenses/utils-2.35.10.jar.sha1 | 1 - solr/licenses/utils-2.41.0.jar.sha1 | 1 + solr/licenses/utils-lite-2.35.10.jar.sha1 | 1 - solr/licenses/utils-lite-2.41.0.jar.sha1 | 1 + .../org/apache/solr/s3/S3StorageClient.java | 86 +++++++++++-------- .../solr/s3/S3BackupRepositoryTest.java | 6 +- .../apache/solr/s3/S3OutputStreamTest.java | 27 ++++-- versions.lock | 64 +++++++------- versions.props | 2 +- 67 files changed, 141 insertions(+), 106 deletions(-) delete mode 100644 solr/licenses/annotations-2.35.10.jar.sha1 create mode 100644 solr/licenses/annotations-2.41.0.jar.sha1 delete mode 100644 solr/licenses/apache-client-2.35.10.jar.sha1 create mode 100644 solr/licenses/apache-client-2.41.0.jar.sha1 delete mode 100644 solr/licenses/arns-2.35.10.jar.sha1 create mode 100644 solr/licenses/arns-2.41.0.jar.sha1 delete mode 100644 solr/licenses/auth-2.35.10.jar.sha1 create mode 100644 solr/licenses/auth-2.41.0.jar.sha1 delete mode 100644 solr/licenses/aws-core-2.35.10.jar.sha1 create mode 100644 solr/licenses/aws-core-2.41.0.jar.sha1 delete mode 100644 solr/licenses/aws-query-protocol-2.35.10.jar.sha1 create mode 100644 solr/licenses/aws-query-protocol-2.41.0.jar.sha1 delete mode 100644 solr/licenses/aws-xml-protocol-2.35.10.jar.sha1 create mode 100644 solr/licenses/aws-xml-protocol-2.41.0.jar.sha1 delete mode 100644 solr/licenses/checksums-2.35.10.jar.sha1 create mode 100644 solr/licenses/checksums-2.41.0.jar.sha1 delete mode 100644 solr/licenses/checksums-spi-2.35.10.jar.sha1 create mode 100644 solr/licenses/checksums-spi-2.41.0.jar.sha1 delete mode 100644 solr/licenses/crt-core-2.35.10.jar.sha1 create mode 100644 solr/licenses/crt-core-2.41.0.jar.sha1 delete mode 100644 solr/licenses/endpoints-spi-2.35.10.jar.sha1 create mode 100644 solr/licenses/endpoints-spi-2.41.0.jar.sha1 delete mode 100644 solr/licenses/http-auth-2.35.10.jar.sha1 create mode 100644 solr/licenses/http-auth-2.41.0.jar.sha1 delete mode 100644 solr/licenses/http-auth-aws-2.35.10.jar.sha1 create mode 100644 solr/licenses/http-auth-aws-2.41.0.jar.sha1 delete mode 100644 solr/licenses/http-auth-aws-eventstream-2.35.10.jar.sha1 create mode 100644 solr/licenses/http-auth-aws-eventstream-2.41.0.jar.sha1 delete mode 100644 solr/licenses/http-auth-spi-2.35.10.jar.sha1 create mode 100644 solr/licenses/http-auth-spi-2.41.0.jar.sha1 delete mode 100644 solr/licenses/http-client-spi-2.35.10.jar.sha1 create mode 100644 solr/licenses/http-client-spi-2.41.0.jar.sha1 delete mode 100644 solr/licenses/identity-spi-2.35.10.jar.sha1 create mode 100644 solr/licenses/identity-spi-2.41.0.jar.sha1 delete mode 100644 solr/licenses/json-utils-2.35.10.jar.sha1 create mode 100644 solr/licenses/json-utils-2.41.0.jar.sha1 delete mode 100644 solr/licenses/metrics-spi-2.35.10.jar.sha1 create mode 100644 solr/licenses/metrics-spi-2.41.0.jar.sha1 delete mode 100644 solr/licenses/profiles-2.35.10.jar.sha1 create mode 100644 solr/licenses/profiles-2.41.0.jar.sha1 delete mode 100644 solr/licenses/protocol-core-2.35.10.jar.sha1 create mode 100644 solr/licenses/protocol-core-2.41.0.jar.sha1 delete mode 100644 solr/licenses/regions-2.35.10.jar.sha1 create mode 100644 solr/licenses/regions-2.41.0.jar.sha1 delete mode 100644 solr/licenses/retries-2.35.10.jar.sha1 create mode 100644 solr/licenses/retries-2.41.0.jar.sha1 delete mode 100644 solr/licenses/retries-spi-2.35.10.jar.sha1 create mode 100644 solr/licenses/retries-spi-2.41.0.jar.sha1 delete mode 100644 solr/licenses/s3-2.35.10.jar.sha1 create mode 100644 solr/licenses/s3-2.41.0.jar.sha1 delete mode 100644 solr/licenses/sdk-core-2.35.10.jar.sha1 create mode 100644 solr/licenses/sdk-core-2.41.0.jar.sha1 delete mode 100644 solr/licenses/sts-2.35.10.jar.sha1 create mode 100644 solr/licenses/sts-2.41.0.jar.sha1 delete mode 100644 solr/licenses/third-party-jackson-core-2.35.10.jar.sha1 create mode 100644 solr/licenses/third-party-jackson-core-2.41.0.jar.sha1 delete mode 100644 solr/licenses/url-connection-client-2.35.10.jar.sha1 create mode 100644 solr/licenses/url-connection-client-2.41.0.jar.sha1 delete mode 100644 solr/licenses/utils-2.35.10.jar.sha1 create mode 100644 solr/licenses/utils-2.41.0.jar.sha1 delete mode 100644 solr/licenses/utils-lite-2.35.10.jar.sha1 create mode 100644 solr/licenses/utils-lite-2.41.0.jar.sha1 diff --git a/solr/licenses/annotations-2.35.10.jar.sha1 b/solr/licenses/annotations-2.35.10.jar.sha1 deleted file mode 100644 index cce2502558b5..000000000000 --- a/solr/licenses/annotations-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ba69d7c4e14e325acaa2b4b27fe3375412ab3d0f diff --git a/solr/licenses/annotations-2.41.0.jar.sha1 b/solr/licenses/annotations-2.41.0.jar.sha1 new file mode 100644 index 000000000000..72a162709a06 --- /dev/null +++ b/solr/licenses/annotations-2.41.0.jar.sha1 @@ -0,0 +1 @@ +38fedba1c0c3f3c65afe4896c3f936ef77129620 diff --git a/solr/licenses/apache-client-2.35.10.jar.sha1 b/solr/licenses/apache-client-2.35.10.jar.sha1 deleted file mode 100644 index 54ee99dbf1d1..000000000000 --- a/solr/licenses/apache-client-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -022ee1cada92c23bc423921df33fd1abbd49137f diff --git a/solr/licenses/apache-client-2.41.0.jar.sha1 b/solr/licenses/apache-client-2.41.0.jar.sha1 new file mode 100644 index 000000000000..c33cef4d4ff0 --- /dev/null +++ b/solr/licenses/apache-client-2.41.0.jar.sha1 @@ -0,0 +1 @@ +908533ce5db358d9b965ec87bdc5de4dd97fc0c9 diff --git a/solr/licenses/arns-2.35.10.jar.sha1 b/solr/licenses/arns-2.35.10.jar.sha1 deleted file mode 100644 index e021ae6c31c0..000000000000 --- a/solr/licenses/arns-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2e5266a36270e0bfad9bfc4157f455c5c39cc958 diff --git a/solr/licenses/arns-2.41.0.jar.sha1 b/solr/licenses/arns-2.41.0.jar.sha1 new file mode 100644 index 000000000000..912b6a604103 --- /dev/null +++ b/solr/licenses/arns-2.41.0.jar.sha1 @@ -0,0 +1 @@ +fc2479d009ff23a6e8b29bfdbc65b097dcdf7451 diff --git a/solr/licenses/auth-2.35.10.jar.sha1 b/solr/licenses/auth-2.35.10.jar.sha1 deleted file mode 100644 index cb91273bcddf..000000000000 --- a/solr/licenses/auth-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -841e6bee4d79a8b9d420991ad54937fde6ef2843 diff --git a/solr/licenses/auth-2.41.0.jar.sha1 b/solr/licenses/auth-2.41.0.jar.sha1 new file mode 100644 index 000000000000..b6084d57e2dd --- /dev/null +++ b/solr/licenses/auth-2.41.0.jar.sha1 @@ -0,0 +1 @@ +3303638f0e18cb52b63b36435dc099dd23bcff38 diff --git a/solr/licenses/aws-core-2.35.10.jar.sha1 b/solr/licenses/aws-core-2.35.10.jar.sha1 deleted file mode 100644 index 461edcd279a2..000000000000 --- a/solr/licenses/aws-core-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ef1bf21d99dc5510b737e6b15bb52c5a0a324a7a diff --git a/solr/licenses/aws-core-2.41.0.jar.sha1 b/solr/licenses/aws-core-2.41.0.jar.sha1 new file mode 100644 index 000000000000..a2e00ca32e34 --- /dev/null +++ b/solr/licenses/aws-core-2.41.0.jar.sha1 @@ -0,0 +1 @@ +8c43ee4145f1566651ddd484a3bece07ccb9af13 diff --git a/solr/licenses/aws-query-protocol-2.35.10.jar.sha1 b/solr/licenses/aws-query-protocol-2.35.10.jar.sha1 deleted file mode 100644 index f5c892016415..000000000000 --- a/solr/licenses/aws-query-protocol-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e2257d00f1220bc39a3612bc3df1f6765c2ccef4 diff --git a/solr/licenses/aws-query-protocol-2.41.0.jar.sha1 b/solr/licenses/aws-query-protocol-2.41.0.jar.sha1 new file mode 100644 index 000000000000..44edb43c2990 --- /dev/null +++ b/solr/licenses/aws-query-protocol-2.41.0.jar.sha1 @@ -0,0 +1 @@ +5eaa9a7ad5037963118d81825328f958bc78a28d diff --git a/solr/licenses/aws-xml-protocol-2.35.10.jar.sha1 b/solr/licenses/aws-xml-protocol-2.35.10.jar.sha1 deleted file mode 100644 index bda39358afcd..000000000000 --- a/solr/licenses/aws-xml-protocol-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2cd28d8b301bfe1ec9a57ee04dbc58dea0ae1a3 diff --git a/solr/licenses/aws-xml-protocol-2.41.0.jar.sha1 b/solr/licenses/aws-xml-protocol-2.41.0.jar.sha1 new file mode 100644 index 000000000000..992c28b95704 --- /dev/null +++ b/solr/licenses/aws-xml-protocol-2.41.0.jar.sha1 @@ -0,0 +1 @@ +6d6437cd2619553e7fce5d0499584d3afb399dde diff --git a/solr/licenses/checksums-2.35.10.jar.sha1 b/solr/licenses/checksums-2.35.10.jar.sha1 deleted file mode 100644 index 3bfb1b98cd9e..000000000000 --- a/solr/licenses/checksums-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ac37b1500f329c12374f76bd32501ee48cdb7ab2 diff --git a/solr/licenses/checksums-2.41.0.jar.sha1 b/solr/licenses/checksums-2.41.0.jar.sha1 new file mode 100644 index 000000000000..f2a74fa3230a --- /dev/null +++ b/solr/licenses/checksums-2.41.0.jar.sha1 @@ -0,0 +1 @@ +65efd61cbbf6e368cc1aa43c11d01d325987d794 diff --git a/solr/licenses/checksums-spi-2.35.10.jar.sha1 b/solr/licenses/checksums-spi-2.35.10.jar.sha1 deleted file mode 100644 index 2f3a2bc62321..000000000000 --- a/solr/licenses/checksums-spi-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -685b825fd9132410dbc6c30bb02be43e59ba7920 diff --git a/solr/licenses/checksums-spi-2.41.0.jar.sha1 b/solr/licenses/checksums-spi-2.41.0.jar.sha1 new file mode 100644 index 000000000000..19d60c81528b --- /dev/null +++ b/solr/licenses/checksums-spi-2.41.0.jar.sha1 @@ -0,0 +1 @@ +7996b92051d6348ef0429f97ac52e3b1c1ede292 diff --git a/solr/licenses/crt-core-2.35.10.jar.sha1 b/solr/licenses/crt-core-2.35.10.jar.sha1 deleted file mode 100644 index a1355d4ba6c8..000000000000 --- a/solr/licenses/crt-core-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0d303bb215277960a6f4edceddb8888bad966378 diff --git a/solr/licenses/crt-core-2.41.0.jar.sha1 b/solr/licenses/crt-core-2.41.0.jar.sha1 new file mode 100644 index 000000000000..33e209d16be9 --- /dev/null +++ b/solr/licenses/crt-core-2.41.0.jar.sha1 @@ -0,0 +1 @@ +ea1ace621f14f62f5677d32b343bb8e95f6ce064 diff --git a/solr/licenses/endpoints-spi-2.35.10.jar.sha1 b/solr/licenses/endpoints-spi-2.35.10.jar.sha1 deleted file mode 100644 index 95060bd0e8a0..000000000000 --- a/solr/licenses/endpoints-spi-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f04d68579874a977cf8da2cb27f8e452ff76e9d2 diff --git a/solr/licenses/endpoints-spi-2.41.0.jar.sha1 b/solr/licenses/endpoints-spi-2.41.0.jar.sha1 new file mode 100644 index 000000000000..70b9cfc6c31e --- /dev/null +++ b/solr/licenses/endpoints-spi-2.41.0.jar.sha1 @@ -0,0 +1 @@ +8b183fc09f4996069d572b92a8ebabc8fd5c5747 diff --git a/solr/licenses/http-auth-2.35.10.jar.sha1 b/solr/licenses/http-auth-2.35.10.jar.sha1 deleted file mode 100644 index 08e732724919..000000000000 --- a/solr/licenses/http-auth-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7322df88e01f97e1e6da2505c4efe2bb9cc51a5 diff --git a/solr/licenses/http-auth-2.41.0.jar.sha1 b/solr/licenses/http-auth-2.41.0.jar.sha1 new file mode 100644 index 000000000000..c65747a003f7 --- /dev/null +++ b/solr/licenses/http-auth-2.41.0.jar.sha1 @@ -0,0 +1 @@ +fad29613fa347c9377c7f05e5bd425ac3f1c7b12 diff --git a/solr/licenses/http-auth-aws-2.35.10.jar.sha1 b/solr/licenses/http-auth-aws-2.35.10.jar.sha1 deleted file mode 100644 index a614a54386fe..000000000000 --- a/solr/licenses/http-auth-aws-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f42d21abf1e09d590e3ebb27c64496afefb45fbd diff --git a/solr/licenses/http-auth-aws-2.41.0.jar.sha1 b/solr/licenses/http-auth-aws-2.41.0.jar.sha1 new file mode 100644 index 000000000000..5ddb859669d9 --- /dev/null +++ b/solr/licenses/http-auth-aws-2.41.0.jar.sha1 @@ -0,0 +1 @@ +31196b2da7f78ad5dd8c00229a9d2666104d96b6 diff --git a/solr/licenses/http-auth-aws-eventstream-2.35.10.jar.sha1 b/solr/licenses/http-auth-aws-eventstream-2.35.10.jar.sha1 deleted file mode 100644 index dc7324caa926..000000000000 --- a/solr/licenses/http-auth-aws-eventstream-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -53d2eed18c752487fbf6aa2369ca52b040fdf8b8 diff --git a/solr/licenses/http-auth-aws-eventstream-2.41.0.jar.sha1 b/solr/licenses/http-auth-aws-eventstream-2.41.0.jar.sha1 new file mode 100644 index 000000000000..adf3997ab395 --- /dev/null +++ b/solr/licenses/http-auth-aws-eventstream-2.41.0.jar.sha1 @@ -0,0 +1 @@ +3ad44b3298ae201982f6c78fd317689116242fab diff --git a/solr/licenses/http-auth-spi-2.35.10.jar.sha1 b/solr/licenses/http-auth-spi-2.35.10.jar.sha1 deleted file mode 100644 index cd9c4eb28903..000000000000 --- a/solr/licenses/http-auth-spi-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a54798be45bff1e7c3cafc76a8a93d43e41c6f4a diff --git a/solr/licenses/http-auth-spi-2.41.0.jar.sha1 b/solr/licenses/http-auth-spi-2.41.0.jar.sha1 new file mode 100644 index 000000000000..1e0f8bcf161c --- /dev/null +++ b/solr/licenses/http-auth-spi-2.41.0.jar.sha1 @@ -0,0 +1 @@ +dd75ccc1d5fb92d4b63847e86b14d56ee01fd470 diff --git a/solr/licenses/http-client-spi-2.35.10.jar.sha1 b/solr/licenses/http-client-spi-2.35.10.jar.sha1 deleted file mode 100644 index 9355d369a849..000000000000 --- a/solr/licenses/http-client-spi-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e9b34eadb312b699290da830f28b7e7393d3f5b8 diff --git a/solr/licenses/http-client-spi-2.41.0.jar.sha1 b/solr/licenses/http-client-spi-2.41.0.jar.sha1 new file mode 100644 index 000000000000..a75f1c6f2d15 --- /dev/null +++ b/solr/licenses/http-client-spi-2.41.0.jar.sha1 @@ -0,0 +1 @@ +39f0d9581cb02d152917af7af1930a52526e0a36 diff --git a/solr/licenses/identity-spi-2.35.10.jar.sha1 b/solr/licenses/identity-spi-2.35.10.jar.sha1 deleted file mode 100644 index 0ea540a86f16..000000000000 --- a/solr/licenses/identity-spi-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -38c3e34170f93f07de740435828098f9aa8d7694 diff --git a/solr/licenses/identity-spi-2.41.0.jar.sha1 b/solr/licenses/identity-spi-2.41.0.jar.sha1 new file mode 100644 index 000000000000..6e6e26a3d950 --- /dev/null +++ b/solr/licenses/identity-spi-2.41.0.jar.sha1 @@ -0,0 +1 @@ +e780d4fd05f77159671a124d1a354b0f1f156fbc diff --git a/solr/licenses/json-utils-2.35.10.jar.sha1 b/solr/licenses/json-utils-2.35.10.jar.sha1 deleted file mode 100644 index ef729bbe9e5b..000000000000 --- a/solr/licenses/json-utils-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1f8cf8870a6eeadc59fc5b53c21148ad97b8ac11 diff --git a/solr/licenses/json-utils-2.41.0.jar.sha1 b/solr/licenses/json-utils-2.41.0.jar.sha1 new file mode 100644 index 000000000000..644d7c43ff43 --- /dev/null +++ b/solr/licenses/json-utils-2.41.0.jar.sha1 @@ -0,0 +1 @@ +28669255117df36fadb30aa20bcfe7d1d7d8554f diff --git a/solr/licenses/metrics-spi-2.35.10.jar.sha1 b/solr/licenses/metrics-spi-2.35.10.jar.sha1 deleted file mode 100644 index 5285da58b4ce..000000000000 --- a/solr/licenses/metrics-spi-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -db2b5f8b8340176eb380e91008a502bfb0ac4b61 diff --git a/solr/licenses/metrics-spi-2.41.0.jar.sha1 b/solr/licenses/metrics-spi-2.41.0.jar.sha1 new file mode 100644 index 000000000000..ddaf388a0740 --- /dev/null +++ b/solr/licenses/metrics-spi-2.41.0.jar.sha1 @@ -0,0 +1 @@ +22db7213d930e6f80b03d8c8df0d5e9cb431008e diff --git a/solr/licenses/profiles-2.35.10.jar.sha1 b/solr/licenses/profiles-2.35.10.jar.sha1 deleted file mode 100644 index abe22c6c52db..000000000000 --- a/solr/licenses/profiles-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -625267d4ddef03df4e49be7a292e6f7430b698de diff --git a/solr/licenses/profiles-2.41.0.jar.sha1 b/solr/licenses/profiles-2.41.0.jar.sha1 new file mode 100644 index 000000000000..7e96af7b2e30 --- /dev/null +++ b/solr/licenses/profiles-2.41.0.jar.sha1 @@ -0,0 +1 @@ +67f147fbfd6ff92d7ff1c5143228643fcd795ce9 diff --git a/solr/licenses/protocol-core-2.35.10.jar.sha1 b/solr/licenses/protocol-core-2.35.10.jar.sha1 deleted file mode 100644 index 8ad567dd0f0e..000000000000 --- a/solr/licenses/protocol-core-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -38960c68eb67b144bf077541f772fc42b3966e5f diff --git a/solr/licenses/protocol-core-2.41.0.jar.sha1 b/solr/licenses/protocol-core-2.41.0.jar.sha1 new file mode 100644 index 000000000000..26db0bfb8c2b --- /dev/null +++ b/solr/licenses/protocol-core-2.41.0.jar.sha1 @@ -0,0 +1 @@ +92e71779441a3045cf6e71aefbc88e96a81c1a5b diff --git a/solr/licenses/regions-2.35.10.jar.sha1 b/solr/licenses/regions-2.35.10.jar.sha1 deleted file mode 100644 index 80a5addcf375..000000000000 --- a/solr/licenses/regions-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d4f3f2c92cd5351069d153ac7b3a346e7dbf00cb diff --git a/solr/licenses/regions-2.41.0.jar.sha1 b/solr/licenses/regions-2.41.0.jar.sha1 new file mode 100644 index 000000000000..5c61f5c3666d --- /dev/null +++ b/solr/licenses/regions-2.41.0.jar.sha1 @@ -0,0 +1 @@ +32927c5905c139b3ebede179084f3a0184f445c7 diff --git a/solr/licenses/retries-2.35.10.jar.sha1 b/solr/licenses/retries-2.35.10.jar.sha1 deleted file mode 100644 index b362a84877ac..000000000000 --- a/solr/licenses/retries-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bee496fc90334ba8c31be4909f8d68a070690b12 diff --git a/solr/licenses/retries-2.41.0.jar.sha1 b/solr/licenses/retries-2.41.0.jar.sha1 new file mode 100644 index 000000000000..f04738d6d23e --- /dev/null +++ b/solr/licenses/retries-2.41.0.jar.sha1 @@ -0,0 +1 @@ +3d08ea778e7c49741f02563020efd3f0fa14189d diff --git a/solr/licenses/retries-spi-2.35.10.jar.sha1 b/solr/licenses/retries-spi-2.35.10.jar.sha1 deleted file mode 100644 index 810b1f97b875..000000000000 --- a/solr/licenses/retries-spi-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -609e15015e3961059e9fe67af38d0378305a45dc diff --git a/solr/licenses/retries-spi-2.41.0.jar.sha1 b/solr/licenses/retries-spi-2.41.0.jar.sha1 new file mode 100644 index 000000000000..7ed5767cd512 --- /dev/null +++ b/solr/licenses/retries-spi-2.41.0.jar.sha1 @@ -0,0 +1 @@ +95dd9d0eed2c678bfd51f3f6d5caee2f1e873c29 diff --git a/solr/licenses/s3-2.35.10.jar.sha1 b/solr/licenses/s3-2.35.10.jar.sha1 deleted file mode 100644 index a5fc5f892f61..000000000000 --- a/solr/licenses/s3-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -543274787737678ba31a9b82aa25a94d2341fa20 diff --git a/solr/licenses/s3-2.41.0.jar.sha1 b/solr/licenses/s3-2.41.0.jar.sha1 new file mode 100644 index 000000000000..f3a042147e19 --- /dev/null +++ b/solr/licenses/s3-2.41.0.jar.sha1 @@ -0,0 +1 @@ +9e8e9a162f502f31ef2020c05d9a379bce583015 diff --git a/solr/licenses/sdk-core-2.35.10.jar.sha1 b/solr/licenses/sdk-core-2.35.10.jar.sha1 deleted file mode 100644 index 4112cf09388d..000000000000 --- a/solr/licenses/sdk-core-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -003f7a2ca5b4f1289f298e9a3db3e3fe905f1b8d diff --git a/solr/licenses/sdk-core-2.41.0.jar.sha1 b/solr/licenses/sdk-core-2.41.0.jar.sha1 new file mode 100644 index 000000000000..35f39a381d50 --- /dev/null +++ b/solr/licenses/sdk-core-2.41.0.jar.sha1 @@ -0,0 +1 @@ +f83030af2f845f036dc1b529b917bdcd3e200234 diff --git a/solr/licenses/sts-2.35.10.jar.sha1 b/solr/licenses/sts-2.35.10.jar.sha1 deleted file mode 100644 index cc6675492181..000000000000 --- a/solr/licenses/sts-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b485ac4f9fd9ae20cf0eb69fb09c99d577374108 diff --git a/solr/licenses/sts-2.41.0.jar.sha1 b/solr/licenses/sts-2.41.0.jar.sha1 new file mode 100644 index 000000000000..840d1e2c2ed7 --- /dev/null +++ b/solr/licenses/sts-2.41.0.jar.sha1 @@ -0,0 +1 @@ +cb69d34e47304181cae83809d07f3dc89fd4fc75 diff --git a/solr/licenses/third-party-jackson-core-2.35.10.jar.sha1 b/solr/licenses/third-party-jackson-core-2.35.10.jar.sha1 deleted file mode 100644 index 8b9514dcca67..000000000000 --- a/solr/licenses/third-party-jackson-core-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -61af899f8ced215b1decca7efe15b2935078b213 diff --git a/solr/licenses/third-party-jackson-core-2.41.0.jar.sha1 b/solr/licenses/third-party-jackson-core-2.41.0.jar.sha1 new file mode 100644 index 000000000000..aa5554c65474 --- /dev/null +++ b/solr/licenses/third-party-jackson-core-2.41.0.jar.sha1 @@ -0,0 +1 @@ +eb418464f58798f68f800aa3e28257e47670e306 diff --git a/solr/licenses/url-connection-client-2.35.10.jar.sha1 b/solr/licenses/url-connection-client-2.35.10.jar.sha1 deleted file mode 100644 index fdf8a5a67f1f..000000000000 --- a/solr/licenses/url-connection-client-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f782233a89a60a611f5d377725e20809be542a19 diff --git a/solr/licenses/url-connection-client-2.41.0.jar.sha1 b/solr/licenses/url-connection-client-2.41.0.jar.sha1 new file mode 100644 index 000000000000..e0d32a85d324 --- /dev/null +++ b/solr/licenses/url-connection-client-2.41.0.jar.sha1 @@ -0,0 +1 @@ +7b0bf0cc123ccc6256796ab298b26b0fb51a3873 diff --git a/solr/licenses/utils-2.35.10.jar.sha1 b/solr/licenses/utils-2.35.10.jar.sha1 deleted file mode 100644 index d5d4647688ea..000000000000 --- a/solr/licenses/utils-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95e82dcd0e69a8c359e60007226aba8792401c97 diff --git a/solr/licenses/utils-2.41.0.jar.sha1 b/solr/licenses/utils-2.41.0.jar.sha1 new file mode 100644 index 000000000000..2d1bd6aa7923 --- /dev/null +++ b/solr/licenses/utils-2.41.0.jar.sha1 @@ -0,0 +1 @@ +ac0fbab3deab0720e168ac08eb9bb82c39972dc5 diff --git a/solr/licenses/utils-lite-2.35.10.jar.sha1 b/solr/licenses/utils-lite-2.35.10.jar.sha1 deleted file mode 100644 index be437e0ec975..000000000000 --- a/solr/licenses/utils-lite-2.35.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6eb0b1c525a2a2df355933a24bda4eee591b4d07 diff --git a/solr/licenses/utils-lite-2.41.0.jar.sha1 b/solr/licenses/utils-lite-2.41.0.jar.sha1 new file mode 100644 index 000000000000..74c6dc8a5d26 --- /dev/null +++ b/solr/licenses/utils-lite-2.41.0.jar.sha1 @@ -0,0 +1 @@ +a08dd775d9257f8680bf426541d9630d29fcd8f8 diff --git a/solr/modules/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java b/solr/modules/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java index 990a051eae88..fbf611b2fde0 100644 --- a/solr/modules/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java +++ b/solr/modules/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java @@ -44,6 +44,7 @@ import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.http.apache.ProxyConfiguration; import software.amazon.awssdk.regions.Region; @@ -92,6 +93,9 @@ public class S3StorageClient { /** The S3 bucket where we read/write all data. */ private final String bucketName; + /** The HTTP client used by the S3 client. Needs to be closed separately. */ + private final SdkHttpClient httpClient; + S3StorageClient( String bucketName, String profile, @@ -100,30 +104,19 @@ public class S3StorageClient { boolean proxyUseSystemSettings, String endpoint, boolean disableRetries) { - this( - createInternalClient( - profile, region, proxyUrl, proxyUseSystemSettings, endpoint, disableRetries), - bucketName); + this.bucketName = bucketName; + this.httpClient = createHttpClient(proxyUrl, proxyUseSystemSettings); + this.s3Client = createInternalClient(profile, region, endpoint, disableRetries, httpClient); } @VisibleForTesting S3StorageClient(S3Client s3Client, String bucketName) { this.s3Client = s3Client; this.bucketName = bucketName; + this.httpClient = null; } - private static S3Client createInternalClient( - String profile, - String region, - String proxyUrl, - boolean proxyUseSystemSettings, - String endpoint, - boolean disableRetries) { - S3Configuration.Builder configBuilder = S3Configuration.builder().pathStyleAccessEnabled(true); - if (StrUtils.isNotNullOrEmpty(profile)) { - configBuilder.profileName(profile); - } - + private static SdkHttpClient createHttpClient(String proxyUrl, boolean proxyUseSystemSettings) { ApacheHttpClient.Builder sdkHttpClientBuilder = ApacheHttpClient.builder(); // If configured, add proxy ProxyConfiguration.Builder proxyConfigurationBuilder = ProxyConfiguration.builder(); @@ -135,6 +128,20 @@ private static S3Client createInternalClient( sdkHttpClientBuilder.proxyConfiguration(proxyConfigurationBuilder.build()); sdkHttpClientBuilder.useIdleConnectionReaper(false); + return sdkHttpClientBuilder.build(); + } + + private static S3Client createInternalClient( + String profile, + String region, + String endpoint, + boolean disableRetries, + SdkHttpClient httpClient) { + S3Configuration.Builder configBuilder = S3Configuration.builder().pathStyleAccessEnabled(true); + if (StrUtils.isNotNullOrEmpty(profile)) { + configBuilder.profileName(profile); + } + /* * Retry logic */ @@ -167,7 +174,7 @@ private static S3Client createInternalClient( .credentialsProvider(credentialsProviderBuilder.build()) .overrideConfiguration(builder -> builder.retryStrategy(retryStrategy)) .serviceConfiguration(configBuilder.build()) - .httpClient(sdkHttpClientBuilder.build()); + .httpClient(httpClient); if (StrUtils.isNotNullOrEmpty(endpoint)) { clientBuilder.endpointOverride(URI.create(endpoint)); @@ -374,23 +381,27 @@ InputStream pullStream(String path) throws S3Exception { GetObjectRequest.Builder getBuilder = GetObjectRequest.builder().bucket(bucketName).key(s3Path); // This InputStream instance needs to be closed by the caller - return s3Client.getObject( - getBuilder.build(), - ResponseTransformer.unmanaged( - (response, inputStream) -> { - final long contentLength = response.contentLength(); - return new ResumableInputStream( - inputStream, - bytesRead -> { - if (contentLength > 0 && bytesRead >= contentLength) { - // No more bytes to read - return null; - } else if (bytesRead > 0) { - getBuilder.range(String.format(Locale.ROOT, "bytes=%d-", bytesRead)); - } - return s3Client.getObject(getBuilder.build()); - }); - })); + // Use Duration.ZERO to disable timeout and prevent response-input-stream-timeout-scheduler + // thread leak (see https://github.com/aws/aws-sdk-java-v2/issues/6567) + software.amazon.awssdk.core.ResponseInputStream< + software.amazon.awssdk.services.s3.model.GetObjectResponse> + responseStream = + s3Client.getObject( + getBuilder.build(), ResponseTransformer.toInputStream(java.time.Duration.ZERO)); + final long contentLength = responseStream.response().contentLength(); + return new ResumableInputStream( + responseStream, + bytesRead -> { + if (contentLength > 0 && bytesRead >= contentLength) { + // No more bytes to read + return null; + } else if (bytesRead > 0) { + getBuilder.range(String.format(Locale.ROOT, "bytes=%d-", bytesRead)); + } + // Use Duration.ZERO to disable timeout on resumed streams as well + return s3Client.getObject( + getBuilder.build(), ResponseTransformer.toInputStream(java.time.Duration.ZERO)); + }); } catch (SdkException sdke) { throw handleAmazonException(sdke); } @@ -419,6 +430,9 @@ OutputStream pushStream(String path) throws S3Exception { /** Override {@link Closeable} since we throw no exception. */ void close() { s3Client.close(); + if (httpClient != null) { + httpClient.close(); + } } /** Any file path that specifies a non-existent file will not be treated as an error. */ @@ -544,7 +558,7 @@ private String getParentDirectory(String path) { } /** Ensures path adheres to some rules: -Doesn't start with a leading slash */ - String sanitizedPath(String path) throws S3Exception { + String sanitizedPath(String path) { // Trim space from start and end String sanitizedPath = path.trim(); @@ -580,7 +594,7 @@ String sanitizedFilePath(String path) throws S3Exception { * Ensures directory path adheres to some rules: -Overall Path rules from `sanitizedPath` -Add a * trailing slash if one does not exist */ - String sanitizedDirPath(String path) throws S3Exception { + String sanitizedDirPath(String path) { // Trim space from start and end String sanitizedPath = sanitizedPath(path); diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java index 498b0788ab09..c53ba3f2c774 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java @@ -338,7 +338,11 @@ private void pushObject(String path, String content) { private File pullObject(String path) throws IOException { try (S3Client s3 = S3_MOCK_RULE.createS3ClientV2()) { File file = temporaryFolder.newFile(); - InputStream input = s3.getObject(b -> b.bucket(BUCKET_NAME).key(path)); + InputStream input = + s3.getObject( + b -> b.bucket(BUCKET_NAME).key(path), + software.amazon.awssdk.core.sync.ResponseTransformer.toInputStream( + java.time.Duration.ZERO)); FileUtils.copyInputStreamToFile(input, file); return file; } diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java index 1116d3103d47..7746703ba8ce 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java @@ -21,11 +21,13 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.time.Duration; import org.apache.solr.SolrTestCaseJ4; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; +import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3Client; public class S3OutputStreamTest extends SolrTestCaseJ4 { @@ -63,7 +65,10 @@ public void testWriteByteByByte() throws IOException { } // Check we can re-read same content - try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("byte-by-byte"))) { + try (InputStream input = + s3.getObject( + b -> b.bucket(BUCKET).key("byte-by-byte"), + ResponseTransformer.toInputStream(Duration.ZERO))) { String read = new String(input.readAllBytes(), StandardCharsets.UTF_8); assertEquals("Contents saved to S3 file did not match expected", "hello", read); } @@ -82,7 +87,10 @@ public void testWriteSmallBuffer() throws IOException { } // Check we can re-read same content - try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("small-buffer"))) { + try (InputStream input = + s3.getObject( + b -> b.bucket(BUCKET).key("small-buffer"), + ResponseTransformer.toInputStream(Duration.ZERO))) { String read = new String(input.readAllBytes(), StandardCharsets.UTF_8); assertEquals("hello", read); } @@ -103,7 +111,10 @@ public void testWriteLargeBuffer() throws IOException { } // Check we can re-read same content - try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("large-buffer"))) { + try (InputStream input = + s3.getObject( + b -> b.bucket(BUCKET).key("large-buffer"), + ResponseTransformer.toInputStream(Duration.ZERO))) { String read = new String(input.readAllBytes(), StandardCharsets.UTF_8); assertEquals(new String(buffer, StandardCharsets.UTF_8), read); } @@ -125,7 +136,10 @@ public void testFlushSmallBuffer() throws IOException { } // Check we can re-read same content - try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-small"))) { + try (InputStream input = + s3.getObject( + b -> b.bucket(BUCKET).key("flush-small"), + ResponseTransformer.toInputStream(Duration.ZERO))) { String read = new String(input.readAllBytes(), StandardCharsets.UTF_8); assertEquals( "Flushing a small frame of an S3OutputStream should not impact data written", @@ -152,7 +166,10 @@ public void testFlushLargeBuffer() throws IOException { } // Check we can re-read same content - try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-large"))) { + try (InputStream input = + s3.getObject( + b -> b.bucket(BUCKET).key("flush-large"), + ResponseTransformer.toInputStream(Duration.ZERO))) { String read = new String(input.readAllBytes(), StandardCharsets.UTF_8); assertEquals( "Flushing a large frame of an S3OutputStream should not impact data written", diff --git a/versions.lock b/versions.lock index a36e34e9d567..62288ab50f17 100644 --- a/versions.lock +++ b/versions.lock @@ -344,37 +344,37 @@ org.slf4j:jul-to-slf4j:2.0.17 (2 constraints: c11bd27e) org.slf4j:slf4j-api:2.0.17 (68 constraints: 68d3e189) org.threeten:threetenbp:1.6.9 (4 constraints: 2833ea68) org.xerial.snappy:snappy-java:1.1.10.8 (5 constraints: c146fa38) -software.amazon.awssdk:annotations:2.35.10 (30 constraints: 76ca7ed2) -software.amazon.awssdk:apache-client:2.35.10 (4 constraints: ed29fedb) -software.amazon.awssdk:arns:2.35.10 (2 constraints: 111824c0) -software.amazon.awssdk:auth:2.35.10 (5 constraints: 24382434) -software.amazon.awssdk:aws-core:2.35.10 (6 constraints: ce4d330c) -software.amazon.awssdk:aws-query-protocol:2.35.10 (3 constraints: 432a18da) -software.amazon.awssdk:aws-xml-protocol:2.35.10 (2 constraints: 111824c0) -software.amazon.awssdk:bom:2.35.10 (1 constraints: 6d05b440) -software.amazon.awssdk:checksums:2.35.10 (4 constraints: 6c3693ed) -software.amazon.awssdk:checksums-spi:2.35.10 (7 constraints: 3e62d5f4) -software.amazon.awssdk:crt-core:2.35.10 (1 constraints: bd0b99f9) -software.amazon.awssdk:endpoints-spi:2.35.10 (5 constraints: e640b367) -software.amazon.awssdk:http-auth:2.35.10 (5 constraints: 803fbc6c) -software.amazon.awssdk:http-auth-aws:2.35.10 (5 constraints: 773fd84f) -software.amazon.awssdk:http-auth-aws-eventstream:2.35.10 (2 constraints: 1d1904fe) -software.amazon.awssdk:http-auth-spi:2.35.10 (8 constraints: 906c33b6) -software.amazon.awssdk:http-client-spi:2.35.10 (15 constraints: ffd99689) -software.amazon.awssdk:identity-spi:2.35.10 (9 constraints: be7c2991) -software.amazon.awssdk:json-utils:2.35.10 (5 constraints: 563f5932) -software.amazon.awssdk:metrics-spi:2.35.10 (7 constraints: 056269f8) -software.amazon.awssdk:profiles:2.35.10 (8 constraints: 27619a23) -software.amazon.awssdk:protocol-core:2.35.10 (5 constraints: 6248c44d) -software.amazon.awssdk:regions:2.35.10 (7 constraints: 1650b09c) -software.amazon.awssdk:retries:2.35.10 (3 constraints: ba28e723) -software.amazon.awssdk:retries-spi:2.35.10 (7 constraints: 66547a82) -software.amazon.awssdk:s3:2.35.10 (4 constraints: d52f98bc) -software.amazon.awssdk:sdk-core:2.35.10 (10 constraints: 3887a8ab) -software.amazon.awssdk:sts:2.35.10 (2 constraints: c111fc13) -software.amazon.awssdk:third-party-jackson-core:2.35.10 (2 constraints: 831b9ea6) -software.amazon.awssdk:utils:2.35.10 (26 constraints: 7783fb97) -software.amazon.awssdk:utils-lite:2.35.10 (2 constraints: 8c1a0271) +software.amazon.awssdk:annotations:2.41.0 (30 constraints: 5ec4fa00) +software.amazon.awssdk:apache-client:2.41.0 (4 constraints: 1d29ca51) +software.amazon.awssdk:arns:2.41.0 (2 constraints: a9179094) +software.amazon.awssdk:auth:2.41.0 (5 constraints: 20377f4d) +software.amazon.awssdk:aws-core:2.41.0 (6 constraints: 964c528b) +software.amazon.awssdk:aws-query-protocol:2.41.0 (3 constraints: a729786a) +software.amazon.awssdk:aws-xml-protocol:2.41.0 (2 constraints: a9179094) +software.amazon.awssdk:bom:2.41.0 (1 constraints: 39053e3b) +software.amazon.awssdk:checksums:2.41.0 (4 constraints: 9c356235) +software.amazon.awssdk:checksums-spi:2.41.0 (7 constraints: d26085c0) +software.amazon.awssdk:crt-core:2.41.0 (1 constraints: 890bd3ed) +software.amazon.awssdk:endpoints-spi:2.41.0 (5 constraints: e23f4b57) +software.amazon.awssdk:http-auth:2.41.0 (5 constraints: 7c3e4b61) +software.amazon.awssdk:http-auth-aws:2.41.0 (5 constraints: 733e9b46) +software.amazon.awssdk:http-auth-aws-eventstream:2.41.0 (2 constraints: b518bcd0) +software.amazon.awssdk:http-auth-spi:2.41.0 (8 constraints: f06a20ed) +software.amazon.awssdk:http-client-spi:2.41.0 (15 constraints: f3d67461) +software.amazon.awssdk:identity-spi:2.41.0 (9 constraints: ea7a98fb) +software.amazon.awssdk:json-utils:2.41.0 (5 constraints: 523ead28) +software.amazon.awssdk:metrics-spi:2.41.0 (7 constraints: 996007be) +software.amazon.awssdk:profiles:2.41.0 (8 constraints: 875fd3af) +software.amazon.awssdk:protocol-core:2.41.0 (5 constraints: 5e479017) +software.amazon.awssdk:regions:2.41.0 (7 constraints: de4e2b0c) +software.amazon.awssdk:retries:2.41.0 (3 constraints: 1e2847ba) +software.amazon.awssdk:retries-spi:2.41.0 (7 constraints: fa529ca3) +software.amazon.awssdk:s3:2.41.0 (4 constraints: 6d2fe06d) +software.amazon.awssdk:sdk-core:2.41.0 (10 constraints: 3085fc69) +software.amazon.awssdk:sts:2.41.0 (2 constraints: 59115df5) +software.amazon.awssdk:third-party-jackson-core:2.41.0 (2 constraints: 1b1bf875) +software.amazon.awssdk:utils:2.41.0 (26 constraints: 637e9131) +software.amazon.awssdk:utils-lite:2.41.0 (2 constraints: 241adc40) software.amazon.eventstream:eventstream:1.0.1 (3 constraints: cd2e5385) ua.net.nlp:morfologik-ukrainian-search:4.9.1 (1 constraints: d5126e1e) @@ -464,5 +464,5 @@ org.springframework.boot:spring-boot-starter-logging:2.7.13 (1 constraints: 6e13 org.springframework.boot:spring-boot-starter-web:2.7.13 (1 constraints: f30a39d6) org.testcontainers:testcontainers:1.20.4 (1 constraints: 3905313b) org.yaml:snakeyaml:1.30 (1 constraints: 0713d91f) -software.amazon.awssdk:url-connection-client:2.35.10 (2 constraints: 3f1f00f7) +software.amazon.awssdk:url-connection-client:2.41.0 (2 constraints: 0b1fb8d7) software.amazon.ion:ion-java:1.0.2 (1 constraints: 720db831) diff --git a/versions.props b/versions.props index f878c2d4dc1f..bdd6156e7201 100644 --- a/versions.props +++ b/versions.props @@ -85,4 +85,4 @@ org.semver4j:semver4j=5.8.0 org.slf4j:*=2.0.17 org.testcontainers:testcontainers*=1.20.4 org.xerial.snappy:snappy-java=1.1.10.8 -software.amazon.awssdk:*=2.35.10 +software.amazon.awssdk:*=2.41.0 From 74d15a51cd013eb4f9186b75405866dea1208afe Mon Sep 17 00:00:00 2001 From: SolrBot Date: Sat, 3 Jan 2026 00:21:06 +0000 Subject: [PATCH 04/83] Add changelog entry --- ...-update-software-amazon-awssdk-to-v2-41-0-branch-9x.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-41-0-branch-9x.yml diff --git a/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-41-0-branch-9x.yml b/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-41-0-branch-9x.yml new file mode 100644 index 000000000000..7b23f13b967d --- /dev/null +++ b/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-41-0-branch-9x.yml @@ -0,0 +1,7 @@ +title: Update software.amazon.awssdk:* to v2.41.0 (branch_9x) +type: dependency_update +authors: +- name: solrbot +links: +- name: PR#3593 + url: https://github.com/apache/solr/pull/3593 From 4cc9a008f3af05c129e35a7e3482402b99410133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 3 Jan 2026 01:22:23 +0100 Subject: [PATCH 05/83] Remove double changelog file --- ...93-update-software-amazon-awssdk-to-v2-35-10-branch.yml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-35-10-branch.yml diff --git a/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-35-10-branch.yml b/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-35-10-branch.yml deleted file mode 100644 index 0b83b53c949d..000000000000 --- a/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-35-10-branch.yml +++ /dev/null @@ -1,7 +0,0 @@ -title: Update software.amazon.awssdk:* to v2.35.10 (branch_9x) - abandoned -type: dependency_update -authors: -- name: solrbot -links: -- name: PR#3593 - url: https://github.com/apache/solr/pull/3593 From 13351850512cda460974f3c430d356f2067dea90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 3 Jan 2026 01:33:22 +0100 Subject: [PATCH 06/83] Add import, fix some test bugs --- .../org/apache/solr/s3/S3BackupRepositoryTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java index c53ba3f2c774..bdb5c2cb5854 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java @@ -42,6 +42,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; @@ -107,18 +108,18 @@ public void testLocalDirectoryFunctions() throws Exception { repo.createDirectory(path); assertTrue(repo.exists(path)); assertEquals(BackupRepository.PathType.DIRECTORY, repo.getPathType(path)); - assertEquals("No files should exist in dir yet", repo.listAll(path).length, 0); + assertEquals("No files should exist in dir yet", 0, repo.listAll(path).length); URI subDir = new URI("/test/dir/"); repo.createDirectory(subDir); assertTrue(repo.exists(subDir)); assertEquals(BackupRepository.PathType.DIRECTORY, repo.getPathType(subDir)); - assertEquals("No files should exist in subdir yet", repo.listAll(subDir).length, 0); + assertEquals("No files should exist in subdir yet", 0, repo.listAll(subDir).length); assertEquals( "subDir should now be returned when listing all in parent dir", - repo.listAll(path).length, - 1); + 1, + repo.listAll(path).length); repo.deleteDirectory(path); assertFalse(repo.exists(path)); @@ -341,8 +342,7 @@ private File pullObject(String path) throws IOException { InputStream input = s3.getObject( b -> b.bucket(BUCKET_NAME).key(path), - software.amazon.awssdk.core.sync.ResponseTransformer.toInputStream( - java.time.Duration.ZERO)); + ResponseTransformer.toInputStream(java.time.Duration.ZERO)); FileUtils.copyInputStreamToFile(input, file); return file; } From 98fb17465b49fee38487eaaafacc400f8137ff5f Mon Sep 17 00:00:00 2001 From: SolrBot Date: Tue, 21 Apr 2026 18:52:51 +0000 Subject: [PATCH 07/83] Update changelog entry --- ...593-update-software-amazon-awssdk-to-v2-41-0-branch.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-41-0-branch.yml diff --git a/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-41-0-branch.yml b/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-41-0-branch.yml new file mode 100644 index 000000000000..7599d972d97e --- /dev/null +++ b/changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-41-0-branch.yml @@ -0,0 +1,7 @@ +title: Update software.amazon.awssdk:* to v2.41.0 (branch_9x) - abandoned +type: dependency_update +authors: +- name: solrbot +links: +- name: PR#3593 + url: https://github.com/apache/solr/pull/3593 From 52a5d6eccb1a6e0aaa58996184b068fbd160ed9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 22 Dec 2025 12:42:36 +0100 Subject: [PATCH 08/83] SOLR-18032 Docker gradle build support multi-platform (#3964) (cherry picked from commit 894fcbe15ccbd21127195855e2c959d92c2fa7c2) --- dev-docs/gradle-help/docker.txt | 53 +++++++++++- solr/docker/README.md | 23 +++++ solr/docker/build.gradle | 146 ++++++++++++++++++++++++++++---- 3 files changed, 204 insertions(+), 18 deletions(-) diff --git a/dev-docs/gradle-help/docker.txt b/dev-docs/gradle-help/docker.txt index bd53b6a864d9..05070e63152d 100644 --- a/dev-docs/gradle-help/docker.txt +++ b/dev-docs/gradle-help/docker.txt @@ -26,18 +26,32 @@ Solr Distribution: (Either Full or Slim, the solr binary distribution to build t EnvVar: SOLR_DOCKER_DIST Gradle Property: -Psolr.docker.dist +Docker Platforms: (Comma-separated list of target platforms) + Default: None (builds for current architecture only) + EnvVar: SOLR_DOCKER_PLATFORM + Gradle Property: -Psolr.docker.platform + Examples: + - "linux/amd64" (single platform - can be used with dockerBuild) + - "linux/arm64,linux/amd64" (multi-platform - only supported by dockerPush) + + Note: Multi-platform builds require Docker Buildx and are ONLY supported + by the dockerPush task. The dockerBuild task does not support multi-platform builds because + Docker cannot load multi-platform images into the local daemon. Use dockerPush to build and push + multi-platform images in a single step. You may need QEMU installed to run multi-platform + builds. + Tagging and Pushing ------- To tag the docker image, run the following command. This will also ensure that the docker image has been built as per the inputs detailed above. -gradlew dockerTag +./gradlew dockerTag And to push the image with the given tag, run the following command. Gradle will ensure that the docker image is built and tagged as the inputs describe before being pushed. -gradlew dockerPush +./gradlew dockerPush The docker image tag can be customized via the following options, all accepted via both Environment Variables and Gradle Properties. @@ -61,13 +75,46 @@ Docker Image Name: (Use this to explicitly set a whole image name. If given, the EnvVar: SOLR_DOCKER_IMAGE_NAME Gradle Property: -Psolr.docker.imageName +Multi-Platform Builds +------- + +To build a multi-platform Docker image, use the SOLR_DOCKER_PLATFORM environment variable or Gradle property. + +IMPORTANT: Multi-platform builds (with multiple platforms) are ONLY supported by dockerPush, not dockerBuild. +This is because Docker cannot load multi-platform images into the local daemon. + +Prerequisites: + - Docker Buildx must be installed (included with Docker Desktop and recent Docker Engine versions) + - A buildx builder must be configured. If you don't have one, create it: + docker buildx create --name solr-builder --use + - For cross-platform builds, QEMU may be required + +Build and push for multiple platforms: + SOLR_DOCKER_PLATFORM=linux/arm64,linux/amd64 ./gradlew dockerPush + +Using Gradle property: + ./gradlew dockerPush -Psolr.docker.platform=linux/arm64,linux/amd64 + +The dockerPush task will build and push the multi-platform image in a single step using Docker Buildx. +The task will validate that a suitable builder exists and that it supports all requested platforms before building. + +Single-Platform Builds with Explicit Platform +-------------------------------------------------- + +You can build for a specific platform and load it into your local Docker daemon: + +Build for a specific platform: + SOLR_DOCKER_PLATFORM=linux/arm64 ./gradlew dockerBuild + +This uses Docker Buildx with --load to make the image available locally for testing. + Testing ------- To test the docker image, run the following command. This will also ensure that the docker image has been built as per the inputs detailed above in the "Building" section. -gradlew testDocker +./gradlew testDocker If a docker image build parameters were used during building, then the same inputs must be used while testing. Otherwise a new docker image will be built for the tests to run with. diff --git a/solr/docker/README.md b/solr/docker/README.md index 7a3b7674bd06..93f953515b22 100644 --- a/solr/docker/README.md +++ b/solr/docker/README.md @@ -50,6 +50,29 @@ When building the image, Solr accepts arguments for customization. Currently onl docker build --build-arg BASE_IMAGE=custom/jdk:17-slim -f solr-X.Y.Z/docker/Dockerfile https://www.apache.org/dyn/closer.lua/solr/X.Y.Z/solr-X.Y.Z.tgz ``` +Multi-Platform Builds +---- + +To build multi-platform Docker images from the binary distribution, use Docker Buildx. + +**Prerequisites**: +- Docker Buildx must be installed (included with Docker Desktop and recent Docker Engine) +- A buildx builder must be configured. Create one if needed: + ```bash + docker buildx create --name solr-builder --use + ``` +- For cross-platform builds, QEMU is be required + +**Important**: When building for multiple platforms, you cannot use `--load` to load the image into your local Docker daemon. You must use `--push` to push directly to a registry. + +```bash +# Build and push for multiple platforms +docker buildx build --platform linux/amd64,linux/arm64 -f solr-X.Y.Z/docker/Dockerfile --tag myrepo/solr:X.Y.Z --push - < solr-X.Y.Z.tgz + +# Build for a single specific platform (can use --load for local testing) +docker buildx build --platform linux/arm64 -f solr-X.Y.Z/docker/Dockerfile --tag myrepo/solr:X.Y.Z-arm64 --load - < solr-X.Y.Z.tgz +``` + Official Image Management ---- diff --git a/solr/docker/build.gradle b/solr/docker/build.gradle index 0efa18dc3093..50170ce8ae9b 100644 --- a/solr/docker/build.gradle +++ b/solr/docker/build.gradle @@ -34,6 +34,19 @@ def dockerImageRepo = "${ -> propertyOrEnvOrDefault("solr.docker.imageRepo", "SO def dockerImageTag = "${ -> propertyOrEnvOrDefault("solr.docker.imageTag", "SOLR_DOCKER_IMAGE_TAG", project.version + dockerImageDistSuffix) }" def dockerImageName = "${ -> propertyOrEnvOrDefault("solr.docker.imageName", "SOLR_DOCKER_IMAGE_NAME", "${dockerImageRepo}:${dockerImageTag}${dockerImageTagSuffix}") }" def baseDockerImage = "${ -> propertyOrEnvOrDefault("solr.docker.baseImage", "SOLR_DOCKER_BASE_IMAGE", 'eclipse-temurin:17-jre-jammy') }" +def dockerImagePlatforms = "${ -> propertyOrEnvOrDefault("solr.docker.platform", "SOLR_DOCKER_PLATFORM", '') }" +def isMultiPlatform = { -> !dockerImagePlatforms.isEmpty() && dockerImagePlatforms.contains(',') } +def validatePlatformFormat = { platformValue -> + if (!platformValue.isEmpty()) { + def platformPattern = ~/^linux\/[\w-]+(\/[\w-]+)?(,\s*linux\/[\w-]+(\/[\w-]+)?)*$/ + if (!platformValue.matches(platformPattern)) { + throw new GradleException("Invalid platform format: ${platformValue}\n" + + "Platform must be in format 'linux/arch' or 'linux/arch/variant' (comma-separated for multiple).\n" + + "Examples: 'linux/amd64', 'linux/arm64', 'linux/arm/v7', 'linux/amd64,linux/arm64'") + } + } +} +def getSolrTgzConfiguration = { -> isImageSlim() ? configurations.solrSlimTgz : configurations.solrFullTgz } def officialDockerImageName = {String dist -> "${ -> propertyOrEnvOrDefault("solr.docker.imageName", "SOLR_DOCKER_IMAGE_NAME", "${dockerImageRepo}-official:${dockerImageTag}${distToSuffix(dist)}${dockerImageTagSuffix}") }" } def releaseGpgFingerprint = "${ -> propertyOrDefault('signing.gnupg.keyName',propertyOrDefault('signing.keyId','')) }" @@ -134,6 +147,32 @@ def checksum = { file -> return new DigestUtils(DigestUtils.sha512Digest).digestAsHex(file).trim() } +// Helper method to get Docker Buildx builder info, throws exception if not available +def getBuildxBuilderInfo = { + def builderCheckOutput = new ByteArrayOutputStream() + def builderCheckResult = exec { + ignoreExitValue = true + standardOutput = builderCheckOutput + errorOutput = new ByteArrayOutputStream() + commandLine "docker", "buildx", "inspect" + } + + if (builderCheckResult.exitValue != 0) { + throw new GradleException("Docker Buildx is not available or no builder is configured.\n" + + "Please ensure Docker Buildx is installed and create a builder:\n" + + " docker buildx create --name solr-builder --use\n" + + "Or use an existing builder:\n" + + " docker buildx use ") + } + + return builderCheckOutput.toString() +} + +// Helper method to validate Docker Buildx builder is available +def validateBuildxBuilder = { + getBuildxBuilderInfo() +} + task assemblePackaging(type: Sync) { description = 'Assemble docker scripts and Dockerfile for Solr Packaging' @@ -158,21 +197,51 @@ task dockerBuild() { // Ensure that the docker image is rebuilt on build-arg changes or changes in the docker context inputs.properties([ - baseDockerImage: baseDockerImage + baseDockerImage: baseDockerImage, + dockerImagePlatforms: dockerImagePlatforms ]) - var solrTgzConfiguration = isImageSlim() ? configurations.solrSlimTgz : configurations.solrFullTgz + var solrTgzConfiguration = getSolrTgzConfiguration() inputs.files(solrTgzConfiguration) inputs.property("isSlimImage", isImageSlim()) + inputs.property("isMultiPlatform", isMultiPlatform()) dependsOn(solrTgzConfiguration) doLast { - exec { - standardInput = solrTgzConfiguration.singleFile.newDataInputStream() - commandLine "docker", "build", - "-f", "solr-${ -> project.version }${dockerImageDistSuffix}/docker/Dockerfile", - "--iidfile", imageIdFile, - "--build-arg", "BASE_IMAGE=${ -> inputs.properties.baseDockerImage}", - "-" + def platformValue = inputs.properties.dockerImagePlatforms.toString().trim() + validatePlatformFormat(platformValue) + def useSinglePlatform = !platformValue.isEmpty() && !platformValue.contains(',') + + // Multi-platform builds are not supported by dockerBuild, only by dockerPush + if (isMultiPlatform()) { + throw new GradleException("Multi-platform builds (SOLR_DOCKER_PLATFORM with multiple platforms) are not supported by dockerBuild.\n" + + "Please use 'dockerPush' instead, which will build and push the multi-platform image in a single step.\n" + + "Example: SOLR_DOCKER_PLATFORM=linux/arm64,linux/amd64 ./gradlew dockerPush") + } + + if (useSinglePlatform) { + // Single-platform build with explicit platform using buildx + validateBuildxBuilder() + + exec { + standardInput = solrTgzConfiguration.singleFile.newDataInputStream() + commandLine "docker", "buildx", "build", + "-f", "solr-${ -> project.version }${dockerImageDistSuffix}/docker/Dockerfile", + "--platform", platformValue, + "--build-arg", "BASE_IMAGE=${ -> inputs.properties.baseDockerImage}", + "--iidfile", imageIdFile, + "--load", + "-" + } + } else { + // Standard build for current architecture + exec { + standardInput = solrTgzConfiguration.singleFile.newDataInputStream() + commandLine "docker", "build", + "-f", "solr-${ -> project.version }${dockerImageDistSuffix}/docker/Dockerfile", + "--iidfile", imageIdFile, + "--build-arg", "BASE_IMAGE=${ -> inputs.properties.baseDockerImage}", + "-" + } } } @@ -180,10 +249,13 @@ task dockerBuild() { doLast { def dockerImageId = file(imageIdFile).text project.logger.lifecycle("Solr Docker Image Created") - project.logger.lifecycle("\tID: \t${ -> dockerImageId }") + project.logger.lifecycle("\tID/Ref: \t${ -> dockerImageId }") project.logger.lifecycle("\tBase Image: \t${ -> baseDockerImage }") project.logger.lifecycle("\tSolr Version: \t${ -> project.version }") project.logger.lifecycle("\tSolr Distribution: \t${isImageSlim() ? "Slim" : "Full"}") + if (!dockerImagePlatforms.isEmpty()) { + project.logger.lifecycle("\tPlatforms: \t${ -> dockerImagePlatforms }") + } } outputs.files(imageIdFile) @@ -239,6 +311,7 @@ task dockerPush(dependsOn: tasks.dockerTag) { // Ensure that the docker image is re-pushed if the image ID or tag changes inputs.properties([ dockerImageName: dockerImageName, + dockerImagePlatforms: dockerImagePlatforms ]) inputs.file(imageIdFile) @@ -246,12 +319,55 @@ task dockerPush(dependsOn: tasks.dockerTag) { mustRunAfter tasks.testDocker doLast { - exec { - commandLine "docker", "push", dockerImageName - } + def platformValue = inputs.properties.dockerImagePlatforms.toString().trim() + validatePlatformFormat(platformValue) + + if (isMultiPlatform()) { + // Multi-platform push requires buildx and rebuilding with --push + + // Get builder info and validate it exists + def builderInfo = getBuildxBuilderInfo() + + // Verify the builder supports the requested platforms (fail fast) + def requestedPlatforms = platformValue.split(",").collect { it.trim() } + def missingPlatforms = [] + requestedPlatforms.each { platform -> + // Check for exact platform match - use delimiters instead of word boundaries + // Match platform at start/end or surrounded by commas/spaces + def platformPattern = "(?:^|[,\\s])" + Pattern.quote(platform) + "(?:[,\\s]|\$)" + if (!(builderInfo =~ platformPattern).find()) { + missingPlatforms.add(platform) + } + } - // Print information on the image after it has been created - project.logger.lifecycle("Solr Docker Image Pushed: \t$dockerImageName") + if (!missingPlatforms.isEmpty()) { + // Extract supported platforms from builder info using matcher + def supportedPlatforms = (builderInfo =~ /linux\/[\w-]+(?:\/[\w-]+)?/).collect().join(', ') + throw new GradleException("Current builder does not support all requested platforms: ${missingPlatforms}\n" + + "Supported platforms: ${supportedPlatforms}\n" + + "To add platform support, Install QEMU with proper binfmt support.") + } + + def solrTgzConfiguration = getSolrTgzConfiguration() + exec { + standardInput = solrTgzConfiguration.singleFile.newDataInputStream() + commandLine "docker", "buildx", "build", + "-f", "solr-${ -> project.version }${dockerImageDistSuffix}/docker/Dockerfile", + "--platform", platformValue, + "--build-arg", "BASE_IMAGE=${ -> baseDockerImage}", + "--tag", dockerImageName, + "--push", + "-" + } + project.logger.lifecycle("Solr Docker Multi-Platform Image Pushed: \t$dockerImageName") + project.logger.lifecycle("\tPlatforms: \t$platformValue") + } else { + // Standard push (works for both standard builds and single-platform buildx builds) + exec { + commandLine "docker", "push", dockerImageName + } + project.logger.lifecycle("Solr Docker Image Pushed: \t$dockerImageName") + } } } From 3fd909ef4dd5d5102c97c8e066dc9e921629b2b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 6 Jan 2026 11:21:09 +0100 Subject: [PATCH 09/83] Fix failing errorprone rule in FilterFunctionTest.java --- .../solr/analytics/function/mapping/FilterFunctionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/modules/analytics/src/test/org/apache/solr/analytics/function/mapping/FilterFunctionTest.java b/solr/modules/analytics/src/test/org/apache/solr/analytics/function/mapping/FilterFunctionTest.java index 62c7d278d9f2..271163efc4a3 100644 --- a/solr/modules/analytics/src/test/org/apache/solr/analytics/function/mapping/FilterFunctionTest.java +++ b/solr/modules/analytics/src/test/org/apache/solr/analytics/function/mapping/FilterFunctionTest.java @@ -715,7 +715,7 @@ public void multiValueFloatTest() { fail("There should be no values to stream"); }); - val.setValues(50.343F, -74.9874F, 2.34233242E+9f); + val.setValues(50.343F, -74.9874F, 2.3423324E+9f); filter.setExists(false); func.streamFloats( value -> { From e065f74e331a63c7bf21571c7da61f5c1e7ecdcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 22 Dec 2025 23:05:56 +0100 Subject: [PATCH 10/83] SOLR-18032 Use SOLR_DOCKER_PLATFORM only for dockerPush command (#3973) (cherry picked from commit 7694064fced5b4d649c87faa5e6c1765a6af306a) --- dev-docs/gradle-help/docker.txt | 30 ++++------------- solr/docker/README.md | 5 +++ solr/docker/build.gradle | 58 ++++++--------------------------- 3 files changed, 22 insertions(+), 71 deletions(-) diff --git a/dev-docs/gradle-help/docker.txt b/dev-docs/gradle-help/docker.txt index 05070e63152d..18ee78654089 100644 --- a/dev-docs/gradle-help/docker.txt +++ b/dev-docs/gradle-help/docker.txt @@ -26,19 +26,15 @@ Solr Distribution: (Either Full or Slim, the solr binary distribution to build t EnvVar: SOLR_DOCKER_DIST Gradle Property: -Psolr.docker.dist -Docker Platforms: (Comma-separated list of target platforms) - Default: None (builds for current architecture only) +Docker Platforms: (Comma-separated list of target platforms for dockerPush) + Default: None (pushes image built for current architecture) EnvVar: SOLR_DOCKER_PLATFORM Gradle Property: -Psolr.docker.platform Examples: - - "linux/amd64" (single platform - can be used with dockerBuild) - - "linux/arm64,linux/amd64" (multi-platform - only supported by dockerPush) + - "linux/amd64" (single platform) + - "linux/arm64,linux/amd64" (multi-platform) - Note: Multi-platform builds require Docker Buildx and are ONLY supported - by the dockerPush task. The dockerBuild task does not support multi-platform builds because - Docker cannot load multi-platform images into the local daemon. Use dockerPush to build and push - multi-platform images in a single step. You may need QEMU installed to run multi-platform - builds. + Note: This setting ONLY affects the dockerPush task Tagging and Pushing ------- @@ -78,10 +74,8 @@ Docker Image Name: (Use this to explicitly set a whole image name. If given, the Multi-Platform Builds ------- -To build a multi-platform Docker image, use the SOLR_DOCKER_PLATFORM environment variable or Gradle property. - -IMPORTANT: Multi-platform builds (with multiple platforms) are ONLY supported by dockerPush, not dockerBuild. -This is because Docker cannot load multi-platform images into the local daemon. +Multi-platform Docker builds are ONLY supported by the dockerPush task. +The dockerBuild task always builds for your current local architecture. Prerequisites: - Docker Buildx must be installed (included with Docker Desktop and recent Docker Engine versions) @@ -98,16 +92,6 @@ Using Gradle property: The dockerPush task will build and push the multi-platform image in a single step using Docker Buildx. The task will validate that a suitable builder exists and that it supports all requested platforms before building. -Single-Platform Builds with Explicit Platform --------------------------------------------------- - -You can build for a specific platform and load it into your local Docker daemon: - -Build for a specific platform: - SOLR_DOCKER_PLATFORM=linux/arm64 ./gradlew dockerBuild - -This uses Docker Buildx with --load to make the image available locally for testing. - Testing ------- diff --git a/solr/docker/README.md b/solr/docker/README.md index 93f953515b22..3dcb0de914a5 100644 --- a/solr/docker/README.md +++ b/solr/docker/README.md @@ -73,6 +73,11 @@ docker buildx build --platform linux/amd64,linux/arm64 -f solr-X.Y.Z/docker/Dock docker buildx build --platform linux/arm64 -f solr-X.Y.Z/docker/Dockerfile --tag myrepo/solr:X.Y.Z-arm64 --load - < solr-X.Y.Z.tgz ``` +**Note**: When building Docker images using Gradle tasks (`./gradlew dockerBuild`, `./gradlew dockerPush`), +the `dockerBuild` task always builds for your current local architecture only. Multi-platform and +single-platform targeting is only supported by the `dockerPush` task via the `SOLR_DOCKER_PLATFORM` +environment variable or property `-Psolr.docker.platform`. See `./gradlew helpDocker` for details. + Official Image Management ---- diff --git a/solr/docker/build.gradle b/solr/docker/build.gradle index 50170ce8ae9b..6d7d287bace2 100644 --- a/solr/docker/build.gradle +++ b/solr/docker/build.gradle @@ -168,11 +168,6 @@ def getBuildxBuilderInfo = { return builderCheckOutput.toString() } -// Helper method to validate Docker Buildx builder is available -def validateBuildxBuilder = { - getBuildxBuilderInfo() -} - task assemblePackaging(type: Sync) { description = 'Assemble docker scripts and Dockerfile for Solr Packaging' @@ -197,51 +192,21 @@ task dockerBuild() { // Ensure that the docker image is rebuilt on build-arg changes or changes in the docker context inputs.properties([ - baseDockerImage: baseDockerImage, - dockerImagePlatforms: dockerImagePlatforms + baseDockerImage: baseDockerImage ]) - var solrTgzConfiguration = getSolrTgzConfiguration() + var solrTgzConfiguration = isImageSlim() ? configurations.solrSlimTgz : configurations.solrFullTgz inputs.files(solrTgzConfiguration) inputs.property("isSlimImage", isImageSlim()) - inputs.property("isMultiPlatform", isMultiPlatform()) dependsOn(solrTgzConfiguration) doLast { - def platformValue = inputs.properties.dockerImagePlatforms.toString().trim() - validatePlatformFormat(platformValue) - def useSinglePlatform = !platformValue.isEmpty() && !platformValue.contains(',') - - // Multi-platform builds are not supported by dockerBuild, only by dockerPush - if (isMultiPlatform()) { - throw new GradleException("Multi-platform builds (SOLR_DOCKER_PLATFORM with multiple platforms) are not supported by dockerBuild.\n" + - "Please use 'dockerPush' instead, which will build and push the multi-platform image in a single step.\n" + - "Example: SOLR_DOCKER_PLATFORM=linux/arm64,linux/amd64 ./gradlew dockerPush") - } - - if (useSinglePlatform) { - // Single-platform build with explicit platform using buildx - validateBuildxBuilder() - - exec { - standardInput = solrTgzConfiguration.singleFile.newDataInputStream() - commandLine "docker", "buildx", "build", - "-f", "solr-${ -> project.version }${dockerImageDistSuffix}/docker/Dockerfile", - "--platform", platformValue, - "--build-arg", "BASE_IMAGE=${ -> inputs.properties.baseDockerImage}", - "--iidfile", imageIdFile, - "--load", - "-" - } - } else { - // Standard build for current architecture - exec { - standardInput = solrTgzConfiguration.singleFile.newDataInputStream() - commandLine "docker", "build", - "-f", "solr-${ -> project.version }${dockerImageDistSuffix}/docker/Dockerfile", - "--iidfile", imageIdFile, - "--build-arg", "BASE_IMAGE=${ -> inputs.properties.baseDockerImage}", - "-" - } + exec { + standardInput = solrTgzConfiguration.singleFile.newDataInputStream() + commandLine "docker", "build", + "-f", "solr-${ -> project.version }${dockerImageDistSuffix}/docker/Dockerfile", + "--iidfile", imageIdFile, + "--build-arg", "BASE_IMAGE=${ -> inputs.properties.baseDockerImage}", + "-" } } @@ -249,13 +214,10 @@ task dockerBuild() { doLast { def dockerImageId = file(imageIdFile).text project.logger.lifecycle("Solr Docker Image Created") - project.logger.lifecycle("\tID/Ref: \t${ -> dockerImageId }") + project.logger.lifecycle("\tID: \t${ -> dockerImageId }") project.logger.lifecycle("\tBase Image: \t${ -> baseDockerImage }") project.logger.lifecycle("\tSolr Version: \t${ -> project.version }") project.logger.lifecycle("\tSolr Distribution: \t${isImageSlim() ? "Slim" : "Full"}") - if (!dockerImagePlatforms.isEmpty()) { - project.logger.lifecycle("\tPlatforms: \t${ -> dockerImagePlatforms }") - } } outputs.files(imageIdFile) From 7c790d36ecc57e70f855f02a8b4710df7ef4fd00 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Tue, 6 Jan 2026 06:19:38 -0500 Subject: [PATCH 11/83] Replace HTTP-method string-handling with enum A number of places in Solr referenced the HTTP method used to make the request, often inspecting the value as a string. This is needlessly brittle (e.g. case sensitivity issues). This commit fixes this problem by parsing the method value into an enum, which can then be used by later inspection. --- .../solrj/embedded/EmbeddedSolrServer.java | 4 +- .../apache/solr/handler/SchemaHandler.java | 46 +++++++++++-------- .../solr/handler/SolrConfigHandler.java | 40 +++++++++------- .../handler/admin/SecurityConfHandler.java | 7 +-- .../apache/solr/request/SolrQueryRequest.java | 3 +- .../solr/servlet/SolrRequestParsers.java | 7 ++- .../admin/SecurityConfHandlerTest.java | 17 +++---- .../solr/servlet/SolrRequestParserTest.java | 22 +++++++++ .../MirroringConfigSetsHandlerTest.java | 2 +- .../apache/solr/client/solrj/SolrRequest.java | 18 +++++++- 10 files changed, 113 insertions(+), 53 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java index 45aaea07d1d0..82edafe2b7b7 100644 --- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java +++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java @@ -171,7 +171,7 @@ public NamedList request(SolrRequest request, String coreName) SolrQueryRequest req = _parser.buildRequestFrom( null, getParams(request), getContentStreams(request), request.getUserPrincipal()); - req.getContext().put("httpMethod", request.getMethod().name()); + req.getContext().put("httpMethod", request.getMethod()); req.getContext().put(PATH, path); SolrQueryResponse resp = new SolrQueryResponse(); handler.handleRequest(req, resp); @@ -224,7 +224,7 @@ public NamedList request(SolrRequest request, String coreName) .buildRequestFrom( core, params, getContentStreams(request), request.getUserPrincipal()); req.getContext().put(PATH, path); - req.getContext().put("httpMethod", request.getMethod().name()); + req.getContext().put("httpMethod", request.getMethod()); SolrQueryResponse rsp = new SolrQueryResponse(); SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java index c81fa8eb5724..86a5c8b2323b 100644 --- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java @@ -36,6 +36,7 @@ import org.apache.solr.api.Api; import org.apache.solr.api.ApiBag; import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.SolrParams; @@ -77,26 +78,33 @@ public class SchemaHandler extends RequestHandlerBase @Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { RequestHandlerUtils.setWt(req, JSON); - String httpMethod = (String) req.getContext().get("httpMethod"); - if ("POST".equals(httpMethod)) { - if (isImmutableConfigSet) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet is immutable"); - } - if (req.getContentStreams() == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no stream"); - } - - try { - List> errs = new SchemaManager(req).performOperations(); - if (!errs.isEmpty()) - throw new ApiBag.ExceptionWithErrObject( - SolrException.ErrorCode.BAD_REQUEST, "error processing commands", errs); - } catch (IOException e) { + final SolrRequest.METHOD httpMethod = (SolrRequest.METHOD) req.getContext().get("httpMethod"); + switch (httpMethod) { + case POST: + if (isImmutableConfigSet) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet is immutable"); + } + if (req.getContentStreams() == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no stream"); + } + try { + List> errs = new SchemaManager(req).performOperations(); + if (!errs.isEmpty()) + throw new ApiBag.ExceptionWithErrObject( + SolrException.ErrorCode.BAD_REQUEST, "error processing commands", errs); + } catch (IOException e) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Error reading input String " + e.getMessage(), + e); + } + break; + case GET: + handleGET(req, rsp); + break; + default: throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Error reading input String " + e.getMessage(), e); - } - } else { - handleGET(req, rsp); + SolrException.ErrorCode.BAD_REQUEST, "Unexpected HTTP method: " + httpMethod); } } diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index 2e82dd375f99..a9cb278b2f05 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -131,24 +131,30 @@ public Lock getReloadLock() { public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { RequestHandlerUtils.setWt(req, CommonParams.JSON); - String httpMethod = (String) req.getContext().get("httpMethod"); - Command command = new Command(req, rsp, httpMethod); - if ("POST".equals(httpMethod)) { - if (configEditing_disabled || isImmutableConfigSet) { - final String reason = - configEditing_disabled - ? "due to " + CONFIGSET_EDITING_DISABLED_ARG - : "because ConfigSet is immutable"; + final var httpMethod = (SolrRequest.METHOD) req.getContext().get("httpMethod"); + Command command = new Command(req, rsp, httpMethod.name()); + switch (httpMethod) { + case POST: + if (configEditing_disabled || isImmutableConfigSet) { + final String reason = + configEditing_disabled + ? "due to " + CONFIGSET_EDITING_DISABLED_ARG + : "because ConfigSet is immutable"; + throw new SolrException( + SolrException.ErrorCode.FORBIDDEN, " solrconfig editing is not enabled " + reason); + } + try { + command.handlePOST(); + } finally { + RequestHandlerUtils.addExperimentalFormatWarning(rsp); + } + break; + case GET: + command.handleGET(); + break; + default: throw new SolrException( - SolrException.ErrorCode.FORBIDDEN, " solrconfig editing is not enabled " + reason); - } - try { - command.handlePOST(); - } finally { - RequestHandlerUtils.addExperimentalFormatWarning(rsp); - } - } else { - command.handleGET(); + SolrException.ErrorCode.BAD_REQUEST, "Unexpected HTTP method: " + httpMethod); } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java index 95e1d10f3510..f5b528f3d703 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java @@ -33,6 +33,7 @@ import org.apache.solr.api.Api; import org.apache.solr.api.ApiBag; import org.apache.solr.api.ApiBag.ReqHandlerToApi; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.common.SolrException; import org.apache.solr.common.SpecProvider; import org.apache.solr.common.params.CommonParams; @@ -81,12 +82,12 @@ public PermissionNameProvider.Name getPermissionName(AuthorizationContext ctx) { @Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { RequestHandlerUtils.setWt(req, CommonParams.JSON); - String httpMethod = (String) req.getContext().get("httpMethod"); + final var httpMethod = (SolrRequest.METHOD) req.getContext().get("httpMethod"); String path = (String) req.getContext().get("path"); String key = path.substring(path.lastIndexOf('/') + 1); - if ("GET".equals(httpMethod)) { + if (SolrRequest.METHOD.GET.equals(httpMethod)) { getConf(rsp, key); - } else if ("POST".equals(httpMethod)) { + } else if (SolrRequest.METHOD.POST.equals(httpMethod)) { Object plugin = getPlugin(key); doEdit(req, rsp, path, key, plugin); } diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java index bf571619e3e0..5c139751f7b1 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java +++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.SolrParams; @@ -165,7 +166,7 @@ default List getCommands(boolean validateInput) { } default String getHttpMethod() { - return (String) getContext().get("httpMethod"); + return ((SolrRequest.METHOD) getContext().get("httpMethod")).name(); } default HttpSolrCall getHttpSolrCall() { diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java index 83dcb65c9b2a..dfc7e0565857 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java @@ -45,6 +45,7 @@ import org.apache.commons.io.input.CloseShieldInputStream; import org.apache.lucene.util.IOUtils; import org.apache.solr.api.V2HttpCall; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.CommonParams; @@ -173,7 +174,11 @@ public SolrQueryRequest parse(SolrCore core, String path, HttpServletRequest req // Handlers and login will want to know the path. If it contains a ':' // the handler could use it for RESTful URLs sreq.getContext().put(PATH, RequestHandlers.normalize(path)); - sreq.getContext().put("httpMethod", req.getMethod()); + + final var methodStr = req.getMethod(); + final var parsedMethod = + SolrRequest.METHOD.fromString(methodStr); // Throws error if method not recognized + sreq.getContext().put("httpMethod", parsedMethod); if (addHttpRequestToContext) { sreq.getContext().put("httpRequest", req); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerTest.java index c9335b2a7b65..5a971eb7dd6a 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerTest.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.ContentStreamBase; @@ -44,7 +45,7 @@ public void testEdit() throws Exception { + "'set-user':{ 'tom':'TomIsUberCool'}\n" + "}"; LocalSolrQueryRequest req = new LocalSolrQueryRequest(null, new ModifiableSolrParams()); - req.getContext().put("httpMethod", "POST"); + req.getContext().put("httpMethod", SolrRequest.METHOD.POST); req.getContext().put("path", "/admin/authentication"); ContentStreamBase.ByteArrayStream o = new ContentStreamBase.ByteArrayStream(command.getBytes(StandardCharsets.UTF_8), ""); @@ -77,7 +78,7 @@ public void testEdit() throws Exception { + "}"; req = new LocalSolrQueryRequest(null, new ModifiableSolrParams()); - req.getContext().put("httpMethod", "POST"); + req.getContext().put("httpMethod", SolrRequest.METHOD.POST); req.getContext().put("path", "/admin/authorization"); o = new ContentStreamBase.ByteArrayStream(command.getBytes(StandardCharsets.UTF_8), ""); req.setContentStreams(Collections.singletonList(o)); @@ -101,7 +102,7 @@ public void testEdit() throws Exception { + " 'role': ['admin','dev']\n" + " }}"; req = new LocalSolrQueryRequest(null, new ModifiableSolrParams()); - req.getContext().put("httpMethod", "POST"); + req.getContext().put("httpMethod", SolrRequest.METHOD.POST); req.getContext().put("path", "/admin/authorization"); o = new ContentStreamBase.ByteArrayStream(command.getBytes(StandardCharsets.UTF_8), ""); req.setContentStreams(Collections.singletonList(o)); @@ -122,7 +123,7 @@ public void testEdit() throws Exception { + " 'role': ['guest','admin']\n" + " }}"; req = new LocalSolrQueryRequest(null, new ModifiableSolrParams()); - req.getContext().put("httpMethod", "POST"); + req.getContext().put("httpMethod", SolrRequest.METHOD.POST); req.getContext().put("path", "/admin/authorization"); o = new ContentStreamBase.ByteArrayStream(command.getBytes(StandardCharsets.UTF_8), ""); req.setContentStreams(Collections.singletonList(o)); @@ -139,7 +140,7 @@ public void testEdit() throws Exception { command = "{\n" + "delete-permission: 1,\n" + " set-user-role : { tom :null}\n" + "}"; req = new LocalSolrQueryRequest(null, new ModifiableSolrParams()); - req.getContext().put("httpMethod", "POST"); + req.getContext().put("httpMethod", SolrRequest.METHOD.POST); req.getContext().put("path", "/admin/authorization"); o = new ContentStreamBase.ByteArrayStream(command.getBytes(StandardCharsets.UTF_8), ""); req.setContentStreams(Collections.singletonList(o)); @@ -163,7 +164,7 @@ public void testEdit() throws Exception { + " 'role': 'admin'\n" + " }}"; req = new LocalSolrQueryRequest(null, new ModifiableSolrParams()); - req.getContext().put("httpMethod", "POST"); + req.getContext().put("httpMethod", SolrRequest.METHOD.POST); req.getContext().put("path", "/admin/authorization"); o = new ContentStreamBase.ByteArrayStream(command.getBytes(StandardCharsets.UTF_8), ""); req.setContentStreams(Collections.singletonList(o)); @@ -237,7 +238,7 @@ protected boolean persistConf(SecurityConfig props) { public String getStandardJson() throws Exception { String command = "{\n" + "'set-user': {'solr':'SolrRocks'}\n" + "}"; LocalSolrQueryRequest req = new LocalSolrQueryRequest(null, new ModifiableSolrParams()); - req.getContext().put("httpMethod", "POST"); + req.getContext().put("httpMethod", SolrRequest.METHOD.POST); req.getContext().put("path", "/admin/authentication"); ContentStreamBase.ByteArrayStream o = new ContentStreamBase.ByteArrayStream(command.getBytes(StandardCharsets.UTF_8), ""); @@ -249,7 +250,7 @@ public String getStandardJson() throws Exception { + "'set-permission':{'name': 'security-edit', 'role': 'admin'}" + "}"; req = new LocalSolrQueryRequest(null, new ModifiableSolrParams()); - req.getContext().put("httpMethod", "POST"); + req.getContext().put("httpMethod", SolrRequest.METHOD.POST); req.getContext().put("path", "/admin/authorization"); o = new ContentStreamBase.ByteArrayStream(command.getBytes(StandardCharsets.UTF_8), ""); req.setContentStreams(Collections.singletonList(o)); diff --git a/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java b/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java index e86e6c12c4b3..2c7c947e2ca4 100644 --- a/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java +++ b/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java @@ -263,6 +263,28 @@ public void testStandardParseParamsAndFillStreams() throws Exception { } } + @Test + public void testReportsErrorForUnexpectedHttpMethod() throws Exception { + final String getParams = "q=hello"; + HttpServletRequest request = getMock("/solr/select", "application/x-www-form-urlencoded", 0); + when(request.getMethod()).thenReturn("UNEXPECTED"); + when(request.getQueryString()).thenReturn(getParams); + + MultipartRequestParser multipart = new MultipartRequestParser(2048); + RawRequestParser raw = new RawRequestParser(); + FormDataRequestParser formdata = new FormDataRequestParser(2048); + StandardRequestParser standard = new StandardRequestParser(multipart, raw, formdata); + + final SolrException thrown = + expectThrows( + SolrException.class, + () -> { + parser.parse(h.getCore(), "/select", request); + }); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, thrown.code()); + assertEquals("Request contained unexpected HTTP method: UNEXPECTED", thrown.getMessage()); + } + static class ByteServletInputStream extends ServletInputStream { final BufferedInputStream in; final int len; diff --git a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/handler/MirroringConfigSetsHandlerTest.java b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/handler/MirroringConfigSetsHandlerTest.java index 21a8d378208d..ad93644e7f3c 100644 --- a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/handler/MirroringConfigSetsHandlerTest.java +++ b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/handler/MirroringConfigSetsHandlerTest.java @@ -91,7 +91,7 @@ private static SolrQueryRequest createRequest( List.of(new ContentStreamBase.ByteArrayStream(content, configSetName, "application/zip")); req.setContentStreams(streams); } - req.getContext().put("httpMethod", method); + req.getContext().put("httpMethod", SolrRequest.METHOD.fromString(method)); return req; } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java index 49c52f18d36b..03b23d5dcdda 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java @@ -23,11 +23,13 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.impl.HttpSolrClientBase; import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStream; @@ -48,9 +50,23 @@ public Principal getUserPrincipal() { public enum METHOD { GET, + HEAD, POST, PUT, - DELETE + DELETE; + + /** + * Returns the METHOD enum value matching the provided string, or 'null' if no match is found. + */ + public static METHOD fromString(String methodStr) { + try { + return METHOD.valueOf(methodStr.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Request contained unexpected HTTP method: " + methodStr); + } + } }; public enum ApiVersion { From 6beb67905adc0109b18f4d28379cee5ea03de25d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 6 Jan 2026 14:22:42 +0100 Subject: [PATCH 12/83] Remove faulty override of renovate packageRules --- .github/renovate.json | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/renovate.json b/.github/renovate.json index 12dd2eaa1fb7..6859a96c7acc 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -10,12 +10,5 @@ "prConcurrentLimit": 100, "prHourlyLimit": 10, "branchPrefix": "renovate-9x/", - "commitMessageSuffix": " (branch_9x)", - "packageRules": [ - { - "description": "Skip errorprone upgrades since newer versions require JDK17", - "matchPackagePrefixes": ["com.google.errorprone"], - "enabled": false - } - ] + "commitMessageSuffix": " (branch_9x)" } From e23688efd2e778bfe49fcca02a94c45a25aab18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 6 Jan 2026 14:46:58 +0100 Subject: [PATCH 13/83] Run --write-locks in renovate for branch_9x --- .github/renovate.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/renovate.json b/.github/renovate.json index 6859a96c7acc..539e69a56700 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -2,7 +2,10 @@ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "includePaths": ["versions.*", "build.gradle", ".github/workflows/*"], "postUpgradeTasks": { - "commands": ["./gradlew updateLicenses"], + "commands": [ + "./gradlew --write-locks", + "./gradlew updateLicenses" + ], "fileFilters": ["solr/licenses/*.sha1"], "executionMode": "branch" }, From 1d5d2e54e43233d73e62c2fa723711f385fe0d89 Mon Sep 17 00:00:00 2001 From: Solr Bot <125606113+solrbot@users.noreply.github.com> Date: Tue, 6 Jan 2026 20:43:37 +0100 Subject: [PATCH 14/83] Update actions/cache action to v5 (branch_9x) (#4030) --- .github/workflows/bin-solr-test.yml | 2 +- .github/workflows/docker-test.yml | 2 +- .github/workflows/gradle-extraction-check.yml | 2 +- .github/workflows/gradle-precommit.yml | 2 +- .github/workflows/solrj-test.yml | 2 +- .../unreleased/PR#4030-update-actions-cache-action.yml | 7 +++++++ 6 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/PR#4030-update-actions-cache-action.yml diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml index 9d93d7ea3720..b3fec816ab03 100644 --- a/.github/workflows/bin-solr-test.yml +++ b/.github/workflows/bin-solr-test.yml @@ -34,7 +34,7 @@ jobs: uses: gradle/actions/setup-gradle@v5 - name: Grant execute permission for gradlew run: chmod +x gradlew - - uses: actions/cache@v4 + - uses: actions/cache@v5 with: path: | ~/.gradle/caches diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index ced7190c49f4..6f58b4e67db9 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -38,7 +38,7 @@ jobs: run: sudo apt-get install acl - name: Grant execute permission for gradlew run: chmod +x gradlew - - uses: actions/cache@v4 + - uses: actions/cache@v5 with: path: | ~/.gradle/caches diff --git a/.github/workflows/gradle-extraction-check.yml b/.github/workflows/gradle-extraction-check.yml index 29a8b07f660e..f77c813913e2 100644 --- a/.github/workflows/gradle-extraction-check.yml +++ b/.github/workflows/gradle-extraction-check.yml @@ -35,7 +35,7 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - - uses: actions/cache@v4 + - uses: actions/cache@v5 with: path: | ~/.gradle/caches diff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.yml index 46050e640759..ad9e3f4850d1 100644 --- a/.github/workflows/gradle-precommit.yml +++ b/.github/workflows/gradle-precommit.yml @@ -32,7 +32,7 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - - uses: actions/cache@v4 + - uses: actions/cache@v5 with: path: | ~/.gradle/caches diff --git a/.github/workflows/solrj-test.yml b/.github/workflows/solrj-test.yml index 8cbbdebea70d..7203a75a69f6 100644 --- a/.github/workflows/solrj-test.yml +++ b/.github/workflows/solrj-test.yml @@ -31,7 +31,7 @@ jobs: uses: gradle/actions/setup-gradle@v5 - name: Grant execute permission for gradlew run: chmod +x gradlew - - uses: actions/cache@v4 + - uses: actions/cache@v5 with: path: | ~/.gradle/caches diff --git a/changelog/unreleased/PR#4030-update-actions-cache-action.yml b/changelog/unreleased/PR#4030-update-actions-cache-action.yml new file mode 100644 index 000000000000..ba67cc21f8ee --- /dev/null +++ b/changelog/unreleased/PR#4030-update-actions-cache-action.yml @@ -0,0 +1,7 @@ +title: Update actions/cache action to v5 +type: dependency_update +authors: +- name: solrbot +links: +- name: PR#4030 + url: https://github.com/apache/solr/pull/4030 From ebd2046fec65eb2d592474e4221dc35f8f2b3601 Mon Sep 17 00:00:00 2001 From: Solr Bot <125606113+solrbot@users.noreply.github.com> Date: Tue, 6 Jan 2026 20:44:44 +0100 Subject: [PATCH 15/83] Update actions/upload-artifact action to v6 (branch_9x) (#4031) --- .github/workflows/bin-solr-test.yml | 2 +- .../PR#4031-update-actions-upload-artifact-action.yml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/PR#4031-update-actions-upload-artifact-action.yml diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml index b3fec816ab03..959523510c71 100644 --- a/.github/workflows/bin-solr-test.yml +++ b/.github/workflows/bin-solr-test.yml @@ -46,7 +46,7 @@ jobs: run: ./gradlew integrationTests - name: Archive logs if: ${{ failure() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: logs path: solr/packaging/build/test-output diff --git a/changelog/unreleased/PR#4031-update-actions-upload-artifact-action.yml b/changelog/unreleased/PR#4031-update-actions-upload-artifact-action.yml new file mode 100644 index 000000000000..77a281b3a23b --- /dev/null +++ b/changelog/unreleased/PR#4031-update-actions-upload-artifact-action.yml @@ -0,0 +1,7 @@ +title: Update actions/upload-artifact action to v6 +type: dependency_update +authors: +- name: solrbot +links: +- name: PR#4031 + url: https://github.com/apache/solr/pull/4031 From 5122624951bd85a95f709f4d80a66a0baa7ec32b Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 8 Jan 2026 07:42:16 -0500 Subject: [PATCH 16/83] SOLR-17438: Remove 'json' from requirements.txt --- dev-tools/scripts/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-tools/scripts/requirements.txt b/dev-tools/scripts/requirements.txt index 845a819fabb4..5709c843fe49 100644 --- a/dev-tools/scripts/requirements.txt +++ b/dev-tools/scripts/requirements.txt @@ -6,4 +6,3 @@ ics~=0.7.2 console-menu~=0.7.1 PyGithub~=1.56 jira~=3.4.1 -json \ No newline at end of file From 7922e83fc68dbea68b28b8f912206649fe258f81 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 8 Jan 2026 09:53:11 -0500 Subject: [PATCH 17/83] Logchange release on branch_9_10 - rm changelog/unreleased --- changelog/unreleased/.gitkeep | 0 .../PR#3926-tika-server-idle-timeout-fix.yml | 9 --------- changelog/unreleased/PR#3963-update-log4j.yml | 9 --------- ...-17888-mitigate-tika-cve-disable-xfa-parsing.yml | 9 --------- .../unreleased/SOLR-17972-distributed-lock-lead.yml | 7 ------- .../SOLR-17985-fix-slow-no-rows-queries.yml | 10 ---------- ...erse dist sorting on LatLonPointSpatialField.yml | 13 ------------- ...allel-http-shard-handler-failure-propagation.yml | 7 ------- 8 files changed, 64 deletions(-) delete mode 100644 changelog/unreleased/.gitkeep delete mode 100644 changelog/unreleased/PR#3926-tika-server-idle-timeout-fix.yml delete mode 100644 changelog/unreleased/PR#3963-update-log4j.yml delete mode 100644 changelog/unreleased/SOLR-17888-mitigate-tika-cve-disable-xfa-parsing.yml delete mode 100644 changelog/unreleased/SOLR-17972-distributed-lock-lead.yml delete mode 100644 changelog/unreleased/SOLR-17985-fix-slow-no-rows-queries.yml delete mode 100644 changelog/unreleased/SOLR-18006-Fix reverse dist sorting on LatLonPointSpatialField.yml delete mode 100644 changelog/unreleased/solr-17983-parallel-http-shard-handler-failure-propagation.yml diff --git a/changelog/unreleased/.gitkeep b/changelog/unreleased/.gitkeep deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/changelog/unreleased/PR#3926-tika-server-idle-timeout-fix.yml b/changelog/unreleased/PR#3926-tika-server-idle-timeout-fix.yml deleted file mode 100644 index f114e4d21e37..000000000000 --- a/changelog/unreleased/PR#3926-tika-server-idle-timeout-fix.yml +++ /dev/null @@ -1,9 +0,0 @@ -# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc -title: When using SolrCell with TikaServer, the connection will no longer timeout after 30s idle, such as during OCR processing -type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other -authors: - - name: Jan Høydahl - url: https://home.apache.org/phonebook.html?uid=janhoy -links: - - name: PR#3926 - url: https://github.com/apache/solr/pull/3926 diff --git a/changelog/unreleased/PR#3963-update-log4j.yml b/changelog/unreleased/PR#3963-update-log4j.yml deleted file mode 100644 index a6ce8a01f348..000000000000 --- a/changelog/unreleased/PR#3963-update-log4j.yml +++ /dev/null @@ -1,9 +0,0 @@ -# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc -title: Upgrade Log4j to 2.25.3 -type: dependency_update -authors: - - name: Piotr P. Karwasz - nick: ppkarwasz - url: https://home.apache.org/phonebook.html?uid=pkarwasz -merge_requests: - - 3963 \ No newline at end of file diff --git a/changelog/unreleased/SOLR-17888-mitigate-tika-cve-disable-xfa-parsing.yml b/changelog/unreleased/SOLR-17888-mitigate-tika-cve-disable-xfa-parsing.yml deleted file mode 100644 index 656a41512f97..000000000000 --- a/changelog/unreleased/SOLR-17888-mitigate-tika-cve-disable-xfa-parsing.yml +++ /dev/null @@ -1,9 +0,0 @@ -# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc -title: Mitigate CVE-2025-54988 by disabling XFA parsing in PDF documents when using SolrCell extraction -type: security # added, changed, fixed, deprecated, removed, dependency_update, security, other -authors: - - name: Jan Høydahl - url: https://home.apache.org/phonebook.html?uid=janhoy -links: - - name: SOLR-17888 - url: https://issues.apache.org/jira/browse/SOLR-17888 diff --git a/changelog/unreleased/SOLR-17972-distributed-lock-lead.yml b/changelog/unreleased/SOLR-17972-distributed-lock-lead.yml deleted file mode 100644 index da5b8d7e5141..000000000000 --- a/changelog/unreleased/SOLR-17972-distributed-lock-lead.yml +++ /dev/null @@ -1,7 +0,0 @@ -title: Retry creation of ZK lock on transient connection loss. -type: fixed -authors: - - name: Pierre Salagnac -links: - - name: SOLR-17972 - url: https://issues.apache.org/jira/browse/SOLR-17972 diff --git a/changelog/unreleased/SOLR-17985-fix-slow-no-rows-queries.yml b/changelog/unreleased/SOLR-17985-fix-slow-no-rows-queries.yml deleted file mode 100644 index f7d5f2cee528..000000000000 --- a/changelog/unreleased/SOLR-17985-fix-slow-no-rows-queries.yml +++ /dev/null @@ -1,10 +0,0 @@ -# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc -title: Make distributed no-rows queries fast again -type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other -authors: - - name: Houston Putman - nick: HoustonPutman - url: https://home.apache.org/phonebook.html?uid=houston -links: - - name: SOLR-17985 - url: https://issues.apache.org/jira/browse/SOLR-17985 diff --git a/changelog/unreleased/SOLR-18006-Fix reverse dist sorting on LatLonPointSpatialField.yml b/changelog/unreleased/SOLR-18006-Fix reverse dist sorting on LatLonPointSpatialField.yml deleted file mode 100644 index 3af69caac4a4..000000000000 --- a/changelog/unreleased/SOLR-18006-Fix reverse dist sorting on LatLonPointSpatialField.yml +++ /dev/null @@ -1,13 +0,0 @@ -title: Fix reverse distance sorting on LatLonPointSpatialField and "SRPT" fields when combined with the filter cache. This is a regression since Solr 9.9. -type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other -authors: - - name: Jan Høydahl - url: https://home.apache.org/phonebook.html?uid=janhoy - - name: Umut Saribiyik - nick: umut-sar - - name: David Smiley -links: - - name: SOLR-18006 - url: https://issues.apache.org/jira/browse/SOLR-18006 - - name: SOLR-18016 - url: https://issues.apache.org/jira/browse/SOLR-18016 diff --git a/changelog/unreleased/solr-17983-parallel-http-shard-handler-failure-propagation.yml b/changelog/unreleased/solr-17983-parallel-http-shard-handler-failure-propagation.yml deleted file mode 100644 index 1c19e9e0bbc9..000000000000 --- a/changelog/unreleased/solr-17983-parallel-http-shard-handler-failure-propagation.yml +++ /dev/null @@ -1,7 +0,0 @@ -title: Ensure ParallelHttpShardHandler records submit failures so distributed requests don’t hang -type: fixed -authors: - - name: Mark Miller -links: - - name: SOLR-17983 - url: https://issues.apache.org/jira/browse/SOLR-17983 From 409fda16a7c402771330c29bd123cd82aa69f088 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 8 Jan 2026 09:54:03 -0500 Subject: [PATCH 18/83] Logchange release on branch_9_10 - add changelog/v9.10.1 --- changelog/v9.10.0/version-summary.md | 2 +- .../PR#3926-tika-server-idle-timeout-fix.yml | 9 ++++++ changelog/v9.10.1/PR#3963-update-log4j.yml | 9 ++++++ ...-mitigate-tika-cve-disable-xfa-parsing.yml | 9 ++++++ .../SOLR-17972-distributed-lock-lead.yml | 7 +++++ .../SOLR-17985-fix-slow-no-rows-queries.yml | 10 +++++++ ...ist sorting on LatLonPointSpatialField.yml | 13 +++++++++ changelog/v9.10.1/release-date.txt | 1 + ...http-shard-handler-failure-propagation.yml | 7 +++++ changelog/v9.10.1/version-summary.md | 29 +++++++++++++++++++ 10 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 changelog/v9.10.1/PR#3926-tika-server-idle-timeout-fix.yml create mode 100644 changelog/v9.10.1/PR#3963-update-log4j.yml create mode 100644 changelog/v9.10.1/SOLR-17888-mitigate-tika-cve-disable-xfa-parsing.yml create mode 100644 changelog/v9.10.1/SOLR-17972-distributed-lock-lead.yml create mode 100644 changelog/v9.10.1/SOLR-17985-fix-slow-no-rows-queries.yml create mode 100644 changelog/v9.10.1/SOLR-18006-Fix reverse dist sorting on LatLonPointSpatialField.yml create mode 100644 changelog/v9.10.1/release-date.txt create mode 100644 changelog/v9.10.1/solr-17983-parallel-http-shard-handler-failure-propagation.yml create mode 100644 changelog/v9.10.1/version-summary.md diff --git a/changelog/v9.10.0/version-summary.md b/changelog/v9.10.0/version-summary.md index 3143991e9e48..2738aea37287 100644 --- a/changelog/v9.10.0/version-summary.md +++ b/changelog/v9.10.0/version-summary.md @@ -88,7 +88,7 @@ ### Other (9 changes) - Deprecate `CloudHttp2SolrClient.Builder#withHttpClient` in favor of `CloudHttp2SolrClient.Builder#withInternalClientBuilder`. Deprecate `LBHttp2SolrClient.Builder#withListenerFactory` in favor of `LBHttp2SolrClient.Builder#withListenerFactories`. [SOLR-17541](https://issues.apache.org/jira/browse/SOLR-17541) (James Dyer) -- Use logchange for changelog management ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy) @janhoy) +- Use logchange for changelog management [SOLR-17619](https://issues.apache.org/jira/browse/SOLR-17619) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy) @janhoy) - SolrCloud "live_node" now has metadata: version of Solr, roles [SOLR-17620](https://issues.apache.org/jira/browse/SOLR-17620) (Yuntong Qu) (David Smiley) - Deprecating waitForFinalState parameter in any SolrCloud command that accepts it. It remains defaulted to false in 9, but will become true and likely removed. [SOLR-17712](https://issues.apache.org/jira/browse/SOLR-17712) (Abhishek Umarjikar) (David Smiley) - Deprecate `CloudSolrClient.Builder` in favor of `CloudHttp2SolrClient.Builder`. [SOLR-17771](https://issues.apache.org/jira/browse/SOLR-17771) (James Dyer) diff --git a/changelog/v9.10.1/PR#3926-tika-server-idle-timeout-fix.yml b/changelog/v9.10.1/PR#3926-tika-server-idle-timeout-fix.yml new file mode 100644 index 000000000000..f114e4d21e37 --- /dev/null +++ b/changelog/v9.10.1/PR#3926-tika-server-idle-timeout-fix.yml @@ -0,0 +1,9 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: When using SolrCell with TikaServer, the connection will no longer timeout after 30s idle, such as during OCR processing +type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Jan Høydahl + url: https://home.apache.org/phonebook.html?uid=janhoy +links: + - name: PR#3926 + url: https://github.com/apache/solr/pull/3926 diff --git a/changelog/v9.10.1/PR#3963-update-log4j.yml b/changelog/v9.10.1/PR#3963-update-log4j.yml new file mode 100644 index 000000000000..a6ce8a01f348 --- /dev/null +++ b/changelog/v9.10.1/PR#3963-update-log4j.yml @@ -0,0 +1,9 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Upgrade Log4j to 2.25.3 +type: dependency_update +authors: + - name: Piotr P. Karwasz + nick: ppkarwasz + url: https://home.apache.org/phonebook.html?uid=pkarwasz +merge_requests: + - 3963 \ No newline at end of file diff --git a/changelog/v9.10.1/SOLR-17888-mitigate-tika-cve-disable-xfa-parsing.yml b/changelog/v9.10.1/SOLR-17888-mitigate-tika-cve-disable-xfa-parsing.yml new file mode 100644 index 000000000000..656a41512f97 --- /dev/null +++ b/changelog/v9.10.1/SOLR-17888-mitigate-tika-cve-disable-xfa-parsing.yml @@ -0,0 +1,9 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Mitigate CVE-2025-54988 by disabling XFA parsing in PDF documents when using SolrCell extraction +type: security # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Jan Høydahl + url: https://home.apache.org/phonebook.html?uid=janhoy +links: + - name: SOLR-17888 + url: https://issues.apache.org/jira/browse/SOLR-17888 diff --git a/changelog/v9.10.1/SOLR-17972-distributed-lock-lead.yml b/changelog/v9.10.1/SOLR-17972-distributed-lock-lead.yml new file mode 100644 index 000000000000..da5b8d7e5141 --- /dev/null +++ b/changelog/v9.10.1/SOLR-17972-distributed-lock-lead.yml @@ -0,0 +1,7 @@ +title: Retry creation of ZK lock on transient connection loss. +type: fixed +authors: + - name: Pierre Salagnac +links: + - name: SOLR-17972 + url: https://issues.apache.org/jira/browse/SOLR-17972 diff --git a/changelog/v9.10.1/SOLR-17985-fix-slow-no-rows-queries.yml b/changelog/v9.10.1/SOLR-17985-fix-slow-no-rows-queries.yml new file mode 100644 index 000000000000..f7d5f2cee528 --- /dev/null +++ b/changelog/v9.10.1/SOLR-17985-fix-slow-no-rows-queries.yml @@ -0,0 +1,10 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Make distributed no-rows queries fast again +type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Houston Putman + nick: HoustonPutman + url: https://home.apache.org/phonebook.html?uid=houston +links: + - name: SOLR-17985 + url: https://issues.apache.org/jira/browse/SOLR-17985 diff --git a/changelog/v9.10.1/SOLR-18006-Fix reverse dist sorting on LatLonPointSpatialField.yml b/changelog/v9.10.1/SOLR-18006-Fix reverse dist sorting on LatLonPointSpatialField.yml new file mode 100644 index 000000000000..3af69caac4a4 --- /dev/null +++ b/changelog/v9.10.1/SOLR-18006-Fix reverse dist sorting on LatLonPointSpatialField.yml @@ -0,0 +1,13 @@ +title: Fix reverse distance sorting on LatLonPointSpatialField and "SRPT" fields when combined with the filter cache. This is a regression since Solr 9.9. +type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Jan Høydahl + url: https://home.apache.org/phonebook.html?uid=janhoy + - name: Umut Saribiyik + nick: umut-sar + - name: David Smiley +links: + - name: SOLR-18006 + url: https://issues.apache.org/jira/browse/SOLR-18006 + - name: SOLR-18016 + url: https://issues.apache.org/jira/browse/SOLR-18016 diff --git a/changelog/v9.10.1/release-date.txt b/changelog/v9.10.1/release-date.txt new file mode 100644 index 000000000000..361e2da5120e --- /dev/null +++ b/changelog/v9.10.1/release-date.txt @@ -0,0 +1 @@ +2026-01-08 \ No newline at end of file diff --git a/changelog/v9.10.1/solr-17983-parallel-http-shard-handler-failure-propagation.yml b/changelog/v9.10.1/solr-17983-parallel-http-shard-handler-failure-propagation.yml new file mode 100644 index 000000000000..1c19e9e0bbc9 --- /dev/null +++ b/changelog/v9.10.1/solr-17983-parallel-http-shard-handler-failure-propagation.yml @@ -0,0 +1,7 @@ +title: Ensure ParallelHttpShardHandler records submit failures so distributed requests don’t hang +type: fixed +authors: + - name: Mark Miller +links: + - name: SOLR-17983 + url: https://issues.apache.org/jira/browse/SOLR-17983 diff --git a/changelog/v9.10.1/version-summary.md b/changelog/v9.10.1/version-summary.md new file mode 100644 index 000000000000..17fed63fab29 --- /dev/null +++ b/changelog/v9.10.1/version-summary.md @@ -0,0 +1,29 @@ + + + + + + + + + +[9.10.1] - 2026-01-08 +--------------------- + +### Fixed (5 changes) + +- When using SolrCell with TikaServer, the connection will no longer timeout after 30s idle, such as during OCR processing [PR#3926](https://github.com/apache/solr/pull/3926) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy)) +- Retry creation of ZK lock on transient connection loss. [SOLR-17972](https://issues.apache.org/jira/browse/SOLR-17972) (Pierre Salagnac) +- Make distributed no-rows queries fast again [SOLR-17985](https://issues.apache.org/jira/browse/SOLR-17985) ([Houston Putman](https://home.apache.org/phonebook.html?uid=houston) @HoustonPutman) +- Fix reverse distance sorting on LatLonPointSpatialField and "SRPT" fields when combined with the filter cache. This is a regression since Solr 9.9. [SOLR-18006](https://issues.apache.org/jira/browse/SOLR-18006) [SOLR-18016](https://issues.apache.org/jira/browse/SOLR-18016) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy)) (Umut Saribiyik @umut-sar) (David Smiley) +- Ensure ParallelHttpShardHandler records submit failures so distributed requests don’t hang [SOLR-17983](https://issues.apache.org/jira/browse/SOLR-17983) (Mark Miller) + +### Dependency Upgrades (1 change) + +- Upgrade Log4j to 2.25.3 !3963 ([Piotr P. Karwasz](https://home.apache.org/phonebook.html?uid=pkarwasz) @ppkarwasz) + +### Security (1 change) + +- Mitigate CVE-2025-54988 by disabling XFA parsing in PDF documents when using SolrCell extraction [SOLR-17888](https://issues.apache.org/jira/browse/SOLR-17888) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy)) + + From 68e4c9b6ebe5d689109378357a51336cde6de9fb Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 8 Jan 2026 13:02:03 -0500 Subject: [PATCH 19/83] CHANGELOG.md generated for release v9.10.1 --- CHANGELOG.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6125889109f..c5819c89389b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,26 @@ This file lists Solr's raw release notes with details of every change to Solr. Most people will find the solr-upgrade-notes.adoc file more approachable. [https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc](https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc) +[9.10.1] - 2026-01-08 +--------------------- + +### Fixed (5 changes) + +- When using SolrCell with TikaServer, the connection will no longer timeout after 30s idle, such as during OCR processing [PR#3926](https://github.com/apache/solr/pull/3926) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy)) +- Retry creation of ZK lock on transient connection loss. [SOLR-17972](https://issues.apache.org/jira/browse/SOLR-17972) (Pierre Salagnac) +- Make distributed no-rows queries fast again [SOLR-17985](https://issues.apache.org/jira/browse/SOLR-17985) ([Houston Putman](https://home.apache.org/phonebook.html?uid=houston) @HoustonPutman) +- Fix reverse distance sorting on LatLonPointSpatialField and "SRPT" fields when combined with the filter cache. This is a regression since Solr 9.9. [SOLR-18006](https://issues.apache.org/jira/browse/SOLR-18006) [SOLR-18016](https://issues.apache.org/jira/browse/SOLR-18016) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy)) (Umut Saribiyik @umut-sar) (David Smiley) +- Ensure ParallelHttpShardHandler records submit failures so distributed requests don’t hang [SOLR-17983](https://issues.apache.org/jira/browse/SOLR-17983) (Mark Miller) + +### Dependency Upgrades (1 change) + +- Upgrade Log4j to 2.25.3 !3963 ([Piotr P. Karwasz](https://home.apache.org/phonebook.html?uid=pkarwasz) @ppkarwasz) + +### Security (1 change) + +- Mitigate CVE-2025-54988 by disabling XFA parsing in PDF documents when using SolrCell extraction [SOLR-17888](https://issues.apache.org/jira/browse/SOLR-17888) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy)) + + [9.10.0] -------- @@ -91,7 +111,7 @@ This file lists Solr's raw release notes with details of every change to Solr. M ### Other (9 changes) - Deprecate `CloudHttp2SolrClient.Builder#withHttpClient` in favor of `CloudHttp2SolrClient.Builder#withInternalClientBuilder`. Deprecate `LBHttp2SolrClient.Builder#withListenerFactory` in favor of `LBHttp2SolrClient.Builder#withListenerFactories`. [SOLR-17541](https://issues.apache.org/jira/browse/SOLR-17541) (James Dyer) -- Use logchange for changelog management ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy) @janhoy) +- Use logchange for changelog management [SOLR-17619](https://issues.apache.org/jira/browse/SOLR-17619) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy) @janhoy) - SolrCloud "live_node" now has metadata: version of Solr, roles [SOLR-17620](https://issues.apache.org/jira/browse/SOLR-17620) (Yuntong Qu) (David Smiley) - Deprecating waitForFinalState parameter in any SolrCloud command that accepts it. It remains defaulted to false in 9, but will become true and likely removed. [SOLR-17712](https://issues.apache.org/jira/browse/SOLR-17712) (Abhishek Umarjikar) (David Smiley) - Deprecate `CloudSolrClient.Builder` in favor of `CloudHttp2SolrClient.Builder`. [SOLR-17771](https://issues.apache.org/jira/browse/SOLR-17771) (James Dyer) From f5da392f125199c5d16fe971e024069d1ad07fa7 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 8 Jan 2026 17:00:44 -0800 Subject: [PATCH 20/83] SOLR-17947: CloudSolrClient refreshes collection state asynchronously using a dedicated thread pool to reduce ZooKeeper blocking under load. (#4005) Co-authored-by: Mark Robert Miller --- ...47-cloudsolrclient async state refresh.yml | 8 + .../solrj/impl/CloudHttp2SolrClient.java | 24 +- .../client/solrj/impl/CloudSolrClient.java | 302 ++++++++++--- .../solrj/impl/CloudSolrClientCacheTest.java | 419 +++++++++++++++++- 4 files changed, 676 insertions(+), 77 deletions(-) create mode 100644 changelog/unreleased/SOLR-17947-cloudsolrclient async state refresh.yml diff --git a/changelog/unreleased/SOLR-17947-cloudsolrclient async state refresh.yml b/changelog/unreleased/SOLR-17947-cloudsolrclient async state refresh.yml new file mode 100644 index 000000000000..da7aa452ee1e --- /dev/null +++ b/changelog/unreleased/SOLR-17947-cloudsolrclient async state refresh.yml @@ -0,0 +1,8 @@ +title: CloudSolrClient now refreshes collection state asynchronously using a dedicated + thread pool, reducing ZooKeeper blocking and improving performance under load. +type: changed +authors: +- name: Mark Miller +links: +- name: SOLR-17947 + url: https://issues.apache.org/jira/browse/SOLR-17947 diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java index 2ddc64797690..4ef42557dd8f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java @@ -53,7 +53,11 @@ public class CloudHttp2SolrClient extends CloudSolrClient { * @param builder a {@link Http2SolrClient.Builder} with the options used to create the client. */ protected CloudHttp2SolrClient(Builder builder) { - super(builder.shardLeadersOnly, builder.parallelUpdates, builder.directUpdatesToLeadersOnly); + super( + builder.shardLeadersOnly, + builder.parallelUpdates, + builder.directUpdatesToLeadersOnly, + builder.parallelCacheRefreshesLocks); this.clientIsInternal = builder.httpClient == null; this.myClient = createOrGetHttpClientFromBuilder(builder); this.stateProvider = createClusterStateProvider(builder); @@ -69,10 +73,6 @@ protected CloudHttp2SolrClient(Builder builder) { this.collectionStateCache.timeToLiveMs = TimeUnit.MILLISECONDS.convert(builder.timeToLiveSeconds, TimeUnit.SECONDS); - // If caches are expired then they are refreshed after acquiring a lock. Set the number of - // locks. - this.locks = objectList(builder.parallelCacheRefreshesLocks); - this.lbClient = new LBHttp2SolrClient.Builder(myClient, new String[0]).build(); } @@ -180,7 +180,7 @@ public static class Builder { private String defaultCollection; private long timeToLiveSeconds = 60; - private int parallelCacheRefreshesLocks = 3; + private int parallelCacheRefreshesLocks = DEFAULT_STATE_REFRESH_PARALLELISM; private int zkConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT; private int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT; private boolean canUseZkACLs = true; @@ -328,10 +328,10 @@ public Builder withParallelUpdates(boolean parallelUpdates) { } /** - * When caches are expired then they are refreshed after acquiring a lock. Use this to set the - * number of locks. + * Configures how many collection state refresh operations may run in parallel using a dedicated + * thread pool. This controls the maximum number of concurrent ZooKeeper/cluster state lookups. * - *

Defaults to 3. + *

Defaults to 5. * * @deprecated Please use {@link #withParallelCacheRefreshes(int)} */ @@ -342,10 +342,10 @@ public Builder setParallelCacheRefreshes(int parallelCacheRefreshesLocks) { } /** - * When caches are expired then they are refreshed after acquiring a lock. Use this to set the - * number of locks. + * Configures how many collection state refresh operations may run in parallel using a dedicated + * thread pool. This controls the maximum number of concurrent ZooKeeper/cluster state lookups. * - *

Defaults to 3. + *

Defaults to 5. */ public Builder withParallelCacheRefreshes(int parallelCacheRefreshesLocks) { this.parallelCacheRefreshesLocks = parallelCacheRefreshesLocks; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index 8dbfe1c9aabf..59a406e35d40 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -37,10 +37,12 @@ import java.util.Optional; import java.util.Random; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; @@ -77,7 +79,6 @@ import org.apache.solr.common.params.UpdateParams; import org.apache.solr.common.util.CollectionUtil; import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.Hash; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SolrNamedThreadFactory; @@ -94,6 +95,7 @@ public abstract class CloudSolrClient extends SolrClient { // no of times collection state to be reloaded if stale state error is received private static final int MAX_STALE_RETRIES = Integer.parseInt(System.getProperty("cloudSolrClientMaxStaleRetries", "5")); + static final int DEFAULT_STATE_REFRESH_PARALLELISM = 5; private final Random rand = new Random(); private final boolean updatesToLeaders; @@ -122,7 +124,11 @@ public abstract class CloudSolrClient extends SolrClient { // UpdateParams.ROLLBACK ); - protected volatile Object[] locks = objectList(3); + private final ConcurrentHashMap> collectionRefreshes = + new ConcurrentHashMap<>(); + private final Semaphore stateRefreshSemaphore; + private final int stateRefreshParallelism; + private volatile boolean closed; /** * Constructs {@link CloudSolrClient} instances from provided configuration. @@ -213,6 +219,10 @@ public ExpiringCachedDocCollection get(Object key) { return val; } + ExpiringCachedDocCollection peek(Object key) { + return super.get(key); + } + @Override public ExpiringCachedDocCollection put(String key, ExpiringCachedDocCollection value) { puts.incrementAndGet(); @@ -278,14 +288,46 @@ boolean shouldRetry() { void setRetriedAt() { retriedAtNano = System.nanoTime(); } + + /** + * Marks this entry as {@code maybeStale} if the provided backoff window has elapsed since the + * last retry. + * + * @return {@code true} if the entry was flagged as maybe stale + */ + boolean markMaybeStaleIfOutsideBackoff(long retryBackoffNano) { + if (maybeStale) { + return true; + } + long lastRetry = retriedAtNano; + if (lastRetry != -1 && (System.nanoTime() - lastRetry) <= retryBackoffNano) { + return false; + } + maybeStale = true; + return true; + } } protected CloudSolrClient( boolean updatesToLeaders, boolean parallelUpdates, boolean directUpdatesToLeadersOnly) { + this( + updatesToLeaders, + parallelUpdates, + directUpdatesToLeadersOnly, + DEFAULT_STATE_REFRESH_PARALLELISM); + } + + protected CloudSolrClient( + boolean updatesToLeaders, + boolean parallelUpdates, + boolean directUpdatesToLeadersOnly, + int stateRefreshThreads) { this.updatesToLeaders = updatesToLeaders; this.parallelUpdates = parallelUpdates; this.directUpdatesToLeadersOnly = directUpdatesToLeadersOnly; this.requestRLTGenerator = new RequestReplicaListTransformerGenerator(); + this.stateRefreshParallelism = Math.max(1, stateRefreshThreads); + this.stateRefreshSemaphore = new Semaphore(this.stateRefreshParallelism); } /** @@ -323,6 +365,8 @@ protected boolean wasCommError(Throwable t) { @Override public void close() throws IOException { + closed = true; + collectionRefreshes.clear(); if (this.threadPool != null && !ExecutorUtil.isShutdown(this.threadPool)) { ExecutorUtil.shutdownAndAwaitTermination(this.threadPool); this.threadPool = null; @@ -842,7 +886,13 @@ public NamedList request(SolrRequest request, String collection) List inputCollections = collection == null ? Collections.emptyList() : StrUtils.splitSmart(collection, ",", true); - return requestWithRetryOnStaleState(request, 0, inputCollections); + return requestWithRetryOnStaleState( + request, + 0, + inputCollections, + /*skipStateVersion*/ false, + Map.of(), + /*waitedForRefresh*/ false); } /** @@ -851,7 +901,12 @@ public NamedList request(SolrRequest request, String collection) * and retried. */ protected NamedList requestWithRetryOnStaleState( - SolrRequest request, int retryCount, List inputCollections) + SolrRequest request, + int retryCount, + List inputCollections, + boolean skipStateVersion, + Map> pendingRefreshes, + boolean waitedForRefresh) throws SolrServerException, IOException { // build up a _stateVer_ param to pass to the server containing all the // external collection state versions involved in this request, which allows @@ -903,7 +958,7 @@ protected NamedList requestWithRetryOnStaleState( if (request.getParams() instanceof ModifiableSolrParams) { ModifiableSolrParams params = (ModifiableSolrParams) request.getParams(); - if (stateVerParam != null) { + if (!skipStateVersion && stateVerParam != null) { params.set(STATE_VERSION, stateVerParam); } else { params.remove(STATE_VERSION); @@ -966,9 +1021,21 @@ protected NamedList requestWithRetryOnStaleState( // in retryExpiryTime time if (requestedCollections != null) { for (DocCollection ext : requestedCollections) { - ExpiringCachedDocCollection cacheEntry = collectionStateCache.get(ext.getName()); - if (cacheEntry == null) continue; - cacheEntry.maybeStale = true; + String name = ext.getName(); + ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(name); + if (cacheEntry != null) { + if (wasCommError) { + cacheEntry.maybeStale = true; + } else { + boolean markedStale = + cacheEntry.markMaybeStaleIfOutsideBackoff(retryExpiryTimeNano); + if (markedStale && cacheEntry.shouldRetry()) { + triggerCollectionRefresh(name); + } + } + } else { + triggerCollectionRefresh(name); + } } } if (retryCount < MAX_STALE_RETRIES) { // if it is a communication error , we must try again @@ -985,7 +1052,13 @@ protected NamedList requestWithRetryOnStaleState( MAX_STALE_RETRIES, wasCommError, errorCode); - return requestWithRetryOnStaleState(request, retryCount + 1, inputCollections); + return requestWithRetryOnStaleState( + request, + retryCount + 1, + inputCollections, + skipStateVersion, + pendingRefreshes, + waitedForRefresh); } } else { log.info("request was not communication error it seems"); @@ -1037,16 +1110,57 @@ protected NamedList requestWithRetryOnStaleState( } } - if (requestedCollections != null) { - requestedCollections.clear(); // done with this - } - // if the state was stale, then we retry the request once with new state pulled from Zk if (stateWasStale) { log.warn( "Re-trying request to collection(s) {} after stale state error from server.", inputCollections); - resp = requestWithRetryOnStaleState(request, retryCount + 1, inputCollections); + + Map> refreshesToWaitFor = pendingRefreshes; + if (!waitedForRefresh && (pendingRefreshes == null || pendingRefreshes.isEmpty())) { + refreshesToWaitFor = new HashMap<>(); + for (DocCollection ext : requestedCollections) { + refreshesToWaitFor.put(ext.getName(), triggerCollectionRefresh(ext.getName())); + } + } + + // First retry without sending state versions so the server does not immediately reject the + // request while we intentionally rely on stale routing (e.g., to allow forwarding to a new + // leader) as the background refresh completes. + if (!skipStateVersion && !waitedForRefresh) { + resp = + requestWithRetryOnStaleState( + request, + retryCount + 1, + inputCollections, + /*skipStateVersion*/ true, + refreshesToWaitFor, + waitedForRefresh); + } else if (!waitedForRefresh + && refreshesToWaitFor != null + && !refreshesToWaitFor.isEmpty()) { + for (Map.Entry> entry : + refreshesToWaitFor.entrySet()) { + waitForCollectionRefresh(entry.getKey(), entry.getValue()); + } + resp = + requestWithRetryOnStaleState( + request, + retryCount + 1, + inputCollections, + /*skipStateVersion*/ false, + Map.of(), + /*waitedForRefresh*/ true); + } else { + resp = + requestWithRetryOnStaleState( + request, + retryCount + 1, + inputCollections, + /*skipStateVersion*/ false, + Map.of(), + /*waitedForRefresh*/ waitedForRefresh); + } } else { if (exc instanceof SolrException || exc instanceof SolrServerException @@ -1056,6 +1170,10 @@ protected NamedList requestWithRetryOnStaleState( throw new SolrServerException(rootCause); } } + + if (requestedCollections != null) { + requestedCollections.clear(); // done with this + } } return resp; @@ -1264,63 +1382,123 @@ public boolean isDirectUpdatesToLeadersOnly() { return directUpdatesToLeadersOnly; } - /** - * If caches are expired they are refreshed after acquiring a lock. use this to set the number of - * locks - * - * @deprecated use {@link CloudHttp2SolrClient.Builder#setParallelCacheRefreshes(int)} instead - */ - @Deprecated - public void setParallelCacheRefreshes(int n) { - locks = objectList(n); + /** Visible for tests so they can assert the configured refresh parallelism. */ + protected int getStateRefreshParallelism() { + return stateRefreshParallelism; } - protected static Object[] objectList(int n) { - Object[] l = new Object[n]; - for (int i = 0; i < n; i++) { - l[i] = new Object(); + protected DocCollection getDocCollection(String collection, Integer expectedVersion) + throws SolrException { + if (expectedVersion == null) { + expectedVersion = -1; + } + if (collection == null) { + return null; + } + + ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(collection); + if (cacheEntry != null && cacheEntry.isExpired(collectionStateCache.timeToLiveMs)) { + collectionStateCache.remove(collection, cacheEntry); + cacheEntry = null; + } + + DocCollection cached = cacheEntry == null ? null : cacheEntry.cached; + + if (cacheEntry != null && cacheEntry.shouldRetry()) { + triggerCollectionRefresh(collection); + } + + if (cached != null && expectedVersion <= cached.getZNodeVersion()) { + return cached; } - return l; + + CompletableFuture refreshFuture = triggerCollectionRefresh(collection); + return waitForCollectionRefresh(collection, refreshFuture); } - protected DocCollection getDocCollection(String collection, Integer expectedVersion) - throws SolrException { - if (expectedVersion == null) expectedVersion = -1; - if (collection == null) return null; - ExpiringCachedDocCollection cacheEntry = collectionStateCache.get(collection); - DocCollection col = cacheEntry == null ? null : cacheEntry.cached; - if (col != null) { - if (expectedVersion <= col.getZNodeVersion() && !cacheEntry.shouldRetry()) return col; + private CompletableFuture triggerCollectionRefresh(String collection) { + if (closed) { + ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(collection); + DocCollection cached = cacheEntry == null ? null : cacheEntry.cached; + return CompletableFuture.completedFuture(cached); } + return collectionRefreshes.computeIfAbsent( + collection, + key -> { + ExecutorService executor = threadPool; + CompletableFuture future; + if (executor == null || ExecutorUtil.isShutdown(executor)) { + future = new CompletableFuture<>(); + try { + future.complete(loadDocCollection(key)); + } catch (Throwable t) { + future.completeExceptionally(t); + } + } else { + future = + CompletableFuture.supplyAsync( + () -> { + stateRefreshSemaphore.acquireUninterruptibly(); + try { + return loadDocCollection(key); + } finally { + stateRefreshSemaphore.release(); + } + }, + executor); + } + future.whenCompleteAsync( + (result, error) -> { + collectionRefreshes.remove(key, future); + }); + return future; + }); + } - Object[] locks = this.locks; - int lockId = - Math.abs(Hash.murmurhash3_x86_32(collection, 0, collection.length(), 0) % locks.length); - final Object lock = locks[lockId]; - synchronized (lock) { - /*we have waited for some time just check once again*/ - cacheEntry = collectionStateCache.get(collection); - col = cacheEntry == null ? null : cacheEntry.cached; - if (col != null) { - if (expectedVersion <= col.getZNodeVersion() && !cacheEntry.shouldRetry()) return col; - } - ClusterState.CollectionRef ref = getCollectionRef(collection); - if (ref == null) { - // no such collection exists - return null; - } - // We are going to fetch a new version - // we MUST try to get a new version - DocCollection fetchedCol = ref.get(); // this is a call to ZK - if (fetchedCol == null) return null; // this collection no more exists - if (col != null && fetchedCol.getZNodeVersion() == col.getZNodeVersion()) { - cacheEntry.setRetriedAt(); // we retried and found that it is the same version - cacheEntry.maybeStale = false; - } else { - collectionStateCache.put(collection, new ExpiringCachedDocCollection(fetchedCol)); + private DocCollection waitForCollectionRefresh( + String collection, CompletableFuture refreshFuture) { + try { + return refreshFuture.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Interrupted while refreshing state for collection " + collection, + e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof SolrException) { + throw (SolrException) cause; } - return fetchedCol; + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Error refreshing state for collection " + collection, + cause); + } + } + + private DocCollection loadDocCollection(String collection) { + ClusterState.CollectionRef ref = getCollectionRef(collection); + if (ref == null) { + collectionStateCache.remove(collection); + return null; } + + DocCollection fetchedCol = ref.get(); + if (fetchedCol == null) { + collectionStateCache.remove(collection); + return null; + } + + ExpiringCachedDocCollection existing = collectionStateCache.peek(collection); + if (existing != null && existing.cached.getZNodeVersion() == fetchedCol.getZNodeVersion()) { + existing.setRetriedAt(); + existing.maybeStale = false; + return existing.cached; + } + + collectionStateCache.put(collection, new ExpiringCachedDocCollection(fetchedCol)); + return fetchedCol; } ClusterState.CollectionRef getCollectionRef(String collection) { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java index 9603dccbac36..4d292d6fd22c 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java @@ -22,24 +22,44 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.IOException; import java.net.ConnectException; import java.net.SocketException; import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import java.util.function.Supplier; import org.apache.http.NoHttpResponseException; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.cloud.DelegatingClusterStateProvider; import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.SimpleSolrResponse; +import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.ContentStream; +import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SolrNamedThreadFactory; import org.junit.BeforeClass; public class CloudSolrClientCacheTest extends SolrTestCaseJ4 { @@ -88,7 +108,7 @@ public DocCollection get() { livenodes.addAll(Set.of("192.168.1.108:7574_solr", "192.168.1.108:8983_solr")); ClusterState cs = ClusterState.createFromJson( - 1, coll1State.getBytes(UTF_8), Collections.emptySet(), Instant.now(), null); + 1, COLL1_STATE.getBytes(UTF_8), Collections.emptySet(), Instant.now(), null); refs.put(collName, new Ref(collName)); colls.put(collName, cs.getCollectionOrNull(collName)); responses.put( @@ -109,7 +129,173 @@ public DocCollection get() { UpdateRequest update = new UpdateRequest().add("id", "123", "desc", "Something 0"); cloudClient.request(update, collName); - assertEquals(2, refs.get(collName).getCount()); + // Async refresh with deduplication means rapid retries can share the same Future. + // Race: sometimes async completes fast enough for 2 fetches, sometimes only 1. + int fetchCount = refs.get(collName).getCount(); + assertTrue("Expected 1 or 2 fetches, got " + fetchCount, fetchCount >= 1 && fetchCount <= 2); + } + } + + public void testStaleStateRetrySkipsStateVersionBeforeWait() throws Exception { + String collName = "gettingstarted"; + Set liveNodes = new HashSet<>(Set.of("192.168.1.108:8983_solr")); + AtomicInteger refGets = new AtomicInteger(); + AtomicReference currentDoc = new AtomicReference<>(loadCollection(collName, 1)); + Map refs = + Map.of(collName, new TestCollectionRef(currentDoc::get, refGets, null, null, -1)); + try (ClusterStateProvider provider = getStateProvider(liveNodes, refs); + RecordingCloudSolrClient client = new RecordingCloudSolrClient(provider, 3)) { + client.enqueue( + (req, cols) -> { + throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "stale"); + }); + client.enqueue((req, cols) -> null); + + DummyRequest request = new DummyRequest(collName); + NamedList resp = client.request(request, collName); + assertNotNull(resp); + + List history = client.getStateVersionHistory(); + assertEquals(2, history.size()); + assertTrue(history.get(0).startsWith(collName + ":")); + assertNull("Second attempt should skip _stateVer_", history.get(1)); + assertTrue("Expected refresh to be triggered", refGets.get() >= 1); + } + } + + public void testDirectUpdatesToLeadersSkipStateVersionBeforeWait() throws Exception { + String collName = "gettingstarted"; + Set liveNodes = new HashSet<>(Set.of("192.168.1.108:8983_solr")); + AtomicInteger refGets = new AtomicInteger(); + AtomicReference currentDoc = new AtomicReference<>(loadCollection(collName, 1)); + Map refs = + Map.of(collName, new TestCollectionRef(currentDoc::get, refGets, null, null, -1)); + try (ClusterStateProvider provider = getStateProvider(liveNodes, refs); + RecordingCloudSolrClient client = + new RecordingCloudSolrClient(provider, true, true, true, 3)) { + client.enqueue( + (req, cols) -> { + throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "stale"); + }); + client.enqueue((req, cols) -> null); + + DummyUpdateRequest request = new DummyUpdateRequest(collName); + NamedList resp = client.request(request, collName); + assertNotNull(resp); + + List history = client.getStateVersionHistory(); + assertEquals(2, history.size()); + assertTrue(history.get(0).startsWith(collName + ":")); + assertNull(history.get(1)); + assertTrue(refGets.get() >= 1); + } + } + + public void testStaleStateRetryWaitsAfterSkipFailure() throws Exception { + String collName = "gettingstarted"; + AtomicReference currentDoc = new AtomicReference<>(loadCollection(collName, 1)); + + // Track when refresh is triggered to ensure the async refresh mechanism is used + AtomicInteger refGets = new AtomicInteger(); + TestCollectionRef ref = new TestCollectionRef(currentDoc::get, refGets, null, null, -1); + Map refs = Map.of(collName, ref); + Set liveNodes = new HashSet<>(Set.of("192.168.1.108:8983_solr")); + + try (ClusterStateProvider provider = getStateProvider(liveNodes, refs); + RecordingCloudSolrClient client = new RecordingCloudSolrClient(provider, 2)) { + // First attempt: returns stale error, triggers skipStateVersion retry + client.enqueue( + (req, cols) -> { + throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "stale-first"); + }); + // Second attempt (skipStateVersion retry): also returns stale error + client.enqueue( + (req, cols) -> { + throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "stale-second"); + }); + // Third attempt (after waiting for refresh): succeeds + client.enqueue((req, cols) -> null); + + DummyRequest request = new DummyRequest(collName); + NamedList resp = client.request(request, collName); + assertNotNull(resp); + + // Verify the retry sequence: + // 1. First attempt with state version + // 2. skipStateVersion retry without state version + // 3. Final retry with refreshed state version + List history = client.getStateVersionHistory(); + assertEquals("Should have 3 attempts", 3, history.size()); + assertTrue( + "First attempt should have state param", history.get(0).startsWith(collName + ":")); + assertNull("skipStateVersion attempt should NOT have state param", history.get(1)); + assertTrue( + "Final attempt should have state param after refresh", + history.get(2).startsWith(collName + ":")); + + // Verify refresh was triggered (at least initial load + refresh after stale errors) + assertTrue("Refresh should have been called", refGets.get() >= 2); + } + } + + public void testStateRefreshThreadsConfiguredViaBuilder() throws Exception { + String collName = "gettingstarted"; + AtomicReference currentDoc = new AtomicReference<>(loadCollection(collName, 1)); + Map refs = + Map.of( + collName, new TestCollectionRef(currentDoc::get, new AtomicInteger(), null, null, -1)); + Set liveNodes = new HashSet<>(Set.of("192.168.1.108:8983_solr")); + + try (ClusterStateProvider provider = getStateProvider(liveNodes, refs); + RecordingCloudSolrClient client = new RecordingCloudSolrClient(provider, 7)) { + assertEquals(7, client.getStateRefreshParallelism()); + } + } + + public void testConcurrentRefreshIsDeduplicated() throws Exception { + String collName = "gettingstarted"; + AtomicReference currentDoc = new AtomicReference<>(loadCollection(collName, 1)); + AtomicInteger refGets = new AtomicInteger(); + CountDownLatch refreshStarted = new CountDownLatch(1); + CountDownLatch releaseRefresh = new CountDownLatch(1); + Map refs = + Map.of( + collName, + new TestCollectionRef(currentDoc::get, refGets, refreshStarted, releaseRefresh, 1)); + Set liveNodes = new HashSet<>(Set.of("192.168.1.108:8983_solr")); + + try (ClusterStateProvider provider = getStateProvider(liveNodes, refs); + RecordingCloudSolrClient client = new RecordingCloudSolrClient(provider, 2)) { + AtomicInteger sendCount = new AtomicInteger(); + client.setDefaultInvocation( + (req, cols) -> { + if (sendCount.incrementAndGet() <= 2) { + throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "stale"); + } + return null; + }); + + DummyRequest request = new DummyRequest(collName); + ExecutorService executor = + ExecutorUtil.newMDCAwareFixedThreadPool( + 2, new SolrNamedThreadFactory("CloudSolrClientCacheTest-parallel")); + try { + Future> first = executor.submit(() -> client.request(request, collName)); + Future> second = executor.submit(() -> client.request(request, collName)); + + assertTrue( + "Refresh should start within timeout", refreshStarted.await(30, TimeUnit.SECONDS)); + assertEquals("Only one refresh should be in flight", 1, refGets.get()); + releaseRefresh.countDown(); + + NamedList firstResp = first.get(30, TimeUnit.SECONDS); + NamedList secondResp = second.get(30, TimeUnit.SECONDS); + + assertNotNull(firstResp); + assertNotNull(secondResp); + } finally { + ExecutorUtil.shutdownAndAwaitTermination(executor); + } } } @@ -159,7 +345,234 @@ public T getClusterProperty(String propertyName, T def) { }; } - private String coll1State = + private DocCollection loadCollection(String collection, int version) throws Exception { + ClusterState state = + ClusterState.createFromJson( + version, COLL1_STATE.getBytes(UTF_8), Collections.emptySet(), Instant.now(), null); + return state.getCollectionOrNull(collection); + } + + private static class RecordingCloudSolrClient extends CloudSolrClient implements AutoCloseable { + private final ClusterStateProvider provider; + private final ConcurrentLinkedQueue invocations = new ConcurrentLinkedQueue<>(); + private volatile Invocation defaultInvocation; + private final List stateHistory = Collections.synchronizedList(new ArrayList<>()); + private final NamedList okResponse; + + RecordingCloudSolrClient(ClusterStateProvider provider, int refreshThreads) { + this(provider, true, true, false, refreshThreads); + } + + RecordingCloudSolrClient( + ClusterStateProvider provider, + boolean updatesToLeaders, + boolean parallelUpdates, + boolean directUpdatesToLeadersOnly, + int refreshThreads) { + super(updatesToLeaders, parallelUpdates, directUpdatesToLeadersOnly, refreshThreads); + this.provider = provider; + NamedList header = new NamedList<>(); + header.add("status", 0); + okResponse = new NamedList<>(); + okResponse.add("responseHeader", header); + } + + void enqueue(Invocation invocation) { + invocations.add(invocation); + } + + void setDefaultInvocation(Invocation invocation) { + this.defaultInvocation = invocation; + } + + List getStateVersionHistory() { + synchronized (stateHistory) { + return new ArrayList<>(stateHistory); + } + } + + @Override + protected NamedList sendRequest(SolrRequest request, List inputCollections) + throws SolrServerException, IOException { + String stateParam = + request.getParams() == null ? null : request.getParams().get(STATE_VERSION); + stateHistory.add(stateParam); + Invocation invocation = invocations.poll(); + if (invocation == null) { + invocation = defaultInvocation; + } + if (invocation == null) { + return okResponse; + } + try { + NamedList rsp = invocation.invoke(request, inputCollections); + return rsp == null ? okResponse : rsp; + } catch (SolrServerException | IOException | SolrException e) { + throw e; + } catch (Exception e) { + throw new SolrServerException(e); + } + } + + @Override + protected LBSolrClient getLbClient() { + throw new UnsupportedOperationException("LB client not used in test harness"); + } + + @Override + public ClusterStateProvider getClusterStateProvider() { + return provider; + } + + @FunctionalInterface + interface Invocation { + NamedList invoke(SolrRequest request, List inputCollections) + throws Exception; + } + } + + private static class DummyRequest extends SolrRequest { + private final ModifiableSolrParams params = new ModifiableSolrParams(); + private final String collection; + + DummyRequest(String collection) { + super(METHOD.GET, "/dummy"); + this.collection = collection; + } + + @Override + public ModifiableSolrParams getParams() { + return params; + } + + @Override + public Collection getContentStreams() { + return null; + } + + @Override + protected SimpleSolrResponse createResponse(SolrClient solrClient) { + return new SimpleSolrResponse(); + } + + @Override + public boolean requiresCollection() { + return true; + } + + @Override + public String getCollection() { + return collection; + } + + @Override + public String getRequestType() { + return SolrRequestType.UNSPECIFIED.toString(); + } + } + + private static class DummyUpdateRequest extends DummyRequest { + DummyUpdateRequest(String collection) { + super(collection); + } + + @Override + public String getRequestType() { + return SolrRequestType.UPDATE.toString(); + } + } + + private static class TestCollectionRef extends ClusterState.CollectionRef { + private final Supplier supplier; + private final AtomicInteger counter; + private final CountDownLatch phaseOneReady; // Signals COUNT=phaseOneCount reached + private final CountDownLatch phaseOneProceed; // Test signals OK to proceed + private final CountDownLatch phaseTwoStarted; // Signals COUNT=phaseTwoCount blocked + private final CountDownLatch phaseTwoProceed; // Test signals OK to complete + private final int phaseOneCount; + private final int phaseTwoCount; + private final AtomicBoolean phaseOneTriggered = new AtomicBoolean(false); + private final AtomicBoolean phaseTwoTriggered = new AtomicBoolean(false); + + // Two-phase constructor for precise control over async refresh ordering + TestCollectionRef( + Supplier supplier, + AtomicInteger counter, + CountDownLatch phaseOneReady, + CountDownLatch phaseOneProceed, + CountDownLatch phaseTwoStarted, + CountDownLatch phaseTwoProceed, + int phaseOneCount, + int phaseTwoCount) { + super(null); + this.supplier = supplier; + this.counter = counter; + this.phaseOneReady = phaseOneReady; + this.phaseOneProceed = phaseOneProceed; + this.phaseTwoStarted = phaseTwoStarted; + this.phaseTwoProceed = phaseTwoProceed; + this.phaseOneCount = phaseOneCount; + this.phaseTwoCount = phaseTwoCount; + } + + // Backward-compatible single-block constructor for existing tests + TestCollectionRef( + Supplier supplier, + AtomicInteger counter, + CountDownLatch startLatch, + CountDownLatch waitLatch, + int blockAtCount) { + this(supplier, counter, null, null, startLatch, waitLatch, -1, blockAtCount); + } + + @Override + public boolean isLazilyLoaded() { + return true; + } + + @Override + public DocCollection get() { + int count = counter.incrementAndGet(); + + // Phase 1: Sync point to control async refresh completion timing + if (phaseOneCount > 0 + && count == phaseOneCount + && phaseOneTriggered.compareAndSet(false, true)) { + if (phaseOneReady != null) { + phaseOneReady.countDown(); + } + if (phaseOneProceed != null) { + try { + phaseOneProceed.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + + // Phase 2: Block for test verification + if (phaseTwoCount > 0 + && count == phaseTwoCount + && phaseTwoTriggered.compareAndSet(false, true)) { + if (phaseTwoStarted != null) { + phaseTwoStarted.countDown(); + } + if (phaseTwoProceed != null) { + try { + phaseTwoProceed.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + + return supplier.get(); + } + } + + private static final String COLL1_STATE = "{'gettingstarted':{\n" + " 'replicationFactor':'2',\n" + " 'router':{'name':'compositeId'},\n" From 984da7c19d145762928182a798734b9a6e1ac6e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 9 Jan 2026 11:02:57 +0100 Subject: [PATCH 21/83] Add changelog release-date for v9.10.0 --- changelog/v9.10.0/release-date.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/v9.10.0/release-date.txt diff --git a/changelog/v9.10.0/release-date.txt b/changelog/v9.10.0/release-date.txt new file mode 100644 index 000000000000..7d0595fe0cc3 --- /dev/null +++ b/changelog/v9.10.0/release-date.txt @@ -0,0 +1 @@ +2025-11-06 From e74d86ba31811c7feadf40872afbc426a79a92f0 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Fri, 9 Jan 2026 12:22:50 -0500 Subject: [PATCH 22/83] SOLR-18058: Tweak "allowPath" checks to avoid NPEs --- .../java/org/apache/solr/core/CoreContainer.java | 11 ++++++++--- .../solr/core/FileSystemConfigSetService.java | 13 +++++++++++++ .../src/java/org/apache/solr/core/SolrPaths.java | 4 ++++ .../solr/response/TestPrometheusResponseWriter.java | 4 ++++ .../test/org/apache/solr/search/TestThinCache.java | 8 +++++++- .../org/apache/solr/servlet/HideStackTraceTest.java | 5 +++++ .../scraper/SolrStandaloneScraperBasicAuthTest.java | 5 ++++- .../scraper/SolrStandaloneScraperTest.java | 6 ++++-- .../client/solrj/impl/Http2SolrClientProxyTest.java | 5 +++++ .../solr/util/EmbeddedSolrServerTestRule.java | 3 +++ 10 files changed, 57 insertions(+), 7 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 2ed0a3fb2d10..149e1c4ffc3c 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.nio.file.Path; -import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Collections; @@ -447,6 +446,7 @@ public CoreContainer(NodeConfig config, CoresLocator locator, boolean asyncSolrC SolrPaths.AllowPathBuilder allowPathBuilder = new SolrPaths.AllowPathBuilder(); allowPathBuilder.addPath(cfg.getSolrHome()); allowPathBuilder.addPath(cfg.getCoreRootDirectory()); + allowPathBuilder.addPath(cfg.getConfigSetBaseDirectory()); if (cfg.getSolrDataHome() != null) { allowPathBuilder.addPath(cfg.getSolrDataHome()); } @@ -1593,6 +1593,10 @@ public SolrCore create( log.warn(msg); throw new SolrException(ErrorCode.CONFLICT, msg); } + + // Validate 'instancePath' prior to instantiating CoreDescriptor, as CD construction attempts + // to read properties from 'instancePath' + assertPathAllowed(instancePath); CoreDescriptor cd = new CoreDescriptor( coreName, instancePath, parameters, getContainerProperties(), getZkController()); @@ -1607,8 +1611,7 @@ public SolrCore create( } // Validate paths are relative to known locations to avoid path traversal - assertPathAllowed(cd.getInstanceDir()); - assertPathAllowed(Paths.get(cd.getDataDir())); + assertPathAllowed(Path.of(cd.getDataDir())); boolean preExistingZkEntry = false; try { @@ -1679,6 +1682,8 @@ public SolrCore create( } } + public static final String ALLOW_PATHS_SYSPROP = "solr.allowPaths"; + /** * Checks that the given path is relative to SOLR_HOME, SOLR_DATA_HOME, coreRootDirectory or one * of the paths specified in solr.xml's allowPaths element. Delegates to {@link diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java index 56f17c395d18..72a98acd4e2c 100644 --- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java @@ -56,15 +56,21 @@ public class FileSystemConfigSetService extends ConfigSetService { public static final String METADATA_FILE = ".metadata.json"; private final Path configSetBase; + // TODO currently it's not really possible to check paths against allowPaths without a + // CoreContainer reference, see SOLR-18059 + private final CoreContainer cc; public FileSystemConfigSetService(CoreContainer cc) { super(cc.getResourceLoader(), cc.getConfig().hasSchemaCache()); + + this.cc = cc; this.configSetBase = cc.getConfig().getConfigSetBaseDirectory(); } /** Testing purpose */ protected FileSystemConfigSetService(Path configSetBase) { super(null, false); + this.cc = null; this.configSetBase = configSetBase; } @@ -317,6 +323,13 @@ protected Path locateInstanceDir(CoreDescriptor cd) { String configSet = cd.getConfigSet(); if (configSet == null) return cd.getInstanceDir(); Path configSetDirectory = configSetBase.resolve(configSet); + + // CoreContainer only null in testing scenarios - bit of a hack, but will go away with + // SOLR-18059 + if (cc != null) { + cc.assertPathAllowed(configSetDirectory); + } + if (!Files.isDirectory(configSetDirectory)) throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, diff --git a/solr/core/src/java/org/apache/solr/core/SolrPaths.java b/solr/core/src/java/org/apache/solr/core/SolrPaths.java index 7906c1e219fe..dc7efe471aa2 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrPaths.java +++ b/solr/core/src/java/org/apache/solr/core/SolrPaths.java @@ -126,6 +126,10 @@ public AllowPathBuilder addPath(String path) { * (not supported as a {@link Path} on Windows), see {@link #addPath(String)}. */ public AllowPathBuilder addPath(Path path) { + if (path == null) { + return this; + } + if (paths != ALL_PATHS) { if (path.equals(ALL_PATH)) { paths = ALL_PATHS; diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java index 7f15a6ae8943..7ea3b352bf60 100644 --- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java @@ -16,6 +16,8 @@ */ package org.apache.solr.response; +import static org.apache.solr.core.CoreContainer.ALLOW_PATHS_SYSPROP; + import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; import com.codahale.metrics.Meter; @@ -33,6 +35,7 @@ import org.apache.solr.client.solrj.impl.NoOpResponseParser; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.util.ExternalPaths; @@ -53,6 +56,7 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 { public static void beforeClass() throws Exception { SharedMetricRegistries.clear(); + EnvUtils.setProperty(ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME); solrClientTestRule.startSolr(LuceneTestCase.createTempDir()); solrClientTestRule.newCollection().withConfigSet(ExternalPaths.DEFAULT_CONFIGSET).create(); var cc = solrClientTestRule.getCoreContainer(); diff --git a/solr/core/src/test/org/apache/solr/search/TestThinCache.java b/solr/core/src/test/org/apache/solr/search/TestThinCache.java index c485b9fe46ac..6de790500ee6 100644 --- a/solr/core/src/test/org/apache/solr/search/TestThinCache.java +++ b/solr/core/src/test/org/apache/solr/search/TestThinCache.java @@ -16,6 +16,8 @@ */ package org.apache.solr.search; +import static org.apache.solr.core.CoreContainer.ALLOW_PATHS_SYSPROP; + import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; @@ -39,6 +41,9 @@ public class TestThinCache extends SolrTestCaseJ4 { @ClassRule public static EmbeddedSolrServerTestRule solrRule = new EmbeddedSolrServerTestRule(); public static final String SOLR_NODE_LEVEL_CACHE_XML = "\n" + + " ${" + + ALLOW_PATHS_SYSPROP + + ":}" + " \n" + " Date: Wed, 14 Jan 2026 12:12:04 -0500 Subject: [PATCH 23/83] Fix 'path' and 'permission' related NPEs Fixes another lingering source of NPEs in v1 and v2 requests. --- .../src/java/org/apache/solr/api/V2HttpCall.java | 6 ++++-- .../org/apache/solr/handler/SchemaHandler.java | 11 ++++++++--- .../apache/solr/handler/SolrConfigHandler.java | 5 ++--- .../solr/handler/admin/CollectionsHandler.java | 4 +++- .../solr/handler/admin/ConfigSetsHandler.java | 3 ++- .../apache/solr/handler/admin/InfoHandler.java | 4 +++- .../solr/handler/admin/SecurityConfHandler.java | 3 ++- .../solr/handler/admin/ZookeeperInfoHandler.java | 9 ++++++++- .../RuleBasedAuthorizationPluginBase.java | 12 ++++++++++++ .../org/apache/solr/servlet/HttpSolrCall.java | 16 ++++++++++++---- 10 files changed, 56 insertions(+), 17 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java index 24cb696aeee2..3bf131883491 100644 --- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java +++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java @@ -169,7 +169,8 @@ public void call(SolrQueryRequest req, SolrQueryResponse rsp) { if (action == REMOTEQUERY) { action = ADMIN_OR_REMOTEQUERY; coreUrl = coreUrl.replace("/solr/", "/solr/____v2/c/"); - this.path = path = path.substring(prefix.length() + collectionName.length() + 2); + normalizeAndSetPath(path.substring(prefix.length() + collectionName.length() + 2)); + path = this.path; return; } } @@ -187,7 +188,8 @@ public void call(SolrQueryRequest req, SolrQueryResponse rsp) { } Thread.currentThread().setContextClassLoader(core.getResourceLoader().getClassLoader()); - this.path = path = path.substring(prefix.length() + pathSegments.get(1).length() + 2); + normalizeAndSetPath(path.substring(prefix.length() + pathSegments.get(1).length() + 2)); + path = this.path; // Core-level API, so populate "collection" template val parts.put(COLLECTION_PROP, origCorename); Api apiInfo = getApiInfo(core.getRequestHandlers(), path, req.getMethod(), fullPath, parts); diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java index 86a5c8b2323b..5988989e3dc1 100644 --- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java @@ -103,8 +103,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw handleGET(req, rsp); break; default: - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Unexpected HTTP method: " + httpMethod); + throw getUnexpectedHttpMethodException(httpMethod.name()); } } @@ -118,10 +117,16 @@ public PermissionNameProvider.Name getPermissionName(AuthorizationContext ctx) { case "POST": return PermissionNameProvider.Name.SCHEMA_EDIT_PERM; default: - return null; + throw getUnexpectedHttpMethodException(ctx.getHttpMethod()); } } + public static SolrException getUnexpectedHttpMethodException(String methodName) + throws SolrException { + return new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Unexpected HTTP method: " + methodName); + } + private void handleGET(SolrQueryRequest req, SolrQueryResponse rsp) { try { String path = (String) req.getContext().get("path"); diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index a9cb278b2f05..e25d661faeb5 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -153,8 +153,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw command.handleGET(); break; default: - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Unexpected HTTP method: " + httpMethod); + throw SchemaHandler.getUnexpectedHttpMethodException(httpMethod.name()); } } @@ -961,7 +960,7 @@ public Name getPermissionName(AuthorizationContext ctx) { case "POST": return Name.CONFIG_EDIT_PERM; default: - return null; + throw SchemaHandler.getUnexpectedHttpMethodException(ctx.getHttpMethod()); } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index f5ee17d60405..b172deeb5580 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -251,7 +251,9 @@ public PermissionNameProvider.Name getPermissionName(AuthorizationContext ctx) { if (action == null) return PermissionNameProvider.Name.COLL_READ_PERM; CollectionParams.CollectionAction collectionAction = CollectionParams.CollectionAction.get(action); - if (collectionAction == null) return null; + if (collectionAction == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown action: " + action); + } return collectionAction.isWrite ? PermissionNameProvider.Name.COLL_EDIT_PERM : PermissionNameProvider.Name.COLL_READ_PERM; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java index 535deb54e445..ab6ca4f6bcb7 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java @@ -205,6 +205,7 @@ public Name getPermissionName(AuthorizationContext ctx) { return Name.CONFIG_READ_PERM; } } - return null; + + throw new SolrException(ErrorCode.BAD_REQUEST, "Required parameter 'action' not provided"); } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java index 455bd361247e..8b587a32fc2c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java @@ -190,7 +190,9 @@ public Name getPermissionName(AuthorizationContext request) { if (handler != null) { return handler.getPermissionName(request); } else { - return null; + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Unable to identify 'info' sub-handler for path " + path); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java index f5b528f3d703..0f3cc35d0624 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java @@ -43,6 +43,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.RequestHandlerUtils; +import org.apache.solr.handler.SchemaHandler; import org.apache.solr.handler.admin.api.GetAuthenticationConfigAPI; import org.apache.solr.handler.admin.api.GetAuthorizationConfigAPI; import org.apache.solr.handler.admin.api.ModifyNoAuthPluginSecurityConfigAPI; @@ -75,7 +76,7 @@ public PermissionNameProvider.Name getPermissionName(AuthorizationContext ctx) { case "POST": return PermissionNameProvider.Name.SECURITY_EDIT_PERM; default: - return null; + throw SchemaHandler.getUnexpectedHttpMethodException(ctx.getHttpMethod()); } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperInfoHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperInfoHandler.java index b5a8234a321b..ed46a25f20f1 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperInfoHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperInfoHandler.java @@ -40,6 +40,7 @@ import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; import org.apache.lucene.util.BytesRef; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; @@ -55,6 +56,7 @@ import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStream; +import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.RequestHandlerBase; @@ -105,7 +107,7 @@ public Category getCategory() { @Override public Name getPermissionName(AuthorizationContext request) { var params = request.getParams(); - String path = params.get(PATH, ""); + String path = normalizePath(params.get(PATH, "")); String detail = params.get(PARAM_DETAIL, "false"); if ("/security.json".equalsIgnoreCase(path) && "true".equalsIgnoreCase(detail)) { return Name.SECURITY_READ_PERM; @@ -425,6 +427,11 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw rsp.getValues().add(RawResponseWriter.CONTENT, printer); } + @SuppressForbidden(reason = "JDK String class doesn't offer a stripEnd equivalent") + private String normalizePath(String path) { + return StringUtils.stripEnd(path, "/"); + } + // -------------------------------------------------------------------------------------- // // -------------------------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPluginBase.java b/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPluginBase.java index 7c37e034c734..e126b453f1d9 100644 --- a/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPluginBase.java +++ b/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPluginBase.java @@ -30,12 +30,14 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.solr.api.AnnotatedApi; import org.apache.solr.api.Api; +import org.apache.solr.common.SolrException; import org.apache.solr.common.SpecProvider; import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.ValidatingJsonMap; @@ -229,6 +231,16 @@ private boolean predefinedPermissionAppliesToRequest( } else { PermissionNameProvider handler = (PermissionNameProvider) context.getHandler(); PermissionNameProvider.Name permissionName = handler.getPermissionName(context); + if (permissionName == null) { + final var errorMessage = + String.format( + Locale.ROOT, + "Unable to find 'predefined' associated with requestHandler [%s] and request [%s %s]", + handler.getClass().getName(), + context.getHttpMethod(), + context.getResource()); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, errorMessage); + } boolean applies = permissionName != null && predefinedPermission.name.equals(permissionName.name); diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 93feb4b5d150..4fab7a6af83c 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -62,6 +62,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import net.jcip.annotations.ThreadSafe; +import org.apache.commons.lang3.StringUtils; import org.apache.http.Header; import org.apache.http.HeaderIterator; import org.apache.http.HttpEntity; @@ -183,6 +184,8 @@ public HttpSolrCall( this.response = response; this.retry = retry; this.requestType = RequestType.UNKNOWN; + normalizeAndSetPath(ServletUtils.getPathAfterContext(req)); + req.setAttribute(HttpSolrCall.class.getName(), this); // set a request timer which can be reused by requests if needed req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimerTree()); @@ -191,6 +194,11 @@ public HttpSolrCall( path = ServletUtils.getPathAfterContext(req); } + @SuppressForbidden(reason = "JDK String class doesn't offer a stripEnd equivalent") + protected void normalizeAndSetPath(String unnormalizedPath) { + this.path = StringUtils.stripEnd(unnormalizedPath, "/"); + } + public String getPath() { return path; } @@ -242,7 +250,7 @@ protected void init() throws Exception { // Try to resolve a Solr core name core = cores.getCore(origCorename); if (core != null) { - path = path.substring(idx); + normalizeAndSetPath(path.substring(idx)); } else { // extra mem barriers, so don't look at this before trying to get core if (cores.isCoreLoading(origCorename)) { @@ -251,7 +259,7 @@ protected void init() throws Exception { // the core may have just finished loading core = cores.getCore(origCorename); if (core != null) { - path = path.substring(idx); + normalizeAndSetPath(path.substring(idx)); } else { if (!cores.isZooKeeperAware()) { core = cores.getCore(""); @@ -280,14 +288,14 @@ protected void init() throws Exception { core = getCoreByCollection(collectionName, isPreferLeader); if (core != null) { if (idx > 0) { - path = path.substring(idx); + normalizeAndSetPath(path.substring(idx)); } } else { // if we couldn't find it locally, look on other nodes if (idx > 0) { extractRemotePath(collectionName, origCorename); if (action == REMOTEQUERY) { - path = path.substring(idx); + normalizeAndSetPath(path.substring(idx)); return; } } From 064183f6710a19045bca4beea4b8a4e3170486ec Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Tue, 20 Jan 2026 10:55:20 -0500 Subject: [PATCH 24/83] DOAP changes for release 9.10.1 --- dev-tools/doap/solr.rdf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dev-tools/doap/solr.rdf b/dev-tools/doap/solr.rdf index d1f949b81470..4b6d0c8d9269 100644 --- a/dev-tools/doap/solr.rdf +++ b/dev-tools/doap/solr.rdf @@ -68,6 +68,13 @@ + + + solr-9.10.1 + 2026-01-20 + 9.10.1 + + solr-9.10.0 From 4233b85cbfb709006702ab34f6fa61bd459d7d85 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Wed, 21 Jan 2026 09:54:14 -0500 Subject: [PATCH 25/83] Fix changelog inconsistencies following 9.10.1 release --- CHANGELOG.md | 10 +++++++--- changelog/v9.10.0/version-summary.md | 4 ++-- .../SOLR-17947-cloudsolrclient async state refresh.yml | 0 changelog/v9.10.1/release-date.txt | 2 +- changelog/v9.10.1/version-summary.md | 6 +++++- 5 files changed, 15 insertions(+), 7 deletions(-) rename changelog/{unreleased => v9.10.1}/SOLR-17947-cloudsolrclient async state refresh.yml (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5819c89389b..3197c70534ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,13 @@ This file lists Solr's raw release notes with details of every change to Solr. Most people will find the solr-upgrade-notes.adoc file more approachable. [https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc](https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc) -[9.10.1] - 2026-01-08 +[9.10.1] - 2026-01-20 --------------------- +### Changed (1 change) + +- CloudSolrClient now refreshes collection state asynchronously using a dedicated thread pool, reducing ZooKeeper blocking and improving performance under load. [SOLR-17947](https://issues.apache.org/jira/browse/SOLR-17947) (Mark Miller) + ### Fixed (5 changes) - When using SolrCell with TikaServer, the connection will no longer timeout after 30s idle, such as during OCR processing [PR#3926](https://github.com/apache/solr/pull/3926) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy)) @@ -30,8 +34,8 @@ This file lists Solr's raw release notes with details of every change to Solr. M - Mitigate CVE-2025-54988 by disabling XFA parsing in PDF documents when using SolrCell extraction [SOLR-17888](https://issues.apache.org/jira/browse/SOLR-17888) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy)) -[9.10.0] --------- +[9.10.0] - 2025-11-06 +--------------------- ### Added (4 changes) diff --git a/changelog/v9.10.0/version-summary.md b/changelog/v9.10.0/version-summary.md index 2738aea37287..e553aafe8788 100644 --- a/changelog/v9.10.0/version-summary.md +++ b/changelog/v9.10.0/version-summary.md @@ -7,8 +7,8 @@ -[9.10.0] --------- +[9.10.0] - 2025-11-06 +--------------------- ### Added (4 changes) diff --git a/changelog/unreleased/SOLR-17947-cloudsolrclient async state refresh.yml b/changelog/v9.10.1/SOLR-17947-cloudsolrclient async state refresh.yml similarity index 100% rename from changelog/unreleased/SOLR-17947-cloudsolrclient async state refresh.yml rename to changelog/v9.10.1/SOLR-17947-cloudsolrclient async state refresh.yml diff --git a/changelog/v9.10.1/release-date.txt b/changelog/v9.10.1/release-date.txt index 361e2da5120e..c629d61e8f23 100644 --- a/changelog/v9.10.1/release-date.txt +++ b/changelog/v9.10.1/release-date.txt @@ -1 +1 @@ -2026-01-08 \ No newline at end of file +2026-01-20 diff --git a/changelog/v9.10.1/version-summary.md b/changelog/v9.10.1/version-summary.md index 17fed63fab29..2256eb7149f6 100644 --- a/changelog/v9.10.1/version-summary.md +++ b/changelog/v9.10.1/version-summary.md @@ -7,9 +7,13 @@ -[9.10.1] - 2026-01-08 +[9.10.1] - 2026-01-20 --------------------- +### Changed (1 change) + +- CloudSolrClient now refreshes collection state asynchronously using a dedicated thread pool, reducing ZooKeeper blocking and improving performance under load. [SOLR-17947](https://issues.apache.org/jira/browse/SOLR-17947) (Mark Miller) + ### Fixed (5 changes) - When using SolrCell with TikaServer, the connection will no longer timeout after 30s idle, such as during OCR processing [PR#3926](https://github.com/apache/solr/pull/3926) ([Jan Høydahl](https://home.apache.org/phonebook.html?uid=janhoy)) From 83d69cdee568eca65671c2828f9027afbe4e345f Mon Sep 17 00:00:00 2001 From: Rahul Goswami Date: Thu, 22 Jan 2026 14:39:14 -0500 Subject: [PATCH 26/83] Fix build breakage on Windows (#4072) --- gradle/documentation/changes-to-html/changes2html.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gradle/documentation/changes-to-html/changes2html.py b/gradle/documentation/changes-to-html/changes2html.py index 67c2bbb592c8..8ecb4832b221 100755 --- a/gradle/documentation/changes-to-html/changes2html.py +++ b/gradle/documentation/changes-to-html/changes2html.py @@ -771,6 +771,7 @@ def main(): html = generator.generate(parser.releases, parser.title, parser.preamble) # Output + sys.stdout.reconfigure(encoding='utf-8') #UTF-8 encoding for Windows compatibility print(html) From 543a0066c990bd82abd39a6391e5d266bb152b35 Mon Sep 17 00:00:00 2001 From: Rahul Goswami Date: Fri, 23 Jan 2026 10:21:46 -0500 Subject: [PATCH 27/83] --cloud option is not honored when running example (-e) on Windows (#4075) Co-authored-by: Eric Pugh --- changelog/unreleased/rahulgoswami-solr_9x.yml | 4 ++++ solr/bin/solr.cmd | 1 + 2 files changed, 5 insertions(+) create mode 100644 changelog/unreleased/rahulgoswami-solr_9x.yml diff --git a/changelog/unreleased/rahulgoswami-solr_9x.yml b/changelog/unreleased/rahulgoswami-solr_9x.yml new file mode 100644 index 000000000000..195916c61126 --- /dev/null +++ b/changelog/unreleased/rahulgoswami-solr_9x.yml @@ -0,0 +1,4 @@ +title: Fixed --cloud option is not honored when running example (-e) on Windows in Solr CLI +type: fixed +authors: + - name: Rahul Goswami diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index aab7eb8d48bb..ac67559faa53 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -510,6 +510,7 @@ goto parse_args :set_cloud_mode set SOLR_MODE=solrcloud +set "PASS_TO_RUN_EXAMPLE=--cloud !PASS_TO_RUN_EXAMPLE!" REM Notify user in 9.x about the default mode change if cloud flag is used. @echo Solr will start in SolrCloud mode by default in version 10, and you will no longer need to pass in -c or --cloud flag. SHIFT From 2aada75d3181636d15d0f7c6a458b783ed946f3e Mon Sep 17 00:00:00 2001 From: jvanneman Date: Sun, 25 Jan 2026 00:53:13 -0500 Subject: [PATCH 28/83] SOLR-18051: HttpJettySolrClient.requestAsync performance/reliability (#3992) Improve HttpJettySolrClient.requestAsync (used in sharded/distributed-search and more) to increase throughput and prevent a rare deadlock. Fix double registration of phaser and semaphore in HttpJettySolrClient, which had detrimental effects. (cherry picked from commit 8ec69db53e4bdf6d5872f376647e09ddf70fc659) --- ...SOLR-18051-fix-double-registration-bug.yml | 8 ++ .../client/solrj/impl/Http2SolrClient.java | 86 +++++++++---------- 2 files changed, 49 insertions(+), 45 deletions(-) create mode 100644 changelog/unreleased/SOLR-18051-fix-double-registration-bug.yml diff --git a/changelog/unreleased/SOLR-18051-fix-double-registration-bug.yml b/changelog/unreleased/SOLR-18051-fix-double-registration-bug.yml new file mode 100644 index 000000000000..918950bc94eb --- /dev/null +++ b/changelog/unreleased/SOLR-18051-fix-double-registration-bug.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Improve HttpJettySolrClient.requestAsync (used in sharded/distributed-search and more) to increase throughput and prevent a rare deadlock. +type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: James Vanneman +links: + - name: SOLR-18051 + url: https://issues.apache.org/jira/browse/SOLR-18051 diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java index b4936da4160a..78bbee8b4e04 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java @@ -524,51 +524,47 @@ public CompletableFuture> requestAsync( future.completeExceptionally(e); return future; } - mrrv.request - .onRequestQueued(asyncTracker.queuedListener) - .onComplete(asyncTracker.completeListener) - .send( - new InputStreamResponseListener() { - // MDC snapshot from requestAsync's thread - MDCCopyHelper mdcCopyHelper = new MDCCopyHelper(); - - @Override - public void onHeaders(Response response) { - super.onHeaders(response); - InputStreamResponseListener listener = this; - executor.execute( - () -> { - InputStream is = listener.getInputStream(); - try { - NamedList body = - processErrorsAndResponse(solrRequest, response, is, url); - mdcCopyHelper.onBegin(null); - log.debug("response processing success"); - future.complete(body); - } catch (CancellationException e) { - mdcCopyHelper.onBegin(null); - log.debug("response processing cancelled", e); - if (!future.isDone()) { - future.cancel(true); - } - } catch (Throwable e) { - mdcCopyHelper.onBegin(null); - log.debug("response processing failed", e); - future.completeExceptionally(e); - } finally { - log.debug("response processing completed"); - mdcCopyHelper.onComplete(null); - } - }); - } - - @Override - public void onFailure(Response response, Throwable failure) { - super.onFailure(response, failure); - future.completeExceptionally( - new SolrServerException(failure.getMessage(), failure)); - } - }); + mrrv.request.send( + new InputStreamResponseListener() { + // MDC snapshot from requestAsync's thread + MDCCopyHelper mdcCopyHelper = new MDCCopyHelper(); + + @Override + public void onHeaders(Response response) { + super.onHeaders(response); + InputStreamResponseListener listener = this; + executor.execute( + () -> { + InputStream is = listener.getInputStream(); + try { + NamedList body = + processErrorsAndResponse(solrRequest, response, is, url); + mdcCopyHelper.onBegin(null); + log.debug("response processing success"); + future.complete(body); + } catch (CancellationException e) { + mdcCopyHelper.onBegin(null); + log.debug("response processing cancelled", e); + if (!future.isDone()) { + future.cancel(true); + } + } catch (Throwable e) { + mdcCopyHelper.onBegin(null); + log.debug("response processing failed", e); + future.completeExceptionally(e); + } finally { + log.debug("response processing completed"); + mdcCopyHelper.onComplete(null); + } + }); + } + + @Override + public void onFailure(Response response, Throwable failure) { + super.onFailure(response, failure); + future.completeExceptionally(new SolrServerException(failure.getMessage(), failure)); + } + }); // SOLR-17819: Disabling request aborting for Solr 9.x (Jetty 10). // Not disabled for Solr 10.x because Jetty 12 does not have the same issue. From 66deb795f16e58ef5f7d1564f5c5160f1ef23ccc Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Mon, 26 Jan 2026 11:25:27 -0500 Subject: [PATCH 29/83] SOLR-12224: Add APIs to read collection properties (#4071) Prior to this PR Solr allowed users to write collection properties but never read them. This commit adds two new APIs to serve this need: the first for listing all properties (`GET /api/collections/someCollName/properties`) and the second for fetching a single property by name (`GET /api/collections/someCollName/properties/somePropName`). Corresponding SolrJ "SolrRequest" and "SolrResponse" classes are also generated based on the OAS definition for these new APIs. --- .../SOLR-12224-add-collprop-read-apis.yml | 8 +++ .../api/endpoint/CollectionPropertyApi.java | 23 ++++++- .../model/GetCollectionPropertyResponse.java | 28 +++++++++ .../ListCollectionPropertiesResponse.java | 29 +++++++++ .../handler/admin/api/CollectionProperty.java | 57 +++++++++++++++++- .../solr/cloud/CollectionsAPISolrJTest.java | 42 +++++++++++-- .../pages/collection-management.adoc | 60 +++++++++++++++++-- 7 files changed, 231 insertions(+), 16 deletions(-) create mode 100644 changelog/unreleased/SOLR-12224-add-collprop-read-apis.yml create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/GetCollectionPropertyResponse.java create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/ListCollectionPropertiesResponse.java diff --git a/changelog/unreleased/SOLR-12224-add-collprop-read-apis.yml b/changelog/unreleased/SOLR-12224-add-collprop-read-apis.yml new file mode 100644 index 000000000000..3372502b7a40 --- /dev/null +++ b/changelog/unreleased/SOLR-12224-add-collprop-read-apis.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Create new v2 APIs for listing and reading collection properties ("collprops") +type: added # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Jason Gerlowski +links: + - name: SOLR-12224 + url: https://issues.apache.org/jira/browse/SOLR-12224 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/CollectionPropertyApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/CollectionPropertyApi.java index 8c69aa3ce4c7..a4fd54de8db5 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/CollectionPropertyApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/CollectionPropertyApi.java @@ -18,16 +18,36 @@ import io.swagger.v3.oas.annotations.Operation; import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; import jakarta.ws.rs.PUT; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; +import org.apache.solr.client.api.model.GetCollectionPropertyResponse; +import org.apache.solr.client.api.model.ListCollectionPropertiesResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.UpdateCollectionPropertyRequestBody; /** V2 API definitions for modifying collection-level properties. */ -@Path("/collections/{collName}/properties/{propName}") +@Path("/collections/{collName}/properties") public interface CollectionPropertyApi { + @GET + @Operation( + summary = "List all properties for the specified collection", + tags = {"collection-properties"}) + ListCollectionPropertiesResponse listCollectionProperties(@PathParam("collName") String collName) + throws Exception; + + @GET + @Path("/{propName}") + @Operation( + summary = "Get the value of a specific collection property", + tags = {"collection-properties"}) + GetCollectionPropertyResponse getCollectionProperty( + @PathParam("collName") String collName, @PathParam("propName") String propName) + throws Exception; + @PUT + @Path("/{propName}") @Operation( summary = "Create or update a collection property", tags = {"collection-properties"}) @@ -38,6 +58,7 @@ SolrJerseyResponse createOrUpdateCollectionProperty( throws Exception; @DELETE + @Path("/{propName}") @Operation( summary = "Delete the specified collection property from the collection", tags = {"collection-properties"}) diff --git a/solr/api/src/java/org/apache/solr/client/api/model/GetCollectionPropertyResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/GetCollectionPropertyResponse.java new file mode 100644 index 000000000000..0da15ee34bb1 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/GetCollectionPropertyResponse.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +/** The Response for the v2 "get collection property" API */ +public class GetCollectionPropertyResponse extends SolrJerseyResponse { + + @Schema(description = "The value of the collection property.") + @JsonProperty("value") + public String value; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ListCollectionPropertiesResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ListCollectionPropertiesResponse.java new file mode 100644 index 000000000000..74479ebf9148 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ListCollectionPropertiesResponse.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.Map; + +/** The Response for the v2 "list collection properties" API */ +public class ListCollectionPropertiesResponse extends SolrJerseyResponse { + + @Schema(description = "The properties for the collection.") + @JsonProperty("properties") + public Map properties; +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CollectionProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CollectionProperty.java index 55da1ea0d53b..c68e6ce21f22 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CollectionProperty.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CollectionProperty.java @@ -18,9 +18,14 @@ package org.apache.solr.handler.admin.api; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM; +import jakarta.inject.Inject; import java.io.IOException; +import java.util.Map; import org.apache.solr.client.api.endpoint.CollectionPropertyApi; +import org.apache.solr.client.api.model.GetCollectionPropertyResponse; +import org.apache.solr.client.api.model.ListCollectionPropertiesResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.UpdateCollectionPropertyRequestBody; import org.apache.solr.common.SolrException; @@ -31,13 +36,13 @@ import org.apache.solr.response.SolrQueryResponse; /** - * V2 API implementations for modifying collection-level properties. + * V2 API implementations for managing collection-level properties. * - *

These APIs (PUT and DELETE /api/collections/collName/properties/propName) are analogous to the - * v1 /admin/collections?action=COLLECTIONPROP command. + *

These APIs are analogous to the v1 /admin/collections?action=COLLECTIONPROP command. */ public class CollectionProperty extends AdminAPIBase implements CollectionPropertyApi { + @Inject public CollectionProperty( CoreContainer coreContainer, SolrQueryRequest solrQueryRequest, @@ -45,6 +50,52 @@ public CollectionProperty( super(coreContainer, solrQueryRequest, solrQueryResponse); } + @Override + @PermissionName(COLL_READ_PERM) + public ListCollectionPropertiesResponse listCollectionProperties(String collName) + throws Exception { + final var response = instantiateJerseyResponse(ListCollectionPropertiesResponse.class); + ensureRequiredParameterProvided("collName", collName); + fetchAndValidateZooKeeperAwareCoreContainer(); + recordCollectionForLogAndTracing(collName, solrQueryRequest); + + String resolvedCollection = coreContainer.getAliases().resolveSimpleAlias(collName); + CollectionProperties cp = + new CollectionProperties(coreContainer.getZkController().getZkClient()); + Map properties = cp.getCollectionProperties(resolvedCollection); + + // Handle null case - return empty map instead of null + response.properties = (properties != null) ? properties : Map.of(); + + return response; + } + + @Override + @PermissionName(COLL_READ_PERM) + public GetCollectionPropertyResponse getCollectionProperty(String collName, String propName) + throws Exception { + final var response = instantiateJerseyResponse(GetCollectionPropertyResponse.class); + ensureRequiredParameterProvided("collName", collName); + ensureRequiredParameterProvided("propName", propName); + fetchAndValidateZooKeeperAwareCoreContainer(); + recordCollectionForLogAndTracing(collName, solrQueryRequest); + + String resolvedCollection = coreContainer.getAliases().resolveSimpleAlias(collName); + CollectionProperties cp = + new CollectionProperties(coreContainer.getZkController().getZkClient()); + Map properties = cp.getCollectionProperties(resolvedCollection); + + if (properties != null && properties.containsKey(propName)) { + response.value = properties.get(propName); + } else { + throw new SolrException( + SolrException.ErrorCode.NOT_FOUND, + "Property '" + propName + "' not found for collection '" + collName + "'"); + } + + return response; + } + @Override @PermissionName(COLL_EDIT_PERM) public SolrJerseyResponse createOrUpdateCollectionProperty( diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java index d4b9148c228c..73abeb3042be 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java @@ -50,6 +50,7 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CollectionPropertiesApi; import org.apache.solr.client.solrj.request.CollectionsApi; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.CoreStatus; @@ -544,7 +545,7 @@ public void testClusterProp() throws IOException, SolrServerException { } @Test - public void testCollectionProp() throws InterruptedException, IOException, SolrServerException { + public void testCollectionProp() throws Exception { String collectionName = getSaferTestName(); final String propName = "testProperty"; @@ -563,18 +564,47 @@ public void testCollectionProp() throws InterruptedException, IOException, SolrS CollectionAdminRequest.setCollectionProperty(collectionName, propName, null) .process(cluster.getSolrClient()); checkCollectionProperty(collectionName, propName, null); + + // Test that "list-properties" returns all properties + CollectionAdminRequest.setCollectionProperty(collectionName, propName + "1", "prop1Val") + .process(cluster.getSolrClient()); + CollectionAdminRequest.setCollectionProperty(collectionName, propName + "2", "prop2Val") + .process(cluster.getSolrClient()); + final var allProperties = + new CollectionPropertiesApi.ListCollectionProperties(collectionName) + .process(cluster.getSolrClient()) + .getParsed() + .properties; + assertEquals(2, allProperties.size()); + assertEquals("prop1Val", allProperties.get(propName + "1")); + assertEquals("prop2Val", allProperties.get(propName + "2")); + + // Test GET single property API + final var prop1Response = + new CollectionPropertiesApi.GetCollectionProperty(collectionName, propName + "1") + .process(cluster.getSolrClient()) + .getParsed(); + assertEquals("prop1Val", prop1Response.value); + + final var prop2Response = + new CollectionPropertiesApi.GetCollectionProperty(collectionName, propName + "2") + .process(cluster.getSolrClient()) + .getParsed(); + assertEquals("prop2Val", prop2Response.value); } private void checkCollectionProperty(String collection, String propertyName, String propertyValue) - throws InterruptedException { + throws Exception { TimeOut timeout = new TimeOut(TIMEOUT, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME); while (!timeout.hasTimedOut()) { - Thread.sleep(10); - if (Objects.equals( - cluster.getZkStateReader().getCollectionProperties(collection).get(propertyName), - propertyValue)) { + final var listCollPropRsp = + new CollectionPropertiesApi.ListCollectionProperties(collection) + .process(cluster.getSolrClient()) + .getParsed(); + if (Objects.equals(listCollPropRsp.properties.get(propertyName), propertyValue)) { return; } + Thread.sleep(10); } fail("Timed out waiting for cluster property value"); diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc index 55b3f8b5d386..f743fb4ea8c7 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc @@ -673,7 +673,8 @@ If the status is anything other than "success", an error message will explain wh [[collectionprop]] == COLLECTIONPROP: Collection Properties -Add, edit or delete a collection property. +Add, update, delete, or retrieve collection properties. +(Listing all collection properties, or fetching an individual property by name are only supported in Solr's v2 API) [tabs#collectionproperty-request] ====== @@ -682,7 +683,7 @@ V1 API:: ==== [source,bash] ---- -http://localhost:8983/solr/admin/collections?action=COLLECTIONPROP&name=techproducts_v2&propertyName=propertyName&propertyValue=propertyValue +http://localhost:8983/solr/admin/collections?action=COLLECTIONPROP&name=techproducts&propertyName=propertyName&propertyValue=propertyValue ---- ==== @@ -690,20 +691,36 @@ V2 API:: + ==== To create or update a collection property: + [source,bash] ---- -curl -X PUT http://localhost:8983/api/collections/techproducts_v2/properties/foo -H 'Content-Type: application/json' -d ' +curl -X PUT http://localhost:8983/api/collections/techproducts_v2/properties/propertyName -H 'Content-Type: application/json' -d ' { - "value": "bar" + "value": "propertyValue" } ' ---- +To list all properties for a collection: + +[source,bash] +---- +curl http://localhost:8983/api/collections/techproducts/properties +---- + +To get a specific collection property by name: + +[source,bash] +---- +curl http://localhost:8983/api/collections/techproducts/properties/propertyName +---- + + To delete an existing collection property: [source,bash] ---- -curl -X DELETE http://localhost:8983/api/collections/techproducts_v2/properties/foo +curl -X DELETE http://localhost:8983/api/collections/techproducts/properties/propertyName ---- ==== ====== @@ -742,9 +759,40 @@ When not provided in v1 requests, the property is deleted. === COLLECTIONPROP Response -The response will include the status of the request and the properties that were updated or removed. +The response will include the status of the request. If the status is anything other than "0", an error message will explain why the request failed. +For GET requests to list all properties, the response includes a `properties` object containing all property name-value pairs: + +[source,json] +---- +{ + "responseHeader": { + "status": 0, + "QTime": 5 + }, + "properties": { + "foo": "bar", + "myProperty": "myValue" + } +} +---- + +For GET requests to retrieve a single property, the response includes a `value` field with the property value: + +[source,json] +---- +{ + "responseHeader": { + "status": 0, + "QTime": 3 + }, + "value": "bar" +} +---- + +If a requested property does not exist, the API will return a 404 error with an appropriate error message. + [[migrate]] == MIGRATE: Migrate Documents to Another Collection From 5f19397466f74ccb524f26ea3c17f05b990ee59e Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 28 Jan 2026 08:31:27 -0500 Subject: [PATCH 30/83] Backport SOLR-18008 to 9x (#4086) --- ...SOLR-18008-simulate_solr_core_remnants.yml | 9 + .../org/apache/solr/core/CoreContainer.java | 4 +- .../solr/core/CorePropertiesLocator.java | 21 ++ .../cloud/DeleteCoreRemnantsOnCreateTest.java | 320 ++++++++++++++++++ .../pages/core-discovery.adoc | 2 + .../pages/collection-management.adoc | 9 + 6 files changed, 363 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml create mode 100644 solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java diff --git a/changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml b/changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml new file mode 100644 index 000000000000..45d804cccd67 --- /dev/null +++ b/changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml @@ -0,0 +1,9 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Add solr.cloud.delete.unknown.cores.enabled setting for removing unknown but existing core data when a core is created in SolrCloud mode. +type: changed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Eric Pugh + - name: David Smiley +links: +- name: SOLR-18008 + url: https://issues.apache.org/jira/browse/SOLR-18008 diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 149e1c4ffc3c..776219293f23 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -1782,7 +1782,7 @@ private SolrCore createFromDescriptor( } catch (Exception e) { coreInitFailures.put(dcore.getName(), new CoreLoadFailure(dcore, e)); if (e instanceof ZkController.NotInClusterStateException && !newCollection) { - // this mostly happens when the core is deleted when this node is down + // this mostly happens when the core is deleted when this node is down, // but it can also happen if connecting to the wrong zookeeper final boolean deleteUnknownCores = Boolean.parseBoolean(System.getProperty("solr.deleteUnknownCores", "false")); @@ -1793,7 +1793,7 @@ private SolrCore createFromDescriptor( (deleteUnknownCores ? " It will be deleted. See SOLR-13396 for more information." : "")); - // We alreday have an ongoing CoreOp, so do not wait to start another one + // We already have an ongoing CoreOp, so do not wait to start another one unloadWithoutCoreOp( dcore.getName(), deleteUnknownCores, deleteUnknownCores, deleteUnknownCores); throw e; diff --git a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java index c060fc5d3cdd..2fe356dfca8a 100644 --- a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java +++ b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java @@ -39,6 +39,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.EnvUtils; import org.apache.solr.util.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +66,7 @@ public CorePropertiesLocator(Path coreDiscoveryRoot) { @Override public void create(CoreContainer cc, CoreDescriptor... coreDescriptors) { for (CoreDescriptor cd : coreDescriptors) { + checkForExistingCore(cd); Path propertiesFile = cd.getInstanceDir().resolve(PROPERTIES_FILENAME); if (Files.exists(propertiesFile)) throw new SolrException( @@ -240,4 +242,23 @@ protected Properties buildCoreProperties(CoreDescriptor cd) { p.putAll(cd.getPersistableUserProperties()); return p; } + + protected void checkForExistingCore(CoreDescriptor cd) { + if (cd.getCloudDescriptor() != null && Files.exists(cd.getInstanceDir())) { + final boolean deleteUnknownCores = + EnvUtils.getPropertyAsBool("solr.cloud.delete.unknown.cores.enabled", false); + if (deleteUnknownCores) { + log.warn( + "Automatically deleting existing directory at [{}] for core [{}] because solr.cloud.delete.unknown.cores.enabled is true", + cd.getInstanceDir().toAbsolutePath(), + cd.getName()); + SolrCore.deleteUnloadedCore(cd, true, true); + } else { + log.warn( + "Directory at [{}] for core[{}] already exists may prevent create operation. Set solr.cloud.delete.unknown.cores.enabled=true to delete directory. (SOLR-18008)", + cd.getInstanceDir().toAbsolutePath(), + cd.getName()); + } + } + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java new file mode 100644 index 000000000000..3f07879a148b --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import org.apache.solr.client.solrj.impl.JsonMapResponseParser; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.core.SolrCore; +import org.apache.solr.embedded.JettySolrRunner; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Test "solr.cloud.delete.unknown.cores.enabled" property that can be used if Solr has an + * inconsistent state with its cores lifecycle where remnant files are left on disk after various + * operations that delete a core. + */ +public class DeleteCoreRemnantsOnCreateTest extends SolrCloudTestCase { + private static final String DELETE_UNKNOWN_CORES_PROP = "solr.cloud.delete.unknown.cores.enabled"; + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(1).addConfig("conf", configset("cloud-minimal")).configure(); + } + + /** + * Shared setup for testing collection creation with remnants. Creates a collection, deletes it, + * and then leaves behind a remnant directory. + */ + private String setupCollectionRemnant(String collectionName) throws Exception { + List jettys = cluster.getJettySolrRunners(); + String primaryNode = jettys.get(0).getNodeName(); + + CollectionAdminRequest.Create createRequest = + CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1); + createRequest.process(cluster.getSolrClient()); + + waitForState( + "Expected collection to be fully active", + collectionName, + (n, c) -> DocCollection.isFullyActive(n, c, 1, 1)); + + Replica primaryReplica = getReplicaOnNode(collectionName, "shard1", primaryNode); + JettySolrRunner primaryJetty = cluster.getReplicaJetty(primaryReplica); + String originalCoreName = primaryReplica.getCoreName(); + Path remnantInstanceDir; + try (SolrCore core = primaryJetty.getCoreContainer().getCore(originalCoreName)) { + CoreDescriptor cd = core.getCoreDescriptor(); + remnantInstanceDir = cd.getInstanceDir(); + } + + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + waitForState("Expected collection deletion", collectionName, (n, c) -> c == null); + + // Simulate a core remnant still exists by creating the directory and core.properties + Files.createDirectories(remnantInstanceDir); + String propertiesContent = ""; + Files.writeString( + remnantInstanceDir.resolve("core.properties"), propertiesContent, StandardCharsets.UTF_8); + + return originalCoreName; + } + + /** + * Shared setup for testing replica addition with remnants. Creates a collection, then simulates a + * remnant directory on the single node that will impact the next addReplica command. + */ + private void setupReplicaRemnant(String collectionName) throws Exception { + List jettys = cluster.getJettySolrRunners(); + String primaryNode = jettys.get(0).getNodeName(); + + CollectionAdminRequest.Create createRequest = + CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1); + createRequest.process(cluster.getSolrClient()); + + waitForState( + "Expected collection to be fully active", + collectionName, + (n, c) -> DocCollection.isFullyActive(n, c, 1, 1)); + + int nextReplicaIndex = 3; // Yep, from 1 to 3 due to how we count in ZK and setup. + String expectedNewReplicaName = collectionName + "_shard1_replica_n" + nextReplicaIndex; + + // Simulate a core remnant on the single node adjacent to the existing replica instance path + Replica existing = getReplicaOnNode(collectionName, "shard1", primaryNode); + try (SolrCore core = + cluster.getReplicaJetty(existing).getCoreContainer().getCore(existing.getCoreName())) { + Path siblingDir = core.getInstancePath().getParent().resolve(expectedNewReplicaName); + Files.createDirectories(siblingDir); + Files.writeString( + siblingDir.resolve("core.properties"), + "name=" + + expectedNewReplicaName + + "_remnant\n" + + "collection=" + + collectionName + + "_remnant\n" + + "shard=shard1\n" + + "coreNodeName=core_node_remnant\n", + StandardCharsets.UTF_8); + } + } + + /** + * Shared setup for testing DeleteCore admin API with remnants. Creates a collection, deletes it, + * and then leaves behind a remnant core directory. + */ + private String setupCoreRemnantForUnloadCoreOperation(String collectionName) throws Exception { + List jettys = cluster.getJettySolrRunners(); + String primaryNode = jettys.get(0).getNodeName(); + + CollectionAdminRequest.Create createRequest = + CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1); + createRequest.process(cluster.getSolrClient()); + + waitForState( + "Expected collection to be fully active", + collectionName, + (n, c) -> DocCollection.isFullyActive(n, c, 1, 1)); + + Replica primaryReplica = getReplicaOnNode(collectionName, "shard1", primaryNode); + JettySolrRunner primaryJetty = cluster.getReplicaJetty(primaryReplica); + String originalCoreName = primaryReplica.getCoreName(); + Path remnantInstanceDir; + try (SolrCore core = primaryJetty.getCoreContainer().getCore(originalCoreName)) { + CoreDescriptor cd = core.getCoreDescriptor(); + remnantInstanceDir = cd.getInstanceDir(); + } + + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + waitForState("Expected collection deletion", collectionName, (n, c) -> c == null); + + // Simulate a core remnant still exists by creating the directory and core.properties + Files.createDirectories(remnantInstanceDir); + Files.writeString( + remnantInstanceDir.resolve("core.properties"), + "name=" + originalCoreName + "\n", + StandardCharsets.UTF_8); + + return originalCoreName; + } + + @Test + public void testCreateCollectionWithRemnantsFailsWithoutSetting() throws Exception { + assertNull( + "Property should not be set by default", System.getProperty(DELETE_UNKNOWN_CORES_PROP)); + + String collectionName = "coreRemnantCreateNoSetting"; + setupCollectionRemnant(collectionName); + + // Try to create the collection again - this demonstrates the behavior without the setting + // In typical environments, this might fail, but behavior depends on configuration + CollectionAdminRequest.Create recreateRequest = + CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1); + + // The request to create a collection SHOULD fail based on the remnant file, if it does not it + // means we've changed Solr's behavior when creating a core and + // remnants exist, and therefore we should rethink the utility of this setting. + Exception e = + assertThrows( + "This request to recreate the collection should have failed due to remnant files.", + Exception.class, + () -> recreateRequest.process(cluster.getSolrClient())); + + assertTrue( + "Verify the exception was due to core creation failed.", + e.getMessage().contains("Underlying core creation failed")); + } + + @Test + public void testCreateCollectionWithRemnantsWithSetting() throws Exception { + System.setProperty(DELETE_UNKNOWN_CORES_PROP, "true"); + + String collectionName = "coreRemnantCreateWithSetting"; + setupCollectionRemnant(collectionName); + + // With the setting enabled, collection creation should succeed despite remnants + CollectionAdminRequest.Create recreateRequest = + CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1); + List jettys = cluster.getJettySolrRunners(); + recreateRequest.process(cluster.getSolrClient()); + + waitForState( + "Expected recreated collection to be fully active", + collectionName, + (n, c) -> DocCollection.isFullyActive(n, c, 1, 1)); + + // Verify collection was created successfully + DocCollection collection = getCollectionState(collectionName); + assertNotNull("Collection should exist", collection); + assertEquals("Should have 1 replica", 1, collection.getReplicas().size()); + + // Verify replica on the node where we had the remnant is active + Replica recreatedReplica = + getReplicaOnNode(collectionName, "shard1", jettys.get(0).getNodeName()); + assertNotNull("Should have a replica on the primary node", recreatedReplica); + assertEquals("Replica should be active", Replica.State.ACTIVE, recreatedReplica.getState()); + } + + @Test + public void testAddReplicaWithRemnantFailsWithoutSetting() throws Exception { + assertNull( + "Property should not be set by default", System.getProperty(DELETE_UNKNOWN_CORES_PROP)); + + String collectionName = "coreRemnantAddReplicaNoSetting"; + setupReplicaRemnant(collectionName); + + List jettys = cluster.getJettySolrRunners(); + String primaryNode = jettys.get(0).getNodeName(); + + // Try to add a new replica - this demonstrates the behavior without the setting + CollectionAdminRequest.AddReplica addReplicaRequest = + CollectionAdminRequest.addReplicaToShard(collectionName, "shard1"); + addReplicaRequest.setNode(primaryNode); + + Exception e = + assertThrows( + "This request to add a replica to the collection should have failed due to remnant files.", + Exception.class, + () -> addReplicaRequest.process(cluster.getSolrClient())); + + assertTrue( + "Verify the exception was due to core creation failed.", + e.getMessage().contains("ADDREPLICA failed to create replica")); + } + + @Test + public void testAddReplicaWithRemnantWithSetting() throws Exception { + System.setProperty(DELETE_UNKNOWN_CORES_PROP, "true"); + + String collectionName = "coreRemnantAddReplicaWithSetting"; + setupReplicaRemnant(collectionName); + + List jettys = cluster.getJettySolrRunners(); + String primaryNode = jettys.get(0).getNodeName(); + + // With the setting enabled, replica addition should succeed despite remnants + CollectionAdminRequest.AddReplica addReplicaRequest = + CollectionAdminRequest.addReplicaToShard(collectionName, "shard1"); + addReplicaRequest.setNode(primaryNode); + addReplicaRequest.process(cluster.getSolrClient()); + + waitForState( + "Expected replica addition to finish", + collectionName, + (n, c) -> DocCollection.isFullyActive(n, c, 1, 2)); + + // Verify collection now has 2 replicas + DocCollection collection = getCollectionState(collectionName); + assertNotNull("Collection should exist", collection); + assertEquals("Should have 2 replicas after adding", 2, collection.getReplicas().size()); + + // Verify the replica was added on the single node and is active + Replica addedReplica = getReplicaOnNode(collectionName, "shard1", primaryNode); + assertNotNull("Should have added a replica on the primary node", addedReplica); + assertEquals("Added replica should be active", Replica.State.ACTIVE, addedReplica.getState()); + } + + /** + * This test demonstrates that you can't call the direct core unload admin operation to get rid of + * a remnant core, not because of the existence of the remnant, but because the core no longer has + * a CoreDescriptor record in ZooKeeper. + */ + @Test + public void testDeleteCoreFailsWhenUnknown() throws Exception { + + String collectionName = "coreRemnantDelete"; + String coreName = setupCollectionRemnant(collectionName); + + // Try to delete a core that only exists as a remnant - and has no record in ZooKeeper + CoreAdminRequest.Unload unloadRequest = new CoreAdminRequest.Unload(true); + unloadRequest.setDeleteIndex(true); + unloadRequest.setDeleteDataDir(true); + unloadRequest.setDeleteInstanceDir(true); + unloadRequest.setCoreName(coreName); + unloadRequest.setResponseParser(new JsonMapResponseParser()); + + Exception e = + assertThrows( + "Expected request to fail.", + Exception.class, + () -> cluster.getSolrClient().request(unloadRequest)); + + assertTrue( + "Verify the exception was due to ZK not knowing about the core existence.", + e.getMessage().contains("Cannot unload non-existent core [" + coreName + "]")); + } + + private Replica getReplicaOnNode(String collectionName, String shard, String nodeName) { + DocCollection collectionState = getCollectionState(collectionName); + Slice slice = collectionState.getSlice(shard); + Optional replica = + slice.getReplicas().stream().filter(r -> nodeName.equals(r.getNodeName())).findFirst(); + return replica.orElseThrow( + () -> new AssertionError("No replica found on node " + nodeName + " for " + shard)); + } +} diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc index 779c0812c3b2..944ede33e3af 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc @@ -18,6 +18,8 @@ Core discovery means that creating a core is as simple as a `core.properties` file located on disk. +TIP: If you are running in SolrCloud mode, you are shielded from this complexity. + == The core.properties File In Solr, the term _core_ is used to refer to a single index and associated transaction log and configuration files (including the `solrconfig.xml` and schema files, among others). diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc index f743fb4ea8c7..1d70836584af 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc @@ -292,6 +292,15 @@ Request ID to track this action which will be xref:configuration-guide:collectio Collections are first created in read-write mode but can be put in `readOnly` mode using the xref:collection-management.adoc#modifycollection[MODIFYCOLLECTION] action. +Solr can occasionally enter an inconsistent state where remnant core files remain on disk after previous collection deletion operations fail. + +The system property `solr.cloud.delete.unknown.cores.enabled` is an expert-level setting designed to handle this situation. +When enabled, Solr automatically deletes any remnant core data on startup that lacks a record in ZooKeeper. +During core creation (whether for new collections or replicas), Solr also deletes preexisting remnant files, allowing operations to complete. +Without this setting, these files would cause new core creation to fail. + +Enable this feature with caution— it indicates an underlying problem in your Solr setup that should be investigated. + === CREATE Response The response will include the status of the request and the new core names. From 4f695d5abf6c9f171206a443a67af41378b090d6 Mon Sep 17 00:00:00 2001 From: Luke Kot-Zaniewski Date: Sat, 31 Jan 2026 14:51:21 -0500 Subject: [PATCH 31/83] SOLR-18071: Support stored fields in ExportWriter (#4053) via a new includeStoredFields parameter (cherry picked from commit 45adb11a9eaed2456244cd03d573d8daf10d92e1) --- ...71-support-stored-fields-export-writer.yml | 8 + .../handler/export/DoubleFieldWriter.java | 8 +- .../solr/handler/export/ExportWriter.java | 87 ++++-- .../solr/handler/export/FieldWriter.java | 12 +- .../solr/handler/export/FloatFieldWriter.java | 8 +- .../solr/handler/export/IntFieldWriter.java | 8 +- .../solr/handler/export/LongFieldWriter.java | 8 +- .../solr/handler/export/MultiFieldWriter.java | 9 +- .../handler/export/StoredFieldsWriter.java | 143 +++++++++ .../handler/export/StringFieldWriter.java | 8 +- .../conf/schema-sortingresponse.xml | 20 +- .../solr/handler/export/TestExportWriter.java | 294 +++++++++++++++++- .../pages/exporting-result-sets.adoc | 73 ++++- 13 files changed, 622 insertions(+), 64 deletions(-) create mode 100644 changelog/unreleased/SOLR-18071-support-stored-fields-export-writer.yml create mode 100644 solr/core/src/java/org/apache/solr/handler/export/StoredFieldsWriter.java diff --git a/changelog/unreleased/SOLR-18071-support-stored-fields-export-writer.yml b/changelog/unreleased/SOLR-18071-support-stored-fields-export-writer.yml new file mode 100644 index 000000000000..dbd1b8c02371 --- /dev/null +++ b/changelog/unreleased/SOLR-18071-support-stored-fields-export-writer.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Support including stored fields in Export Writer output. +type: added # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Luke Kot-Zaniewski +links: + - name: SOLR-18071 + url: https://issues.apache.org/jira/browse/SOLR-18071 diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java index e439560894b4..561d03366786 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java @@ -34,8 +34,7 @@ public DoubleFieldWriter( } @Override - public boolean write( - SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew, int fieldIndex) + public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew) throws IOException { double val; SortValue sortValue = sortDoc.getSortValue(this.field); @@ -43,7 +42,7 @@ public boolean write( if (sortValue.isPresent()) { val = (double) sortValue.getCurrentValue(); } else { // empty-value - return false; + return; } } else { // field is not part of 'sort' param, but part of 'fl' param @@ -53,10 +52,9 @@ public boolean write( if (vals != null) { val = Double.longBitsToDouble(vals.longValue()); } else { - return false; + return; } } ew.put(this.field, val); - return true; } } diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java index 209680aee120..9307f167b48f 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java @@ -28,8 +28,11 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeSet; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -99,15 +102,15 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { public static final String BATCH_SIZE_PARAM = "batchSize"; public static final String QUEUE_SIZE_PARAM = "queueSize"; + public static final String INCLUDE_STORED_FIELDS_PARAM = "includeStoredFields"; public static final int DEFAULT_BATCH_SIZE = 30000; public static final int DEFAULT_QUEUE_SIZE = 150000; private static final FieldWriter EMPTY_FIELD_WRITER = new FieldWriter() { @Override - public boolean write( - SortDoc sortDoc, LeafReaderContext readerContext, EntryWriter out, int fieldIndex) { - return false; + public void write(SortDoc sortDoc, LeafReaderContext readerContext, EntryWriter out) { + // do nothing } }; @@ -482,45 +485,72 @@ void writeDoc( throws IOException { int ord = sortDoc.ord; LeafReaderContext context = leaves.get(ord); - int fieldIndex = 0; for (FieldWriter fieldWriter : writers) { - if (fieldWriter.write(sortDoc, context, ew, fieldIndex)) { - ++fieldIndex; - } + fieldWriter.write(sortDoc, context, ew); } } public List getFieldWriters(String[] fields, SolrQueryRequest req) throws IOException { DocValuesIteratorCache dvIterCache = new DocValuesIteratorCache(req.getSearcher(), false); - SolrReturnFields solrReturnFields = new SolrReturnFields(fields, req); + boolean includeStoredFields = req.getParams().getBool(INCLUDE_STORED_FIELDS_PARAM, false); List writers = new ArrayList<>(); + Set docValueFields = new LinkedHashSet<>(); + Map storedFields = new LinkedHashMap<>(); + for (String field : req.getSearcher().getFieldNames()) { if (!solrReturnFields.wantsField(field)) { continue; } SchemaField schemaField = req.getSchema().getField(field); - if (!schemaField.hasDocValues()) { - throw new IOException(schemaField + " must have DocValues to use this feature."); - } - boolean multiValued = schemaField.multiValued(); FieldType fieldType = schemaField.getType(); - FieldWriter writer; - if (fieldType instanceof SortableTextField && !schemaField.useDocValuesAsStored()) { - if (solrReturnFields.getRequestedFieldNames() != null - && solrReturnFields.getRequestedFieldNames().contains(field)) { - // Explicitly requested field cannot be used due to not having useDocValuesAsStored=true, - // throw exception + Set requestFieldNames = + solrReturnFields.getRequestedFieldNames() == null + ? Set.of() + : solrReturnFields.getRequestedFieldNames(); + + if (canUseDocValues(schemaField, fieldType)) { + // Prefer DocValues when available + docValueFields.add(schemaField); + } else if (schemaField.stored()) { + // Field is stored-only (no usable DocValues) + if (includeStoredFields) { + storedFields.put(field, schemaField); + } else if (requestFieldNames.contains(field)) { + // Explicitly requested field without DocValues and includeStoredFields=false + throw new IOException( + schemaField + + " must have DocValues to use this feature. " + + "Try setting includeStoredFields=true to retrieve this field from stored values."); + } + // Else: glob matched stored-only field without includeStoredFields - silently skip + } else if (requestFieldNames.contains(field)) { + // Explicitly requested field that has neither DocValues nor stored + if (fieldType instanceof SortableTextField && !schemaField.useDocValuesAsStored()) { throw new IOException( schemaField + " Must have useDocValuesAsStored='true' to be used with export writer"); } else { - // Glob pattern matched field cannot be used due to not having useDocValuesAsStored=true - continue; + throw new IOException( + schemaField + " must have DocValues or be stored to use this feature."); } } + // Else: glob matched field with neither DocValues nor stored - silently skip + } + + for (SchemaField schemaField : docValueFields) { + String field = schemaField.getName(); + boolean multiValued = schemaField.multiValued(); + FieldType fieldType = schemaField.getType(); + FieldWriter writer; + + if (schemaField.stored() && !storedFields.isEmpty()) { + // if we're reading StoredFields *anyway*, then we might as well avoid this extra DV lookup + storedFields.put(field, schemaField); + continue; + } DocValuesIteratorCache.FieldDocValuesSupplier docValuesCache = dvIterCache.getSupplier(field); @@ -574,9 +604,24 @@ public List getFieldWriters(String[] fields, SolrQueryRequest req) } writers.add(writer); } + + if (!storedFields.isEmpty()) { + writers.add(new StoredFieldsWriter(storedFields)); + } + return writers; } + private static boolean canUseDocValues(SchemaField schemaField, FieldType fieldType) { + return schemaField.hasDocValues() + // Special handling for SortableTextField: unlike other field types, it requires + // useDocValuesAsStored=true to be included via glob patterns in /export. This + // matches the behavior of /select (which requires useDocValuesAsStored=true for + // all globbed fields) and avoids performance issues. The requirement cannot be + // extended to other field types in /export for backward compatibility reasons. + && (!(fieldType instanceof SortableTextField) || schemaField.useDocValuesAsStored()); + } + SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IOException { SortValue[] sortValues = new SortValue[sortFields.length]; IndexSchema schema = searcher.getSchema(); @@ -591,7 +636,7 @@ SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IO throw new IOException(field + " must have DocValues to use this feature."); } - if (ft instanceof SortableTextField && schemaField.useDocValuesAsStored() == false) { + if (ft instanceof SortableTextField && !schemaField.useDocValuesAsStored()) { throw new IOException( schemaField + " Must have useDocValuesAsStored='true' to be used with export writer"); } diff --git a/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java index 1923afb410f7..4b7cf7eb47b6 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java @@ -22,7 +22,15 @@ import org.apache.solr.common.MapWriter; abstract class FieldWriter { - public abstract boolean write( - SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter out, int fieldIndex) + /** + * Writes field values from the document to the output. + * + * @param sortDoc the document being exported + * @param readerContext the leaf reader context for accessing field values + * @param out the output writer to write field values to + * @throws IOException if an I/O error occurs while reading or writing field values + */ + public abstract void write( + SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter out) throws IOException; } diff --git a/solr/core/src/java/org/apache/solr/handler/export/FloatFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/FloatFieldWriter.java index a60c14e6b0ad..68a36f84b717 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/FloatFieldWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/FloatFieldWriter.java @@ -34,8 +34,7 @@ public FloatFieldWriter( } @Override - public boolean write( - SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew, int fieldIndex) + public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew) throws IOException { float val; SortValue sortValue = sortDoc.getSortValue(this.field); @@ -43,7 +42,7 @@ public boolean write( if (sortValue.isPresent()) { val = (float) sortValue.getCurrentValue(); } else { // empty-value - return false; + return; } } else { // field is not part of 'sort' param, but part of 'fl' param @@ -53,10 +52,9 @@ public boolean write( if (vals != null) { val = Float.intBitsToFloat((int) vals.longValue()); } else { - return false; + return; } } ew.put(this.field, val); - return true; } } diff --git a/solr/core/src/java/org/apache/solr/handler/export/IntFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/IntFieldWriter.java index bf0396d4ab87..fc7c2d174ab8 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/IntFieldWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/IntFieldWriter.java @@ -34,8 +34,7 @@ public IntFieldWriter( } @Override - public boolean write( - SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew, int fieldIndex) + public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew) throws IOException { int val; SortValue sortValue = sortDoc.getSortValue(this.field); @@ -43,7 +42,7 @@ public boolean write( if (sortValue.isPresent()) { val = (int) sortValue.getCurrentValue(); } else { // empty-value - return false; + return; } } else { // field is not part of 'sort' param, but part of 'fl' param @@ -53,10 +52,9 @@ public boolean write( if (vals != null) { val = (int) vals.longValue(); } else { - return false; + return; } } ew.put(this.field, val); - return true; } } diff --git a/solr/core/src/java/org/apache/solr/handler/export/LongFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/LongFieldWriter.java index 7961549477cf..38997e5a495c 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/LongFieldWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/LongFieldWriter.java @@ -35,8 +35,7 @@ public LongFieldWriter( } @Override - public boolean write( - SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew, int fieldIndex) + public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew) throws IOException { long val; SortValue sortValue = sortDoc.getSortValue(this.field); @@ -44,7 +43,7 @@ public boolean write( if (sortValue.isPresent()) { val = (long) sortValue.getCurrentValue(); } else { // empty-value - return false; + return; } } else { // field is not part of 'sort' param, but part of 'fl' param @@ -54,11 +53,10 @@ public boolean write( if (vals != null) { val = vals.longValue(); } else { - return false; + return; } } doWrite(ew, val); - return true; } protected void doWrite(MapWriter.EntryWriter ew, long val) throws IOException { diff --git a/solr/core/src/java/org/apache/solr/handler/export/MultiFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/MultiFieldWriter.java index 7f5bdee4899f..51ea833f8526 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/MultiFieldWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/MultiFieldWriter.java @@ -61,15 +61,14 @@ public MultiFieldWriter( } @Override - public boolean write( - SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter out, int fieldIndex) + public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter out) throws IOException { if (this.fieldType.isPointField()) { SortedNumericDocValues vals = docValuesCache.getSortedNumericDocValues( sortDoc.docId, readerContext.reader(), readerContext.ord); if (vals == null) { - return false; + return; } final SortedNumericDocValues docVals = vals; @@ -82,13 +81,12 @@ public boolean write( w.add(bitsToValue.apply(docVals.nextValue())); } }); - return true; } else { SortedSetDocValues vals = docValuesCache.getSortedSetDocValues( sortDoc.docId, readerContext.reader(), readerContext.ord); if (vals == null) { - return false; + return; } final SortedSetDocValues docVals = vals; @@ -105,7 +103,6 @@ public boolean write( else w.add(fieldType.toObject(f)); } }); - return true; } } diff --git a/solr/core/src/java/org/apache/solr/handler/export/StoredFieldsWriter.java b/solr/core/src/java/org/apache/solr/handler/export/StoredFieldsWriter.java new file mode 100644 index 000000000000..58d502e2579d --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/export/StoredFieldsWriter.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.export; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.StoredFields; +import org.apache.solr.common.MapWriter.EntryWriter; +import org.apache.solr.schema.BoolField; +import org.apache.solr.schema.DateValueFieldType; +import org.apache.solr.schema.SchemaField; + +class StoredFieldsWriter extends FieldWriter { + + private static final ThreadLocal> + STORED_FIELDS_MAP = ThreadLocal.withInitial(WeakHashMap::new); + private final Map schemaFields; + + public StoredFieldsWriter(Map fieldsToRead) { + this.schemaFields = fieldsToRead; + } + + @Override + public void write(SortDoc sortDoc, LeafReaderContext readerContext, EntryWriter out) + throws IOException { + WeakHashMap map = STORED_FIELDS_MAP.get(); + LeafReader reader = readerContext.reader(); + StoredFields storedFields = map.get(reader.getReaderCacheHelper().getKey()); + if (storedFields == null) { + storedFields = reader.storedFields(); + map.put(reader.getReaderCacheHelper().getKey(), storedFields); + } + ExportVisitor visitor = new ExportVisitor(out); + storedFields.document(sortDoc.docId, visitor); + visitor.flush(); + } + + class ExportVisitor extends StoredFieldVisitor { + + final EntryWriter out; + String lastFieldName; + List multiValue = null; + int fieldsVisited; + + public ExportVisitor(EntryWriter out) { + this.out = out; + } + + @Override + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + var schemaField = schemaFields.get(fieldInfo.name); + var fieldType = schemaField == null ? null : schemaField.getType(); + if (fieldType instanceof BoolField) { + // Convert "T"/"F" stored value to boolean true/false + addField(fieldInfo.name, Boolean.valueOf(fieldType.indexedToReadable(value))); + } else { + addField(fieldInfo.name, value); + } + } + + @Override + public void intField(FieldInfo fieldInfo, int value) throws IOException { + addField(fieldInfo.name, value); + } + + @Override + public void longField(FieldInfo fieldInfo, long value) throws IOException { + var schemaField = schemaFields.get(fieldInfo.name); + var fieldType = schemaField == null ? null : schemaField.getType(); + if (fieldType instanceof DateValueFieldType) { + Date date = new Date(value); + addField(fieldInfo.name, date); + } else { + addField(fieldInfo.name, value); + } + } + + @Override + public void floatField(FieldInfo fieldInfo, float value) throws IOException { + addField(fieldInfo.name, value); + } + + @Override + public void doubleField(FieldInfo fieldInfo, double value) throws IOException { + addField(fieldInfo.name, value); + } + + @Override + public Status needsField(FieldInfo fieldInfo) { + return schemaFields.containsKey(fieldInfo.name) ? Status.YES : Status.NO; + } + + private void addField(String fieldName, T value) throws IOException { + if (fieldName.equals(lastFieldName)) { + // assume adding another value to a multi-value field + multiValue.add(value); + return; + } + // new/different field... + flush(); // completes the previous field if there's something to do + fieldsVisited++; + lastFieldName = fieldName; + + if (schemaFields.get(fieldName).multiValued()) { + multiValue = new ArrayList<>(); + multiValue.add(value); + } else { + out.put(fieldName, value); + } + } + + private void flush() throws IOException { + if (multiValue != null) { + out.put(lastFieldName, multiValue); + multiValue = null; + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java index 2f8d0963e3a1..228f3c1c743a 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java @@ -59,8 +59,7 @@ public StringFieldWriter( } @Override - public boolean write( - SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew, int fieldIndex) + public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew) throws IOException { StringValue stringValue = (StringValue) sortDoc.getSortValue(this.field); BytesRef ref = null; @@ -74,7 +73,7 @@ public boolean write( if (stringValue.currentOrd == -1) { // Null sort value - return false; + return; } if (this.lastOrd == stringValue.currentOrd) { @@ -89,7 +88,7 @@ public boolean write( docValuesCache.getSortedDocValues( sortDoc.docId, readerContext.reader(), readerContext.ord); if (vals == null) { - return false; + return; } int ord = vals.ordValue(); @@ -102,7 +101,6 @@ public boolean write( } writeBytes(ew, ref, fieldType); - return true; } protected void writeBytes(MapWriter.EntryWriter ew, BytesRef ref, FieldType fieldType) diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml index d821c3935f2b..5674b1dd7b2f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml @@ -33,7 +33,7 @@ - + @@ -105,7 +105,7 @@ - + @@ -128,6 +128,22 @@ + + + + + + + + + + + + + + + + id diff --git a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java index f4836a93ce2f..8829d7e5c605 100644 --- a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java +++ b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java @@ -17,6 +17,7 @@ package org.apache.solr.handler.export; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -952,7 +953,7 @@ private void testSortingOutput() throws Exception { s.contains("\"status\":400}")); assertTrue( "Should have a cause when exporting sortabledv_m, it does not have useDocValuesAsStored='true'", - s.contains("Must have useDocValuesAsStored='true' to be used with export writer")); + s.contains("includeStoredFields=true")); s = h.query( @@ -970,7 +971,7 @@ private void testSortingOutput() throws Exception { s.contains("\"status\":400}")); assertTrue( "Should have a cause when exporting sortabledv, it does not have useDocValuesAsStored='true'", - s.contains("Must have useDocValuesAsStored='true' to be used with export writer")); + s.contains("includeStoredFields=true")); } private void assertJsonEquals(String actual, String expected) { @@ -1292,9 +1293,7 @@ public void testExpr() throws Exception { assertTrue("doc doesn't have exception", doc.containsKey(StreamParams.EXCEPTION)); assertTrue( "wrong exception message", - doc.get(StreamParams.EXCEPTION) - .toString() - .contains("Must have useDocValuesAsStored='true'")); + doc.get(StreamParams.EXCEPTION).toString().contains("includeStoredFields=true")); } @Test @@ -1476,4 +1475,289 @@ private void addField(SolrInputDocument doc, String type, String value, boolean doc.addField("number_" + type + (mv ? "s" : "") + "_ni_t", value); doc.addField("number_" + type + (mv ? "s" : "") + "_ni_p", value); } + + @Test + public void testIncludeStoredFieldsExplicitRequest() throws Exception { + // Test that stored-only fields are returned when includeStoredFields=true + clearIndex(); + + assertU( + adoc( + "id", "1", + "intdv", "1", + "str_s_stored", "hello", + "num_i_stored", "42", + "num_l_stored", "1234567890123", + "num_f_stored", "3.14", + "num_d_stored", "2.71828", + "date_dt_stored", "2024-01-15T10:30:00Z", + "bool_b_stored", "true")); + assertU(commit()); + + String resp = + h.query( + req( + "qt", "/export", + "q", "*:*", + "fl", + "id,str_s_stored,num_i_stored,num_l_stored,num_f_stored,num_d_stored,date_dt_stored,bool_b_stored", + "sort", "intdv asc", + "includeStoredFields", "true")); + + assertJsonEquals( + resp, + "{" + + " \"responseHeader\":{\"status\":0}," + + " \"response\":{" + + " \"numFound\":1," + + " \"docs\":[{" + + " \"id\":\"1\"," + + " \"str_s_stored\":\"hello\"," + + " \"num_i_stored\":42," + + " \"num_l_stored\":1234567890123," + + " \"num_f_stored\":3.14," + + " \"num_d_stored\":2.71828," + + " \"date_dt_stored\":\"2024-01-15T10:30:00Z\"," + + " \"bool_b_stored\":true}]}}"); + } + + @Test + public void testIncludeStoredFieldsErrorWithoutParam() throws Exception { + // Test that error with hint is thrown when requesting stored-only field without + // includeStoredFields + clearIndex(); + + assertU(adoc("id", "1", "intdv", "1", "str_s_stored", "hello")); + assertU(commit()); + + // Request stored-only field without includeStoredFields=true should error + String resp = + h.query( + req( + "qt", "/export", + "q", "*:*", + "fl", "id,str_s_stored", + "sort", "intdv asc")); + + assertTrue( + "Expected error message to contain hint about includeStoredFields", + resp.contains("includeStoredFields=true")); + assertTrue("Expected error message to mention the field", resp.contains("str_s_stored")); + } + + @Test + public void testIncludeStoredFieldsGlobSkipsWithoutParam() throws Exception { + // Test that glob pattern silently skips stored-only fields when includeStoredFields=false + clearIndex(); + + assertU( + adoc( + "id", "1", + "intdv", "1", + "stringdv", "docvalue_string", + "str_s_stored", "stored_string")); + assertU(commit()); + + // Explicit fl with stored-only field should error + String resp = + h.query( + req( + "qt", "/export", + "q", "*:*", + "fl", "id,intdv,stringdv,str_s_stored", + "sort", "intdv asc")); + + // Should error because str_s_stored is explicitly requested + assertTrue( + "Expected error for explicitly requested stored-only field", resp.contains("str_s_stored")); + assertTrue( + "Expected hint about includeStoredFields", resp.contains("includeStoredFields=true")); + + // Now test with glob - should silently skip stored-only fields and succeed + resp = + h.query( + req( + "qt", "/export", + "q", "*:*", + "fl", "intdv,*", + "sort", "intdv asc")); + + assertJsonEquals( + resp, + "{" + + " \"responseHeader\":{\"status\":0}," + + " \"response\":{" + + " \"numFound\":1," + + " \"docs\":[{" + + " \"id\":\"1\"," + + " \"intdv\":1," + + " \"stringdv\":\"docvalue_string\"}]}}"); + } + + @Test + public void testIncludeStoredFieldsGlobIncludesWithParam() throws Exception { + // Test that glob pattern includes stored-only fields when includeStoredFields=true + clearIndex(); + + assertU( + adoc( + "id", "1", + "intdv", "1", + "stringdv", "docvalue_string", + "str_s_stored", "stored_string")); + assertU(commit()); + + // Glob fl=* with includeStoredFields=true should include stored-only fields + String resp = + h.query( + req( + "qt", "/export", + "q", "*:*", + "fl", "*", + "sort", "intdv asc", + "includeStoredFields", "true")); + + assertJsonEquals( + resp, + "{" + + " \"responseHeader\":{\"status\":0}," + + " \"response\":{" + + " \"numFound\":1," + + " \"docs\":[{" + + " \"intdv\":1," + + " \"stringdv\":\"docvalue_string\"," + + " \"id\":\"1\"," + + " \"str_s_stored\":\"stored_string\"}]}}"); + } + + @Test + public void testIncludeStoredFieldsMultiValued() throws Exception { + // Test that multi-valued stored-only fields work correctly + clearIndex(); + + assertU( + adoc( + "id", "1", + "intdv", "1", + "strs_ss_stored", "value1", + "strs_ss_stored", "value2", + "strs_ss_stored", "value3", + "nums_is_stored", "10", + "nums_is_stored", "20", + "nums_is_stored", "30")); + assertU(commit()); + + String resp = + h.query( + req( + "qt", "/export", + "q", "*:*", + "fl", "id,strs_ss_stored,nums_is_stored", + "sort", "intdv asc", + "includeStoredFields", "true")); + + assertJsonEquals( + resp, + "{" + + " \"responseHeader\":{\"status\":0}," + + " \"response\":{" + + " \"numFound\":1," + + " \"docs\":[{" + + " \"id\":\"1\"," + + " \"strs_ss_stored\":[\"value1\",\"value2\",\"value3\"]," + + " \"nums_is_stored\":[10,20,30]}]}}"); + } + + @Test + public void testIncludeStoredFieldsAllTypes() throws Exception { + // Test all supported stored field types including Date + clearIndex(); + + assertU( + adoc( + "id", "1", + "intdv", "1", + "str_s_stored", "test_string", + "num_i_stored", "123", + "num_l_stored", "9876543210", + "num_f_stored", "1.5", + "num_d_stored", "2.5", + "date_dt_stored", "2025-12-25T00:00:00Z", + "bool_b_stored", "false")); + assertU( + adoc( + "id", "2", + "intdv", "2", + "str_s_stored", "another_string", + "num_i_stored", "456", + "num_l_stored", "1234567890", + "num_f_stored", "2.5", + "num_d_stored", "3.5", + "date_dt_stored", "2025-06-15T12:30:00Z", + "bool_b_stored", "true")); + assertU(commit()); + + String resp = + h.query( + req( + "qt", "/export", + "q", "*:*", + "fl", + "id,str_s_stored,num_i_stored,num_l_stored,num_f_stored,num_d_stored,date_dt_stored,bool_b_stored", + "sort", "intdv asc", + "includeStoredFields", "true")); + + assertJsonEquals( + resp, + "{" + + " \"responseHeader\":{\"status\":0}," + + " \"response\":{" + + " \"numFound\":2," + + " \"docs\":[{" + + " \"id\":\"1\"," + + " \"str_s_stored\":\"test_string\"," + + " \"num_i_stored\":123," + + " \"num_l_stored\":9876543210," + + " \"num_f_stored\":1.5," + + " \"num_d_stored\":2.5," + + " \"date_dt_stored\":\"2025-12-25T00:00:00Z\"," + + " \"bool_b_stored\":false}," + + " {" + + " \"id\":\"2\"," + + " \"str_s_stored\":\"another_string\"," + + " \"num_i_stored\":456," + + " \"num_l_stored\":1234567890," + + " \"num_f_stored\":2.5," + + " \"num_d_stored\":3.5," + + " \"date_dt_stored\":\"2025-06-15T12:30:00Z\"," + + " \"bool_b_stored\":true}]}}"); + } + + @Test + public void testSortingWithoutDocValues() throws Exception { + // Attempting to sort on a field without DocValues should fail + clearIndex(); + + assertU( + adoc( + "id", "1", + "sorted_i_stored", "0")); + assertU(commit()); + + IOException ex = + expectThrows( + IOException.class, + () -> + h.query( + req( + "qt", "/export", + "q", "*:*", + "fl", "id", + "sort", "sorted_i_stored asc", + "includeStoredFields", "true"))); + + assertTrue( + "Error message should mention DocValues requirement", + ex.getMessage().contains("DocValues")); + } } diff --git a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc index bbd31c7b358f..fc6f4d6a7ef3 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc @@ -25,7 +25,9 @@ The cases where this functionality may be useful include: session analysis, dist == Field Requirements -All the fields being sorted and exported must have docValues set to `true`. +All the fields being sorted must have docValues set to `true`. +By default, fields in the field list (`fl`) must also have docValues. +However, you can include stored-only fields (fields without docValues) by setting the `includeStoredFields` parameter to `true`. For more information, see the section on xref:indexing-guide:docvalues.adoc[]. == The /export RequestHandler @@ -44,6 +46,12 @@ Filter queries are also supported. An optional parameter `batchSize` determines the size of the internal buffers for partial results. The default value is `30000` but users may want to specify smaller values to limit the memory use (at the cost of degraded performance) or higher values to improve export performance (the relationship is not linear and larger values don't bring proportionally larger performance increases). +An optional parameter `includeStoredFields` (default `false`) enables exporting fields that only have stored values (no docValues). +When set to `true`, fields without docValues but with stored values can be included in the field list (`fl`). +Note that retrieving stored fields may significantly impact export performance compared to docValues fields, as stored fields require additional I/O operations. +If all requested fields are `docValues=true` then the data will only be read from docValues. +This behavior applies to fields that are also `stored=true` and does not depend on the value of the `includeStoredFields` parameter. + The supported response writers are `json` and `javabin`. For backward compatibility reasons `wt=xsort` is also supported as input, but `wt=xsort` behaves same as `wt=json`. The default output format is `json`. @@ -58,8 +66,8 @@ http://localhost:8983/solr/core_name/export?q=my-query&sort=severity+desc,timest === Specifying the Sort Criteria The `sort` property defines how documents will be sorted in the exported result set. -Results can be sorted by any field that has a field type of int,long, float, double, string. -The sort fields must be single valued fields. +Results can be sorted by any field that has a field type of int, long, float, double, string. +The sort fields must be single valued fields and must have docValues enabled. The export performance will get slower as you add more sort fields. If there is enough physical memory available outside of the JVM to load up the sort fields then the performance will be linearly slower with addition of sort fields. @@ -71,6 +79,10 @@ The `fl` property defines the fields that will be exported with the result set. Any of the field types that can be sorted (i.e., int, long, float, double, string, date, boolean) can be used in the field list. The fields can be single or multi-valued. +By default, fields in the field list must have docValues enabled. +However, when the `includeStoredFields` parameter is set to `true`, fields with only stored values (no docValues) can also be included. +Note that sort fields still require docValues, regardless of this setting. + Wildcard patterns can be used for the field list (e.g. `fl=*_i`) and will be expanded to the list of fields that match the pattern and are able to be exported, see <>. Returning scores is not supported at this time. @@ -105,6 +117,61 @@ http://localhost:8983/solr/core_name/export?q=my-query&sort=reporter+desc,&fl=re (Note that the `over` parameter must use one of the fields requested in the `fl` parameter). +== Comparison with Cursors + +The `/export` handler and xref:pagination-of-results.adoc#fetching-a-large-number-of-sorted-results-cursors[cursor-based pagination] offer different trade-offs for streaming large result sets. + +[cols="h,2,2"] +|=== +| |Export |Cursors + +|Advantages +a| +* Query executed once -- efficient +* Consistent snapshot (no duplicates or missing docs) +* Lower latency to the first document (typically) +* Decoupled reader and writer creates smoother flow +a| +* Sharded collection support, intrinsically supported +* Flexible sort criteria +* Resumable across requests and restarts +* Full `SearchHandler` features (highlighting, etc.) + +|Disadvantages +a| +* Requires streaming expressions for distributed queries +* Sort criteria can only be fields with docValues; no score +* Must consume in a single session +* A long session may retain old segments from being removed in a timely manner +a| +* Query re-executed for each page -- inefficient +* Possible duplicates or missing docs with concurrent updates +* Higher latency to the first document (typically) +* Uneven flow; large batches needed for throughput +|=== + +=== Details + +With cursors, the query is re-executed for each page of results. +In contrast, `/export` runs the filter query once and the resulting segment-level bitmasks are applied once per segment, after which the documents are simply iterated over. +Additionally, the segments that existed when the stream was opened are held open for the duration of the export, eliminating the disappearing or duplicate document issues that can occur with cursors. +However, this means IndexReaders are kept around for longer periods of time, which delays cleanup of memory and disk resources until the export completes. + +The `/export` handler has significantly lower latency until the first document is returned, because the internal batch size is decoupled from the response message size. +With cursors, you typically need to set the `rows` parameter to a high value (e.g., 10k-100k depending on `fl`/document size) to achieve decent throughput, and provided you have enough memory (rows * shards * `fl`-size). +However, this creates a "glugging" effect: when you request a large batch, Solr must build the entire payload and send it over the wire while your client waits (assuming a sharded-collection). +Only after receiving and decoding this large payload can the client request the next batch, but in the interim Solr sits idle on this request. +With the `/export` handler, these steps are decoupled; Solr can continue sorting and decoding/encoding documents while waiting for more demand from the client. + +The advantage of cursors is _flexibility_. +Cursors impose no constraints on the sort criteria except that you must include a unique key, which isn't a real constraint. +Cursors work as part of `SearchHandler` and thus can include most/all capabilities of it like highlighting. +A `cursorMark` can be persisted and resumed later, even across restarts, or never continued if enough results were consumed to satisfy the use-case. +An `/export` stream must be consumed in a single session. +Cursors also support distributed queries by default while `/export` does not, although they can be achieved using +xref:streaming-expressions.adoc[streaming expressions] which are built on top of the `/export` handler. + == Distributed Support See the section xref:streaming-expressions.adoc[] for distributed support. + From be726e06889a363b5fb4c5b9963d3ce14dde7657 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sat, 31 Jan 2026 18:27:21 -0500 Subject: [PATCH 32/83] modules/analytics: build.gradle: warnings shouldn't error Suppressing intermittent warnings (that fail the build) relating to floating point constants in FilterFunctionTest.multiValueFloatTest --- solr/modules/analytics/build.gradle | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/solr/modules/analytics/build.gradle b/solr/modules/analytics/build.gradle index 393174449b91..25a98bf3325c 100644 --- a/solr/modules/analytics/build.gradle +++ b/solr/modules/analytics/build.gradle @@ -35,3 +35,8 @@ dependencies { testImplementation 'junit:junit' testImplementation 'org.slf4j:slf4j-api' } + +// Disable treating warnings as errors for this module +tasks.withType(JavaCompile) { + options.compilerArgs -= "-Werror" +} From d41b0a7f7bedef66111ba3d2733138e5294bfa2a Mon Sep 17 00:00:00 2001 From: Rahul Goswami Date: Sat, 31 Jan 2026 22:36:53 -0500 Subject: [PATCH 33/83] SOLR-18096: /admin/cores?action=UPGRADECOREINDEX (#3903) Provides a /admin/cores api (action=UPGRADECOREINDEX) to upgrade older index segments to the latest version (by targeted reindexing). Calling this endpoint on an index created in version X-1 (assuming you are on Solr version X) would reindex documents in any segments of the older version (X-1) creating new segments, resulting in the old ones getting deleted. This would help prepare the index for when Solr is upgraded to X+1 without having to recreate the index from source (as is the requirement today). Various limitations apply. (cherry picked from commit 6cc80a0f39c4155553c25c774e7d9b9462d7639f) --- ...Admin-API-to-upgrade-an-index-in-place.yml | 9 + .../model/UpgradeCoreIndexRequestBody.java | 33 ++ .../api/model/UpgradeCoreIndexResponse.java | 38 ++ .../handler/admin/CoreAdminOperation.java | 4 +- .../handler/admin/UpgradeCoreIndexOp.java | 82 ++++ .../handler/admin/api/UpgradeCoreIndex.java | 434 ++++++++++++++++++ .../apache/solr/update/DocumentBuilder.java | 30 ++ .../admin/UpgradeCoreIndexActionTest.java | 392 ++++++++++++++++ .../pages/coreadmin-api.adoc | 88 ++++ .../solr/common/params/CoreAdminParams.java | 3 +- 10 files changed, 1111 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/SOLR-18096-CoreAdmin-API-to-upgrade-an-index-in-place.yml create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/UpgradeCoreIndexRequestBody.java create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/UpgradeCoreIndexResponse.java create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/UpgradeCoreIndexOp.java create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/api/UpgradeCoreIndex.java create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/UpgradeCoreIndexActionTest.java diff --git a/changelog/unreleased/SOLR-18096-CoreAdmin-API-to-upgrade-an-index-in-place.yml b/changelog/unreleased/SOLR-18096-CoreAdmin-API-to-upgrade-an-index-in-place.yml new file mode 100644 index 000000000000..0fa7000bf593 --- /dev/null +++ b/changelog/unreleased/SOLR-18096-CoreAdmin-API-to-upgrade-an-index-in-place.yml @@ -0,0 +1,9 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: CoreAdmin API (/admin/cores?action=UPGRADECOREINDEX) to upgrade an index in-place +type: added +authors: + - name: Rahul Goswami +links: + - name: SOLR-18096 + url: https://issues.apache.org/jira/browse/SOLR-18096 + diff --git a/solr/api/src/java/org/apache/solr/client/api/model/UpgradeCoreIndexRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/UpgradeCoreIndexRequestBody.java new file mode 100644 index 000000000000..ecc3081014e9 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/UpgradeCoreIndexRequestBody.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +public class UpgradeCoreIndexRequestBody { + + @Schema(description = "Request ID to track this action which will be processed asynchronously.") + @JsonProperty + public String async; + + @Schema( + description = + "updateChain to be used for reindexing during index upgrade if you don't want to use the one used by /update by default") + @JsonProperty + public String updateChain; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/UpgradeCoreIndexResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/UpgradeCoreIndexResponse.java new file mode 100644 index 000000000000..09e0ba1e8b22 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/UpgradeCoreIndexResponse.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +public class UpgradeCoreIndexResponse extends SolrJerseyResponse { + @Schema(description = "The name of the core.") + @JsonProperty + public String core; + + @Schema(description = "The total number of segments eligible for upgrade.") + @JsonProperty + public Integer numSegmentsEligibleForUpgrade; + + @Schema(description = "The number of segments successfully upgraded.") + @JsonProperty + public Integer numSegmentsUpgraded; + + @Schema(description = "Status of the core index upgrade operation.") + @JsonProperty + public String upgradeStatus; +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java index 0aafba7c70b9..7fe36f5092f7 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java @@ -40,6 +40,7 @@ import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.STATUS; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.SWAP; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.UNLOAD; +import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.UPGRADECOREINDEX; import static org.apache.solr.common.params.CoreAdminParams.REPLICA; import static org.apache.solr.common.params.CoreAdminParams.REPLICA_TYPE; import static org.apache.solr.common.params.CoreAdminParams.SHARD; @@ -295,7 +296,8 @@ public enum CoreAdminOperation implements CoreAdminOp { final ListCoreSnapshotsResponse response = coreSnapshotAPI.listSnapshots(coreName); V2ApiUtils.squashIntoSolrResponseWithoutHeader(it.rsp, response); - }); + }), + UPGRADECOREINDEX_OP(UPGRADECOREINDEX, new UpgradeCoreIndexOp()); final CoreAdminParams.CoreAdminAction action; final CoreAdminOp fun; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/UpgradeCoreIndexOp.java b/solr/core/src/java/org/apache/solr/handler/admin/UpgradeCoreIndexOp.java new file mode 100644 index 000000000000..8fff5c93d310 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/UpgradeCoreIndexOp.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import org.apache.solr.client.api.model.UpgradeCoreIndexRequestBody; +import org.apache.solr.client.api.model.UpgradeCoreIndexResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonAdminParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.UpdateParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.api.UpgradeCoreIndex; +import org.apache.solr.handler.api.V2ApiUtils; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +class UpgradeCoreIndexOp implements CoreAdminHandler.CoreAdminOp { + @FunctionalInterface + public interface UpgradeCoreIndexFactory { + UpgradeCoreIndex create( + CoreContainer coreContainer, + CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker, + SolrQueryRequest req, + SolrQueryResponse rsp); + } + + static UpgradeCoreIndexFactory UPGRADE_CORE_INDEX_FACTORY = UpgradeCoreIndex::new; + + @Override + public boolean isExpensive() { + return true; + } + + @Override + public void execute(CoreAdminHandler.CallInfo it) throws Exception { + + assert it.handler.coreContainer != null; + if (it.handler.coreContainer.isZooKeeperAware()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "action=UPGRADECOREINDEX is not supported in SolrCloud mode. As an alternative, in order to upgrade index, configure LatestVersionMergePolicyFactory in solrconfig.xml and reindex the data in your collection."); + } + + SolrParams params = it.req.getParams(); + String cname = params.required().get(CoreAdminParams.CORE); + final boolean isAsync = params.get(CommonAdminParams.ASYNC) != null; + final var requestBody = new UpgradeCoreIndexRequestBody(); + requestBody.updateChain = params.get(UpdateParams.UPDATE_CHAIN); + + UpgradeCoreIndex upgradeCoreIndexApi = + UPGRADE_CORE_INDEX_FACTORY.create( + it.handler.coreContainer, it.handler.coreAdminAsyncTracker, it.req, it.rsp); + final UpgradeCoreIndexResponse response = + upgradeCoreIndexApi.upgradeCoreIndex(cname, requestBody); + V2ApiUtils.squashIntoSolrResponseWithoutHeader(it.rsp, response); + + if (isAsync) { + final var opResponse = new NamedList<>(); + V2ApiUtils.squashIntoNamedListWithoutHeader(opResponse, response); + // REQUESTSTATUS is returning the inner response NamedList as a positional array + // ([k1,v1,k2,v2...]). + // so converting to a map + it.rsp.addResponse(opResponse.asMap(1)); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/UpgradeCoreIndex.java b/solr/core/src/java/org/apache/solr/handler/admin/api/UpgradeCoreIndex.java new file mode 100644 index 000000000000..a525b0f75956 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/UpgradeCoreIndex.java @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.List; +import java.util.Set; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FilterLeafReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.StoredFields; +import org.apache.lucene.index.Terms; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.Version; +import org.apache.solr.client.api.model.UpgradeCoreIndexRequestBody; +import org.apache.solr.client.api.model.UpgradeCoreIndexResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.UpdateParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.index.LatestVersionMergePolicy; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.DocValuesIteratorCache; +import org.apache.solr.search.SolrDocumentFetcher; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.update.CommitUpdateCommand; +import org.apache.solr.update.DocumentBuilder; +import org.apache.solr.update.processor.UpdateRequestProcessor; +import org.apache.solr.update.processor.UpdateRequestProcessorChain; +import org.apache.solr.util.RefCounted; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implements the UPGRADECOREINDEX CoreAdmin action, which upgrades an existing core's index + * in-place by reindexing documents from segments belonging to older Lucene versions, so that they + * get written into latest version segments. + * + *

The upgrade process: + * + *

    + *
  • Temporarily installs {@link LatestVersionMergePolicy} to prevent older-version segments + * from participating in merges during reindexing. + *
  • Iterates each segment whose {@code minVersion} is older than the current Lucene major + * version. For each live document, rebuilds a {@link SolrInputDocument} from stored fields, + * decorates it with non-stored DocValues fields (excluding copyField targets), and re-adds it + * through Solr's update pipeline. + *
  • Commits the changes and validates that no older-format segments remain. + *
  • Restores the original merge policy. + *
+ * + * @see LatestVersionMergePolicy + * @see UpgradeCoreIndexRequestBody + * @see UpgradeCoreIndexResponse + */ +public class UpgradeCoreIndex extends CoreAdminAPIBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public enum CoreIndexUpgradeStatus { + UPGRADE_SUCCESSFUL, + ERROR, + NO_UPGRADE_NEEDED; + } + + private static final int RETRY_COUNT_FOR_SEGMENT_DELETION = 5; + + public UpgradeCoreIndex( + CoreContainer coreContainer, + CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker, + SolrQueryRequest req, + SolrQueryResponse rsp) { + super(coreContainer, coreAdminAsyncTracker, req, rsp); + } + + @Override + public boolean isExpensive() { + return true; + } + + public UpgradeCoreIndexResponse upgradeCoreIndex( + String coreName, UpgradeCoreIndexRequestBody requestBody) throws Exception { + ensureRequiredParameterProvided("coreName", coreName); + + final UpgradeCoreIndexResponse response = + instantiateJerseyResponse(UpgradeCoreIndexResponse.class); + + return handlePotentiallyAsynchronousTask( + response, + coreName, + requestBody.async, + "upgrade-index", + () -> performUpgrade(coreName, requestBody, response)); + } + + private UpgradeCoreIndexResponse performUpgrade( + String coreName, UpgradeCoreIndexRequestBody requestBody, UpgradeCoreIndexResponse response) { + + try (SolrCore core = coreContainer.getCore(coreName)) { + return performUpgradeImpl(core, requestBody, response); + } + } + + private UpgradeCoreIndexResponse performUpgradeImpl( + SolrCore core, UpgradeCoreIndexRequestBody requestBody, UpgradeCoreIndexResponse response) { + + RefCounted iwRef = null; + MergePolicy originalMergePolicy = null; + int numSegmentsEligibleForUpgrade = 0, numSegmentsUpgraded = 0; + String coreName = core.getName(); + try { + iwRef = core.getSolrCoreState().getIndexWriter(core); + IndexWriter iw = iwRef.get(); + + RefCounted searcherRef = core.getSearcher(); + try { + // Check for nested documents before processing - we don't support them + if (indexContainsNestedDocs(searcherRef.get())) { + throw new SolrException( + BAD_REQUEST, + "UPGRADECOREINDEX does not support indexes containing nested documents. " + + " Consider reindexing your data " + + "from the original source."); + } + + /* Set LatestVersionMergePolicy to prevent older segments from + participating in merges while we reindex. This is to prevent any older version + segments from + merging with any newly formed segments created due to reindexing and undoing the work + we are doing. */ + originalMergePolicy = iw.getConfig().getMergePolicy(); + iw.getConfig() + .setMergePolicy( + new LatestVersionMergePolicy( + iw.getConfig().getMergePolicy())); // prevent older segments from merging + + List leafContexts = searcherRef.get().getIndexReader().leaves(); + DocValuesIteratorCache dvICache = new DocValuesIteratorCache(searcherRef.get()); + + UpdateRequestProcessorChain updateProcessorChain = + getUpdateProcessorChain(core, requestBody.updateChain); + + for (LeafReaderContext lrc : leafContexts) { + if (!shouldUpgradeSegment(lrc)) { + continue; + } + numSegmentsEligibleForUpgrade++; + processSegment(lrc, updateProcessorChain, core, searcherRef.get(), dvICache); + numSegmentsUpgraded++; + } + + if (numSegmentsEligibleForUpgrade == 0) { + response.core = coreName; + response.upgradeStatus = CoreIndexUpgradeStatus.NO_UPGRADE_NEEDED.toString(); + response.numSegmentsEligibleForUpgrade = 0; + return response; + } + } catch (Exception e) { + log.error("Error while processing core: [{}}]", coreName, e); + throw new CoreAdminAPIBaseException(e); + } finally { + // important to decrement searcher ref count after use since we obtained it via + // SolrCore.getSearcher() + searcherRef.decref(); + } + + try { + doCommit(core); + } catch (IOException e) { + throw new CoreAdminAPIBaseException(e); + } + + boolean indexUpgraded = isIndexUpgraded(core); + + if (!indexUpgraded) { + log.error( + "Validation failed for core '{}'. Some data is still present in the older (<{}.x) Lucene index format.", + coreName, + Version.LATEST.major); + throw new CoreAdminAPIBaseException( + new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Validation failed for core '" + + coreName + + "'. Some data is still present in the older (<" + + Version.LATEST.major + + ".x) Lucene index format.")); + } + + response.core = coreName; + response.upgradeStatus = CoreIndexUpgradeStatus.UPGRADE_SUCCESSFUL.toString(); + response.numSegmentsEligibleForUpgrade = numSegmentsEligibleForUpgrade; + response.numSegmentsUpgraded = numSegmentsUpgraded; + } catch (Exception ioEx) { + // Avoid double-wrapping if already a CoreAdminAPIBaseException + if (ioEx instanceof CoreAdminAPIBaseException) { + throw (CoreAdminAPIBaseException) ioEx; + } + throw new CoreAdminAPIBaseException(ioEx); + + } finally { + // Restore original merge policy + if (iwRef != null) { + IndexWriter iw = iwRef.get(); + if (originalMergePolicy != null) { + iw.getConfig().setMergePolicy(originalMergePolicy); + } + iwRef.decref(); + } + } + + return response; + } + + private boolean shouldUpgradeSegment(LeafReaderContext lrc) { + Version segmentMinVersion = null; + + LeafReader leafReader = lrc.reader(); + leafReader = FilterLeafReader.unwrap(leafReader); + + SegmentCommitInfo si = ((SegmentReader) leafReader).getSegmentInfo(); + segmentMinVersion = si.info.getMinVersion(); + + return (segmentMinVersion == null || segmentMinVersion.major < Version.LATEST.major); + } + + private boolean indexContainsNestedDocs(SolrIndexSearcher searcher) throws IOException { + IndexSchema schema = searcher.getSchema(); + + // First check if schema supports nested docs + if (!schema.isUsableForChildDocs()) { + return false; + } + + // Check if _root_ field has fewer unique values than documents with that field. + // This indicates multiple docs share the same _root_ (i.e., child docs exist) + IndexReader reader = searcher.getIndexReader(); + for (LeafReaderContext leaf : reader.leaves()) { + Terms terms = leaf.reader().terms(IndexSchema.ROOT_FIELD_NAME); + if (terms != null) { + long uniqueRootValues = terms.size(); + int docsWithRoot = terms.getDocCount(); + + if (uniqueRootValues == -1 || uniqueRootValues < docsWithRoot) { + return true; // Codec doesn't store number of terms (so a safe fallback), or multiple docs + // share same _root_ (aka nested docs exist) + } + } + } + return false; + } + + @SuppressWarnings({"rawtypes"}) + private UpdateRequestProcessorChain getUpdateProcessorChain( + SolrCore core, String requestedUpdateChain) { + + // Try explicitly requested chain first + if (requestedUpdateChain != null) { + UpdateRequestProcessorChain resolvedChain = + core.getUpdateProcessingChain(requestedUpdateChain); + if (resolvedChain != null) { + return resolvedChain; + } + throw new SolrException( + BAD_REQUEST, + "Requested update chain '" + + requestedUpdateChain + + "' not found for core " + + core.getName()); + } + + // Try to find chain configured in /update handler + String updateChainName = null; + SolrRequestHandler reqHandler = core.getRequestHandler("/update"); + + NamedList initArgs = ((RequestHandlerBase) reqHandler).getInitArgs(); + + if (initArgs != null) { + // Check invariants first + Object invariants = initArgs.get("invariants"); + if (invariants instanceof NamedList) { + updateChainName = (String) ((NamedList) invariants).get(UpdateParams.UPDATE_CHAIN); + } + + // Check defaults if not found in invariants + if (updateChainName == null) { + Object defaults = initArgs.get("defaults"); + if (defaults instanceof NamedList) { + updateChainName = (String) ((NamedList) defaults).get(UpdateParams.UPDATE_CHAIN); + } + } + } + + // default chain is returned if updateChainName is null + return core.getUpdateProcessingChain(updateChainName); + } + + private boolean isIndexUpgraded(SolrCore core) throws IOException { + + Directory dir = + core.getDirectoryFactory() + .get( + core.getIndexDir(), + DirectoryFactory.DirContext.DEFAULT, + core.getSolrConfig().indexConfig.lockType); + + try (IndexReader reader = DirectoryReader.open(dir)) { + List leaves = reader.leaves(); + if (leaves == null || leaves.isEmpty()) { + // no segments to process/validate + return true; + } + + for (LeafReaderContext lrc : leaves) { + LeafReader leafReader = lrc.reader(); + leafReader = FilterLeafReader.unwrap(leafReader); + if (leafReader instanceof SegmentReader) { + SegmentReader segmentReader = (SegmentReader) leafReader; + SegmentCommitInfo si = segmentReader.getSegmentInfo(); + Version segMinVersion = si.info.getMinVersion(); + if (segMinVersion == null || segMinVersion.major != Version.LATEST.major) { + log.warn( + "isIndexUpgraded(): Core: {}, Segment [{}] is still at minVersion [{}] and is not updated to the latest version [{}]; numLiveDocs: [{}]", + core.getName(), + si.info.name, + (segMinVersion == null ? 6 : segMinVersion.major), + Version.LATEST.major, + segmentReader.numDocs()); + return false; + } + } + } + return true; + } catch (Exception e) { + log.error("Error while opening segmentInfos for core [{}]", core.getName(), e); + throw e; + } finally { + if (dir != null) { + core.getDirectoryFactory().release(dir); + } + } + } + + private void doCommit(SolrCore core) throws IOException { + try (LocalSolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams())) { + CommitUpdateCommand cmd = new CommitUpdateCommand(req, false); // optimize=false + core.getUpdateHandler().commit(cmd); + } catch (IOException ioEx) { + log.warn("Error committing on core [{}] during index upgrade", core.getName(), ioEx); + throw ioEx; + } + } + + private void processSegment( + LeafReaderContext leafReaderContext, + UpdateRequestProcessorChain processorChain, + SolrCore core, + SolrIndexSearcher solrIndexSearcher, + DocValuesIteratorCache dvICache) + throws Exception { + + String coreName = core.getName(); + IndexSchema indexSchema = core.getLatestSchema(); + + LeafReader leafReader = leafReaderContext.reader(); + Bits liveDocs = leafReader.getLiveDocs(); + SolrDocumentFetcher docFetcher = solrIndexSearcher.getDocFetcher(); + + // Exclude copy field targets to avoid duplicating values on reindex + Set nonStoredDVFields = docFetcher.getNonStoredDVsWithoutCopyTargets(); + + try (LocalSolrQueryRequest solrRequest = + new LocalSolrQueryRequest(core, new ModifiableSolrParams())) { + SolrQueryResponse rsp = new SolrQueryResponse(); + UpdateRequestProcessor processor = processorChain.createProcessor(solrRequest, rsp); + try { + StoredFields storedFields = leafReader.storedFields(); + for (int luceneDocId = 0; luceneDocId < leafReader.maxDoc(); luceneDocId++) { + if (liveDocs != null && !liveDocs.get(luceneDocId)) { + continue; + } + Document doc = storedFields.document(luceneDocId); + SolrInputDocument solrDoc = DocumentBuilder.toSolrInputDocument(doc, indexSchema); + + docFetcher.decorateDocValueFields( + solrDoc, leafReaderContext.docBase + luceneDocId, nonStoredDVFields, dvICache); + + AddUpdateCommand currDocCmd = new AddUpdateCommand(solrRequest); + currDocCmd.solrDoc = solrDoc; + processor.processAdd(currDocCmd); + } + } finally { + // finish() must be called before close() to flush pending operations + processor.finish(); + processor.close(); + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java index c11502672f73..8508ce8472fa 100644 --- a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java +++ b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java @@ -435,4 +435,34 @@ private static void moveLargestFieldLast(Document doc) { } } } + + /** Convert a lucene Document to a SolrInputDocument */ + public static SolrInputDocument toSolrInputDocument( + org.apache.lucene.document.Document doc, IndexSchema schema) { + SolrInputDocument out = new SolrInputDocument(); + for (IndexableField f : doc.getFields()) { + String fname = f.name(); + SchemaField sf = schema.getFieldOrNull(f.name()); + Object val = null; + if (sf != null) { + if ((!sf.hasDocValues() && !sf.stored()) || schema.isCopyFieldTarget(sf)) { + continue; + } + val = sf.getType().toObject(f); + } else { + val = f.stringValue(); + if (val == null) { + val = f.numericValue(); + } + if (val == null) { + val = f.binaryValue(); + } + if (val == null) { + val = f; + } + } + out.addField(fname, val); + } + return out; + } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/UpgradeCoreIndexActionTest.java b/solr/core/src/test/org/apache/solr/handler/admin/UpgradeCoreIndexActionTest.java new file mode 100644 index 000000000000..96fc1bba7976 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/UpgradeCoreIndexActionTest.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import static org.hamcrest.CoreMatchers.containsString; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.lucene.index.FilterLeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.util.Version; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonAdminParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.util.RefCounted; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class UpgradeCoreIndexActionTest extends SolrTestCaseJ4 { + private static final int DOCS_PER_SEGMENT = 3; + private static final String DV_FIELD = "dvonly_i_dvo"; + + private static VarHandle segmentInfoMinVersionHandle; + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-nomergepolicyfactory.xml", "schema.xml"); + segmentInfoMinVersionHandle = + MethodHandles.privateLookupIn(SegmentInfo.class, MethodHandles.lookup()) + .findVarHandle(SegmentInfo.class, "minVersion", Version.class); + } + + @Before + public void resetIndex() { + assertU(delQ("*:*")); + assertU(commit("openSearcher", "true")); + } + + @Test + public void testUpgradeCoreIndexSelectiveReindexDeletesOldSegments() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + final SegmentLayout layout = buildThreeSegments(coreName); + final Version simulatedOldMinVersion = Version.fromBits(Version.LATEST.major - 1, 0, 0); + + // Simulate: + // - seg1: "pure 9x" (minVersion=9) + // - seg2: "pure 10x" (minVersion=10) + // - seg3: "minVersion 9x, version 10x" (merged segment; minVersion=9) + setMinVersionForSegments(core, Set.of(layout.seg1, layout.seg3), simulatedOldMinVersion); + + final Set segmentsBeforeUpgrade = listSegmentNames(core); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.UPGRADECOREINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp); + + assertNull("Unexpected exception: " + resp.getException(), resp.getException()); + assertEquals(coreName, resp.getValues().get("core")); + assertEquals(2, resp.getValues().get("numSegmentsEligibleForUpgrade")); + assertEquals(2, resp.getValues().get("numSegmentsUpgraded")); + assertEquals("UPGRADE_SUCCESSFUL", resp.getValues().get("upgradeStatus")); + } finally { + admin.shutdown(); + admin.close(); + } + + // The action commits internally and reopens the searcher; verify segments on disk. + final Set segmentsAfter = listSegmentNames(core); + final Set newSegments = new HashSet<>(segmentsAfter); + newSegments.removeAll(segmentsBeforeUpgrade); + assertFalse( + "Expected at least one new segment to be created by reindexing", newSegments.isEmpty()); + assertTrue("Expected seg2 to remain", segmentsAfter.contains(layout.seg2)); + assertFalse("Expected seg1 to be dropped", segmentsAfter.contains(layout.seg1)); + assertFalse("Expected seg3 to be dropped", segmentsAfter.contains(layout.seg3)); + + // Searcher was reopened by the action's commit; verify document count and field values. + assertQ(req("q", "*:*"), "//result[@numFound='" + (3 * DOCS_PER_SEGMENT) + "']"); + + // Validate docValues-only (non-stored) fields were preserved for reindexed documents. + // seg1 and seg3 were reindexed; seg2 was not. + assertDocValuesOnlyFieldPreserved(); + } + + @Test + @SuppressWarnings({"unchecked"}) + public void testUpgradeCoreIndexAsyncRequestStatusContainsOperationResponse() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + final SegmentLayout layout = buildThreeSegments(coreName); + final Version simulatedOldMinVersion = Version.fromBits(Version.LATEST.major - 1, 0, 0); + setMinVersionForSegments(core, Set.of(layout.seg1, layout.seg3), simulatedOldMinVersion); + + final Set segmentsBeforeUpgrade = listSegmentNames(core); + + final String requestId = "upgradecoreindex_async_1"; + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + SolrQueryResponse submitResp = new SolrQueryResponse(); + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.UPGRADECOREINDEX.toString(), + CoreAdminParams.CORE, + coreName, + CommonAdminParams.ASYNC, + requestId), + submitResp); + assertNull(submitResp.getException()); + + SolrQueryResponse statusResp = new SolrQueryResponse(); + int maxRetries = 60; + while (maxRetries-- > 0) { + statusResp = new SolrQueryResponse(); + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.REQUESTSTATUS.toString(), + CoreAdminParams.REQUESTID, + requestId), + statusResp); + + if ("completed".equals(statusResp.getValues().get("STATUS"))) { + break; + } + Thread.sleep(250); + } + + assertEquals("completed", statusResp.getValues().get("STATUS")); + Object opResponse = statusResp.getValues().get("response"); + assertNotNull(opResponse); + assertTrue("Expected map response, got: " + opResponse.getClass(), opResponse instanceof Map); + + Map opResponseMap = (Map) opResponse; + assertEquals(coreName, opResponseMap.get("core")); + assertEquals(2, ((Number) opResponseMap.get("numSegmentsEligibleForUpgrade")).intValue()); + assertEquals(2, ((Number) opResponseMap.get("numSegmentsUpgraded")).intValue()); + assertEquals("UPGRADE_SUCCESSFUL", opResponseMap.get("upgradeStatus")); + } finally { + admin.shutdown(); + admin.close(); + } + + final Set segmentsAfter = listSegmentNames(core); + final Set newSegments = new HashSet<>(segmentsAfter); + newSegments.removeAll(segmentsBeforeUpgrade); + assertFalse( + "Expected at least one new segment to be created by reindexing", newSegments.isEmpty()); + assertTrue("Expected seg2 to remain", segmentsAfter.contains(layout.seg2)); + assertFalse("Expected seg1 to be dropped", segmentsAfter.contains(layout.seg1)); + assertFalse("Expected seg3 to be dropped", segmentsAfter.contains(layout.seg3)); + + // Validate docValues-only (non-stored) fields were preserved for reindexed documents. + assertDocValuesOnlyFieldPreserved(); + } + + @Test + public void testNoUpgradeNeededWhenAllSegmentsCurrent() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + // Index documents and commit - all segments will be at the current Lucene version + for (int i = 0; i < DOCS_PER_SEGMENT; i++) { + assertU(adoc("id", Integer.toString(i))); + } + assertU(commit("openSearcher", "true")); + + final Set segmentsBefore = listSegmentNames(core); + assertFalse("Expected at least one segment", segmentsBefore.isEmpty()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.UPGRADECOREINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp); + + assertNull("Unexpected exception: " + resp.getException(), resp.getException()); + assertEquals(coreName, resp.getValues().get("core")); + assertEquals(0, resp.getValues().get("numSegmentsEligibleForUpgrade")); + assertEquals("NO_UPGRADE_NEEDED", resp.getValues().get("upgradeStatus")); + } finally { + admin.shutdown(); + admin.close(); + } + + // Verify no segments were modified + final Set segmentsAfter = listSegmentNames(core); + assertEquals("Segments should remain unchanged", segmentsBefore, segmentsAfter); + + // Verify documents are still queryable + assertQ(req("q", "*:*"), "//result[@numFound='" + DOCS_PER_SEGMENT + "']"); + } + + private SegmentLayout buildThreeSegments(String coreName) throws Exception { + final SolrCore core = h.getCore(); + + Set segmentsBefore = listSegmentNames(core); + indexDocs(0); + final String seg1 = commitAndGetNewSegment(core, segmentsBefore); + segmentsBefore = listSegmentNames(core); + + indexDocs(1000); + final String seg2 = commitAndGetNewSegment(core, segmentsBefore); + segmentsBefore = listSegmentNames(core); + + indexDocs(2000); + final String seg3 = commitAndGetNewSegment(core, segmentsBefore); + + Set allSegments = listSegmentNames(core); + assertTrue(allSegments.contains(seg1)); + assertTrue(allSegments.contains(seg2)); + assertTrue(allSegments.contains(seg3)); + + return new SegmentLayout(coreName, seg1, seg2, seg3); + } + + private void indexDocs(int baseId) { + for (int i = 0; i < DOCS_PER_SEGMENT; i++) { + // schema.xml copies id into numeric fields; use numeric IDs to avoid parsing errors + final String id = Integer.toString(baseId + i); + assertU(adoc("id", id, DV_FIELD, Integer.toString(baseId + i + 10_000), "title", "t" + id)); + } + } + + private void assertDocValuesOnlyFieldPreserved() { + // Assert one doc that must have been reindexed (seg1) and one from seg3. + assertDocHasDvFieldValue(0, 10_000); + assertDocHasDvFieldValue(2000, 12_000); + + // Also sanity-check a doc from the untouched segment (seg2) still has its value. + assertDocHasDvFieldValue(1000, 11_000); + } + + private void assertDocHasDvFieldValue(int id, int expected) { + assertQ( + req("q", "id:" + id, "fl", "id," + DV_FIELD), + "//result[@numFound='1']", + "//result/doc/int[@name='" + DV_FIELD + "'][.='" + expected + "']"); + } + + private String commitAndGetNewSegment(SolrCore core, Set segmentsBefore) + throws Exception { + assertU(commit("openSearcher", "true")); + Set segmentsAfter = new HashSet<>(listSegmentNames(core)); + segmentsAfter.removeAll(new HashSet<>(segmentsBefore)); + assertEquals("Expected exactly one new segment", 1, segmentsAfter.size()); + return segmentsAfter.iterator().next(); + } + + private Set listSegmentNames(SolrCore core) throws Exception { + return core.withSearcher( + searcher -> { + final Set segmentNames = new HashSet<>(); + for (LeafReaderContext ctx : searcher.getTopReaderContext().leaves()) { + SegmentReader segmentReader = (SegmentReader) FilterLeafReader.unwrap(ctx.reader()); + segmentNames.add(segmentReader.getSegmentName()); + } + return segmentNames; + }); + } + + private void setMinVersionForSegments(SolrCore core, Set segments, Version minVersion) + throws Exception { + RefCounted searcherRef = core.getSearcher(); + try { + final List leaves = searcherRef.get().getTopReaderContext().leaves(); + for (LeafReaderContext ctx : leaves) { + SegmentReader segmentReader = (SegmentReader) FilterLeafReader.unwrap(ctx.reader()); + if (!segments.contains(segmentReader.getSegmentName())) { + continue; + } + final SegmentInfo segmentInfo = segmentReader.getSegmentInfo().info; + segmentInfoMinVersionHandle.set(segmentInfo, minVersion); + } + } finally { + searcherRef.decref(); + } + } + + private static class SegmentLayout { + final String coreName; + final String seg1; + final String seg2; + final String seg3; + + SegmentLayout(String coreName, String seg1, String seg2, String seg3) { + this.coreName = coreName; + this.seg1 = seg1; + this.seg2 = seg2; + this.seg3 = seg3; + } + } + + @Test + public void testUpgradeCoreIndexFailsWithNestedDocuments() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + // Create a parent document with a child document (nested doc) + SolrInputDocument parentDoc = new SolrInputDocument(); + parentDoc.addField("id", "100"); + parentDoc.addField("title", "Parent Document"); + + SolrInputDocument childDoc = new SolrInputDocument(); + childDoc.addField("id", "101"); + childDoc.addField("title", "Child Document"); + + parentDoc.addChildDocument(childDoc); + + // Index the nested document + LocalSolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams()); + try { + AddUpdateCommand cmd = new AddUpdateCommand(req); + cmd.solrDoc = parentDoc; + core.getUpdateHandler().addDoc(cmd); + } finally { + req.close(); + } + assertU(commit("openSearcher", "true")); + + // Verify documents were indexed (parent + child = 2 docs) + assertQ(req("q", "*:*"), "//result[@numFound='2']"); + + // Attempt to upgrade the index - should fail because of nested documents + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + SolrException thrown = + assertThrows( + SolrException.class, + () -> + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.UPGRADECOREINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp)); + + // Verify the exception message indicates nested documents are not supported + assertThat( + thrown.getMessage(), + containsString("does not support indexes containing nested documents")); + } finally { + admin.shutdown(); + admin.close(); + } + } +} diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc index 81315d11b05c..4b4e48c67006 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc @@ -780,6 +780,94 @@ This command is used as part of SolrCloud's xref:deployment-guide:shard-manageme When used against a core in a user-managed cluster without `split.key` parameter, this action will split the source index and distribute its documents alternately so that each split piece contains an equal number of documents. If the `split.key` parameter is specified then only documents having the same route key will be split from the source index. +[[coreadmin-upgradecoreindex]] +== UPGRADECOREINDEX + +The `UPGRADECOREINDEX` action upgrades an existing core's index in-place after a Solr major-version upgrade by reindexing documents from older-format segments. +If a core is upgraded by this action, it ensures index compatibility with the next Solr major version (upon a future Solr upgrade) without having to re-create the index from source. + +This action is expensive and can take a while to complete on large indexes. Consider running with `async` option in such cases. + +Note: + +* Only stored fields and fields with docValues enabled can be preserved during upgrade. +Fields that are neither stored nor docValues-backed will lose their data, unless they are `copyField` targets. +* Not supported in SolrCloud mode. In order to achieve the same purpose in SolrCloud mode (aka upgrade index for compatibility with next Solr version), configure the `LatestVersionMergePolicyFactory` for the collection and reindex all documents through a client utility. +* Indexes containing child/nested documents are not supported. + +It is recommended to test on a copy and have a backup before running on production data. + +=== UPGRADECOREINDEX Parameters + +`core`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The name of the core whose index should be upgraded. + +`async`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Request ID to track this action which will be processed asynchronously. +Use <> with the provided `requestid` to poll for completion and retrieve the operation response. + +`update.chain`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The update processor chain to use for reindexing. +If omitted, Solr uses the chain configured for the `/update` handler, or the default update chain for the core (in that order). + +=== UPGRADECOREINDEX Response + +On success, the response includes: + +`core`:: +The core name. + +`numSegmentsEligibleForUpgrade`:: +The number of segments with an older Lucene format that were targeted. + +`numSegmentsUpgraded`:: +The number of segments successfully processed. + +`upgradeStatus`:: +One of `UPGRADE_SUCCESSFUL` or `NO_UPGRADE_NEEDED`. +On failure, an exception is thrown with error details. + +=== UPGRADECOREINDEX Examples + +*Synchronous:* + +[source,bash] +---- +http://localhost:8983/solr/admin/cores?action=UPGRADECOREINDEX&core=techproducts +---- + +*Asynchronous (recommended for large cores):* + +[source,bash] +---- +http://localhost:8983/solr/admin/cores?action=UPGRADECOREINDEX&core=techproducts&async=upgrade_1 +---- + +Then poll status: + +[source,bash] +---- +http://localhost:8983/solr/admin/cores?action=REQUESTSTATUS&requestid=upgrade_1 +---- + [[coreadmin-requeststatus]] == REQUESTSTATUS diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java index 15d4397bd392..4385d9a2ee4b 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java @@ -188,7 +188,8 @@ public enum CoreAdminAction { INSTALLCOREDATA, CREATESNAPSHOT, DELETESNAPSHOT, - LISTSNAPSHOTS; + LISTSNAPSHOTS, + UPGRADECOREINDEX; public final boolean isRead; From b9eb47855357b783be1c3fa16b869dcbc9360d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 4 Feb 2026 15:52:57 +0100 Subject: [PATCH 34/83] SOLR-18073 JWT Authentication plugin now supports matching non-string claims (#4049) (cherry picked from commit 507859f3dae209dbe39a4609992cf738f9226e95) --- .../unreleased/SOLR-18073-jwt-bool-claims.yml | 9 +++++ .../solr/security/jwt/JWTAuthPlugin.java | 6 ++-- .../solr/security/jwt/JWTAuthPluginTest.java | 34 +++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/SOLR-18073-jwt-bool-claims.yml diff --git a/changelog/unreleased/SOLR-18073-jwt-bool-claims.yml b/changelog/unreleased/SOLR-18073-jwt-bool-claims.yml new file mode 100644 index 000000000000..0053d2c78de6 --- /dev/null +++ b/changelog/unreleased/SOLR-18073-jwt-bool-claims.yml @@ -0,0 +1,9 @@ +title: JWT Authentication plugin now supports matching non-string claims such as boolean +type: fixed +authors: + - name: Jan Høydahl + url: https://home.apache.org/phonebook.html?uid=janhoy + - name: Tony Panza +links: + - name: SOLR-18073 + url: https://issues.apache.org/jira/browse/SOLR-18073 diff --git a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTAuthPlugin.java b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTAuthPlugin.java index bb64f458f4e1..819f2293f9be 100644 --- a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTAuthPlugin.java +++ b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTAuthPlugin.java @@ -591,13 +591,15 @@ protected JWTAuthenticationResponse authenticate(String authorizationHeader) { for (Map.Entry entry : claimsMatchCompiled.entrySet()) { String claim = entry.getKey(); if (jwtClaims.hasClaim(claim)) { - if (!entry.getValue().matcher(jwtClaims.getStringClaimValue(claim)).matches()) { + Object claimValue = jwtClaims.getClaimValue(claim); + String claimValueStr = (claimValue != null) ? String.valueOf(claimValue) : ""; + if (!entry.getValue().matcher(claimValueStr).matches()) { return new JWTAuthenticationResponse( AuthCode.CLAIM_MISMATCH, "Claim " + claim + "=" - + jwtClaims.getStringClaimValue(claim) + + claimValueStr + " does not match required regular expression " + entry.getValue().pattern()); } diff --git a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java index a5642b9f6cdf..7327008a24a1 100644 --- a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java +++ b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java @@ -141,6 +141,8 @@ protected static JwtClaims generateClaims() { claims.setClaim("claim1", "foo"); // additional claims/attributes about the subject can be added claims.setClaim("claim2", "bar"); // additional claims/attributes about the subject can be added claims.setClaim("claim3", "foo"); // additional claims/attributes about the subject can be added + claims.setClaim("email_verified", true); // boolean claim as per OIDC spec + claims.setClaim("admin", false); // another boolean claim List roles = Arrays.asList("group-one", "other-group", "group-three"); claims.setStringListClaim( "roles", roles); // multi-valued claims work too and will end up as a JSON array @@ -336,6 +338,38 @@ public void claimMatch() { assertEquals(CLAIM_MISMATCH, resp.getAuthCode()); } + @Test + public void claimMatchWithBooleanClaim() { + // Test that boolean claims work correctly with claimsMatch + Map shouldMatch = new HashMap<>(); + shouldMatch.put("email_verified", "true"); + testConfig.put("claimsMatch", shouldMatch); + plugin.init(testConfig); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); + assertTrue(resp.getErrorMessage(), resp.isAuthenticated()); + + // Test matching false boolean value + shouldMatch.clear(); + shouldMatch.put("admin", "false"); + plugin.init(testConfig); + resp = plugin.authenticate(testHeader); + assertTrue(resp.getErrorMessage(), resp.isAuthenticated()); + + // Test mismatch with boolean claim + shouldMatch.clear(); + shouldMatch.put("email_verified", "false"); + plugin.init(testConfig); + resp = plugin.authenticate(testHeader); + assertEquals(CLAIM_MISMATCH, resp.getAuthCode()); + + // Test regex pattern with boolean claim + shouldMatch.clear(); + shouldMatch.put("email_verified", "true|false"); + plugin.init(testConfig); + resp = plugin.authenticate(testHeader); + assertTrue(resp.getErrorMessage(), resp.isAuthenticated()); + } + @Test public void missingIssAudExp() { testConfig.put("requireIss", "false"); From be27be10357795068f33e2dd01571092f55d490a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 4 Feb 2026 15:50:50 +0100 Subject: [PATCH 35/83] Improve writeChangelog gradle task (#4007) (cherry picked from commit a29f0472fde9978ea0c1039e2ac978c94736f681) --- gradle/changelog.gradle | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle index 661c8cfb54fb..00770b0688db 100644 --- a/gradle/changelog.gradle +++ b/gradle/changelog.gradle @@ -44,7 +44,11 @@ task writeChangelog { def jiraRef = jiraMatcher ? jiraMatcher[0].toUpperCase() : "SOLR-XXXX" def jiraUrl = "https://issues.apache.org/jira/browse/${jiraRef}" def jiraLinks = jiraMatcher ? "links:\n - name: ${jiraRef}\n url: ${jiraUrl}" : "" - def title = gitBranch.replaceFirst(/(?i)SOLR-\d\d\d+-/, "").replace("-", " ").capitalize() + def title = gitBranch.replaceFirst(/(?i)SOLR-\d\d\d+\b-?/, "").replace("-", " ").capitalize() + // Warn user when generating empty title + if (title.isEmpty()) { + println "WARNING: Title in generated changelog is empty, please edit" + } // Strip everything before and including '/', then sanitize special characters def sanitizedBranchName = gitBranch.replaceAll(/^.*\//, "").replaceAll(/[^a-zA-Z0-9._-]/, "-") def fileName = "changelog/unreleased/${sanitizedBranchName}.yml" @@ -58,7 +62,7 @@ authors: ${jiraLinks} """ - println "Generated file: ${fileName} -- open it" + println "Generated file: ${fileName} -- open and edit before committing" } } From 9a5bb5f918166e967b9cb68df77aa5a9c8d9d25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 12 Feb 2026 14:48:46 +0100 Subject: [PATCH 36/83] SOLR-18076 Agents.md for solr (#4103) Co-authored-by: David Smiley (cherry picked from commit ed4edf3a574e09864d0136ec22f0e78021bb1cdf) --- AGENTS.md | 64 +++++++++++++++++++++++++++++++++ dev-docs/how-to-contribute.adoc | 10 ++++++ 2 files changed, 74 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000000..4395a2a4ee5d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,64 @@ + +# AGENTS.md for Apache Solr + +While README.md and CONTRIBUTING.md are mainly written for humans, this file is a condensed knowledge base for LLM coding agents on the Solr codebase. See https://agents.md for more info and how to make various coding assistants consume this file. Also see `dev-docs/how-to-contribute.adoc` for some guidelines when using genAI to contribute to Solr. + +## Licensing and Dependencies + +- Follow Apache Software Foundation licensing rules, avoid adding a dependency with a banned license +- Always apply the Apache License to new source files +- All versions must be delcared in `gradle/libs.versions.toml`, never build.gradle files +- Try first declaring a dependency without a version (the version might already be in a BOM); and if fails to resolve _then_ specify a version +- The build may complain with "Dependency analysis found issues.", a category like "usedUndeclaredArtifacts", and a dependency list. Declare or undeclare these dependencies, as the category will imply. The special 'permit*' configurations are a choice of last resort. +- Always run `gradlew updateLicenses resolveAndLockAll --write-locks` after adding or changing a dependency. See `dev-docs/gradle-help/dependencies.txt` for more info + +## Build and Development Workflow + +- When done or preparing to commit changes to java source files, be sure to run `gradlew tidy` to format the code +- Always run `gradlew check -x test` before declaring a feature done + +## Code Quality and Best Practices + +- Use the project's custom `EnvUtils` to read system properties. It auto converts env.var SOLR_FOO_BAR to system property solr.foo.bar +- Be careful to not add non-essential logging! If you add slf4j log calls, make sure to wrap debug/trace level calls in `logger.isXxxEnabled()` clause +- Validate user input. For file paths, always call `myCoreContainer.assertPathAllowed(myPath)` before using + +## Testing + +- When adding a test to an existing suite/file, keep the same style / design choices +- When adding a *new* Java test suite/file: + - Subclass SolrTestCase, or if SolrCloud is needed then SolrCloudTestCase + - If SolrTestCase and need to embed Solr, use either EmbeddedSolrServerTestRule (doesn't use HTTP) or SolrJettyTestRule if HTTP/Jetty is relevant to what is being tested. + - Avoid SolrTestCaseJ4 for new tests + +- See `dev-docs/gradle-help/tests.txt` for hints on running tests + +## Documentation + +- For major or breaking changes, add a prominent note in reference guide major-changes-in-solr-X.adoc +- Always consider whether a reference-guide page needs updating due to the new/changed features. Target audience is end user +- For changes to build system and other developer-focused changes, consider updating or adding docs in dev-docs/ folder +- Keep all documentation including javadoc concise +- New classes should have some javadocs +- Changes should not have code comments communicating the change, which are instead great comments to leave for code review / commentary + +## Changelog + +- We use the "logchange" tooling to manage our changelog. See `dev-docs/changelog.adoc` for details and conventions +- To scaffold a new changelog entry, run `gradlew writeChangelog`, and then edit the new file located in `changelog/unreleased/`. +- Do not add a changelog entry before a JIRA issue or a Github PR is assigned, as one is required. diff --git a/dev-docs/how-to-contribute.adoc b/dev-docs/how-to-contribute.adoc index 7621eb7b8e7d..6e0ebfc9d530 100644 --- a/dev-docs/how-to-contribute.adoc +++ b/dev-docs/how-to-contribute.adoc @@ -59,6 +59,16 @@ Please do: * update documentation (e.g., package.html files, this wiki, etc.) * try to provide a unit test that shows a bug was indeed fixed or the new functionality truly works * use the "draft state" for PRs which are work in progress +* carefully review and take responsibility for any AI-assisted code or documentation (see below) + +## Use of AI Coding Assistants + +AI-powered tools or agents (like GitHub Copilot, ChatGPT, etc.) can assist with contributions, but human contributors remain fully responsible for all submitted code and documentation. See the [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html) for the foundation's policy. If you use AI tools: + +- Do include our `AGENTS.md` file in the LLM's context for best result. +- Carefully review all generated content for correctness, security, and alignment with Solr's architecture +- For major AI-assisted contributions, disclose the use of AI tools in your PR description +- For documentation contributions, prefer concise, human-readable content over verbose AI-generated text ## Getting Your Contribution Merged From 6fb59a55b5e0516cde04f6009e6dce55142673c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 12 Feb 2026 15:12:15 +0100 Subject: [PATCH 37/83] Add AI coding agents to gitignore (#4104) Co-authored-by: David Smiley (cherry picked from commit c27f96674dedc053e91c5271083f1e72f4d2bdc1) --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 05199687470f..73bd851ebbdd 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,13 @@ gradle/wrapper/gradle-wrapper.jar # Kotlin .kotlin/ +# AI Coding Assistants, we only want AGENTS.md checked in +CLAUDE.md +.claude/ +.cursor/ +GEMINI.md +.gemini/ + # WANT TO ADD MORE? You can tell Git without adding to this file: # See https://git-scm.com/docs/gitignore # In particular, if you have tools you use, add to $GIT_DIR/info/exclude or use core.excludesFile From 83c915aa3b391e72309678dcd02e0a8e601f145e Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 6 Feb 2026 23:39:22 -0500 Subject: [PATCH 38/83] build, GHA workflow: remove DEVELOCITY_ACCESS_KEY (#4077) Don't want GHA PR workflows to use Develocity. (cherry picked from commit f5da9a62d4c66a25289b340011b852be61693977) --- .github/workflows/bin-solr-test.yml | 3 --- .github/workflows/docker-test.yml | 1 - .github/workflows/gradle-extraction-check.yml | 3 --- .github/workflows/gradle-precommit.yml | 5 +---- .github/workflows/solrj-test.yml | 3 --- 5 files changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml index 959523510c71..f885b7da2366 100644 --- a/.github/workflows/bin-solr-test.yml +++ b/.github/workflows/bin-solr-test.yml @@ -18,9 +18,6 @@ jobs: runs-on: ubuntu-latest - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - steps: # Setup - uses: actions/checkout@v5 diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 6f58b4e67db9..5db95b0eb6f9 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -21,7 +21,6 @@ jobs: env: SOLR_DOCKER_IMAGE_REPO: github-pr/solr SOLR_DOCKER_IMAGE_TAG: ${{github.event.number}} - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} steps: # Setup diff --git a/.github/workflows/gradle-extraction-check.yml b/.github/workflows/gradle-extraction-check.yml index f77c813913e2..e86c7b6f7433 100644 --- a/.github/workflows/gradle-extraction-check.yml +++ b/.github/workflows/gradle-extraction-check.yml @@ -15,9 +15,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.SOLR_DEVELOCITY_ACCESS_KEY }} - steps: - name: Checkout code uses: actions/checkout@v5 diff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.yml index ad9e3f4850d1..88c47be0f5c7 100644 --- a/.github/workflows/gradle-precommit.yml +++ b/.github/workflows/gradle-precommit.yml @@ -1,6 +1,6 @@ name: Gradle Precommit -on: +on: pull_request: branches: - 'main' @@ -12,9 +12,6 @@ jobs: runs-on: ubuntu-latest - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - steps: # Setup - uses: actions/checkout@v5 diff --git a/.github/workflows/solrj-test.yml b/.github/workflows/solrj-test.yml index 7203a75a69f6..60a8f84483d1 100644 --- a/.github/workflows/solrj-test.yml +++ b/.github/workflows/solrj-test.yml @@ -15,9 +15,6 @@ jobs: runs-on: ubuntu-latest - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - steps: # Setup - uses: actions/checkout@v5 From 5bc0fdef67b25dd3bcc2829b28c76c7664e8e1f0 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 7 Nov 2025 17:40:15 -0500 Subject: [PATCH 39/83] New gradle runDev alternative to "dev" (#3816) with modifications for Solr 9 (cherry picked from commit d6c8095371ed571545ca9a566a237a2a695cfbab) --- dev-docs/gradle-help/workflow.txt | 11 ++++++---- gradle/help.gradle | 2 +- solr/packaging/build.gradle | 34 +++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/dev-docs/gradle-help/workflow.txt b/dev-docs/gradle-help/workflow.txt index a50063334ab5..47cde864f536 100644 --- a/dev-docs/gradle-help/workflow.txt +++ b/dev-docs/gradle-help/workflow.txt @@ -31,15 +31,18 @@ Put together a local Solr binary "distribution" folder: gradlew -p solr/packaging assemble ls solr/packaging/build/solr-* # expanded directory -For quick local development -gradlew -p solr/packaging dev -ls solr/packaging/build/dev # expanded directory +Build & run Solr directly from Gradle, skipping "bin/solr", retaining data between runs: +gradlew -p solr/packaging runDev + +Build & run Solr, retaining data between runs: +gradlew -p solr/packaging devSlim +(cd solr/packaging/build/dev-slim && bin/solr start -f) Generate the release tar archive (see publishing.txt for details) gradlew -p solr/distribution assembleRelease ls solr/distribution/build/release # release archives -Build a docker image from the local repository (see docker/gradle-help.txt for more) +Build & run a docker image from the local repository (see docker/gradle-help.txt for more) gradlew dockerBuild dockerTag docker run --rm -p 8983:8983 apache/solr:9.0.0-SNAPSHOT diff --git a/gradle/help.gradle b/gradle/help.gradle index f17b4067efb1..3e3e5f957d83 100644 --- a/gradle/help.gradle +++ b/gradle/help.gradle @@ -46,7 +46,7 @@ configure(rootProject) { doLast { println "" println "This is the Solr gradle build. See some guidelines, " - println "ant-equivalent commands, etc. under help/*; or type:" + println "ant-equivalent commands, etc. under dev-docs/gradle-help/*; or type:" println "" helpFiles.each { section, path, sectionInfo -> println String.format(Locale.ROOT, diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index fe4edb04b073..ad3d6501c0e3 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -229,6 +229,40 @@ artifacts { solrSlimTgz(slimDistTar) } +tasks.register('runDev', JavaExec) { + group = 'application' + description = 'Starts Solr for local experimentation. Does NOT use the bin/solr script.' + dependsOn 'devSlim' + + workingDir = file("$slimDevDir/server") + + // Intentionally not synchronizing the logic here with bin/solr; don't want the burden + + doFirst { + println "To attach a debugger, at the CLI pass: --debug-jvm" + println "To pass JVM args, at the CLI pass: -PjvmArgs='--add-modules jdk.incubator.vector'" + println "To pass system properties, at the CLI pass: -Djetty.port=8983" + } + + maxHeapSize = project.findProperty('maxHeapSize') ?: '1G' + + jvmArgs = [] // don't use defaults from our build that assume this is a typical build task + jvmArgs((project.findProperty('jvmArgs') ?: '').split()) + + systemProperty 'host', '127.0.0.1' + systemProperty 'jetty.port', '8983' + systemProperty 'solr.install.dir', file("$workingDir/..") + systemProperty 'solr.log.dir', file("$workingDir/logs") + systemProperty 'zkRun', '' // SolrCloud; embedded ZK + + // Propagate CLI system properties (override defaults above) + systemProperties += gradle.startParameter.systemPropertiesArgs + + classpath = files("$workingDir/start.jar") + + args((project.findProperty('args') ?: '--module=http').split()) +} + task downloadBats(type: NpmTask) { group = 'Build Dependency Download' args = ["install", "https://github.com/bats-core/bats-core#v1.8.2", From c0404cca09e364992e7ce8dd4b64b745ccf1a479 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 20 Feb 2026 23:44:26 -0500 Subject: [PATCH 40/83] SOLR-18107: Test fix testLogLevelHandlerOutput (#4146) (cherry picked from commit 88c7edd1299f64a36752730de5080e98172cb4e8) --- .../handler/admin/LoggingHandlerTest.java | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/handler/admin/LoggingHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/LoggingHandlerTest.java index 0726bec92b44..a4063b25f0ed 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/LoggingHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/LoggingHandlerTest.java @@ -21,6 +21,7 @@ import java.util.Map; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.Configuration; import org.apache.solr.SolrTestCaseJ4; @@ -66,9 +67,16 @@ public void testLogLevelHandlerOutput() throws Exception { final Configuration config = ctx.getConfiguration(); + // Create and hold strong references to loggers to prevent GC from removing them + // during test execution (Log4j2 uses weak references internally) + Logger parentLogger = LogManager.getLogger(PARENT_LOGGER_NAME); + Logger aLogger = LogManager.getLogger(A_LOGGER_NAME); + Logger bLogger = LogManager.getLogger(B_LOGGER_NAME); + Logger bxLogger = LogManager.getLogger(BX_LOGGER_NAME); + { // sanity check our setup... - // did anybody break the anotations? + // did anybody break the annotations? assertNotNull(this.getClass().getAnnotation(LogLevel.class)); final String annotationConfig = this.getClass().getAnnotation(LogLevel.class).value(); assertTrue("WTF: " + annotationConfig, annotationConfig.startsWith(PARENT_LOGGER_NAME)); @@ -88,11 +96,11 @@ public void testLogLevelHandlerOutput() throws Exception { config.getLoggerConfig(logger)); } - // Either explicit, or inherited, effictive logger values... - for (String logger : - Arrays.asList(PARENT_LOGGER_NAME, A_LOGGER_NAME, B_LOGGER_NAME, BX_LOGGER_NAME)) { - assertEquals(Level.DEBUG, LogManager.getLogger(logger).getLevel()); - } + // Either explicit, or inherited, effective logger values... + assertEquals(Level.DEBUG, parentLogger.getLevel()); + assertEquals(Level.DEBUG, aLogger.getLevel()); + assertEquals(Level.DEBUG, bLogger.getLevel()); + assertEquals(Level.DEBUG, bxLogger.getLevel()); } SolrClient client = new EmbeddedSolrServer(h.getCore()); @@ -130,18 +138,18 @@ public void testLogLevelHandlerOutput() throws Exception { assertLoggerLevel(updatedLoggerLevel, B_LOGGER_NAME, "TRACE", true); assertLoggerLevel(updatedLoggerLevel, BX_LOGGER_NAME, "TRACE", false); - // check directly with log4j what it's (updated) config has... + // check directly with log4j what its (updated) config has... assertEquals(Level.DEBUG, config.getLoggerConfig(PARENT_LOGGER_NAME).getExplicitLevel()); assertEquals(Level.TRACE, config.getLoggerConfig(B_LOGGER_NAME).getExplicitLevel()); assertEquals( "Unexpected config for BX ... expected B's config", config.getLoggerConfig(B_LOGGER_NAME), config.getLoggerConfig(BX_LOGGER_NAME)); - // ...and what it's effective values - assertEquals(Level.DEBUG, LogManager.getLogger(PARENT_LOGGER_NAME).getLevel()); - assertEquals(Level.DEBUG, LogManager.getLogger(A_LOGGER_NAME).getLevel()); - assertEquals(Level.TRACE, LogManager.getLogger(B_LOGGER_NAME).getLevel()); - assertEquals(Level.TRACE, LogManager.getLogger(BX_LOGGER_NAME).getLevel()); + // ...and what its effective values are + assertEquals(Level.DEBUG, parentLogger.getLevel()); + assertEquals(Level.DEBUG, aLogger.getLevel()); + assertEquals(Level.TRACE, bLogger.getLevel()); + assertEquals(Level.TRACE, bxLogger.getLevel()); } { // UNSET @@ -161,7 +169,7 @@ public void testLogLevelHandlerOutput() throws Exception { assertLoggerLevel(removedLoggerLevel, B_LOGGER_NAME, "DEBUG", false); assertLoggerLevel(removedLoggerLevel, BX_LOGGER_NAME, "DEBUG", false); - // check directly with log4j what it's (updated) config has... + // check directly with log4j what its (updated) config has... // // NOTE: LoggingHandler must not actually "remove" the LoggerConfig for B on 'unset' // (it might have already been defined in log4j's original config for some other reason, @@ -176,11 +184,11 @@ public void testLogLevelHandlerOutput() throws Exception { "Unexpected config for BX ... expected B's config", config.getLoggerConfig(B_LOGGER_NAME), config.getLoggerConfig(BX_LOGGER_NAME)); - // ...and what it's effective values - assertEquals(Level.DEBUG, LogManager.getLogger(PARENT_LOGGER_NAME).getLevel()); - assertEquals(Level.DEBUG, LogManager.getLogger(A_LOGGER_NAME).getLevel()); - assertEquals(Level.DEBUG, LogManager.getLogger(B_LOGGER_NAME).getLevel()); - assertEquals(Level.DEBUG, LogManager.getLogger(BX_LOGGER_NAME).getLevel()); + // ...and what its effective values are + assertEquals(Level.DEBUG, parentLogger.getLevel()); + assertEquals(Level.DEBUG, aLogger.getLevel()); + assertEquals(Level.DEBUG, bLogger.getLevel()); + assertEquals(Level.DEBUG, bxLogger.getLevel()); } } From cb1bfc38fe153435b96a10a75d881743adb53b01 Mon Sep 17 00:00:00 2001 From: Vishnu Priya <9874772+vishnupriyachandrasekar@users.noreply.github.com> Date: Thu, 26 Feb 2026 22:50:06 -0500 Subject: [PATCH 41/83] SOLR-18056: CloudSolrClient better urlScheme detection (from URL or solr.ssl.enabled) (#4162) Improved CloudSolrClient's urlScheme detection by using the scheme of provided Solr URLs, or looking at "solr.ssl.enabled". Co-authored-by: David Smiley (cherry picked from commit 1962fc69a1d8949a98d4dcd2b668ff0aae37de31) --- .../unreleased/SOLR-18056-urlScheme-csp.yml | 8 +++ .../impl/ZkClientClusterStateProvider.java | 19 +++++- .../cloud/DelegatingClusterStateProvider.java | 8 +++ .../impl/BaseHttpClusterStateProvider.java | 8 ++- .../client/solrj/impl/CloudSolrClient.java | 2 +- .../solrj/impl/ClusterStateProvider.java | 3 + .../solrj/impl/ClusterStateProviderTest.java | 63 +++++++++++++++++++ 7 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/SOLR-18056-urlScheme-csp.yml diff --git a/changelog/unreleased/SOLR-18056-urlScheme-csp.yml b/changelog/unreleased/SOLR-18056-urlScheme-csp.yml new file mode 100644 index 000000000000..f7a3d2690b1e --- /dev/null +++ b/changelog/unreleased/SOLR-18056-urlScheme-csp.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Improved CloudSolrClient's urlScheme detection by using the scheme of provided Solr URLs, or looking at "solr.ssl.enabled". +type: changed +authors: + - name: Vishnu Priya Chandra Sekar +links: + - name: SOLR-18056 + url: https://issues.apache.org/jira/browse/SOLR-18056 diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java index f9d202f59498..0a1ee9ab439a 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java @@ -35,6 +35,7 @@ import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZooKeeperException; +import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.Utils; import org.apache.zookeeper.KeeperException; import org.noggit.JSONWriter; @@ -45,7 +46,7 @@ public class ZkClientClusterStateProvider implements ClusterStateProvider, SolrZkClientTimeoutAware { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - + private static final String SOLR_SSL_ENABLED = "solr.ssl.enabled"; volatile ZkStateReader zkStateReader; private boolean closeZkStateReader = true; private final String zkHost; @@ -339,4 +340,20 @@ public int getZkClientTimeout() { public void setZkClientTimeout(int zkClientTimeout) { this.zkClientTimeout = zkClientTimeout; } + + /** + * @return url scheme with the help of cluster property or environment variable. + */ + @Override + public String getUrlScheme() { + final Boolean isSolrSslEnabled = EnvUtils.getPropertyAsBool(SOLR_SSL_ENABLED); + if (isSolrSslEnabled != null) { + return isSolrSslEnabled ? "https" : "http"; + } + final Object urlSchemeClusterProperty = getClusterProperty(ClusterState.URL_SCHEME); + if (urlSchemeClusterProperty != null) { + return urlSchemeClusterProperty.toString(); + } + return "http"; + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java index 4177f0a017a2..0ccaa7b25e99 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java @@ -141,4 +141,12 @@ public String getQuorumHosts() { } return null; } + + @Override + public String getUrlScheme() { + if (delegate != null) { + return delegate.getUrlScheme(); + } + return null; + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java index a7d6543c9b96..c7319ffaee9b 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java @@ -74,11 +74,12 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid Executors.newSingleThreadScheduledExecutor( new SolrNamedThreadFactory("liveNodeReloadingExecutor")); - protected void initConfiguredNodes(List solrUrls) throws Exception { + protected void initConfiguredNodes(List solrUrls) { this.configuredNodes = solrUrls.stream() .map(BaseHttpClusterStateProvider::stringToUrl) .collect(Collectors.toList()); + this.urlScheme = this.configuredNodes.get(0).getProtocol(); } private static URL stringToUrl(String solrUrl) { @@ -454,6 +455,11 @@ public void close() throws IOException { liveNodeReloadingService.shutdown(); } + @Override + public String getUrlScheme() { + return this.urlScheme; + } + private enum ClusterStateRequestType { FETCH_LIVE_NODES, FETCH_CLUSTER_PROP, diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index 59a406e35d40..09c87decf2d9 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -1215,7 +1215,7 @@ protected NamedList sendRequest(SolrRequest request, List inp requestRLTGenerator.getReplicaListTransformer(reqParams); final ClusterStateProvider provider = getClusterStateProvider(); - final String urlScheme = provider.getClusterProperty(ClusterState.URL_SCHEME, "http"); + final String urlScheme = provider.getUrlScheme(); final Set liveNodes = provider.getLiveNodes(); final List theUrlList = new ArrayList<>(); // we populate this as follows... diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java index ce7c7f9c2e4d..439363f91494 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java @@ -127,4 +127,7 @@ default T getClusterProperty(String key, T defaultValue) { void connect(); String getQuorumHosts(); + + /** Get url scheme like http or https but never null. */ + String getUrlScheme(); } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java index 488d816767da..fa2c6a32fee6 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.reflect.InvocationTargetException; +import java.nio.file.Path; import java.time.Instant; import java.util.List; import java.util.Map; @@ -38,8 +39,13 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.cloud.ZkController; +import org.apache.solr.cloud.ZkTestServer; +import org.apache.solr.common.cloud.ClusterProperties; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.NamedList; import org.eclipse.jetty.http.HttpField; import org.eclipse.jetty.http.HttpHeader; @@ -49,6 +55,7 @@ import org.junit.Test; public class ClusterStateProviderTest extends SolrCloudTestCase { + private static final String SOLR_SSL_ENABLED = "solr.ssl.enabled"; @BeforeClass public static void setupCluster() throws Exception { @@ -372,6 +379,62 @@ public void testClusterStateProviderLiveNodesWithNewNode() throws Exception { } } + @Test + public void testHttpClusterStateProviderUrlScheme() throws Exception { + final List solrUrls = + List.of("https://localhost:8983/solr", "https://localhost:8984/solr"); + try (var httpClient = new Http2SolrClient.Builder().build(); + ClusterStateProvider clusterStateProvider = + new Http2ClusterStateProvider(solrUrls, httpClient)) { + assertEquals("https", clusterStateProvider.getUrlScheme()); + } + } + + @Test + public void testZkClusterStateProviderUrlScheme() throws Exception { + final Path zkDir = createTempDir("zkData"); + + final ZkTestServer server = new ZkTestServer(zkDir); + try { + server.run(); + SolrZkClient client = new SolrZkClient.Builder().withUrl(server.getZkAddress()).build(); + ZkController.createClusterZkNodes(client); + testUrlSchemeWithClusterProperties(client); + testUrlSchemeDefault(client); + testUrlSchemeWithSystemProperties(client); + client.close(); + } finally { + server.shutdown(); + } + } + + private void testUrlSchemeDefault(SolrZkClient client) throws Exception { + try (var zkStateReader = new ZkStateReader(client); + var clusterStateProvider = new ZkClientClusterStateProvider(zkStateReader)) { + assertEquals("http", clusterStateProvider.getUrlScheme()); + } + } + + private void testUrlSchemeWithSystemProperties(SolrZkClient client) throws Exception { + System.setProperty(SOLR_SSL_ENABLED, "true"); + try (var zkStateReader = new ZkStateReader(client); + var clusterStateProvider = new ZkClientClusterStateProvider(zkStateReader)) { + assertEquals("https", clusterStateProvider.getUrlScheme()); + } finally { + System.clearProperty(SOLR_SSL_ENABLED); + } + } + + private void testUrlSchemeWithClusterProperties(SolrZkClient client) throws Exception { + ClusterProperties cp = new ClusterProperties(client); + cp.setClusterProperty("urlScheme", "ftp"); + try (var zkStateReader = new ZkStateReader(client); + var clusterStateProvider = new ZkClientClusterStateProvider(zkStateReader)) { + zkStateReader.createClusterStateWatchersAndUpdate(); + assertEquals("ftp", clusterStateProvider.getUrlScheme()); + } + } + private void waitForCSPCacheTimeout() throws InterruptedException { Thread.sleep(2000); } From 865ef83a91324eaf9bdc849ac8f9a59ad7d1b9fc Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Tue, 30 Jul 2024 13:59:14 -0400 Subject: [PATCH 42/83] Default to MockDirectoryFactory in test configs (#2598) This was already the effective default value, by virtue of it being set in 'randomization.gradle'. But setting the value there instead of in the config files themselves meant that the value wasn't being properly set for IDE test runners. This commit fixes this by removing the 'randomization.gradle' setting and instead setting MockDirectoryFactory as the default (via ${} expansion syntax) in each individual test solrconfig file. (cherry picked from commit c7630fe13ff9e5475c0c358beb9e92c12826e599) --- gradle/testing/randomization.gradle | 1 - .../test-files/solr/collection1/conf/bad-error-solrconfig.xml | 2 +- .../test-files/solr/collection1/conf/bad-mpf-solrconfig.xml | 2 +- .../src/test-files/solr/collection1/conf/bad_solrconfig.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-SOLR-749.xml | 2 +- .../solrconfig-add-schema-fields-update-processor-chains.xml | 2 +- .../solr/collection1/conf/solrconfig-analytics-query.xml | 2 +- .../src/test-files/solr/collection1/conf/solrconfig-basic.xml | 2 +- .../solr/collection1/conf/solrconfig-cache-enable-disable.xml | 2 +- .../solr/collection1/conf/solrconfig-classification.xml | 2 +- .../solr/collection1/conf/solrconfig-collapseqparser.xml | 2 +- .../solr/collection1/conf/solrconfig-components-name.xml | 2 +- .../collection1/conf/solrconfig-concurrentmergescheduler.xml | 2 +- .../solr/collection1/conf/solrconfig-coreproperties.xml | 2 +- .../solr/collection1/conf/solrconfig-delaying-component.xml | 2 +- .../solr/collection1/conf/solrconfig-delpolicy1.xml | 2 +- .../solr/collection1/conf/solrconfig-delpolicy2.xml | 2 +- .../conf/solrconfig-distrib-update-processor-chains.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-elevate.xml | 2 +- .../collection1/conf/solrconfig-externalversionconstraint.xml | 2 +- .../solr/collection1/conf/solrconfig-follower-auth.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-follower.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-follower1.xml | 2 +- .../solr/collection1/conf/solrconfig-functionquery.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-headers.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-highlight.xml | 2 +- .../solr/collection1/conf/solrconfig-implicitproperties.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-leader.xml | 2 +- .../collection1/conf/solrconfig-leader1-keepOneBackup.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-leader1.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-leader2.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-leader3.xml | 2 +- .../collection1/conf/solrconfig-legacy-circuitbreaker.xml | 2 +- .../collection1/conf/solrconfig-logmergepolicyfactory.xml | 2 +- .../solr/collection1/conf/solrconfig-lucene-codec.xml | 2 +- .../solr/collection1/conf/solrconfig-managed-schema-test.xml | 2 +- .../solr/collection1/conf/solrconfig-mergepolicy-defaults.xml | 2 +- .../solr/collection1/conf/solrconfig-mergepolicy-legacy.xml | 2 +- .../collection1/conf/solrconfig-mergepolicyfactory-nocfs.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-minhash.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-nocache.xml | 2 +- .../solr/collection1/conf/solrconfig-nomergepolicyfactory.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-noopregen.xml | 2 +- .../solr/collection1/conf/solrconfig-parallel-commit.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-paramset.xml | 2 +- .../conf/solrconfig-parsing-update-processor-chains.xml | 2 +- .../solr/collection1/conf/solrconfig-phrasesuggest.xml | 2 +- .../solr/collection1/conf/solrconfig-plugcollector.xml | 2 +- .../collection1/conf/solrconfig-pluggable-circuitbreaker.xml | 2 +- .../solr/collection1/conf/solrconfig-query-parser-init.xml | 2 +- .../solr/collection1/conf/solrconfig-querysender-noquery.xml | 2 +- .../solr/collection1/conf/solrconfig-querysender.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-repeater.xml | 2 +- .../solr/collection1/conf/solrconfig-replication-legacy.xml | 2 +- .../collection1/conf/solrconfig-response-log-component.xml | 2 +- .../solr/collection1/conf/solrconfig-searcher-listeners1.xml | 2 +- .../collection1/conf/solrconfig-sortingmergepolicyfactory.xml | 2 +- .../solr/collection1/conf/solrconfig-sortingresponse.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-spatial.xml | 2 +- .../solr/collection1/conf/solrconfig-spellcheckcomponent.xml | 2 +- .../solr/collection1/conf/solrconfig-spellchecker.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-tagger.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-test-misc.xml | 2 +- .../solr/collection1/conf/solrconfig-testxmlparser.xml | 2 +- .../collection1/conf/solrconfig-tieredmergepolicyfactory.xml | 2 +- .../solr/collection1/conf/solrconfig-transformers.xml | 2 +- .../collection1/conf/solrconfig-update-processor-chains.xml | 2 +- .../conf/solrconfig-warmer-randommergepolicyfactory.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-xinclude.xml | 2 +- solr/core/src/test-files/solr/collection1/conf/solrconfig.xml | 2 +- .../solr/collection1/conf/solrconfig_SimpleTextCodec.xml | 2 +- .../src/test-files/solr/collection1/conf/solrconfig_codec.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig_codec2.xml | 2 +- .../src/test-files/solr/collection1/conf/solrconfig_perf.xml | 2 +- .../solr/configsets/bad-mergepolicy/conf/solrconfig.xml | 2 +- .../solr/configsets/cloud-minimal/conf/solrconfig.xml | 2 +- .../solr/configsets/exitable-directory/conf/solrconfig.xml | 2 +- .../test-files/solr/configsets/minimal/conf/solrconfig.xml | 2 +- .../solr/collection1/conf/solrconfig-icucollate.xml | 2 +- .../solr/collection1/conf/solrconfig-opennlp-extract.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-analytics.xml | 2 +- .../clustering/solr/collection1/conf/solrconfig.xml | 2 +- .../extraction/solr/collection1/conf/solrconfig-minimal.xml | 2 +- .../extraction/solr/collection1/conf/solrconfig.xml | 2 +- .../solr/collection1/conf/solrconfig-languageidentifier.xml | 2 +- .../src/test-files/solr/collection1/conf/solrconfig-ltr.xml | 2 +- .../solr/collection1/conf/solrconfig-ltr_Th10_10.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-multiseg.xml | 2 +- .../collection1/conf/solrconfig-script-updateprocessor.xml | 2 +- .../conf/stateless-solrconfig-script-updateprocessor.xml | 2 +- .../solrj/solr/collection1/conf/solrconfig-follower1.xml | 2 +- .../solrj/solr/configsets/shared/conf/solrconfig.xml | 2 +- .../src/java/org/apache/solr/SolrTestCaseJ4.java | 4 ---- .../src/test-files/solr/collection1/conf/solrconfig.xml | 2 +- 94 files changed, 92 insertions(+), 97 deletions(-) diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradle index 115555951ac9..4bb30c48ed74 100644 --- a/gradle/testing/randomization.gradle +++ b/gradle/testing/randomization.gradle @@ -128,7 +128,6 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr") }) { plugins.withType(JavaPlugin) { project.ext { testOptions += [ - [propName: 'solr.directoryFactory', value: "org.apache.solr.core.MockDirectoryFactory", description: "Solr directory factory."], [propName: 'tests.src.home', value: null, description: "See SOLR-14023."], [propName: 'solr.tests.use.numeric.points', value: null, description: "Point implementation to use (true=numerics, false=trie)."], ] diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-error-solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/bad-error-solrconfig.xml index c8bb8cf66e37..af4edbddcaf6 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-error-solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-error-solrconfig.xml @@ -19,7 +19,7 @@ - + ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-mpf-solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/bad-mpf-solrconfig.xml index 19d786056ad7..bfaf24a0cfae 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-mpf-solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-mpf-solrconfig.xml @@ -19,7 +19,7 @@ - + ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml index e24df5846fdc..70edc2d47234 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${unset.sys.property} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml index 9c2411bde335..502146d9591f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-SOLR-749.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml index 34ee4f7d0385..d95c8a740c34 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml @@ -24,7 +24,7 @@ ${tests.luceneMatchVersion:LATEST} - + true diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-analytics-query.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-analytics-query.xml index cda45dfe364b..b6d4a1cb673a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-analytics-query.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-analytics-query.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml index d98ecacf454b..d28e6a6d052a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml @@ -23,7 +23,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-cache-enable-disable.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-cache-enable-disable.xml index d51f8050c561..96e0b6d754ce 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-cache-enable-disable.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-cache-enable-disable.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml index 3370600574dd..038813e8dd31 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml @@ -24,7 +24,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-collapseqparser.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-collapseqparser.xml index 8d242f46a54e..d104e8d80af0 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-collapseqparser.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-collapseqparser.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml index 6a14a8cc91c5..0ca0e186db66 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml @@ -29,7 +29,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml index 140c4cfeedca..3c7c7959ed04 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-coreproperties.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-coreproperties.xml index fc707d461fda..13bada25dd7a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-coreproperties.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-coreproperties.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delaying-component.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delaying-component.xml index cfb812fa25b5..7cabbd19c6fa 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delaying-component.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delaying-component.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml index 424783beef58..2bd4d26b150c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml index bb4d3f85ba05..8a57b7fb8e1b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml index b23a7dc9ff6b..9b56a39a52f5 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.hdfs.blockcache.enabled:true} ${solr.hdfs.blockcache.blocksperbank:1024} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml index d814a538a50c..63bb8f84ffaa 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml @@ -28,7 +28,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml index 0e785ce0adc7..5e366166ea12 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.hdfs.blockcache.enabled:true} ${solr.hdfs.blockcache.blocksperbank:1024} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml index 1635cfb099bf..236a0997b5a0 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower.xml index fc0298ef5fd2..f009ac9e59cd 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower1.xml index 71f4157ccb2c..9ae3600b2005 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower1.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower1.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml index ef0c39ae8e43..2086a6c4ba9b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml @@ -25,7 +25,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml index 328fc9bd56a7..695cd36e2a4c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml index c714a4148d70..7688b3e0a7e3 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml @@ -25,7 +25,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-implicitproperties.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-implicitproperties.xml index b9a72c5634a9..072cae1d8691 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-implicitproperties.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-implicitproperties.xml @@ -26,7 +26,7 @@ ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}"/> diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader.xml index 89c06ad3a54c..ea2e8e332daf 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1-keepOneBackup.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1-keepOneBackup.xml index 36065b43ae7f..40e93b8e5c57 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1-keepOneBackup.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1-keepOneBackup.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1.xml index 4e4e39992246..e4afd9b9f8f7 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader2.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader2.xml index f6c2a77dde37..933337f0a8c4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader2.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader2.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader3.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader3.xml index 49d1ed31b5a3..b46fb393cf3b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader3.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader3.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-legacy-circuitbreaker.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-legacy-circuitbreaker.xml index fb00756b86cc..275e8961f020 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-legacy-circuitbreaker.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-legacy-circuitbreaker.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-logmergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-logmergepolicyfactory.xml index 539fd5c4d5cb..528ea8795684 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-logmergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-logmergepolicyfactory.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-lucene-codec.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-lucene-codec.xml index 1ee427dc40d0..9c1312638500 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-lucene-codec.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-lucene-codec.xml @@ -29,7 +29,7 @@ ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema-test.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema-test.xml index 666132f8e743..e3f4612e7e15 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema-test.xml @@ -22,6 +22,6 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-defaults.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-defaults.xml index 3e0cf1927b63..76684eaebe8f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-defaults.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-defaults.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-legacy.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-legacy.xml index b67d6645f31c..d02b9b018276 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-legacy.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-legacy.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicyfactory-nocfs.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicyfactory-nocfs.xml index b93fabd7c3a9..f9276ba9298c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicyfactory-nocfs.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicyfactory-nocfs.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml index 9fa236dda0b6..0d32632fc947 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml index fb891f822fdf..f4ffe8a8502d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml @@ -24,7 +24,7 @@ - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml index 62fb05b03c48..6e5545be7c41 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-noopregen.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-noopregen.xml index e2a20a3379ca..1799508b1150 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-noopregen.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-noopregen.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parallel-commit.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parallel-commit.xml index 3e619948e189..662089c170a6 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parallel-commit.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parallel-commit.xml @@ -24,7 +24,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml index a9e71f63ebc1..1b41d2b949a3 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml @@ -23,7 +23,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml index 43f2d285a9fb..0416d3e796e5 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml @@ -24,7 +24,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml index 0f79d7c4572f..65f1e9f3a4ab 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml index 845998ec2f4a..63f2ce29b7ed 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-pluggable-circuitbreaker.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-pluggable-circuitbreaker.xml index 52956f608241..4c8d441b2d30 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-pluggable-circuitbreaker.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-pluggable-circuitbreaker.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-query-parser-init.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-query-parser-init.xml index eb538fa620b0..0dd520baca9d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-query-parser-init.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-query-parser-init.xml @@ -24,7 +24,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml index 9d4f83d68c31..e6e94a2aa67f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml @@ -23,7 +23,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml index 1404f8abb4fa..f8bc376ad29e 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml @@ -23,7 +23,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml index f5571f9827f2..b426ac82f930 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml index 43c42ffb089b..ddd116be38a7 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml index 643d9a62fc72..f3bb105bd25d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml @@ -31,7 +31,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-searcher-listeners1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-searcher-listeners1.xml index c71f8baf9d41..ebceb1485790 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-searcher-listeners1.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-searcher-listeners1.xml @@ -30,7 +30,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml index 5920348551ae..5d1e7242e70c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingresponse.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingresponse.xml index 1bb3c97a225d..ee27bec61444 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingresponse.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingresponse.xml @@ -23,7 +23,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml index a3d09a034a54..53cd9fa6c2d8 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml index 0a92e46115ba..1b3815ff2d5b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml @@ -57,7 +57,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml index a876fb6f27c8..5349edc7c8ca 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml @@ -21,7 +21,7 @@ - + ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml index c97ce085660d..de0b35692709 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml @@ -26,7 +26,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml index 00d49db32851..1020db8319ee 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml @@ -25,7 +25,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-testxmlparser.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-testxmlparser.xml index 401710e8d325..f9f8427707b4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-testxmlparser.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-testxmlparser.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tieredmergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tieredmergepolicyfactory.xml index 4d2efd9ec00c..8f66b4a55498 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tieredmergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tieredmergepolicyfactory.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml index ef38e8099e9c..b4e4846426d5 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml @@ -23,7 +23,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml index e22ad6958599..4e6d7bdd3702 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml @@ -27,7 +27,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer-randommergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer-randommergepolicyfactory.xml index 21101b1d9e65..3c6056a3fc12 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer-randommergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer-randommergepolicyfactory.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml index 17df214cb71d..48f6676645f4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml index 5e6c9c82412c..856079f329ed 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig_SimpleTextCodec.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig_SimpleTextCodec.xml index 7b0c3e368bab..a971665edee0 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig_SimpleTextCodec.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig_SimpleTextCodec.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml index ad080961d9fd..991d2a50f612 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec2.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec2.xml index c4a8ae73dfe9..94f934dbcbef 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec2.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec2.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml index f2b5ef8a381d..0c6b57c4e77c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml @@ -28,7 +28,7 @@ - + diff --git a/solr/core/src/test-files/solr/configsets/bad-mergepolicy/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/bad-mergepolicy/conf/solrconfig.xml index 3ef080dcca39..c286b018cef4 100644 --- a/solr/core/src/test-files/solr/configsets/bad-mergepolicy/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/bad-mergepolicy/conf/solrconfig.xml @@ -19,7 +19,7 @@ - + ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml index 853ba6562416..baef69fde907 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml @@ -24,7 +24,7 @@ ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}"/> ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml index 515ad64d7f55..7e1f3434e96c 100644 --- a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.hdfs.blockcache.enabled:true} ${solr.hdfs.blockcache.blocksperbank:1024} diff --git a/solr/core/src/test-files/solr/configsets/minimal/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/minimal/conf/solrconfig.xml index 346b0448318c..4e90243daf6a 100644 --- a/solr/core/src/test-files/solr/configsets/minimal/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/minimal/conf/solrconfig.xml @@ -24,7 +24,7 @@ ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}"/> ${tests.luceneMatchVersion:LATEST} diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml index 90c52d71cbe9..44ef65307526 100644 --- a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml +++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml @@ -23,5 +23,5 @@ ${useCompoundFile:false} - + diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-opennlp-extract.xml b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-opennlp-extract.xml index 7fd793e94d39..1bf101f0db16 100644 --- a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-opennlp-extract.xml +++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-opennlp-extract.xml @@ -22,7 +22,7 @@ - + diff --git a/solr/modules/analytics/src/test-files/solr/collection1/conf/solrconfig-analytics.xml b/solr/modules/analytics/src/test-files/solr/collection1/conf/solrconfig-analytics.xml index 4c359a67c6aa..366e8114e562 100644 --- a/solr/modules/analytics/src/test-files/solr/collection1/conf/solrconfig-analytics.xml +++ b/solr/modules/analytics/src/test-files/solr/collection1/conf/solrconfig-analytics.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/modules/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml b/solr/modules/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml index 4d268b0efd89..7376c7a07d19 100644 --- a/solr/modules/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml +++ b/solr/modules/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml @@ -25,7 +25,7 @@ If replication is in use, this should match the replication configuration. --> ${solr.data.dir:} - + diff --git a/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig-minimal.xml b/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig-minimal.xml index dc3ca73d54d8..a39eee3fb9c5 100644 --- a/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig-minimal.xml +++ b/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig-minimal.xml @@ -24,7 +24,7 @@ ${solr.data.dir:} - + diff --git a/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml b/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml index 724b9335b68d..159d5e161627 100644 --- a/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml +++ b/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml @@ -27,7 +27,7 @@ It defaults to "index" if not present, and should probably not be changed if replication is in use. --> ${solr.data.dir:} - + diff --git a/solr/modules/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml b/solr/modules/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml index f03387a170b0..a951ea4d80da 100644 --- a/solr/modules/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml +++ b/solr/modules/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml @@ -31,7 +31,7 @@ - + ${tests.luceneMatchVersion:LATEST} diff --git a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml index 1b3985970173..c2ae34d1f57f 100644 --- a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml +++ b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml @@ -14,7 +14,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}" /> diff --git a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml index 66bc42f46227..33ace9850d87 100644 --- a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml +++ b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml @@ -14,7 +14,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}" /> diff --git a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml index bccfd2bd652d..4f1a03a051bd 100644 --- a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml +++ b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml @@ -14,7 +14,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}" /> diff --git a/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/solrconfig-script-updateprocessor.xml b/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/solrconfig-script-updateprocessor.xml index afa5a7c56d74..82ce0331eda2 100644 --- a/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/solrconfig-script-updateprocessor.xml +++ b/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/solrconfig-script-updateprocessor.xml @@ -25,7 +25,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/stateless-solrconfig-script-updateprocessor.xml b/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/stateless-solrconfig-script-updateprocessor.xml index 58fbb8628605..d8c6fd2bdbb1 100644 --- a/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/stateless-solrconfig-script-updateprocessor.xml +++ b/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/stateless-solrconfig-script-updateprocessor.xml @@ -25,7 +25,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-follower1.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-follower1.xml index ab2773d6eb4d..43ff0410a8ca 100644 --- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-follower1.xml +++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-follower1.xml @@ -23,7 +23,7 @@ ${useCompoundFile:false} ${solr.data.dir:} - + diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/solrconfig.xml b/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/solrconfig.xml index 11ea52b8da2b..fb89854b08c5 100644 --- a/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/solrconfig.xml +++ b/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/solrconfig.xml @@ -26,7 +26,7 @@ ${useCompoundFile:false} ${tempDir}/data/${l10n:}-${version:} - + diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index bdb2cff914b0..34bdcce7e6cb 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -760,10 +760,6 @@ public static void initCore() throws Exception { log.info("####initCore"); ignoreException("ignore_exception"); - factoryProp = System.getProperty("solr.directoryFactory"); - if (factoryProp == null) { - System.setProperty("solr.directoryFactory", "solr.RAMDirectoryFactory"); - } // other methods like starting a jetty instance need these too System.setProperty("solr.test.sys.prop1", "propone"); diff --git a/solr/test-framework/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/test-framework/src/test-files/solr/collection1/conf/solrconfig.xml index 82dca6384d8b..ea95918291c1 100644 --- a/solr/test-framework/src/test-files/solr/collection1/conf/solrconfig.xml +++ b/solr/test-framework/src/test-files/solr/collection1/conf/solrconfig.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 From 4661f21718c5bd51306b23aa3a7946e667962e42 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sat, 7 Mar 2026 15:58:01 -0500 Subject: [PATCH 43/83] SOLR-18142: Fix CloudSolrClient cache state refresh; regression. (#4176) CloudSolrClient- fixed state refresh race; didn't refresh. Regression from 9.10.1/10.0 Found by flaky test: CloudSolrClientCacheTest.testStaleStateRetryWaitsAfterSkipFailure (cherry picked from commit 19fbd97f02d01d927405efcdf59176cb56209304) --- changelog/unreleased/SOLR-18142.yml | 7 +++ .../client/solrj/impl/CloudSolrClient.java | 62 ++++++++----------- 2 files changed, 34 insertions(+), 35 deletions(-) create mode 100644 changelog/unreleased/SOLR-18142.yml diff --git a/changelog/unreleased/SOLR-18142.yml b/changelog/unreleased/SOLR-18142.yml new file mode 100644 index 000000000000..b8b20bfea78e --- /dev/null +++ b/changelog/unreleased/SOLR-18142.yml @@ -0,0 +1,7 @@ +title: CloudSolrClient- fixed state refresh race; didn't refresh. Regression from 9.10.1/10.0. +type: fixed +authors: + - name: David Smiley +links: + - name: SOLR-18142 + url: https://issues.apache.org/jira/browse/SOLR-18142 diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index 09c87decf2d9..b4000eaf1efa 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -102,7 +102,7 @@ public abstract class CloudSolrClient extends SolrClient { private final boolean directUpdatesToLeadersOnly; private final RequestReplicaListTransformerGenerator requestRLTGenerator; private final boolean parallelUpdates; - private ExecutorService threadPool = + private final ExecutorService threadPool = ExecutorUtil.newMDCAwareCachedThreadPool( new SolrNamedThreadFactory("CloudSolrClient ThreadPool")); @@ -367,9 +367,8 @@ protected boolean wasCommError(Throwable t) { public void close() throws IOException { closed = true; collectionRefreshes.clear(); - if (this.threadPool != null && !ExecutorUtil.isShutdown(this.threadPool)) { + if (!ExecutorUtil.isShutdown(this.threadPool)) { ExecutorUtil.shutdownAndAwaitTermination(this.threadPool); - this.threadPool = null; } } @@ -1417,41 +1416,34 @@ protected DocCollection getDocCollection(String collection, Integer expectedVers } private CompletableFuture triggerCollectionRefresh(String collection) { - if (closed) { - ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(collection); - DocCollection cached = cacheEntry == null ? null : cacheEntry.cached; - return CompletableFuture.completedFuture(cached); - } - return collectionRefreshes.computeIfAbsent( + return collectionRefreshes.compute( collection, - key -> { - ExecutorService executor = threadPool; - CompletableFuture future; - if (executor == null || ExecutorUtil.isShutdown(executor)) { - future = new CompletableFuture<>(); - try { - future.complete(loadDocCollection(key)); - } catch (Throwable t) { - future.completeExceptionally(t); - } + (key, existingFuture) -> { + // A refresh is still in progress; return it. + if (existingFuture != null && !existingFuture.isDone()) { + return existingFuture; + } + // No refresh is in-progress, so trigger it. + + if (ExecutorUtil.isShutdown(threadPool)) { + assert closed; // see close() for the sequence + ExpiringCachedDocCollection cacheEntry = collectionStateCache.peek(key); + DocCollection cached = cacheEntry == null ? null : cacheEntry.cached; + return CompletableFuture.completedFuture(cached); } else { - future = - CompletableFuture.supplyAsync( - () -> { - stateRefreshSemaphore.acquireUninterruptibly(); - try { - return loadDocCollection(key); - } finally { - stateRefreshSemaphore.release(); - } - }, - executor); + return CompletableFuture.supplyAsync( + () -> { + stateRefreshSemaphore.acquireUninterruptibly(); + try { + return loadDocCollection(key); + } finally { + stateRefreshSemaphore.release(); + // Remove the entry in case of many collections + collectionRefreshes.remove(key); + } + }, + threadPool); } - future.whenCompleteAsync( - (result, error) -> { - collectionRefreshes.remove(key, future); - }); - return future; }); } From 35ec600c512e72248bb6f6f944899633513c4f41 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Mon, 9 Mar 2026 15:12:41 -0400 Subject: [PATCH 44/83] SOLR-18146: Fix race in CircuitBreakerRegistry (#4189) Global CB loading is now only done by a single thread, and skipped by whichever threads "lose" that race on startup. --- ...OLR-18146-fix-global-cb-race-condition.yml | 7 + .../CircuitBreakerRegistry.java | 31 ++-- .../solr/util/TestGlobalCircuitBreaker.java | 3 + .../TestGlobalCircuitBreakerRegistration.java | 137 ++++++++++++++++++ .../util/circuitbreaker/package-info.java | 22 +++ 5 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 changelog/unreleased/SOLR-18146-fix-global-cb-race-condition.yml create mode 100644 solr/core/src/test/org/apache/solr/util/circuitbreaker/TestGlobalCircuitBreakerRegistration.java create mode 100644 solr/core/src/test/org/apache/solr/util/circuitbreaker/package-info.java diff --git a/changelog/unreleased/SOLR-18146-fix-global-cb-race-condition.yml b/changelog/unreleased/SOLR-18146-fix-global-cb-race-condition.yml new file mode 100644 index 000000000000..f9056b84814f --- /dev/null +++ b/changelog/unreleased/SOLR-18146-fix-global-cb-race-condition.yml @@ -0,0 +1,7 @@ +title: Fix race conditions in "global" CircuitBreaker registration +type: fixed +authors: + - name: Jason Gerlowski +links: + - name: SOLR-18146 + url: https://issues.apache.org/jira/browse/SOLR-18146 diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java index 91e284b823f1..075aefc23dd8 100644 --- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java +++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -50,9 +51,6 @@ */ public class CircuitBreakerRegistry implements Closeable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final Map> circuitBreakerMap = new HashMap<>(); - private static final Map> globalCircuitBreakerMap = - new ConcurrentHashMap<>(); private static final Pattern SYSPROP_REGEX = Pattern.compile("solr.circuitbreaker\\.(update|query)\\.(cpu|mem|loadavg)"); public static final String SYSPROP_PREFIX = "solr.circuitbreaker."; @@ -64,6 +62,12 @@ public class CircuitBreakerRegistry implements Closeable { public static final String SYSPROP_QUERY_LOADAVG = SYSPROP_PREFIX + "query.loadavg"; public static final String SYSPROP_WARN_ONLY_SUFFIX = ".warnonly"; + private static boolean globalsInitialized = false; + private static final Map> globalCircuitBreakerMap = + new ConcurrentHashMap<>(); + + private final Map> circuitBreakerMap = new HashMap<>(); + public CircuitBreakerRegistry(CoreContainer coreContainer) { initGlobal(coreContainer); } @@ -118,7 +122,8 @@ private static String buildCircuitBreakerKey(String metricType, String threshold return metricType + ":" + System.getProperty(thresholdProp) + ":" + warnOnly; } - private static void initGlobal(CoreContainer coreContainer) { + private static synchronized void initGlobal(CoreContainer coreContainer) { + if (globalsInitialized) return; final var parsedBreakers = parseCircuitBreakersFromProperties(coreContainer); for (CircuitBreaker breaker : parsedBreakers) { registerGlobal(breaker); @@ -129,6 +134,7 @@ private static void initGlobal(CoreContainer coreContainer) { breaker.getRequestTypes()); } } + globalsInitialized = true; } /** List all registered circuit breakers for global context */ @@ -165,7 +171,7 @@ public static void registerGlobal(CircuitBreaker circuitBreaker) { .forEach( r -> { List list = - globalCircuitBreakerMap.computeIfAbsent(r, k -> new ArrayList<>()); + globalCircuitBreakerMap.computeIfAbsent(r, k -> new CopyOnWriteArrayList<>()); list.add(circuitBreaker); }); } @@ -235,14 +241,13 @@ public void close() throws IOException { } } - private static void closeGlobal() { - synchronized (globalCircuitBreakerMap) { - closeCircuitBreakers( - globalCircuitBreakerMap.values().stream() - .flatMap(List::stream) - .collect(Collectors.toList())); - globalCircuitBreakerMap.clear(); - } + private static synchronized void closeGlobal() { + closeCircuitBreakers( + globalCircuitBreakerMap.values().stream() + .flatMap(List::stream) + .collect(Collectors.toList())); + globalCircuitBreakerMap.clear(); + globalsInitialized = false; } /** diff --git a/solr/core/src/test/org/apache/solr/util/TestGlobalCircuitBreaker.java b/solr/core/src/test/org/apache/solr/util/TestGlobalCircuitBreaker.java index a50e2162856b..b99d3df908ad 100644 --- a/solr/core/src/test/org/apache/solr/util/TestGlobalCircuitBreaker.java +++ b/solr/core/src/test/org/apache/solr/util/TestGlobalCircuitBreaker.java @@ -34,6 +34,9 @@ public static void setUpClass() throws Exception { System.setProperty("queryResultCache.enabled", "false"); System.setProperty("documentCache.enabled", "true"); + // Deregister existing CBs so that they're re-populated on the next core load. + CircuitBreakerRegistry.deregisterGlobal(); + // Set a global update breaker for a low CPU, which will trip during indexing System.setProperty(CircuitBreakerRegistry.SYSPROP_UPDATE_LOADAVG, "0.1"); diff --git a/solr/core/src/test/org/apache/solr/util/circuitbreaker/TestGlobalCircuitBreakerRegistration.java b/solr/core/src/test/org/apache/solr/util/circuitbreaker/TestGlobalCircuitBreakerRegistration.java new file mode 100644 index 000000000000..fe04fa34170d --- /dev/null +++ b/solr/core/src/test/org/apache/solr/util/circuitbreaker/TestGlobalCircuitBreakerRegistration.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util.circuitbreaker; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.junit.After; +import org.junit.Test; + +/** + * Attempts to reproduce the race condition described in SOLR-18146 + * + *

The race: {@link CircuitBreakerRegistry} relies on a static map of global (i.e. + * non-core-specific) CircuitBreakers. If Solr was loading multiple cores on startup, these + * concurrent core-loads could race when initializing this static state leading to + * ConcurrentModificationExceptions, ArrayIndexOutOfBoundsExceptions, dropped circuit breaker + * entries, etc. + * + *

These issues should be resolved by SOLR-18146, but the test remains to catch them if they + * recur. Often useful to run in larger iterations with {@code -Ptests.iters=20}. + */ +public class TestGlobalCircuitBreakerRegistration extends SolrTestCaseJ4 { + + @After + public void cleanup() { + CircuitBreakerRegistry.deregisterGlobal(); + } + + @Test + public void testConcurrentGlobalRegistrationDoesNotThrow() throws Exception { + final int threadCount = 50; + final int itersPerThread = 20; + final int total = threadCount * itersPerThread; + final CyclicBarrier barrier = new CyclicBarrier(threadCount); + final List failures = Collections.synchronizedList(new ArrayList<>()); + + // Pre-create all circuit breakers before the barrier to make the race as "hot" as possible + List> breakersPerThread = new ArrayList<>(threadCount); + for (int t = 0; t < threadCount; t++) { + List threadBreakers = new ArrayList<>(itersPerThread); + for (int i = 0; i < itersPerThread; i++) { + CircuitBreaker b = + new CircuitBreaker() { + @Override + public boolean isTripped() { + return false; + } + + @Override + public String getErrorMessage() { + return "test"; + } + }; + b.setRequestTypes(List.of("query")); + threadBreakers.add(b); + } + breakersPerThread.add(threadBreakers); + } + + ExecutorService executor = + ExecutorUtil.newMDCAwareFixedThreadPool( + threadCount, new SolrNamedThreadFactory("test-concurrent-cb-registration")); + try { + List> futures = new ArrayList<>(); + for (int t = 0; t < threadCount; t++) { + final List myBreakers = breakersPerThread.get(t); + futures.add( + executor.submit( + () -> { + try { + // Release all threads simultaneously to maximize contention on + // the shared ArrayList for the QUERY key in globalCircuitBreakerMap. + barrier.await(); + for (CircuitBreaker b : myBreakers) { + CircuitBreakerRegistry.registerGlobal(b); + } + } catch (BrokenBarrierException | InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable ex) { + // Captures ArrayIndexOutOfBoundsException (or similar) from + // concurrent ArrayList.add() when the race is triggered. + failures.add(ex); + } + })); + } + for (Future f : futures) { + f.get(); + } + } finally { + ExecutorUtil.shutdownAndAwaitTermination(executor); + } + + assertTrue( + "Exceptions thrown during concurrent registerGlobal: " + failures, failures.isEmpty()); + + // Even without a thrown exception the backing ArrayList can be silently + // corrupted: concurrent adds may produce null slots or overwrite each other, + // losing registrations. Verify the invariant that all registered breakers are + // non-null and that none were lost. + Set registered = CircuitBreakerRegistry.listGlobal(); + assertFalse( + "globalCircuitBreakerMap contains null entries — ArrayList corrupted by concurrent add()", + registered.contains(null)); + assertEquals( + "Expected " + + total + + " registered circuit breakers but found " + + registered.size() + + " — data was lost due to concurrent ArrayList.add() corruption", + total, + registered.size()); + } +} diff --git a/solr/core/src/test/org/apache/solr/util/circuitbreaker/package-info.java b/solr/core/src/test/org/apache/solr/util/circuitbreaker/package-info.java new file mode 100644 index 000000000000..f093cf77bfaa --- /dev/null +++ b/solr/core/src/test/org/apache/solr/util/circuitbreaker/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Tests for {@link org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry} and other CB + * functionality. + */ +package org.apache.solr.util.circuitbreaker; From e7ac003deb620b1f0c99816e928f8794cb368c4a Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 11 Mar 2026 21:54:45 -0400 Subject: [PATCH 45/83] SOLR-18144: fix schema designer to auto-create .system in 9x (#4202) Moved auto-creation logic from HttpSolrCall to ZkController --- changelog/unreleased/SOLR-18144.yml | 9 ++++ .../org/apache/solr/cloud/ZkController.java | 44 +++++++++++++++++ .../SchemaDesignerConfigSetHelper.java | 32 +++--------- .../org/apache/solr/servlet/HttpSolrCall.java | 49 ++----------------- .../TestSchemaDesignerConfigSetHelper.java | 4 -- 5 files changed, 63 insertions(+), 75 deletions(-) create mode 100644 changelog/unreleased/SOLR-18144.yml diff --git a/changelog/unreleased/SOLR-18144.yml b/changelog/unreleased/SOLR-18144.yml new file mode 100644 index 000000000000..04ee5eaec3c7 --- /dev/null +++ b/changelog/unreleased/SOLR-18144.yml @@ -0,0 +1,9 @@ +title: Fixed schema designer to create a missing .system collection. This is a regression specific to 9.x. +type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: David Smiley + - name: Eric Pugh + - name: Jan Høydahl +links: + - name: SOLR-18144 + url: https://issues.apache.org/jira/browse/SOLR-18144 diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 78d3267023c0..26ba27b82e36 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -26,8 +26,13 @@ import static org.apache.solr.common.cloud.ZkStateReader.LIVE_NODE_SOLR_VERSION; import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.SYSTEM_COLL; import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; +import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.common.params.CoreAdminParams.ACTION; import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE; import java.io.Closeable; @@ -104,6 +109,7 @@ import org.apache.solr.common.cloud.ZooKeeperException; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.Compressor; import org.apache.solr.common.util.EnvUtils; @@ -125,6 +131,8 @@ import org.apache.solr.handler.component.HttpShardHandler; import org.apache.solr.handler.component.HttpShardHandlerFactory; import org.apache.solr.logging.MDCLoggingContext; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.update.UpdateLog; import org.apache.solr.util.AddressUtils; @@ -3143,4 +3151,40 @@ private static void ensureRegisteredSearcher(SolrCore core) throws InterruptedEx } } } + + /** Creates the .system collection if it doesn't exist. Returns true iff this created it. */ + public boolean createSystemColl() throws Exception { + if (getClusterState().hasCollection(SYSTEM_COLL)) { + return false; + } + log.info("Going to auto-create {} collection", SYSTEM_COLL); + SolrQueryResponse rsp = new SolrQueryResponse(); + String repFactor = String.valueOf(Math.min(3, getClusterState().getLiveNodes().size())); + cc.getCollectionsHandler() + .handleRequestBody( + new LocalSolrQueryRequest( + null, + new ModifiableSolrParams() + .add(ACTION, CREATE.toString()) + .add(NAME, SYSTEM_COLL) + .add(REPLICATION_FACTOR, repFactor)), + rsp); + if (rsp.getValues().get("success") == null) { + throw new SolrException( + ErrorCode.SERVER_ERROR, + "Could not auto-create " + + SYSTEM_COLL + + " collection: " + + Utils.toJSONString(rsp.getValues())); + } + + try { + getZkStateReader().waitForState(SYSTEM_COLL, 3, TimeUnit.SECONDS, Objects::nonNull); + } catch (TimeoutException e) { + throw new SolrException( + ErrorCode.SERVER_ERROR, + "Could not find " + SYSTEM_COLL + " collection even after 3 seconds"); + } + return true; + } } diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java index f24fe26ed61c..efbd8805fec9 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java @@ -45,7 +45,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -72,7 +71,6 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZkStateReader; @@ -521,29 +519,6 @@ protected void postDataToBlobStore(CloudSolrClient cloudClient, String blobName, } } - private String getBaseUrl(final String collection) { - String baseUrl = null; - try { - Set liveNodes = zkStateReader().getClusterState().getLiveNodes(); - DocCollection docColl = zkStateReader().getCollection(collection); - if (docColl != null && !liveNodes.isEmpty()) { - Optional maybeActive = - docColl.getReplicas().stream().filter(r -> r.isActive(liveNodes)).findAny(); - if (maybeActive.isPresent()) { - baseUrl = maybeActive.get().getBaseUrl(); - } - } - } catch (Exception exc) { - log.warn("Failed to lookup base URL for collection {}", collection, exc); - } - - if (baseUrl == null) { - baseUrl = zkStateReader().getBaseUrlForNodeName(cc.getZkController().getNodeName()); - } - - return baseUrl; - } - protected String getManagedSchemaZkPath(final String configSet) { return getConfigSetZkPath(configSet, DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME); } @@ -658,6 +633,13 @@ void createCollection( } protected CloudSolrClient cloudClient() { + // create the system collection if it doesn't exist + try { + cc.getZkController().createSystemColl(); + } catch (Exception e) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Error creating system collection", e); + } + return cc.getZkController().getSolrClient(); } diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 4fab7a6af83c..c894dc431585 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -19,11 +19,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; import static org.apache.solr.common.params.CollectionAdminParams.SYSTEM_COLL; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE; -import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.common.params.CoreAdminParams.ACTION; import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN; import static org.apache.solr.servlet.SolrDispatchFilter.Action.FORWARD; import static org.apache.solr.servlet.SolrDispatchFilter.Action.PASSTHROUGH; @@ -55,8 +51,6 @@ import java.util.Objects; import java.util.Random; import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; @@ -111,7 +105,6 @@ import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.ContentStreamHandlerBase; -import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; import org.apache.solr.request.SolrRequestHandler; @@ -375,46 +368,10 @@ protected DocCollection resolveDocCollection(String collectionName) { } protected void autoCreateSystemColl(String corename) throws Exception { - if (core == null - && SYSTEM_COLL.equals(corename) - && "POST".equals(req.getMethod()) - && !cores.getZkController().getClusterState().hasCollection(SYSTEM_COLL)) { - log.info("Going to auto-create {} collection", SYSTEM_COLL); - SolrQueryResponse rsp = new SolrQueryResponse(); - String repFactor = - String.valueOf( - Math.min(3, cores.getZkController().getClusterState().getLiveNodes().size())); - cores - .getCollectionsHandler() - .handleRequestBody( - new LocalSolrQueryRequest( - null, - new ModifiableSolrParams() - .add(ACTION, CREATE.toString()) - .add(NAME, SYSTEM_COLL) - .add(REPLICATION_FACTOR, repFactor)), - rsp); - if (rsp.getValues().get("success") == null) { - throw new SolrException( - ErrorCode.SERVER_ERROR, - "Could not auto-create " - + SYSTEM_COLL - + " collection: " - + Utils.toJSONString(rsp.getValues())); - } - - try { - cores - .getZkController() - .getZkStateReader() - .waitForState(SYSTEM_COLL, 3, TimeUnit.SECONDS, Objects::nonNull); - } catch (TimeoutException e) { - throw new SolrException( - ErrorCode.SERVER_ERROR, - "Could not find " + SYSTEM_COLL + " collection even after 3 seconds"); + if (core == null && SYSTEM_COLL.equals(corename) && "POST".equals(req.getMethod())) { + if (cores.getZkController().createSystemColl()) { + action = RETRY; } - - action = RETRY; } } diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java index 861261c99049..3f666039fa1b 100644 --- a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java +++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java @@ -32,7 +32,6 @@ import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.util.SimpleOrderedMap; @@ -60,9 +59,6 @@ public static void createCluster() throws Exception { configureCluster(1) .addConfig(DEFAULT_CONFIGSET_NAME, new File(ExternalPaths.DEFAULT_CONFIGSET).toPath()) .configure(); - // SchemaDesignerConfigSetHelper depends on the blob store - CollectionAdminRequest.createCollection(BLOB_STORE_ID, 1, 1).process(cluster.getSolrClient()); - cluster.waitForActiveCollection(BLOB_STORE_ID, 1, 1); } @AfterClass From 658038bab416a7c79ff940209969e8ec320a6ac2 Mon Sep 17 00:00:00 2001 From: Pierre Salagnac Date: Wed, 18 Mar 2026 10:27:55 +0100 Subject: [PATCH 46/83] SOLR-18155: Abort shard leader election if container shutdown has started (#4224) This is mostly for tests. It makes sure a replica cannot be elected leader for a very short time while all nodes are shutting down. (cherry picked from commit 8873cf1242be0787503530079be23f30c55916a0) --- .../unreleased/SOLR-18155-election-leak.yml | 7 +++++++ .../solr/cloud/ShardLeaderElectionContext.java | 18 +++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/SOLR-18155-election-leak.yml diff --git a/changelog/unreleased/SOLR-18155-election-leak.yml b/changelog/unreleased/SOLR-18155-election-leak.yml new file mode 100644 index 000000000000..a43436d09805 --- /dev/null +++ b/changelog/unreleased/SOLR-18155-election-leak.yml @@ -0,0 +1,7 @@ +title: Abort shard leader election if container shutdown sequence has started, so we don't have leaders elected very late and not properly closed. +type: fixed +authors: + - name: Pierre Salagnac +links: + - name: SOLR-18155 + url: https://issues.apache.org/jira/browse/SOLR-18155 diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index edee0f0c6e7e..5d4a8a365d9e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -75,6 +75,14 @@ public void close() { syncStrategy.close(); } + /** + * Internally check whether we should abort the election process. This returns true if either this + * context was explicitly closed, or Solr server is being shut down. + */ + private boolean shouldAbort() { + return isClosed || cc.isShutDown(); + } + @Override public void cancelElection() throws InterruptedException, KeeperException { String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP); @@ -151,7 +159,7 @@ void runLeaderProcess(boolean weAreReplacement, int pauseBeforeStart) areAllReplicasParticipating(); } - if (isClosed) { + if (shouldAbort()) { // Solr is shutting down or the ZooKeeper session expired while waiting for replicas. If the // later, we cannot be sure we are still the leader, so we should bail out. The OnReconnect // handler will re-register the cores and handle a new leadership election. @@ -182,7 +190,7 @@ void runLeaderProcess(boolean weAreReplacement, int pauseBeforeStart) } } - if (isClosed) { + if (shouldAbort()) { return; } @@ -264,7 +272,7 @@ void runLeaderProcess(boolean weAreReplacement, int pauseBeforeStart) } } - if (!isClosed) { + if (!shouldAbort()) { try { if (replicaType == Replica.Type.TLOG) { // stop replicate from old leader @@ -356,7 +364,7 @@ private boolean waitForEligibleBecomeLeaderAfterTimeout( ZkShardTerms zkShardTerms, String coreNodeName, int timeout) throws InterruptedException { long timeoutAt = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS); - while (!isClosed && !cc.isShutDown()) { + while (!shouldAbort()) { if (System.nanoTime() > timeoutAt) { log.warn( "After waiting for {}ms, no other potential leader was found, {} try to become leader anyway (core_term:{}, highest_term:{})", @@ -441,7 +449,7 @@ private boolean waitForReplicasToComeUp(int timeoutms) throws InterruptedExcepti DocCollection docCollection = zkController.getClusterState().getCollectionOrNull(collection); Slice slices = (docCollection == null) ? null : docCollection.getSlice(shardId); int cnt = 0; - while (!isClosed && !cc.isShutDown()) { + while (!shouldAbort()) { // wait for everyone to be up if (slices != null) { int found = 0; From f610af44ba13e93ebd40b27f572d5b5ddf29e767 Mon Sep 17 00:00:00 2001 From: Pierre Salagnac Date: Thu, 19 Mar 2026 11:12:31 +0100 Subject: [PATCH 47/83] SOLR-18157: Optimize buffer allocation in JavaBinCodec (#4208) Implements a smarter allocation strategy using powers of 2 for the internal buffer of JavaBinCodec. (cherry picked from commit f3fe28b872aae7a91d2d17d9a94c63e91f643a38) --- .../unreleased/SOLR-18157-javabin-buffer.yml | 8 ++ .../solr/bench/search/RequestWriters.java | 121 ++++++++++++++++++ .../apache/solr/common/util/JavaBinCodec.java | 36 +++++- .../solr/common/util/TestJavaBinCodec.java | 10 ++ 4 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/SOLR-18157-javabin-buffer.yml create mode 100644 solr/benchmark/src/java/org/apache/solr/bench/search/RequestWriters.java diff --git a/changelog/unreleased/SOLR-18157-javabin-buffer.yml b/changelog/unreleased/SOLR-18157-javabin-buffer.yml new file mode 100644 index 000000000000..dc5999cc7f00 --- /dev/null +++ b/changelog/unreleased/SOLR-18157-javabin-buffer.yml @@ -0,0 +1,8 @@ +title: Optimize the size of the internal buffer of JavaBin codec to reduce the number of allocations. This reduces GC pressure on SolrJ client under high indexing load. +type: changed +authors: + - name: Pierre Salagnac +links: + - name: SOLR-18157 + url: https://issues.apache.org/jira/browse/SOLR-18157 + diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/RequestWriters.java b/solr/benchmark/src/java/org/apache/solr/bench/search/RequestWriters.java new file mode 100644 index 000000000000..09c82fea3c1e --- /dev/null +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/RequestWriters.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench.search; + +import static org.apache.solr.bench.Docs.docs; +import static org.apache.solr.bench.generators.SourceDSL.integers; +import static org.apache.solr.bench.generators.SourceDSL.longs; +import static org.apache.solr.bench.generators.SourceDSL.strings; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Iterator; +import java.util.function.Supplier; +import org.apache.commons.io.output.NullOutputStream; +import org.apache.solr.bench.Docs; +import org.apache.solr.client.solrj.impl.BinaryRequestWriter; +import org.apache.solr.client.solrj.impl.XMLRequestWriter; +import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.common.SolrInputDocument; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Benchmark for serialization of requests by the client. This only focuses on the serialization + * itself, ignoring sending the request over the network (and for sure ignoring processing the + * request). + */ +@Fork(value = 1) +@BenchmarkMode(Mode.Throughput) +@Warmup(time = 2, iterations = 1) +@Measurement(time = 5, iterations = 5) +@Threads(value = 1) +public class RequestWriters { + + @State(Scope.Benchmark) + public static class BenchState { + + @Param({"xml", "binary"}) + String type; + + @Param({"10", "100", "1000", "10000"}) + int batchSize; + + private final int docCount = 50000; + + private Supplier writerSupplier; + private Iterator docIterator; + + @Setup(Level.Trial) + public void setup() throws Exception { + preGenerateDocs(); + + switch (type) { + case "xml": + writerSupplier = XMLRequestWriter::new; + break; + case "javabin": + writerSupplier = BinaryRequestWriter::new; + break; + + default: + throw new Error("Unsupported type: " + type); + } + } + + private void preGenerateDocs() throws Exception { + Docs docs = + docs() + .field("id", integers().incrementing()) + .field(strings().basicLatinAlphabet().ofLengthBetween(10, 64)) + .field(strings().basicLatinAlphabet().ofLengthBetween(10, 64)) + .field(strings().basicLatinAlphabet().multi(312).ofLengthBetween(10, 64)) + .field(strings().basicLatinAlphabet().multi(312).ofLengthBetween(10, 64)) + .field(integers().all()) + .field(integers().all()) + .field(longs().all()); + + docs.preGenerate(docCount); + docIterator = docs.generatedDocsCircularIterator(); + } + } + + @Benchmark + public void writeUpdate(BenchState state) throws IOException { + + OutputStream sink = NullOutputStream.INSTANCE; + + UpdateRequest request = new UpdateRequest(); + for (int i = 0; i < state.batchSize; i++) { + request.add(state.docIterator.next()); + } + + RequestWriter writer = state.writerSupplier.get(); + writer.write(request, sink); + } +} diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java index 67d32565b218..7e4a96e50d9d 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java @@ -113,9 +113,10 @@ public class JavaBinCodec implements PushWriter { NAMED_LST = (byte) (6 << 5), // NamedList EXTERN_STRING = (byte) (7 << 5); + private static final int MIN_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY = 512; private static final int MAX_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY = 65536; - private static byte VERSION = 2; + private static final byte VERSION = 2; private final ObjectResolver resolver; protected FastOutputStream daos; private StringCache stringCache; @@ -1073,7 +1074,11 @@ public void writeStr(CharSequence s) throws IOException { int maxSize = end * ByteUtils.MAX_UTF8_BYTES_PER_CHAR; if (maxSize <= MAX_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY) { - if (bytes == null || bytes.length < maxSize) bytes = new byte[maxSize]; + if (bytes == null || bytes.length < maxSize) { + int bufferSize = getBufferSize(maxSize); + bytes = new byte[bufferSize]; + } + int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0); writeTag(STR, sz); daos.write(bytes, 0, sz); @@ -1106,7 +1111,10 @@ public CharSequence readStr( private CharSequence _readStr(DataInputInputStream dis, StringCache stringCache, int sz) throws IOException { - if (bytes == null || bytes.length < sz) bytes = new byte[sz]; + if (bytes == null || bytes.length < sz) { + int bufferSize = getBufferSize(sz); + bytes = new byte[bufferSize]; + } dis.readFully(bytes, 0, sz); if (stringCache != null) { return stringCache.get(bytesRef.reset(bytes, 0, sz)); @@ -1117,6 +1125,28 @@ private CharSequence _readStr(DataInputInputStream dis, StringCache stringCache, } } + /** + * Compute the buffer size for given required size. This returns the next power of 2 that is + * greater than or equal to the given size. + * + *

This is a trade-off so we don't start with a useless too big buffer, but we don't do too + * many allocations. + */ + static int getBufferSize(int required) { + + if (required < MIN_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY) { + return MIN_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY; + } + + int oneBit = Integer.highestOneBit(required); + + if (oneBit == required) { + return oneBit; + } else { + return oneBit << 1; + } + } + /////////// code to optimize reading UTF8 static final int MAX_UTF8_SZ = 1024 * 64; // too big strings can cause too much memory allocation private Function stringProvider; diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java b/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java index a965109b226e..f7fa57d899dd 100644 --- a/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java @@ -494,6 +494,16 @@ public void testStringCaching() throws Exception { assertSame(l1.get(1), l2.get(1)); } + @Test + public void testBufferSize() { + assertEquals(512, JavaBinCodec.getBufferSize(1)); + assertEquals(512, JavaBinCodec.getBufferSize(200)); + assertEquals(512, JavaBinCodec.getBufferSize(500)); + assertEquals(512, JavaBinCodec.getBufferSize(512)); + assertEquals(1024, JavaBinCodec.getBufferSize(513)); + assertEquals(2048, JavaBinCodec.getBufferSize(1500)); + } + public void genBinaryFiles() throws IOException { Object data = generateAllDataTypes(); From 40038329d38ca1689618ef9d835811a6b223ba35 Mon Sep 17 00:00:00 2001 From: openworld-maker Date: Mon, 9 Mar 2026 06:16:11 -0700 Subject: [PATCH 48/83] SOLR-18089: Configure zookeeper.maxCnxns for embedded ZkServer (#4173) Thus avoiding the warning log from o.a.z.s.ServerCnxnFactory about this not being configured. Co-authored-by: Kamlendra (cherry picked from commit 7d9b437237dea4670abc402b3409e800dea30e28) --- .../core/src/java/org/apache/solr/cloud/SolrZkServer.java | 8 ++++++++ .../src/java/org/apache/solr/cloud/ZkTestServer.java | 1 + 2 files changed, 9 insertions(+) diff --git a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java index ee7a14733edd..9d1ef26ff1a1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java +++ b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java @@ -41,6 +41,9 @@ public class SolrZkServer { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String ZK_WHITELIST_PROPERTY = "zookeeper.4lw.commands.whitelist"; + public static final String ZK_MAX_CNXNS_PROPERTY = "zookeeper.maxCnxns"; + // Per ZooKeeper, "0" means no limit for max client connections. + public static final String ZK_MAX_CNXNS_DEFAULT = "0"; String zkRun; String zkHost; @@ -128,6 +131,7 @@ public Map getServers() { public void start() { if (zkRun == null) return; + ensureZkMaxCnxnsConfigured(); if (System.getProperty(ZK_WHITELIST_PROPERTY) == null) { System.setProperty(ZK_WHITELIST_PROPERTY, "ruok, mntr, conf"); } @@ -199,6 +203,10 @@ public void stop() { if (zkRun == null) return; zkThread.interrupt(); } + + static void ensureZkMaxCnxnsConfigured() { + System.getProperties().putIfAbsent(ZK_MAX_CNXNS_PROPERTY, ZK_MAX_CNXNS_DEFAULT); + } } // Allows us to set a default for the data dir before parsing diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java index 479507b6f5ef..9e99eff4798c 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java @@ -541,6 +541,7 @@ public void run() throws InterruptedException, IOException { public void run(boolean solrFormat) throws InterruptedException, IOException { log.info("STARTING ZK TEST SERVER"); + SolrZkServer.ensureZkMaxCnxnsConfigured(); ensureStatCommandWhitelisted(); AtomicReference zooError = new AtomicReference<>(); From c84ec608b1d037782100bf565dedcab1565abd44 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Thu, 12 Mar 2026 22:10:24 -0400 Subject: [PATCH 49/83] ZkTestServer: log.error -> log.warn for 'stat' Wasn't error-worthy (cherry picked from commit 7b0208db9bb3fdabc7c68eda6dd7c4df5be96960) --- .../src/java/org/apache/solr/cloud/ZkTestServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java index 9e99eff4798c..c7dc2c1dfd2b 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java @@ -887,7 +887,7 @@ private static void ensureStatCommandWhitelisted() { if (!FourLetterCommands.isEnabled(stat)) { final String original = System.getProperty(ZK_WHITELIST_PROPERTY); try { - log.error( + log.warn( "ZkTestServer requires the 'stat' command, temporarily manipulating your whitelist"); System.setProperty(ZK_WHITELIST_PROPERTY, "*"); FourLetterCommands.resetWhiteList(); From 4aed807ab23a1f5428cec252a4482fffa823e9b6 Mon Sep 17 00:00:00 2001 From: Khush Jain Date: Tue, 17 Mar 2026 20:57:16 -0400 Subject: [PATCH 50/83] SOLR-17973: Fix `shards.preference` not respected for cross-collection join queries (#4218) backport: had to ignore 2 tests (cherry picked from commit 6f41a7884bac47be908635798351e636f603b7d9) --- ...hards-preference-cross-collection-join.yml | 7 + .../join/CrossCollectionJoinQParser.java | 7 + .../search/join/CrossCollectionJoinQuery.java | 22 +- .../join/CrossCollectionJoinQueryTest.java | 301 ++++++++++++++++++ 4 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/SOLR-17973-fix-shards-preference-cross-collection-join.yml diff --git a/changelog/unreleased/SOLR-17973-fix-shards-preference-cross-collection-join.yml b/changelog/unreleased/SOLR-17973-fix-shards-preference-cross-collection-join.yml new file mode 100644 index 000000000000..1620764ea063 --- /dev/null +++ b/changelog/unreleased/SOLR-17973-fix-shards-preference-cross-collection-join.yml @@ -0,0 +1,7 @@ +title: "SOLR-17973: Fix `shards.preference` not respected for cross-collection join queries" +type: fixed +authors: + - name: khushjain +links: + - name: SOLR-17973 + url: https://issues.apache.org/jira/browse/SOLR-17973 diff --git a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQParser.java b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQParser.java index 1b78a0cb2e07..2e4675a880d2 100644 --- a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQParser.java +++ b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQParser.java @@ -23,6 +23,7 @@ import java.util.Set; import org.apache.lucene.search.Query; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.QParser; @@ -102,6 +103,12 @@ public Query parse() throws SyntaxError { } } + // Propagate shards.preference from request-level params if not already set in localParams + String shardsPreference = req.getParams().get(ShardParams.SHARDS_PREFERENCE); + if (shardsPreference != null && otherParams.get(ShardParams.SHARDS_PREFERENCE) == null) { + otherParams.set(ShardParams.SHARDS_PREFERENCE, shardsPreference); + } + return new CrossCollectionJoinQuery( query, zkHost, solrUrl, collection, fromField, toField, routedByJoinKey, ttl, otherParams); } diff --git a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java index 619025800ac0..c4e5087356b1 100644 --- a/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/CrossCollectionJoinQuery.java @@ -47,11 +47,14 @@ import org.apache.solr.client.solrj.io.stream.UniqueStream; import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter; +import org.apache.solr.client.solrj.routing.RequestReplicaListTransformerGenerator; import org.apache.solr.cloud.CloudDescriptor; +import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocRouter; import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; @@ -217,11 +220,13 @@ private String createHashRangeFq() { } private TupleStream createCloudSolrStream(SolrClientCache solrClientCache) throws IOException { + ZkController zkController = searcher.getCore().getCoreContainer().getZkController(); + String streamZkHost; if (zkHost != null) { streamZkHost = zkHost; } else { - streamZkHost = searcher.getCore().getCoreContainer().getZkController().getZkServerAddress(); + streamZkHost = zkController.getZkServerAddress(); } ModifiableSolrParams params = new ModifiableSolrParams(otherParams); @@ -237,6 +242,21 @@ private TupleStream createCloudSolrStream(SolrClientCache solrClientCache) throw StreamContext streamContext = new StreamContext(); streamContext.setSolrClientCache(solrClientCache); + streamContext.setRequestParams(new ModifiableSolrParams(otherParams)); + if (zkController != null) { + RequestReplicaListTransformerGenerator rltg = + new RequestReplicaListTransformerGenerator( + zkController + .getZkStateReader() + .getClusterProperties() + .getOrDefault(ZkStateReader.DEFAULT_SHARD_PREFERENCES, "") + .toString(), + zkController.getNodeName(), + zkController.getBaseUrl(), + zkController.getHostName(), + zkController.getSysPropsCacher()); + streamContext.setRequestReplicaListTransformerGenerator(rltg); + } TupleStream cloudSolrStream = new CloudSolrStream(streamZkHost, collection, params); TupleStream uniqueStream = new UniqueStream(cloudSolrStream, new FieldEqualitor(fromField)); diff --git a/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java b/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java index 5a6f87ffe6c6..913722cccba4 100644 --- a/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java +++ b/solr/core/src/test/org/apache/solr/search/join/CrossCollectionJoinQueryTest.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import org.apache.lucene.search.Query; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -30,9 +31,17 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.ShardParams; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.search.QueryParsing; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; public class CrossCollectionJoinQueryTest extends SolrCloudTestCase { @@ -307,6 +316,298 @@ public void testAllowSolrUrlsList() throws Exception { } } + @Test + public void testShardsPreferenceRequestParamPropagation() throws Exception { + // shards.preference set as a request-level param (not in localParams) should be + // propagated to the query's otherParams + ModifiableSolrParams requestParams = new ModifiableSolrParams(); + requestParams.set(ShardParams.SHARDS_PREFERENCE, "replica.leader:false"); + + ModifiableSolrParams localParams = new ModifiableSolrParams(); + localParams.set(QueryParsing.V, "*:*"); + localParams.set(CrossCollectionJoinQParser.FROM_INDEX, "products"); + localParams.set(CrossCollectionJoinQParser.FROM, "product_id_s"); + localParams.set(CrossCollectionJoinQParser.TO, "product_id_s"); + localParams.set(CrossCollectionJoinQParser.ROUTED_BY_JOIN_KEY, "false"); + + try (SolrQueryRequest req = new SolrQueryRequestBase(null, requestParams) {}) { + CrossCollectionJoinQParser parser = + new CrossCollectionJoinQParser( + null, localParams, requestParams, req, "product_id_s", null); + Query query = parser.parse(); + + CrossCollectionJoinQuery ccjQuery = (CrossCollectionJoinQuery) query; + assertEquals("replica.leader:false", ccjQuery.otherParams.get(ShardParams.SHARDS_PREFERENCE)); + } + } + + @Test + public void testShardsPreferenceLocalParamTakesPrecedence() throws Exception { + // When shards.preference is set in both localParams and request params, + // the localParams value should take precedence + ModifiableSolrParams requestParams = new ModifiableSolrParams(); + requestParams.set(ShardParams.SHARDS_PREFERENCE, "replica.leader:false"); + + ModifiableSolrParams localParams = new ModifiableSolrParams(); + localParams.set(QueryParsing.V, "*:*"); + localParams.set(CrossCollectionJoinQParser.FROM_INDEX, "products"); + localParams.set(CrossCollectionJoinQParser.FROM, "product_id_s"); + localParams.set(CrossCollectionJoinQParser.TO, "product_id_s"); + localParams.set(CrossCollectionJoinQParser.ROUTED_BY_JOIN_KEY, "false"); + localParams.set(ShardParams.SHARDS_PREFERENCE, "replica.leader:true"); + + try (SolrQueryRequest req = new SolrQueryRequestBase(null, requestParams) {}) { + CrossCollectionJoinQParser parser = + new CrossCollectionJoinQParser( + null, localParams, requestParams, req, "product_id_s", null); + Query query = parser.parse(); + + CrossCollectionJoinQuery ccjQuery = (CrossCollectionJoinQuery) query; + // localParams value should take precedence over request-level param + assertEquals("replica.leader:true", ccjQuery.otherParams.get(ShardParams.SHARDS_PREFERENCE)); + } + } + + @Ignore("not completely backported") + @Test + public void testShardsPreferenceWithCrossCollectionJoin() throws Exception { + // Use 1 shard with 2 replicas for the "from" collection so there is exactly + // 1 leader and 1 non-leader, each on a different node. This lets us verify + // via per-node /export metrics which replica actually served the stream request. + final String fromCollection = "products_pref_test"; + final String toCollection = "parts_pref_test"; + try { + CollectionAdminRequest.createCollection(fromCollection, "ccjoin", 1, 2) + .process(cluster.getSolrClient()); + CollectionAdminRequest.createCollection(toCollection, "ccjoin", 1, 1) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(fromCollection, 1, 2); + cluster.waitForActiveCollection(toCollection, 1, 1); + + // Index test data + List productDocs = new ArrayList<>(); + List partDocs = new ArrayList<>(); + for (int productId = 0; productId < NUM_PRODUCTS; ++productId) { + int sizeNum = productId % SIZES.length; + String size = SIZES[sizeNum]; + productDocs.add( + new SolrInputDocument( + "id", String.valueOf(productId), + "product_id_s", String.valueOf(productId), + "size_s", size)); + for (int partNum = 0; partNum <= sizeNum; partNum++) { + String partId = String.format(Locale.ROOT, "%d_%d", productId, partNum); + partDocs.add( + new SolrInputDocument("id", partId, "product_id_s", String.valueOf(productId))); + } + } + indexDocs(fromCollection, productDocs); + cluster.getSolrClient().commit(fromCollection); + indexDocs(toCollection, partDocs); + cluster.getSolrClient().commit(toCollection); + + // Identify leader and non-leader replicas for the "from" collection's single shard + DocCollection fromDocCollection = + cluster.getSolrClient().getClusterState().getCollection(fromCollection); + Slice shard = fromDocCollection.getSlices().iterator().next(); + Replica leader = shard.getLeader(); + assertNotNull("Leader should exist for shard", leader); + Replica nonLeader = + shard.getReplicas().stream() + .filter(r -> !r.getName().equals(leader.getName())) + .findFirst() + .orElseThrow(() -> new AssertionError("Expected a non-leader replica")); + + String leaderBaseUrl = leader.getBaseUrl(); + String nonLeaderBaseUrl = nonLeader.getBaseUrl(); + assertNotEquals( + "Leader and non-leader should be on different nodes for this test to be meaningful", + leaderBaseUrl, + nonLeaderBaseUrl); + + // --- Test 1: replica.leader:false should route /export to the non-leader --- + double leaderCountBefore = getNumExportRequests(leaderBaseUrl, fromCollection); + double nonLeaderCountBefore = getNumExportRequests(nonLeaderBaseUrl, fromCollection); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set( + "q", + String.format( + Locale.ROOT, + "{!join method=crossCollection fromIndex=%s from=product_id_s to=product_id_s routed=false ttl=0}size_s:M", + fromCollection)); + params.set("rows", "0"); + params.set(ShardParams.SHARDS_PREFERENCE, "replica.leader:false"); + + QueryResponse resp = cluster.getSolrClient().query(toCollection, params); + assertEquals(NUM_PRODUCTS / 2, resp.getResults().getNumFound()); + + double leaderCountAfter = getNumExportRequests(leaderBaseUrl, fromCollection); + double nonLeaderCountAfter = getNumExportRequests(nonLeaderBaseUrl, fromCollection); + + assertTrue( + "Non-leader replica should have received the /export request" + + " (before=" + + nonLeaderCountBefore + + ", after=" + + nonLeaderCountAfter + + ")", + nonLeaderCountAfter > nonLeaderCountBefore); + assertEquals( + "Leader replica should NOT have received the /export request", + leaderCountBefore, + leaderCountAfter, + 0.0); + + // --- Test 2: replica.leader:true should route /export to the leader --- + leaderCountBefore = leaderCountAfter; + nonLeaderCountBefore = nonLeaderCountAfter; + + params.set(ShardParams.SHARDS_PREFERENCE, "replica.leader:true"); + resp = cluster.getSolrClient().query(toCollection, params); + assertEquals(NUM_PRODUCTS / 2, resp.getResults().getNumFound()); + + leaderCountAfter = getNumExportRequests(leaderBaseUrl, fromCollection); + nonLeaderCountAfter = getNumExportRequests(nonLeaderBaseUrl, fromCollection); + + assertTrue( + "Leader replica should have received the /export request" + + " (before=" + + leaderCountBefore + + ", after=" + + leaderCountAfter + + ")", + leaderCountAfter > leaderCountBefore); + assertEquals( + "Non-leader replica should NOT have received the /export request", + nonLeaderCountBefore, + nonLeaderCountAfter, + 0.0); + } finally { + CollectionAdminRequest.deleteCollection(toCollection).process(cluster.getSolrClient()); + CollectionAdminRequest.deleteCollection(fromCollection).process(cluster.getSolrClient()); + } + } + + @Ignore("not completely backported") + @Test + public void testShardsPreferenceLocationLocal() throws Exception { + // Test that replica.location:local routes the join's /export stream to a replica + // on the same node where the join query is processed. This validates that the + // RequestReplicaListTransformerGenerator is initialized with full node context + // (nodeName, baseUrl, hostName). + final String fromCollection = "products_local_test"; + final String toCollection = "parts_local_test"; + try { + // "from" collection: 1 shard, NUM_NODES replicas → one replica on every node + CollectionAdminRequest.createCollection(fromCollection, "ccjoin", 1, NUM_NODES) + .process(cluster.getSolrClient()); + // "to" collection: 1 shard, 1 replica → on exactly one node + CollectionAdminRequest.createCollection(toCollection, "ccjoin", 1, 1) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(fromCollection, 1, NUM_NODES); + cluster.waitForActiveCollection(toCollection, 1, 1); + + // Index test data + List productDocs = new ArrayList<>(); + List partDocs = new ArrayList<>(); + for (int productId = 0; productId < NUM_PRODUCTS; ++productId) { + int sizeNum = productId % SIZES.length; + String size = SIZES[sizeNum]; + productDocs.add( + new SolrInputDocument( + "id", String.valueOf(productId), + "product_id_s", String.valueOf(productId), + "size_s", size)); + for (int partNum = 0; partNum <= sizeNum; partNum++) { + String partId = String.format(Locale.ROOT, "%d_%d", productId, partNum); + partDocs.add( + new SolrInputDocument("id", partId, "product_id_s", String.valueOf(productId))); + } + } + indexDocs(fromCollection, productDocs); + cluster.getSolrClient().commit(fromCollection); + indexDocs(toCollection, partDocs); + cluster.getSolrClient().commit(toCollection); + + // Find the node hosting the "to" collection's shard. The join stream will execute + // on this node, so replica.location:local should prefer the "from" replica here. + DocCollection toDocCollection = + cluster.getSolrClient().getClusterState().getCollection(toCollection); + Slice toShard = toDocCollection.getSlices().iterator().next(); + String toNodeBaseUrl = toShard.getReplicas().iterator().next().getBaseUrl(); + + // Collect all "from" replica base URLs + DocCollection fromDocCollection = + cluster.getSolrClient().getClusterState().getCollection(fromCollection); + Slice fromShard = fromDocCollection.getSlices().iterator().next(); + List fromBaseUrls = new ArrayList<>(); + for (Replica r : fromShard.getReplicas()) { + fromBaseUrls.add(r.getBaseUrl()); + } + assertTrue( + "The 'from' collection should have a replica on the same node as the 'to' collection", + fromBaseUrls.contains(toNodeBaseUrl)); + + // Get baseline /export request counts for the "from" collection on all nodes + double localCountBefore = getNumExportRequests(toNodeBaseUrl, fromCollection); + List remoteCountsBefore = new ArrayList<>(); + for (String baseUrl : fromBaseUrls) { + if (!baseUrl.equals(toNodeBaseUrl)) { + remoteCountsBefore.add(new double[] {getNumExportRequests(baseUrl, fromCollection)}); + } + } + + // Execute join query with replica.location:local + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set( + "q", + String.format( + Locale.ROOT, + "{!join method=crossCollection fromIndex=%s from=product_id_s to=product_id_s routed=false ttl=0}size_s:M", + fromCollection)); + params.set("rows", "0"); + params.set(ShardParams.SHARDS_PREFERENCE, "replica.location:local"); + + QueryResponse resp = cluster.getSolrClient().query(toCollection, params); + assertEquals(NUM_PRODUCTS / 2, resp.getResults().getNumFound()); + + // Verify the local node's "from" replica received the /export request + double localCountAfter = getNumExportRequests(toNodeBaseUrl, fromCollection); + assertTrue( + "Local 'from' replica should have received the /export request" + + " (before=" + + localCountBefore + + ", after=" + + localCountAfter + + ")", + localCountAfter > localCountBefore); + + // Verify remote nodes did NOT receive /export requests + int remoteIdx = 0; + for (String baseUrl : fromBaseUrls) { + if (!baseUrl.equals(toNodeBaseUrl)) { + double remoteCountAfter = getNumExportRequests(baseUrl, fromCollection); + assertEquals( + "Remote 'from' replica on " + baseUrl + " should NOT have received /export request", + remoteCountsBefore.get(remoteIdx)[0], + remoteCountAfter, + 0.0); + remoteIdx++; + } + } + } finally { + CollectionAdminRequest.deleteCollection(toCollection).process(cluster.getSolrClient()); + CollectionAdminRequest.deleteCollection(fromCollection).process(cluster.getSolrClient()); + } + } + + private static double getNumExportRequests(String baseUrl, String collectionName) + throws SolrServerException, IOException { + throw new UnsupportedOperationException("TODO; not fully backported"); + // return SolrJMetricTestUtils.getNumCoreRequests(baseUrl, collectionName, "QUERY", "/export"); + } + public void testCcJoinQuery(String query, boolean expectFullResults) throws Exception { assertResultCount("parts", query, NUM_PRODUCTS / 2, expectFullResults); } From c00f4ee798041ce2f8a93a593364ed2cfe10ee6c Mon Sep 17 00:00:00 2001 From: David Smiley Date: Tue, 17 Mar 2026 22:38:17 -0400 Subject: [PATCH 51/83] TikaServerExtractionBackendTest: ignore on s390x (cherry picked from commit f2c6343392ea04fe149b0d2451fdc10ecf206b22) --- .../extraction/ExtractingRequestHandlerTikaServerTest.java | 3 +++ .../handler/extraction/TikaServerExtractionBackendTest.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java index d0105b26ac7d..af22c6ac6d54 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java @@ -38,6 +38,9 @@ public class ExtractingRequestHandlerTikaServerTest extends ExtractingRequestHan @BeforeClass @SuppressWarnings("resource") public static void beforeClassTika() { + Assume.assumeFalse( + "Skipping on s390x", "s390x".equalsIgnoreCase(System.getProperty("os.arch"))); + String baseUrl; try { tika = diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java index 4edceddfb229..4f2b901dd448 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java @@ -65,6 +65,9 @@ public boolean reject(Thread t) { @SuppressWarnings("resource") @BeforeClass public static void startTikaServer() { + Assume.assumeFalse( + "Skipping on s390x", "s390x".equalsIgnoreCase(System.getProperty("os.arch"))); + try { tika = new GenericContainer<>("apache/tika:3.2.3.0-full").withExposedPorts(9998); tika.start(); From 7275070fd311cc3d7953ff34c7464754fa9542a4 Mon Sep 17 00:00:00 2001 From: Shiming Li Date: Thu, 19 Mar 2026 08:41:12 +0800 Subject: [PATCH 52/83] SOLR-18136: fix multiThreaded=true with rerank & sort (#4164) When multi-threaded segment-parallel search is enabled (`indexSearcherExecutorThreads > 0` and `multiThreaded=true`) and a query uses both reranking (via `RankQuery` / `ReRankCollector`) and a sort, an `ArrayStoreException` is thrown during the merge phase if some segments have matching documents and others do not. (cherry picked from commit 9603aa22a5389a6ea7b8f63cb01d7ad5edfe41ae) --- ...rank-multithreaded-arraystoreexception.yml | 7 + .../solr/search/MultiThreadedSearcher.java | 2 +- .../search/TestMultiThreadedSearcher.java | 199 ++++++++++++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/SOLR-18136-rerank-multithreaded-arraystoreexception.yml create mode 100644 solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java diff --git a/changelog/unreleased/SOLR-18136-rerank-multithreaded-arraystoreexception.yml b/changelog/unreleased/SOLR-18136-rerank-multithreaded-arraystoreexception.yml new file mode 100644 index 000000000000..78e59fc8562b --- /dev/null +++ b/changelog/unreleased/SOLR-18136-rerank-multithreaded-arraystoreexception.yml @@ -0,0 +1,7 @@ +title: Fix ArrayStoreException when combining rerank with sort under multi-threaded segment-parallel search +type: fixed +authors: + - name: Shiming Li +links: + - name: SOLR-18136 + url: https://issues.apache.org/jira/browse/SOLR-18136 diff --git a/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java b/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java index 80300d0bdb38..065feb91e140 100644 --- a/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java @@ -350,7 +350,7 @@ public Object reduce(Collection collectors) throws IOException { TopDocs mergedTopDocs = null; if (topDocs.length > 0 && topDocs[0] != null) { - if (topDocs[0] instanceof TopFieldDocs) { + if (Arrays.stream(topDocs).allMatch(td -> td instanceof TopFieldDocs)) { TopFieldDocs[] topFieldDocs = Arrays.copyOf(topDocs, topDocs.length, TopFieldDocs[].class); mergedTopDocs = TopFieldDocs.merge(searcher.weightSort(cmd.getSort()), len, topFieldDocs); diff --git a/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java b/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java new file mode 100644 index 000000000000..2b83390158f9 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import java.io.IOException; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.Rescorer; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopDocsCollector; +import org.apache.lucene.search.Weight; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.core.NodeConfig; +import org.apache.solr.handler.component.MergeStrategy; +import org.apache.solr.index.NoMergePolicyFactory; +import org.apache.solr.update.UpdateShardHandlerConfig; +import org.apache.solr.util.TestHarness; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** Tests for {@link MultiThreadedSearcher}. */ +public class TestMultiThreadedSearcher extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + systemSetPropertySolrTestsMergePolicyFactory(NoMergePolicyFactory.class.getName()); + + NodeConfig nodeConfig = + new NodeConfig.NodeConfigBuilder("testNode", TEST_PATH()) + .setUseSchemaCache(Boolean.getBoolean("shareSchema")) + .setUpdateShardHandlerConfig(UpdateShardHandlerConfig.TEST_DEFAULT) + .setIndexSearcherExecutorThreads(4) + .build(); + createCoreContainer( + nodeConfig, + new TestHarness.TestCoresLocator( + DEFAULT_TEST_CORENAME, + createTempDir("data").toAbsolutePath().toString(), + "solrconfig-minimal.xml", + "schema.xml")); + h.coreName = DEFAULT_TEST_CORENAME; + + // Non-matching segments first, matching segment last. + // This ensures different slices see different result counts during parallel search. + for (int seg = 0; seg < 7; seg++) { + for (int i = 0; i < 10; i++) { + assertU( + adoc( + "id", String.valueOf(20000 + seg * 100 + i), + "field1_s", "nomatchterm", + "field4_t", "nomatchterm")); + } + assertU(commit()); + } + + // Matching segment last + for (int i = 0; i < 10; i++) { + assertU( + adoc( + "id", String.valueOf(10000 + i), + "field1_s", "xyzrareterm", + "field4_t", "xyzrareterm")); + } + assertU(commit()); + } + + @AfterClass + public static void afterClass() { + System.clearProperty(SYSTEM_PROPERTY_SOLR_TESTS_MERGEPOLICYFACTORY); + } + + public void testReRankWithMultiThreadedSearch() throws Exception { + float fixedScore = 5.0f; + h.getCore() + .withSearcher( + searcher -> { + int numSegments = searcher.getTopReaderContext().leaves().size(); + assertTrue("Expected > 5 segments, got " + numSegments, numSegments > 5); + assertTrue( + "Expected > 1 slice, got " + searcher.getSlices().length, + searcher.getSlices().length > 1); + + final QueryCommand cmd = new QueryCommand(); + cmd.setFlags(SolrIndexSearcher.GET_SCORES); + cmd.setLen(10); + cmd.setMultiThreaded(true); + cmd.setSort( + new Sort(SortField.FIELD_SCORE, new SortField("id", SortField.Type.STRING))); + cmd.setQuery( + new SimpleReRankQuery( + new TermQuery(new Term("field1_s", "xyzrareterm")), fixedScore)); + + final QueryResult qr = searcher.search(cmd); + + assertTrue(qr.getDocList().matches() >= 1); + final DocIterator iter = qr.getDocList().iterator(); + assertTrue(iter.hasNext()); + iter.next(); + assertEquals(fixedScore, iter.score(), 0); + return null; + }); + } + + private static final class SimpleReRankQuery extends RankQuery { + + private Query q; + private final float reRankScore; + + SimpleReRankQuery(Query q, float reRankScore) { + this.q = q; + this.reRankScore = reRankScore; + } + + @Override + public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, float boost) + throws IOException { + return q.createWeight(indexSearcher, scoreMode, boost); + } + + @Override + public void visit(QueryVisitor visitor) { + q.visit(visitor); + } + + @Override + public boolean equals(Object obj) { + return this == obj; + } + + @Override + public int hashCode() { + return q.hashCode(); + } + + @Override + public String toString(String field) { + return q.toString(field); + } + + @Override + public TopDocsCollector getTopDocsCollector( + int len, QueryCommand cmd, IndexSearcher searcher) throws IOException { + return new ReRankCollector( + len, + len, + new Rescorer() { + @Override + public TopDocs rescore(IndexSearcher searcher, TopDocs firstPassTopDocs, int topN) { + for (ScoreDoc scoreDoc : firstPassTopDocs.scoreDocs) { + scoreDoc.score = reRankScore; + } + return firstPassTopDocs; + } + + @Override + public Explanation explain( + IndexSearcher searcher, Explanation firstPassExplanation, int docID) { + return firstPassExplanation; + } + }, + cmd, + searcher, + null); + } + + @Override + public MergeStrategy getMergeStrategy() { + return null; + } + + @Override + public RankQuery wrap(Query q) { + this.q = q; + return this; + } + } +} From 931f7ecf8360a12a95a182357dd41f33486e1371 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Thu, 19 Mar 2026 22:29:27 -0400 Subject: [PATCH 53/83] Benchmark fixes (#4194)s Backport had some issues. (cherry picked from commit 9b0309d3932a27c315b8f7c51eb2038c7d652776) --- .../solr/bench/generators/IntegersDSL.java | 17 ++++++---- .../apache/solr/bench/search/FilterCache.java | 33 +++++++++++-------- .../bench/search/QueryResponseWriters.java | 29 ++++++++-------- .../solr/bench/search/StreamingSearch.java | 21 ++++-------- .../configs/cloud-minimal/conf/schema.xml | 2 +- 5 files changed, 51 insertions(+), 51 deletions(-) diff --git a/solr/benchmark/src/java/org/apache/solr/bench/generators/IntegersDSL.java b/solr/benchmark/src/java/org/apache/solr/bench/generators/IntegersDSL.java index 6a41c10c317b..89e6d3ad24fb 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/generators/IntegersDSL.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/generators/IntegersDSL.java @@ -116,7 +116,7 @@ private static class IntegerMaxCardinalitySolrGen extends SolrGen { private final Gen integers; /** The Cardinality start. */ - Integer cardinalityStart; + volatile Integer cardinalityStart; /** * Instantiates a new Integer max cardinality solr gen. @@ -132,13 +132,18 @@ public IntegerMaxCardinalitySolrGen(int maxCardinality, Gen integers) { @Override public Integer generate(SolrRandomnessSource in) { - if (cardinalityStart == null) { - cardinalityStart = - SolrGenerate.range(0, Integer.MAX_VALUE - maxCardinality - 1).generate(in); + Integer localStart = cardinalityStart; + if (localStart == null) { + synchronized (this) { + localStart = cardinalityStart; + if (localStart == null) { + localStart = SolrGenerate.range(0, Integer.MAX_VALUE - maxCardinality - 1).generate(in); + cardinalityStart = localStart; + } + } } - long seed = - SolrGenerate.range(cardinalityStart, cardinalityStart + maxCardinality - 1).generate(in); + long seed = SolrGenerate.range(localStart, localStart + maxCardinality - 1).generate(in); return integers.generate(new SplittableRandomSource(new SplittableRandom(seed))); } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java b/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java index 4b1bf9485e81..e7bc4ae4e060 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/FilterCache.java @@ -19,9 +19,11 @@ import static org.apache.solr.bench.generators.SourceDSL.integers; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.URI; -import java.nio.charset.StandardCharsets; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.stream.Collectors; import org.apache.solr.bench.BaseBenchState; import org.apache.solr.bench.Docs; import org.apache.solr.bench.MiniClusterState; @@ -116,19 +118,22 @@ public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterSta public void dumpMetrics(MiniClusterState.MiniClusterBenchState miniClusterState) { // TODO add a verbose flag - String url = - miniClusterState.nodes.get(0) - + "/admin/metrics?prefix=CACHE.searcher.filterCache&omitHeader=true"; - HttpURLConnection conn = null; + String url = miniClusterState.nodes.get(0) + "/admin/metrics?category=CACHE?wt=prometheus"; + HttpClient client = HttpClient.newHttpClient(); try { - conn = (HttpURLConnection) URI.create(url).toURL().openConnection(); - conn.connect(); - BaseBenchState.log( - new String(conn.getInputStream().readAllBytes(), StandardCharsets.UTF_8)); - } catch (IOException e) { - // ignored - } finally { - if (conn != null) conn.disconnect(); + HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + // Filter to only lines containing filterCache metrics + String filteredMetrics = + response + .body() + .lines() + .filter(line -> line.contains("filterCache")) + .collect(Collectors.joining("\n")); + BaseBenchState.log(filteredMetrics); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); } } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java index 30cd848d0ee4..eff4de79dea0 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java @@ -22,15 +22,18 @@ import static org.apache.solr.bench.generators.SourceDSL.strings; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import org.apache.solr.bench.Docs; import org.apache.solr.bench.MiniClusterState; import org.apache.solr.bench.MiniClusterState.MiniClusterBenchState; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.InputStreamResponseParser; import org.apache.solr.client.solrj.impl.NoOpResponseParser; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.core.SolrCore; +import org.apache.solr.common.util.NamedList; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -58,18 +61,7 @@ public class QueryResponseWriters { @State(Scope.Benchmark) public static class BenchState { - /** See {@link SolrCore#DEFAULT_RESPONSE_WRITERS} */ - @Param({ - CommonParams.JAVABIN, - CommonParams.JSON, - "cbor", - "smile", - "xml", - "python", - "phps", - "ruby", - "raw" - }) + @Param({CommonParams.JAVABIN, CommonParams.JSON, "cbor", "smile", "xml"}) String wt; private int docs = 100; @@ -102,9 +94,14 @@ public void setup(MiniClusterBenchState miniClusterState) throws Exception { } @Benchmark - public Object query( - BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState) + public Object query(BenchState benchState, MiniClusterState.MiniClusterBenchState solrBenchState) throws SolrServerException, IOException { - return miniClusterState.client.request(benchState.q, collection); + NamedList response = solrBenchState.client.request(benchState.q, collection); + // consume the stream completely + try (InputStream responseStream = + (InputStream) response.get(InputStreamResponseParser.STREAM_KEY)) { + responseStream.transferTo(OutputStream.nullOutputStream()); + } + return response; } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java index 14046644c467..0e8a8573de23 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java @@ -26,9 +26,9 @@ import org.apache.solr.bench.Docs; import org.apache.solr.bench.MiniClusterState; import org.apache.solr.bench.MiniClusterState.MiniClusterBenchState; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.stream.CloudSolrStream; @@ -36,6 +36,7 @@ import org.apache.solr.client.solrj.io.stream.TupleStream; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.IOUtils; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -69,7 +70,7 @@ public static class BenchState { private String zkHost; private ModifiableSolrParams params; private StreamContext streamContext; - private Http2SolrClient http2SolrClient; + private SolrClient httpSolrClient; @Setup(Level.Trial) public void setup(MiniClusterBenchState miniClusterState) throws Exception { @@ -97,15 +98,9 @@ public void setup(MiniClusterBenchState miniClusterState) throws Exception { @Setup(Level.Iteration) public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState) throws SolrServerException, IOException { - SolrClientCache solrClientCache; - if (useHttp1) { - var httpClient = HttpClientUtil.createClient(null); // TODO tune params? - solrClientCache = new SolrClientCache(httpClient); - } else { - http2SolrClient = newHttp2SolrClient(); - solrClientCache = new SolrClientCache(http2SolrClient); - } - + var httpSolrClient = new Http2SolrClient.Builder().useHttp1_1(useHttp1).build(); + this.httpSolrClient = httpSolrClient; + SolrClientCache solrClientCache = new SolrClientCache(httpSolrClient); streamContext = new StreamContext(); streamContext.setSolrClientCache(solrClientCache); } @@ -113,9 +108,7 @@ public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterSta @TearDown(Level.Iteration) public void teardownIt() { streamContext.getSolrClientCache().close(); - if (http2SolrClient != null) { - http2SolrClient.close(); - } + IOUtils.closeQuietly(httpSolrClient); } } diff --git a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml index e517aea59307..b1851c3c2906 100644 --- a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml +++ b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml @@ -36,7 +36,7 @@ - + From 124654c7a355d918d97da22872d55fa6cad4add5 Mon Sep 17 00:00:00 2001 From: Matthew Biscocho Date: Wed, 25 Mar 2026 22:42:58 -0400 Subject: [PATCH 54/83] [SOLR-18114] CloudSolrClient fails deleteById with directUpdatesToLeadersOnly but no route passed (#4219) Fixed a bug where CloudSolrClient throws an exception when deleteById is sent a to collection that has router field configured and has directUpdatesToLeadersOnly=true but is sent without routing info. Also added a log.warn when request is sent but no routing info is passed causing request to be sent in the general unoptimized route. --- changelog/unreleased/SOLR-18114.yml | 8 +++ .../client/solrj/impl/CloudSolrClient.java | 17 ++++++ .../impl/CloudSolrClientRoutingTest.java | 59 +++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 changelog/unreleased/SOLR-18114.yml diff --git a/changelog/unreleased/SOLR-18114.yml b/changelog/unreleased/SOLR-18114.yml new file mode 100644 index 000000000000..cd0e66a96cb7 --- /dev/null +++ b/changelog/unreleased/SOLR-18114.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Fixed CloudSolrClient deleteById failure when routing info is not passed with compositeId router, router.field, and directUpdatesToLeadersOnly enabled +type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Matthew Biscocho +links: + - name: SOLR-18114 + url: https://issues.apache.org/jira/browse/SOLR-18114 diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index b4000eaf1efa..37e4ac3c6bc0 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -532,6 +532,9 @@ private NamedList directUpdate(AbstractUpdateRequest request, String col "directUpdatesToLeadersOnly==true but could not find leader(s)"); } else { // we could not find a leader or routes yet - use unoptimized general path + log.warn( + "No routing info found for update to collection '{}', broadcasting to all shards.", + collection); return null; } } @@ -1560,6 +1563,10 @@ public Map getShardReplicationFactor(String collection, NamedLi return results; } + /** + * Determines whether an UpdateRequest contains sufficient routing information to identify shard + * leaders for direct updates when directUpdatesToLeadersOnly is enabled. + */ private static boolean hasInfoToFindLeaders(UpdateRequest updateRequest, String idField) { final Map> documents = updateRequest.getDocumentsMap(); final Map> deleteById = updateRequest.getDeleteByIdMap(); @@ -1582,6 +1589,16 @@ private static boolean hasInfoToFindLeaders(UpdateRequest updateRequest, String } } + if (deleteById != null) { + for (final Map.Entry> entry : deleteById.entrySet()) { + final Map params = entry.getValue(); + if (params == null || params.get(ShardParams._ROUTE_) == null) { + // deleteById entry lacks explicit route parameter, can't find leader for it + return false; + } + } + } + return true; } } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientRoutingTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientRoutingTest.java index f2824b9e5502..8d18813afce4 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientRoutingTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientRoutingTest.java @@ -18,17 +18,22 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Optional; +import org.apache.lucene.tests.util.TestUtil; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.UpdateParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.util.LogListener; import org.junit.BeforeClass; import org.junit.Test; @@ -130,4 +135,58 @@ public void routeParamHandling() throws IOException, SolrServerException { assertEquals(0, numForwardedWithRoute); assertEquals(100, numForwardedWithoutRoute); } + + @Test + public void testDeleteWithoutRouteWithDirectUpdatesToLeadersOnly() throws Exception { + final String collectionName = "delete_without_route_collection"; + + final String docId = String.valueOf(random().nextInt(1000)); + final String routeValue = TestUtil.randomRealisticUnicodeString(random()); + + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) + .setRouterField("router_field_s") + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(collectionName, 2, 2); + + try (CloudSolrClient client = + new CloudSolrClient.Builder( + Collections.singletonList(cluster.getZkServer().getZkAddress()), Optional.empty()) + .withDefaultCollection(collectionName) + .sendUpdatesOnlyToShardLeaders() + .sendDirectUpdatesToShardLeadersOnly() + .build()) { + + UpdateRequest addRequest = new UpdateRequest(); + addRequest.add("id", docId, "router_field_s", routeValue); + addRequest.process(client); + client.commit(); + + assertEquals(1, client.query(new SolrQuery("id:" + docId)).getResults().getNumFound()); + + try (LogListener warnLog = + LogListener.warn(CloudSolrClient.class) + .substring( + "No routing info found for update to collection '" + + collectionName + + "', broadcasting to all shards.")) { + + // Delete by ID without providing explicit route + // Should still delete via sending to all shards and log a warning + UpdateRequest deleteRequest = new UpdateRequest(); + deleteRequest.deleteById(docId); + deleteRequest.process(client); + client.commit(); + + assertEquals(0, client.query(new SolrQuery("id:" + docId)).getResults().getNumFound()); + + // Verify the warning was logged when routing info was missing + assertNotNull( + "Expected at least one warning about missing routing info for deleteById", + warnLog.pollMessage()); + + // Clear any remaining messages + warnLog.clearQueue(); + } + } + } } From bd2001b4e5f697402feda5bf159bba2fd428bc39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 26 Mar 2026 15:35:10 +0100 Subject: [PATCH 55/83] Add clean to cherrypick.sh before runnign (precommit) check (#4238) (cherry picked from commit ec51d13bc14821ac13041c7461abe66fc12b5483) --- dev-tools/scripts/cherrypick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/scripts/cherrypick.sh b/dev-tools/scripts/cherrypick.sh index a1ad6ac60412..ba8bd855b10c 100755 --- a/dev-tools/scripts/cherrypick.sh +++ b/dev-tools/scripts/cherrypick.sh @@ -180,7 +180,7 @@ for BRANCH in "${BRANCHES[@]}"; do if [[ "$PRECOMMIT" ]] || [[ "$TEST" ]]; then LOG "INFO" "Testing the cherry-pick on $BRANCH by running 'gradlew check ${TESTARG}'" # shellcheck disable=SC2086 - ./gradlew check -q $TESTARG -Pvalidation.errorprone=true + ./gradlew clean check -q $TESTARG -Pvalidation.errorprone=true if [ $? -gt 0 ]; then LOG "WARN" "Tests failed. Please fix and push manually" exit 2 From e0d99a5e3b33c33c53b320795d09fed7645aecbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 26 Mar 2026 16:27:02 +0100 Subject: [PATCH 56/83] Fix a broken refguide link --- .../deployment-guide/pages/solr-control-script-reference.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc index 11e54230ae25..c6e66f7a8805 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc @@ -1022,7 +1022,7 @@ The `bin/solr` script allows enabling or disabling Authentication, allowing you Currently this command is only available when using SolrCloud mode and must be run on the machine hosting Solr. -For Basic Authentication the script provides https://github.com/apache/solr/blob/main/solr/core/resources/security.json[user roles and permission mappings], and maps the created user to the `superadmin` role. +For Basic Authentication the script provides https://github.com/apache/solr/blob/branch_9x/solr/core/src/resources/security.json[user roles and permission mappings], and maps the created user to the `superadmin` role. For Kerberos it only enables the security.json, it doesn't set up any users or role mappings. From ee7c6c1c911dc29fe7cbd578a94a6daf9197aacc Mon Sep 17 00:00:00 2001 From: Matthew Biscocho Date: Thu, 26 Mar 2026 11:30:09 -0400 Subject: [PATCH 57/83] [NO-JIRA] analyzeClassesDependencies failure from unused org.apache.httpcomponents:httpclient (#4241) Remove org.apache.httpcomponents:httpclient dependency from benchmark as it was no longer used after commit a70dcf3 --- solr/benchmark/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/solr/benchmark/build.gradle b/solr/benchmark/build.gradle index 63cea7af01f8..30afbda1561e 100644 --- a/solr/benchmark/build.gradle +++ b/solr/benchmark/build.gradle @@ -49,7 +49,6 @@ dependencies { implementation project(':solr:solrj-streaming') implementation 'org.apache.lucene:lucene-core' - implementation 'org.apache.httpcomponents:httpclient' implementation 'commons-io:commons-io' implementation 'io.dropwizard.metrics:metrics-core' implementation 'org.apache.commons:commons-math3' From dcf2849033e1dd8ba24e08346ba79f851dc6534e Mon Sep 17 00:00:00 2001 From: Renato Haeberli Date: Fri, 27 Mar 2026 18:35:28 +0100 Subject: [PATCH 58/83] Solr Admin Ui: search with contains in solr core/collection drop downs (#4121) --- .../unreleased/GITHUB#4121-improve-search-in-dropdown.yml | 7 +++++++ solr/webapp/web/index.html | 2 ++ solr/webapp/web/partials/analysis.html | 8 +++++++- solr/webapp/web/partials/schema.html | 3 ++- 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/GITHUB#4121-improve-search-in-dropdown.yml diff --git a/changelog/unreleased/GITHUB#4121-improve-search-in-dropdown.yml b/changelog/unreleased/GITHUB#4121-improve-search-in-dropdown.yml new file mode 100644 index 000000000000..c9f6f9f0685b --- /dev/null +++ b/changelog/unreleased/GITHUB#4121-improve-search-in-dropdown.yml @@ -0,0 +1,7 @@ +title: Dropdowns for collection/core, for fields on the schema-page and for field-types on the analyze page are now using a contains-filtering +type: changed +authors: + - name: Renato Haeberli +links: +- name: GITHUB#4121 + url: https://github.com/apache/solr/pull/4121 diff --git a/solr/webapp/web/index.html b/solr/webapp/web/index.html index dbe9a840c1aa..71e9a3c045f0 100644 --- a/solr/webapp/web/index.html +++ b/solr/webapp/web/index.html @@ -202,6 +202,7 @@

Connection recovered...

@@ -229,6 +230,7 @@

Connection recovered...

diff --git a/solr/webapp/web/partials/analysis.html b/solr/webapp/web/partials/analysis.html index 23527f7351ca..116859eccc5f 100644 --- a/solr/webapp/web/partials/analysis.html +++ b/solr/webapp/web/partials/analysis.html @@ -49,7 +49,13 @@
  • - + Schema Browser  diff --git a/solr/webapp/web/partials/schema.html b/solr/webapp/web/partials/schema.html index 9913dbd2f2cf..0b60154a05e4 100644 --- a/solr/webapp/web/partials/schema.html +++ b/solr/webapp/web/partials/schema.html @@ -390,6 +390,7 @@

    ng-model="fieldOrType" chosen data-placeholder="Please select ..." + search-contains="true" ng-change="selectFieldOrType()" ng-options="f.value as f.label group by f.group for f in fieldsAndTypes"> @@ -445,7 +446,7 @@

    - +
    Dynamic Field
    {{field}}
    From 859c54a9b67dc79e3d66cc2263e0f7f4317ac6dd Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 27 Mar 2026 22:13:04 -0400 Subject: [PATCH 59/83] build: GHA: sync from main (#4242) Replaced .github/workflows and actions with contents from main branch. Using Java 11, and need to reference the Prometheus Exporter in one place. I didn't include "labeler" or "stale" or docker publish. --- .github/actions/prepare-for-build/action.yml | 40 +++++++++++++++++++ .github/workflows/bin-solr-test.yml | 30 ++++---------- .github/workflows/docker-test.yml | 33 ++++----------- .github/workflows/gradle-extraction-check.yml | 22 +--------- .github/workflows/gradle-precommit.yml | 37 ++++------------- .github/workflows/solrj-test.yml | 31 ++++---------- .github/workflows/tests-via-crave.yml | 5 +-- .github/workflows/validate-changelog.yml | 23 +---------- 8 files changed, 75 insertions(+), 146 deletions(-) create mode 100644 .github/actions/prepare-for-build/action.yml diff --git a/.github/actions/prepare-for-build/action.yml b/.github/actions/prepare-for-build/action.yml new file mode 100644 index 000000000000..ebddc8d5ca71 --- /dev/null +++ b/.github/actions/prepare-for-build/action.yml @@ -0,0 +1,40 @@ +# This composite action is included in other workflows to have a shared setup +# for java, gradle, caches, etc. + +name: Prepare build +description: Creates a shared setup for other workflows + +inputs: + java-version: + required: false + default: "11" + description: "The default JDK version to set up." + + java-distribution: + required: false + default: "temurin" + description: "The default JDK distribution type" + +runs: + using: "composite" + steps: + - name: Set up Java (${{ inputs.java-distribution }}, ${{ inputs.java-version }})" + uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 + with: + distribution: ${{ inputs.java-distribution }} + java-version: ${{ inputs.java-version }} + java-package: jdk + + - name: Cache gradle-wrapper.jar + uses: actions/cache@v4 + with: + path: gradle/wrapper/gradle-wrapper.jar + key: gradle-wrapper-${{ hashFiles('gradle/wrapper/gradle-wrapper.jar.sha256') }} + + # This includes "smart" caching of gradle dependencies. + - name: Set up Gradle + uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + with: + # increase expiry time for the temp. develocity token. + # https://github.com/gradle/actions/blob/main/docs/setup-gradle.md#increasing-the-expiry-time-for-develocity-access-tokens + develocity-token-expiry: 8 diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml index f885b7da2366..fea033015ab9 100644 --- a/.github/workflows/bin-solr-test.yml +++ b/.github/workflows/bin-solr-test.yml @@ -3,8 +3,7 @@ name: Solr Script Tests on: pull_request: branches: - - 'main' - - 'branch_*' + - '*' paths: - '.github/workflows/bin-solr-test.yml' - 'solr/bin/**' @@ -17,30 +16,17 @@ jobs: name: Run Solr Script Tests runs-on: ubuntu-latest + timeout-minutes: 40 steps: - # Setup - - uses: actions/checkout@v5 - - name: Set up JDK 11 - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: 11 - java-package: jdk - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v5 - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - uses: actions/cache@v5 - with: - path: | - ~/.gradle/caches - key: ${{ runner.os }}-gradle-binsolr-${{ hashFiles('versions.lock') }} - restore-keys: | - ${{ runner.os }}-gradle-binsolr- - ${{ runner.os }}-gradle- + - name: Checkout code + uses: actions/checkout@v5 + + - uses: ./.github/actions/prepare-for-build + - name: Test the bin/solr script run: ./gradlew integrationTests + - name: Archive logs if: ${{ failure() }} uses: actions/upload-artifact@v6 diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 5db95b0eb6f9..aebbde1efb35 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -3,12 +3,10 @@ name: Docker Build & Test on: pull_request: branches: - - 'main' - - 'branch_*' + - '*' paths: - '.github/workflows/docker-test.yml' - 'solr/bin/**' - - 'solr/prometheus-exporter/bin/**' - 'solr/docker/**' - 'solr/packaging/**' @@ -17,35 +15,20 @@ jobs: name: Build and test Docker image runs-on: ubuntu-latest + timeout-minutes: 15 env: SOLR_DOCKER_IMAGE_REPO: github-pr/solr SOLR_DOCKER_IMAGE_TAG: ${{github.event.number}} steps: - # Setup - - uses: actions/checkout@v5 - - name: Set up JDK 11 - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: 11 - java-package: jdk - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v5 - - name: Install ACL - run: sudo apt-get install acl - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - uses: actions/cache@v5 - with: - path: | - ~/.gradle/caches - key: ${{ runner.os }}-gradle-docker-${{ hashFiles('versions.lock') }} - restore-keys: | - ${{ runner.os }}-gradle-docker- - ${{ runner.os }}-gradle- + - name: Checkout code + uses: actions/checkout@v5 + + - uses: ./.github/actions/prepare-for-build + - name: Build Docker image with Gradle run: ./gradlew solr:docker:docker + - name: Run tests on Docker image run: ./gradlew solr:docker:testDocker diff --git a/.github/workflows/gradle-extraction-check.yml b/.github/workflows/gradle-extraction-check.yml index e86c7b6f7433..9c18a01c6807 100644 --- a/.github/workflows/gradle-extraction-check.yml +++ b/.github/workflows/gradle-extraction-check.yml @@ -19,27 +19,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - - name: Set up JDK 11 - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: 11 - java-package: jdk - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v5 - - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - - uses: actions/cache@v5 - with: - path: | - ~/.gradle/caches - key: ${{ runner.os }}-gradle-precommit-${{ hashFiles('versions.lock') }} - restore-keys: | - ${{ runner.os }}-gradle-precommit- - ${{ runner.os }}-gradle- + - uses: ./.github/actions/prepare-for-build - name: Run extraction module tests run: ./gradlew --no-daemon solr:modules:extraction:check diff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.yml index 88c47be0f5c7..ac56df84a17a 100644 --- a/.github/workflows/gradle-precommit.yml +++ b/.github/workflows/gradle-precommit.yml @@ -3,43 +3,20 @@ name: Gradle Precommit on: pull_request: branches: - - 'main' - - 'branch_*' + - '*' jobs: test: - name: gradle check w/ Java 11 + name: gradle check runs-on: ubuntu-latest + timeout-minutes: 15 steps: - # Setup - - uses: actions/checkout@v5 + - name: Checkout code + uses: actions/checkout@v5 - - name: Set up JDK 11 - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: 11 - java-package: jdk - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v5 - - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - - uses: actions/cache@v5 - with: - path: | - ~/.gradle/caches - key: ${{ runner.os }}-gradle-precommit-${{ hashFiles('versions.lock') }} - restore-keys: | - ${{ runner.os }}-gradle-precommit- - ${{ runner.os }}-gradle- + - uses: ./.github/actions/prepare-for-build - name: Run gradle check (without tests) - run: ./gradlew check -x test -Ptask.times=true - - - name: Validate Gradle wrapper - uses: gradle/actions/wrapper-validation@v5 + run: ./gradlew check -x test -Ptask.times=true --continue diff --git a/.github/workflows/solrj-test.yml b/.github/workflows/solrj-test.yml index 60a8f84483d1..c81246b41be8 100644 --- a/.github/workflows/solrj-test.yml +++ b/.github/workflows/solrj-test.yml @@ -3,8 +3,7 @@ name: SolrJ Tests on: pull_request: branches: - - 'main' - - 'branch_*' + - '*' paths: - '.github/workflows/solrj-test.yml' - 'solr/solrj/**' @@ -14,27 +13,13 @@ jobs: name: Run SolrJ Tests runs-on: ubuntu-latest + timeout-minutes: 15 steps: - # Setup - - uses: actions/checkout@v5 - - name: Set up JDK 11 - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: 11 - java-package: jdk - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v5 - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - uses: actions/cache@v5 - with: - path: | - ~/.gradle/caches - key: ${{ runner.os }}-gradle-solrj-${{ hashFiles('versions.lock') }} - restore-keys: | - ${{ runner.os }}-gradle-solrj- - ${{ runner.os }}-gradle- - - name: Test the SolrJ Package + - name: Checkout code + uses: actions/checkout@v5 + + - uses: ./.github/actions/prepare-for-build + + - name: Test SolrJ module run: ./gradlew solr:solrj:test diff --git a/.github/workflows/tests-via-crave.yml b/.github/workflows/tests-via-crave.yml index f9fbc61c9a38..1476d9c96601 100644 --- a/.github/workflows/tests-via-crave.yml +++ b/.github/workflows/tests-via-crave.yml @@ -3,8 +3,7 @@ name: Solr Tests on: pull_request: branches: - - 'main' - - 'branch_*' + - '*' jobs: test: @@ -26,7 +25,7 @@ jobs: - name: Initialize, build, test run: | cd /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER} - crave run --clean + crave run --clean --message "PR: ${GITHUB_REF_NAME}" -- ./gradlew --console=plain test - name: Cleanup if: ${{ always() }} run: | diff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.yml index 495b23802db8..186eec1e3abc 100644 --- a/.github/workflows/validate-changelog.yml +++ b/.github/workflows/validate-changelog.yml @@ -28,27 +28,6 @@ jobs: echo "skip=false" >> $GITHUB_OUTPUT fi - - name: Check for CHANGES.txt edits - if: steps.check-label.outputs.skip == 'false' - run: | - # Get the list of changed files - CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) - - if echo "$CHANGED_FILES" | grep -q "^solr/CHANGES\.txt$"; then - echo "::error::Use of solr/CHANGES.txt is deprecated. Please create a changelog yaml file instead." - echo "" - echo "Instead of editing CHANGES.txt, please:" - echo "1. Run: ./gradlew writeChangelog" - echo "2. Edit the generated YAML file in changelog/unreleased/" - echo "3. Commit both the code change and the YAML file" - echo "" - echo "For more information, see: dev-docs/changelog.adoc" - echo "" - echo "If this PR should not have a changelog entry (e.g., documentation-only changes)," - echo "add the 'no-changelog' label to this PR." - exit 1 - fi - - name: Check for changelog entry if: steps.check-label.outputs.skip == 'false' run: | @@ -106,7 +85,7 @@ jobs: VALIDATION_FAILED=false while IFS= read -r file; do - if [ -z "$file" ]; then + if [ -z "$file" ] || [ ! -f "$file" ]; then continue fi From e0042415dbf9a02718b396e73b4966465535b436 Mon Sep 17 00:00:00 2001 From: raviranjanjha Date: Mon, 6 Apr 2026 13:33:40 +0530 Subject: [PATCH 60/83] Fix for defect SOLR-18186: Solr heap used percentage uses committed heap instead of max heap which raises false alarm (#4257) --- .../SOLR-18186-fix-heap-used-percent-adminui.yml | 7 +++++++ solr/webapp/web/js/angular/controllers/cloud.js | 4 +++- solr/webapp/web/partials/cloud.html | 4 ++-- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/SOLR-18186-fix-heap-used-percent-adminui.yml diff --git a/changelog/unreleased/SOLR-18186-fix-heap-used-percent-adminui.yml b/changelog/unreleased/SOLR-18186-fix-heap-used-percent-adminui.yml new file mode 100644 index 000000000000..2541d69a6a94 --- /dev/null +++ b/changelog/unreleased/SOLR-18186-fix-heap-used-percent-adminui.yml @@ -0,0 +1,7 @@ +title: Fixed Admin UI to use max heap (-Xmx) value instead of committed heap to compute heap used percentage. +type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Ravi Ranjan Jha +links: + - name: SOLR-18186 + url: https://issues.apache.org/jira/browse/SOLR-18186 diff --git a/solr/webapp/web/js/angular/controllers/cloud.js b/solr/webapp/web/js/angular/controllers/cloud.js index b766c9a6e6bc..f87d40043921 100644 --- a/solr/webapp/web/js/angular/controllers/cloud.js +++ b/solr/webapp/web/js/angular/controllers/cloud.js @@ -367,12 +367,14 @@ var nodesSubController = function($scope, Collections, System, Metrics) { nodes[node]['memFree'] = bytesToSize(memFree); nodes[node]['memUsed'] = bytesToSize(memTotal - memFree); + var heapMax = s.jvm.memory.raw.max; var heapTotal = s.jvm.memory.raw.total; var heapFree = s.jvm.memory.raw.free; - var heapPercentage = Math.floor((heapTotal - heapFree) / heapTotal * 100); + var heapPercentage = Math.floor((heapTotal - heapFree) / heapMax * 100); nodes[node]['heapUsed'] = bytesToSize(heapTotal - heapFree); nodes[node]['heapUsedPct'] = heapPercentage; nodes[node]['heapUsedPctStyle'] = styleForPct(heapPercentage); + nodes[node]['heapMax'] = bytesToSize(heapMax); nodes[node]['heapTotal'] = bytesToSize(heapTotal); nodes[node]['heapFree'] = bytesToSize(heapFree); diff --git a/solr/webapp/web/partials/cloud.html b/solr/webapp/web/partials/cloud.html index 01d63499ee3f..4a0e3bb0b44d 100644 --- a/solr/webapp/web/partials/cloud.html +++ b/solr/webapp/web/partials/cloud.html @@ -192,11 +192,11 @@ -
    +
    {{n.heapUsedPct}}%
    - Max: {{n.heapTotal}}
    + Max: {{n.heapMax}}
    Used: {{n.heapUsed}}
    From cd661e3b7350be52358d35e0672eb07e5502d82e Mon Sep 17 00:00:00 2001 From: Matthew Biscocho Date: Thu, 9 Apr 2026 15:31:05 -0400 Subject: [PATCH 61/83] SOLR-18176: HttpShardHandler query throughput bottleneck from ZooKeeper (#4237) CloudReplicaSource was making a clusterstate call to ZooKeeper for every distributed request if you search over multiple collections, and when the coordinator has no local replica for some of them. This is because the get call was bypassing state cache. This created a severe bottleneck in query throughput so small fix made to just enable cached state lookups. --- .../SOLR-18176-shardhandler-bottleneck.yml | 9 +++ .../handler/component/CloudReplicaSource.java | 20 +++-- ...ributedQueryComponentOptimizationTest.java | 73 +++++++++++++++++++ 3 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 changelog/unreleased/SOLR-18176-shardhandler-bottleneck.yml diff --git a/changelog/unreleased/SOLR-18176-shardhandler-bottleneck.yml b/changelog/unreleased/SOLR-18176-shardhandler-bottleneck.yml new file mode 100644 index 000000000000..7ce9084e0cac --- /dev/null +++ b/changelog/unreleased/SOLR-18176-shardhandler-bottleneck.yml @@ -0,0 +1,9 @@ +title: Increased query throughput by removing a call to ZooKeeper for cluster state that should have been cached. Happens when Solr does distributed search over multiple collections, and when the coordinator has no local replica for some of them. +type: changed +authors: + - name: Matthew Biscocho +links: + - name: SOLR-18176 + url: https://issues.apache.org/jira/browse/SOLR-18176 + - name: SOLR-15352 + url: https://issues.apache.org/jira/browse/SOLR-15352 diff --git a/solr/core/src/java/org/apache/solr/handler/component/CloudReplicaSource.java b/solr/core/src/java/org/apache/solr/handler/component/CloudReplicaSource.java index 5e333c8e6d33..e340821743cf 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/CloudReplicaSource.java +++ b/solr/core/src/java/org/apache/solr/handler/component/CloudReplicaSource.java @@ -109,12 +109,13 @@ private void withShardsParam(Builder builder, String shardsParam) { if (sliceOrUrl.indexOf('/') < 0) { // this is a logical shard this.slices[i] = sliceOrUrl; - replicas[i] = - findReplicas( - builder, - shardsParam, - clusterState, - clusterState.getCollection(builder.collection).getSlice(sliceOrUrl)); + DocCollection coll = clusterState.getCollectionOrNull(builder.collection, true); + if (coll == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Could not find collection to resolve replicas: " + builder.collection); + } + replicas[i] = findReplicas(builder, shardsParam, clusterState, coll.getSlice(sliceOrUrl)); } else { // this has urls this.replicas[i] = StrUtils.splitSmart(sliceOrUrl, "|", true); @@ -189,7 +190,12 @@ private void addSlices( String collectionName, String shardKeys, boolean multiCollection) { - DocCollection coll = state.getCollection(collectionName); + DocCollection coll = state.getCollectionOrNull(collectionName, true); + if (coll == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Could not find collection to add slices: " + collectionName); + } Collection slices = coll.getRouter().getSearchSlices(shardKeys, params, coll); ClientUtils.addSlices(target, collectionName, slices, multiCollection); } diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentOptimizationTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentOptimizationTest.java index 8df32f408373..7b65c88caacb 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentOptimizationTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentOptimizationTest.java @@ -24,16 +24,23 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.solr.BaseDistributedSearchTestCase; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; +import org.apache.solr.embedded.JettySolrRunner; import org.junit.BeforeClass; import org.junit.Test; @@ -708,6 +715,72 @@ private QueryResponse queryWithAsserts(String... q) throws Exception { return response; } + /** + * When a node resolves collection state for a collection it doesn't host, queries should use + * cached state and not make ZK calls on every query. + */ + @Test + public void testDistributedQueryDoesNotReadFromZk() throws Exception { + final String secondColl = "secondColl"; + + // Create a collection on only 1 node so the other node uses LazyCollectionRef for state + List jettys = cluster.getJettySolrRunners(); + CollectionAdminRequest.createCollection(secondColl, "conf", 1, 1) + .setCreateNodeSet(jettys.get(0).getNodeName()) + .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT); + cluster + .getZkStateReader() + .waitForState( + secondColl, + DEFAULT_TIMEOUT, + TimeUnit.SECONDS, + (n, c) -> DocCollection.isFullyActive(n, c, 1, 1)); + + try { + // Node 1 hosts COLLECTION but not secondColl. + // Send a multi-collection query to trigger LazyCollectionRef get call + JettySolrRunner nodeWithoutSecondColl = jettys.get(1); + try (SolrClient client = + new Http2SolrClient.Builder(nodeWithoutSecondColl.getBaseUrl().toString()).build()) { + + String collectionsParameter = COLLECTION + "," + secondColl; + + // Warm up LazyCollectionRef state cache with query + client.query(COLLECTION, new SolrQuery("q", "*:*", "collection", collectionsParameter)); + + // Get ZK metrics from the coordinator node (the one we're querying) + long existsBefore = getZkExistsChecks(client); + + // Query again and assert that exists call is not made + client.query(COLLECTION, new SolrQuery("q", "*:*", "collection", collectionsParameter)); + + long existsAfter = getZkExistsChecks(client); + + assertEquals( + "Query should not cause ZK exists checks as collection state should be cached", + existsBefore, + existsAfter); + } + } finally { + CollectionAdminRequest.deleteCollection(secondColl) + .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT); + } + } + + @SuppressWarnings("unchecked") + private long getZkExistsChecks(SolrClient client) throws Exception { + NamedList response = + client.request( + new GenericSolrRequest( + SolrRequest.METHOD.GET, + "/admin/metrics", + new MapSolrParams(Map.of("key", "solr.node:CONTAINER.zkClient")))); + NamedList metrics = (NamedList) response.get("metrics"); + Map zkMetrics = + (Map) metrics.get("solr.node:CONTAINER.zkClient"); + return ((Number) zkMetrics.get("existsChecks")).longValue(); + } + private int getNumRequests( Map> requests) { int beforeNumRequests = 0; From 82524e176e0692de3d575cc4e2357370b77afa18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 16 Apr 2026 22:58:39 +0200 Subject: [PATCH 62/83] SOLR-17353 Remove gosu binary from the Docker image (#4285) (branch_9x) (#4287) --- .../SOLR-17353-remove-gosu-from-docker.yml | 8 ++ .../docker/templates/Dockerfile.body.template | 2 +- solr/docker/tests/cases/gosu/test.sh | 81 ------------------- .../pages/major-changes-in-solr-9.adoc | 4 + 4 files changed, 13 insertions(+), 82 deletions(-) create mode 100644 changelog/unreleased/SOLR-17353-remove-gosu-from-docker.yml delete mode 100755 solr/docker/tests/cases/gosu/test.sh diff --git a/changelog/unreleased/SOLR-17353-remove-gosu-from-docker.yml b/changelog/unreleased/SOLR-17353-remove-gosu-from-docker.yml new file mode 100644 index 000000000000..cad475b362c9 --- /dev/null +++ b/changelog/unreleased/SOLR-17353-remove-gosu-from-docker.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Remove gosu from the Docker image. The Solr Docker image no longer installs the gosu binary. +type: removed +authors: + - name: Jan Høydahl +links: +- name: SOLR-17353 + url: https://issues.apache.org/jira/browse/SOLR-17353 diff --git a/solr/docker/templates/Dockerfile.body.template b/solr/docker/templates/Dockerfile.body.template index 777d051ca193..0d08e2c9f2f0 100644 --- a/solr/docker/templates/Dockerfile.body.template +++ b/solr/docker/templates/Dockerfile.body.template @@ -71,7 +71,7 @@ RUN set -ex; \ RUN set -ex; \ apt-get update; \ - apt-get -y --no-install-recommends install acl lsof procps wget netcat gosu tini jattach; \ + apt-get -y --no-install-recommends install acl lsof procps wget netcat tini jattach; \ rm -rf /var/lib/apt/lists/*; VOLUME /var/solr diff --git a/solr/docker/tests/cases/gosu/test.sh b/solr/docker/tests/cases/gosu/test.sh deleted file mode 100755 index e7d0e85d57f5..000000000000 --- a/solr/docker/tests/cases/gosu/test.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A simple test of gosu. We create a myvarsolr, and chown it - -if [[ "$OSTYPE" == "darwin"* ]]; then - # TODO: Fix this test on Mac - echo "WARNING: Ignoring test 'gosu' on macOS" - exit 0 -fi - -set -euo pipefail - -TEST_DIR="${TEST_DIR:-$(dirname -- "${BASH_SOURCE[0]}")}" -source "${TEST_DIR}/../../shared.sh" - -myvarsolr="${BUILD_DIR}/myvarsolr-${container_name}" -prepare_dir_to_mount 8983 "$myvarsolr" - -echo "Running $container_name" -docker run --user 0:0 --name "$container_name" -d -e VERBOSE=yes \ - -v "$myvarsolr:/var/solr" "$tag" \ - bash -c "chown -R solr:solr /var/solr; touch /var/solr/root_was_here; exec gosu solr:solr solr-precreate gettingstarted" - -wait_for_container_and_solr "$container_name" - -echo "Loading data" -docker exec --user=solr "$container_name" bin/solr post -c gettingstarted example/exampledocs/manufacturers.xml -sleep 1 -echo "Checking data" -data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8983/solr/gettingstarted/select?q=id%3Adell') -if ! grep -E -q 'One Dell Way Round Rock, Texas 78682' <<<"$data"; then - echo "Test $TEST_NAME $tag failed; data did not load" - exit 1 -fi - -# check test file was created by root -data=$(docker exec --user=root "$container_name" stat -c %U /var/solr/root_was_here ) -if [[ "$data" == *'No such file or directory' ]]; then - echo "Missing /var/solr/root_was_here" - exit 1 -fi -if [[ "$data" != root ]]; then - echo "/var/solr/root_was_here is owned by $data" - exit 1 -fi - -# check core is created by solr -data=$(docker exec --user=root "$container_name" stat -c %U /var/solr/data/gettingstarted/core.properties ) -if [[ "$data" == *'No such file or directory' ]]; then - echo "Missing /var/solr/data/gettingstarted/core.properties" - exit 1 -fi -if [[ "$data" != solr ]]; then - echo "/var/solr/data/gettingstarted/core.properties is owned by $data" - exit 1 -fi - -container_cleanup "$container_name" - -# chown it back -docker run --rm --user 0:0 -e VERBOSE=yes \ - -v "$myvarsolr:/myvarsolr" "$tag" \ - bash -c "chown -R $(id -u):$(id -g) /myvarsolr; ls -ld /myvarsolr" - -rm -fr "$myvarsolr" - -echo "Test $TEST_NAME $tag succeeded" diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc index 7db51f45c24e..c9d32d36194e 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc @@ -77,6 +77,10 @@ Due to changes in Lucene 9, that isn't possible any more. == Solr 9.11 +=== Docker + +The `gosu` binary is no longer installed in the Solr Docker image. See https://github.com/tianon/gosu[gosu github page] for alternatives, such as `runuser`, `setpriv` or `chroot`. + === Removing 'local' Tika The project normally doesn't remove functionality in a minor release, but we made an exception to avoid shipping Tika 1.x with known vulnerabilities. From 99bdf2982ad509df2fd7e9f6ca7fd0a127918711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 18 Apr 2026 18:26:09 +0200 Subject: [PATCH 63/83] SOLR-18192: Pin all GitHub Actions to full commit SHAs as per ASF policy (branch_9x) (#4290) --- .github/actions/prepare-for-build/action.yml | 2 +- .github/workflows/bin-solr-test.yml | 4 ++-- .github/workflows/docker-test.yml | 2 +- .github/workflows/gradle-extraction-check.yml | 2 +- .github/workflows/gradle-precommit.yml | 2 +- .github/workflows/solrj-test.yml | 2 +- .github/workflows/validate-changelog.yml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/actions/prepare-for-build/action.yml b/.github/actions/prepare-for-build/action.yml index ebddc8d5ca71..c348f5446c07 100644 --- a/.github/actions/prepare-for-build/action.yml +++ b/.github/actions/prepare-for-build/action.yml @@ -26,7 +26,7 @@ runs: java-package: jdk - name: Cache gradle-wrapper.jar - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: gradle/wrapper/gradle-wrapper.jar key: gradle-wrapper-${{ hashFiles('gradle/wrapper/gradle-wrapper.jar.sha256') }} diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml index fea033015ab9..242626ecb0d4 100644 --- a/.github/workflows/bin-solr-test.yml +++ b/.github/workflows/bin-solr-test.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - uses: ./.github/actions/prepare-for-build @@ -29,7 +29,7 @@ jobs: - name: Archive logs if: ${{ failure() }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: logs path: solr/packaging/build/test-output diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index aebbde1efb35..74557283d92d 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - uses: ./.github/actions/prepare-for-build diff --git a/.github/workflows/gradle-extraction-check.yml b/.github/workflows/gradle-extraction-check.yml index 9c18a01c6807..61b21cb4ca42 100644 --- a/.github/workflows/gradle-extraction-check.yml +++ b/.github/workflows/gradle-extraction-check.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - uses: ./.github/actions/prepare-for-build diff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.yml index ac56df84a17a..7cc7cd95cf9f 100644 --- a/.github/workflows/gradle-precommit.yml +++ b/.github/workflows/gradle-precommit.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - uses: ./.github/actions/prepare-for-build diff --git a/.github/workflows/solrj-test.yml b/.github/workflows/solrj-test.yml index c81246b41be8..68942e7b87df 100644 --- a/.github/workflows/solrj-test.yml +++ b/.github/workflows/solrj-test.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - uses: ./.github/actions/prepare-for-build diff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.yml index 186eec1e3abc..028ddd8c1763 100644 --- a/.github/workflows/validate-changelog.yml +++ b/.github/workflows/validate-changelog.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 From 1eb6d2a37f80e4c852afb9f84b9c279ba396c376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Apr 2026 00:28:14 +0200 Subject: [PATCH 64/83] SOLR 18174 AsyncTracker Semaphore permit leak fix (branch_9x) (#4292) --- ...SOLR-18174-prevent-double-registration.yml | 8 + .../component/HttpShardHandlerFactory.java | 14 + .../AsyncTrackerSemaphoreLeakTest.java | 407 ++++++++++++++++++ .../conf/solr-exporter-config.xml | 14 + .../pages/metrics-reporting.adoc | 13 + .../pages/major-changes-in-solr-9.adoc | 8 + .../client/solrj/impl/Http2SolrClient.java | 87 +++- 7 files changed, 542 insertions(+), 9 deletions(-) create mode 100644 changelog/unreleased/SOLR-18174-prevent-double-registration.yml create mode 100644 solr/core/src/test/org/apache/solr/handler/component/AsyncTrackerSemaphoreLeakTest.java diff --git a/changelog/unreleased/SOLR-18174-prevent-double-registration.yml b/changelog/unreleased/SOLR-18174-prevent-double-registration.yml new file mode 100644 index 000000000000..d31f038ea1b2 --- /dev/null +++ b/changelog/unreleased/SOLR-18174-prevent-double-registration.yml @@ -0,0 +1,8 @@ +title: Fix semaphore permit leaks in Http2SolrClient's AsyncTracker. Avoid IO-thread deadlock on connection failure retries. Add a new metric gauge solr_client_request_async_permits +type: fixed +authors: + - name: Jan Høydahl + url: https://home.apache.org/phonebook.html?uid=janhoy +links: + - name: SOLR-18174 + url: https://issues.apache.org/jira/browse/SOLR-18174 diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java index 9e112b54c52f..9c78d51d4cc6 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java @@ -445,5 +445,19 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) { null, solrMetricsContext.getMetricRegistry(), SolrMetricManager.mkName("httpShardExecutor", expandedScope, "threadPool")); + if (defaultClient != null) { + solrMetricsContext.gauge( + defaultClient::asyncTrackerAvailablePermits, + true, + "asyncPermits.available", + expandedScope, + "threadPool"); + solrMetricsContext.gauge( + defaultClient::asyncTrackerMaxPermits, + true, + "asyncPermits.max", + expandedScope, + "threadPool"); + } } } diff --git a/solr/core/src/test/org/apache/solr/handler/component/AsyncTrackerSemaphoreLeakTest.java b/solr/core/src/test/org/apache/solr/handler/component/AsyncTrackerSemaphoreLeakTest.java new file mode 100644 index 000000000000..11e34dd3257f --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/component/AsyncTrackerSemaphoreLeakTest.java @@ -0,0 +1,407 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.component; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.lucene.util.SuppressForbidden; +import org.apache.solr.client.solrj.impl.Http2SolrClient; +import org.apache.solr.client.solrj.impl.LBHttp2SolrClient; +import org.apache.solr.client.solrj.impl.LBSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.params.SolrParams; +import org.eclipse.jetty.client.api.Request; +import org.eclipse.jetty.client.api.Response; +import org.eclipse.jetty.client.api.Result; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests for two semaphore-permit leak bugs in {@link Http2SolrClient}'s {@code AsyncTracker} that + * cause distributed queries to hang permanently. + * + *

    Pattern A – HTTP/2 GOAWAY double-queue leak

    + * + *

    Jetty HTTP/2 can re-queue the same exchange after a GOAWAY/connection race, firing {@code + * onRequestQueued} twice for one logical request. Because {@code onComplete} fires only once, one + * permit is permanently consumed per occurrence, gradually draining the semaphore over hours or + * days until Pattern B triggers. + * + *

    Pattern B – IO-thread deadlock on LB retry when permits depleted

    + * + *

    When a connection-level failure causes {@link + * org.apache.solr.client.solrj.impl.LBHttp2SolrClient} to retry synchronously inside a {@code + * whenComplete} callback on the Jetty IO selector thread, the retry calls {@code acquire()} on that + * same IO thread before the original request's {@code onComplete} can call {@code release()}. No + * permits are permanently lost — the deadlock simply requires two permits to be available + * simultaneously — but if the semaphore is at zero, {@code acquire()} blocks the IO thread + * permanently and distributed queries hang forever. + */ +public class AsyncTrackerSemaphoreLeakTest extends SolrCloudTestCase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String COLLECTION = "semaphore_leak_test"; + + /** Reduced semaphore size so we can observe the drain without needing thousands of requests. */ + private static final int MAX_PERMITS = 40; + + /** + * Number of concurrent requests. Set equal to MAX_PERMITS so that all permits are exhausted + * before any retry can acquire, triggering the IO-thread deadlock. + */ + private static final int NUM_RETRY_REQUESTS = MAX_PERMITS; + + @BeforeClass + public static void setupCluster() throws Exception { + // Reduce the semaphore size so we can observe drain with few requests. + // This property is read when Http2SolrClient is constructed, so it must + // be set BEFORE the cluster (and its HttpShardHandlerFactory) starts up. + System.setProperty(Http2SolrClient.ASYNC_REQUESTS_MAX_SYSPROP, String.valueOf(MAX_PERMITS)); + + configureCluster(1).addConfig("conf", configset("cloud-dynamic")).configure(); + + CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1) + .process(cluster.getSolrClient()); + + waitForState("Expected 2 active shards with 1 replica each", COLLECTION, clusterShape(2, 2)); + } + + @AfterClass + public static void cleanup() { + System.clearProperty(Http2SolrClient.ASYNC_REQUESTS_MAX_SYSPROP); + } + + /** + * Demonstrates the permanent IO-thread deadlock (Pattern B) caused by {@link + * org.apache.solr.client.solrj.impl.LBHttp2SolrClient} retrying a request synchronously inside a + * {@link CompletableFuture#whenComplete} callback that runs on the Jetty IO selector thread. + * + *

    This test passes with the {@code failureDispatchExecutor} fix in this branch. Without + * the fix, the IO thread would block forever in {@code semaphore.acquire()} and this test would + * time out. + */ + @Test + @SuppressForbidden( + reason = + "Reflection needed to access Http2SolrClient's package-private getHttpClient() to force-stop it during timeout recovery") + public void testSemaphoreLeakOnLBRetry() throws Exception { + // Dedicated client so that permanently deadlocked IO threads don't affect the cluster's client. + Http2SolrClient testClient = + new Http2SolrClient.Builder() + .withConnectionTimeout(5, TimeUnit.SECONDS) + .withIdleTimeout(30, TimeUnit.SECONDS) + .useHttp1_1(true) // HTTP/1.1: every request gets its own TCP connection + .build(); + + String realBaseUrl = + cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/" + COLLECTION; + + List> futures = new ArrayList<>(); + + try (FakeTcpServer fakeServer = new FakeTcpServer(NUM_RETRY_REQUESTS); + LBHttp2SolrClient lbClient = + new LBHttp2SolrClient.Builder(testClient, new LBSolrClient.Endpoint[0]).build()) { + + assertEquals( + "All permits should be available before the test (verifies sysprop was applied)", + MAX_PERMITS, + testClient.asyncTrackerAvailablePermits()); + + // Submit NUM_RETRY_REQUESTS async requests. + // Each request has two endpoints: fakeBaseUrl (first) and realBaseUrl (second/retry). + // Each requestAsync() call acquires a semaphore permit synchronously during send(). + // After NUM_RETRY_REQUESTS calls, the semaphore is at 0. + for (int i = 0; i < NUM_RETRY_REQUESTS; i++) { + QueryRequest qr = new QueryRequest(SolrParams.of("q", "*:*")); + LBSolrClient.Req req = + new LBSolrClient.Req( + qr, + List.of( + new LBSolrClient.Endpoint(fakeServer.baseUrl()), + new LBSolrClient.Endpoint(realBaseUrl))); + futures.add(lbClient.requestAsync(req)); + } + + log.info( + "Queued {} requests (semaphore now at 0). Waiting for all TCP connections...", + NUM_RETRY_REQUESTS); + + // Wait until the fake server has accepted all NUM_RETRY_REQUESTS connections. + // At this point all semaphore permits are consumed and no onComplete has fired yet. + assertTrue( + "All " + + NUM_RETRY_REQUESTS + + " connections should be established within 15 s, but only " + + fakeServer.connectionCount() + + " were.", + fakeServer.awaitAllConnected(15, TimeUnit.SECONDS)); + + assertEquals( + "Semaphore should be fully consumed after queuing all requests", + 0, + testClient.asyncTrackerAvailablePermits()); + + // Close all fake connections simultaneously with TCP RST. + // onFailure fires on the IO thread → LBHttp2SolrClient retry → acquire() blocks + // (semaphore=0). + int connCount = fakeServer.connectionCount(); + log.info("Closing {} fake connections via RST...", connCount); + fakeServer.rstAll(); + + try { + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .get(30, TimeUnit.SECONDS); + } catch (ExecutionException e) { + // Individual request failure is fine; permits are released by onComplete regardless. + log.warn("Some requests failed during retry", e); + } catch (TimeoutException e) { + // Force-stop the HttpClient to unblock any threads stuck in semaphore.acquire() + // before asserting failure, so the finally block can close the client without hanging. + try { + Method getHttpClient = Http2SolrClient.class.getDeclaredMethod("getHttpClient"); + getHttpClient.setAccessible(true); + ((org.eclipse.jetty.client.HttpClient) getHttpClient.invoke(testClient)).stop(); + } catch (Exception ignored) { + log.debug("Failed to stop HttpClient"); + } + fail( + "BUG (LBHttp2SolrClient retry deadlock): futures did not complete within 30s." + + " IO threads are permanently blocked in semaphore.acquire() because the retry" + + " fires synchronously on the IO thread before onComplete can release()."); + } + + int permitsAfterFailures = testClient.asyncTrackerAvailablePermits(); + log.info("Permits after retries: {}/{}", permitsAfterFailures, MAX_PERMITS); + assertEquals( + "All permits should be restored after retries complete", + MAX_PERMITS, + permitsAfterFailures); + } finally { + try { + testClient.close(); + } catch (Exception ignored) { + log.debug("Failed to close LBHttp2SolrClient"); + } + for (CompletableFuture f : futures) { + f.cancel(true); + } + } + } + + /** + * Verifies that the {@code PERMIT_ACQUIRED_ATTR} idempotency guard prevents the Pattern A permit + * leak where Jetty HTTP/2 re-queues the same exchange after a GOAWAY/connection race, firing + * {@code onRequestQueued} twice for one logical request while {@code onComplete} fires only once. + * + *

    Rather than setting up a real HTTP/2 server, this test uses reflection to invoke {@code + * AsyncTracker.queuedListener} twice and {@code AsyncTracker.completeListener} once for the same + * {@code Request} object. Without the guard the semaphore count drops by one; with the guard the + * second queued call is a no-op and the count is unchanged. + */ + @Test + @SuppressForbidden( + reason = + "Reflection needed to access AsyncTracker's private fields for white-box testing without exposing them in the production API") + public void testPermitLeakOnHttp2GoAwayDoubleQueuedListener() throws Exception { + assumeWorkingMockito(); + + Http2SolrClient testClient = + new Http2SolrClient.Builder() + .withConnectionTimeout(5, TimeUnit.SECONDS) + .withIdleTimeout(30, TimeUnit.SECONDS) + // HTTP/2 is the default transport where this GOAWAY race occurs. + .build(); + + // Capture asyncTracker and its class for reflection-based listener access and cleanup. + Field asyncTrackerField = Http2SolrClient.class.getDeclaredField("asyncTracker"); + asyncTrackerField.setAccessible(true); + Object asyncTracker = asyncTrackerField.get(testClient); + Class asyncTrackerClass = asyncTracker.getClass(); + + try { + int maxPermits = testClient.asyncTrackerMaxPermits(); + assertEquals( + "All permits available before test", + maxPermits, + testClient.asyncTrackerAvailablePermits()); + + // Access the raw listeners via reflection to simulate Jetty's internal double-fire. + Field queuedListenerField = asyncTrackerClass.getDeclaredField("queuedListener"); + queuedListenerField.setAccessible(true); + Request.QueuedListener queuedListener = + (Request.QueuedListener) queuedListenerField.get(asyncTracker); + + Field completeListenerField = asyncTrackerClass.getDeclaredField("completeListener"); + completeListenerField.setAccessible(true); + Response.CompleteListener completeListener = + (Response.CompleteListener) completeListenerField.get(asyncTracker); + + // Fake Request that supports the attribute get/set used by the idempotency guard. + Map reqAttributes = new HashMap<>(); + Request fakeRequest = Mockito.mock(Request.class); + Mockito.when(fakeRequest.getAttributes()).thenReturn(reqAttributes); + Mockito.when(fakeRequest.attribute(ArgumentMatchers.anyString(), ArgumentMatchers.any())) + .thenAnswer( + inv -> { + reqAttributes.put(inv.getArgument(0), inv.getArgument(1)); + return fakeRequest; + }); + + // Simulate the GOAWAY double-fire: 1st call acquires a permit; 2nd is the bug trigger. + queuedListener.onQueued(fakeRequest); + queuedListener.onQueued(fakeRequest); + + Result fakeResult = Mockito.mock(Result.class); + Mockito.when(fakeResult.getRequest()).thenReturn(fakeRequest); + // Only one onComplete fires for the logical request (regardless of internal retries). + completeListener.onComplete(fakeResult); + + int permitsAfter = testClient.asyncTrackerAvailablePermits(); + log.info("Permits after double-queued + single complete: {}/{}", permitsAfter, maxPermits); + + assertEquals( + "BUG (Jetty HTTP/2 GOAWAY retry permit leak): onRequestQueued fired twice for the" + + " same Request object but onComplete fired only once. The second acquire()" + + " was not matched by a release(), permanently leaking one permit per" + + " occurrence. In production this causes gradual semaphore depletion over" + + " hours/days until Pattern B IO-thread deadlock triggers.", + maxPermits, + permitsAfter); + + } finally { + // Force-terminate the Phaser as a safety net; without the fix the phaser would be unbalanced. + try { + Field phaserField = asyncTrackerClass.getDeclaredField("phaser"); + phaserField.setAccessible(true); + Phaser phaser = (Phaser) phaserField.get(asyncTracker); + phaser.forceTermination(); + } catch (Exception ignored) { + log.debug("Failed to force-terminate Phaser"); + } + + try { + testClient.close(); + } catch (Exception ignored) { + log.debug("Failed to close Http2SolrClient"); + } + } + } + + /** + * A minimal fake TCP server that accepts a fixed number of connections and holds them open, + * allowing tests to simulate connection-level failures by RST-ing all sockets at once. + * + *

    Implements {@link AutoCloseable} so that the server socket and any open connections are + * always cleaned up when used in a try-with-resources block, even if the test fails or throws. + */ + private static class FakeTcpServer implements AutoCloseable { + private final ServerSocket serverSocket; + private final List connections = Collections.synchronizedList(new ArrayList<>()); + private final CountDownLatch allConnected; + private final AtomicBoolean closed = new AtomicBoolean(false); + + FakeTcpServer(int expectedConnections) throws IOException { + this.serverSocket = new ServerSocket(0); + this.allConnected = new CountDownLatch(expectedConnections); + Thread acceptThread = + new Thread( + () -> { + try { + while (connections.size() < expectedConnections && !serverSocket.isClosed()) { + Socket s = serverSocket.accept(); + connections.add(s); + allConnected.countDown(); + } + } catch (IOException ioe) { + log.warn("Failed to accept connection", ioe); + } + }, + "fake-tcp-server"); + acceptThread.setDaemon(true); + acceptThread.start(); + } + + /** Returns the base URL clients should connect to, e.g. {@code http://127.0.0.1:PORT/solr}. */ + String baseUrl() { + return "http://127.0.0.1:" + serverSocket.getLocalPort() + "/solr"; + } + + /** Waits until all expected connections have been accepted. */ + boolean awaitAllConnected(long timeout, TimeUnit unit) throws InterruptedException { + return allConnected.await(timeout, unit); + } + + /** Returns the number of connections accepted so far. */ + int connectionCount() { + return connections.size(); + } + + /** + * Closes all accepted connections with TCP RST, triggering onFailure on the Jetty IO thread. + */ + void rstAll() { + for (Socket s : connections) { + try { + s.setSoLinger(true, 0); // send RST instead of FIN + s.close(); + } catch (IOException ignored) { + log.debug("Failed to close connection"); + } + } + } + + /** + * RSTs any remaining open connections and closes the server socket, stopping the accept thread. + * Safe to call multiple times. + */ + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + rstAll(); + try { + serverSocket.close(); + } catch (IOException ignored) { + log.debug("Failed to close server socket"); + } + } + } + } +} diff --git a/solr/prometheus-exporter/conf/solr-exporter-config.xml b/solr/prometheus-exporter/conf/solr-exporter-config.xml index 275ab46f4f17..7a36356d49ce 100644 --- a/solr/prometheus-exporter/conf/solr-exporter-config.xml +++ b/solr/prometheus-exporter/conf/solr-exporter-config.xml @@ -563,6 +563,20 @@ value : $value } + + .metrics["solr.node"] | to_entries | .[] | + select(.key | startswith("QUERY.httpShardHandler.threadPool.asyncPermits.")) as $object | + $object.key | split(".") | last as $state | + $object.value as $value | + { + name : "solr_client_request_async_permits", + type : "GAUGE", + help : "See following URL: https://solr.apache.org/guide/solr/latest/deployment-guide/metrics-reporting.html", + label_names : ["state"], + label_values : [$state], + value : $value + } +