SOLR-15560: Optimize JavaBinCodec and related Jetty end points.

dev-docs/overseer/overseer.adoc

@@ -322,7 +322,7 @@ These messages are received from the Collection API queue (there’s a single qu
 
 The `createConfigSet()` call implementing `CREATE` copies all the files of an existing config set (by default the `_default` config set) into a new config set, merges the existing config set properties if any with new ones specified in the message (config set properties in the message are properties that start with `"configSetProp."`, for example `configSetProp.immutable` is a property that prevents the config set from ever being deleted) and writes the resulting properties into `/configs/<config set name>/configsetprops.json` (note: creating a config set based on an `immutable` config set makes the new one immutable as well unless it explicitly specifies that `configSetProp.immutable` is false).
 
-Note the `_default` config set is defined in the SolrCloud distribution and copied if absent into Zookeeper when SolrCloud starts (`ZkController.bootstrapDefaultConfigSet`) then used only from Zookeeper. This config set has a directory `lang` with language specific stop words, contractions and other, and it contains files `managed-schema`, `params.json`, `protwords.txt`, `solrconfig.xml`, `stopwords.txt` and `synonyms.txt`.
+Note the `_default` config set is defined in the SolrCloud distribution and copied if absent into Zookeeper when SolrCloud starts (`ZkController.bootstrapDefaultConfigSet`) then used only from Zookeeper. This config set has a directory `lang` with language specific stop words, contractions and other, and it contains files `managed-schema.xml`, `params.json`, `protwords.txt`, `solrconfig.xml`, `stopwords.txt` and `synonyms.txt`.
 
 deleteConfigSet() deletes the whole znode structure at `/configs/_<config set name>_` assuming the config set is not used by any collection and is not immutable (the only case where an immutable config set can be deleted is when its creation has failed midway).
 

dev-tools/scripts/releaseWizard.py

@@ -116,7 +116,7 @@ def expand_jinja(text, vars=None):
         'set_java_home': set_java_home,
         'latest_version': state.get_latest_version(),
         'latest_lts_version': state.get_latest_lts_version(),
-        'master_version': state.get_master_version(),
+        'main_version': state.get_main_version(),
         'mirrored_versions': state.get_mirrored_versions(),
         'mirrored_versions_to_delete': state.get_mirrored_versions_to_delete(),
         'home': os.path.expanduser("~")
@@ -365,7 +365,7 @@ def get_mirrored_versions_to_delete(self):
             raise Exception("Release version %s must have same major version as current minor or lts release")
         return [ver for ver in versions if ver not in to_keep]
 
-    def get_master_version(self):
+    def get_main_version(self):
         v = Version.parse(self.get_latest_version())
         return "%s.%s.%s" % (v.major + 1, 0, 0)
 
@@ -394,18 +394,18 @@ def validate_release_version(self, branch_type, branch, release_version):
             if not ver.is_minor_release():
                 sys.exit("You can only release minor releases from an existing stable branch")
         elif branch_type == BranchType.unstable:
-            if not branch == 'master':
+            if not branch == 'main':
                 sys.exit("Incompatible branch and branch_type")
             if not ver.is_major_release():
-                sys.exit("You can only release a new major version from master branch")
+                sys.exit("You can only release a new major version from main branch")
         if not getScriptVersion() == release_version:
             print("WARNING: Expected release version %s when on branch %s, but got %s" % (
                 getScriptVersion(), branch, release_version))
 
     def get_base_branch_name(self):
         v = Version.parse(self.release_version)
         if v.is_major_release():
-            return 'master'
+            return 'main'
         elif v.is_minor_release():
             return self.get_stable_branch_name()
         elif v.major == Version.parse(self.get_latest_version()).major:
@@ -573,7 +573,7 @@ def get_minor_branch_name(self):
 
     def get_stable_branch_name(self):
         if self.release_type == 'major':
-            v = Version.parse(self.get_master_version())
+            v = Version.parse(self.get_main_version())
         else:
             v = Version.parse(self.get_latest_version())
         return "branch_%sx" % v.major

dev-tools/scripts/releaseWizard.yaml

@@ -410,7 +410,7 @@ groups:
         cmd: "{{ gradle_cmd }} clean check -x test"
   - !Todo
     id: create_stable_branch
-    title: Create a new stable branch, off from master
+    title: Create a new stable branch, off from main
     description: In our case we'll create {{ stable_branch }}
     types:
     - major
@@ -420,7 +420,7 @@ groups:
       commands_text: Run these commands to create a stable branch
       commands:
       - !Command
-        cmd: git checkout master
+        cmd: git checkout main
         tee: true
       - !Command
         cmd: git pull --ff-only
@@ -467,18 +467,18 @@ groups:
         tee: true
   - !Todo
     id: add_version_major
-    title: Add a new major version on master branch
+    title: Add a new major version on main branch
     types:
     - major
     depends: clean_git_checkout
     vars:
       next_version: "{{ release_version_major + 1 }}.0.0"
     commands: !Commands
       root_folder: '{{ git_checkout_folder }}'
-      commands_text: Run these commands to add the new major version {{ next_version }} to the master branch
+      commands_text: Run these commands to add the new major version {{ next_version }} to the main branch
       commands:
       - !Command
-        cmd: git checkout master
+        cmd: git checkout main
         tee: true
       - !Command
         cmd: python3 -u dev-tools/scripts/addVersion.py {{ next_version }}
@@ -632,7 +632,7 @@ groups:
       Go to the JIRA "Manage Versions" Administration pages and add the new version:
 
       {% if release_type == 'major' -%}
-      . Change name of version `master ({{ release_version_major }}.0)` into `{{ release_version_major }}.0`
+      . Change name of version `main ({{ release_version_major }}.0)` into `{{ release_version_major }}.0`
       {%- endif %}
       . Create a new (unreleased) version `{{ get_next_version }}`
 
@@ -972,7 +972,7 @@ groups:
         logfile: svn_rm_containing.log
         comment: Clean up containing folder on the staging repo
         tee: true
-    post_description: 'Note at this point you will see the Jenkins job "Lucene-Solr-SmokeRelease-master" begin to fail, until you run the "Generate Backcompat Indexes" '
+    post_description: 'Note at this point you will see the Jenkins job "Lucene-Solr-SmokeRelease-main" begin to fail, until you run the "Generate Backcompat Indexes" '
   - !Todo
     id: stage_maven
     title: Stage the maven artifacts for publishing
@@ -1190,7 +1190,7 @@ groups:
     depends:
     - prepare_announce_solr
     description: |
-      Push the website changes to 'master' branch, and check the staging site.
+      Push the website changes to 'main' branch, and check the staging site.
       You will get a chance to preview the diff of all changes before you push.
       If you need to do changes, do the changes (e.g. by re-running previous step 'Update rest of webpage')
       and commit your changes. Then re-run this step and push when everything is OK.
@@ -1201,7 +1201,7 @@ groups:
         You have to exit the editor after review to continue.
       commands:
       - !Command
-        cmd: git checkout master && git status
+        cmd: git checkout main && git status
         stdout: true
       - !Command
         cmd: git diff
@@ -1240,7 +1240,7 @@ groups:
           cmd: git checkout production && git pull --ff-only
           stdout: true
         - !Command
-          cmd: git merge master
+          cmd: git merge main
           stdout: true
         - !Command
           cmd: git push origin
@@ -1269,9 +1269,9 @@ groups:
       commands_text: Edit DOAP files
       commands:
       - !Command
-        cmd: git checkout master && git pull --ff-only
+        cmd: git checkout main && git pull --ff-only
         stdout: true
-        comment: Goto master branch
+        comment: Goto main branch
       - !Command
         cmd: "{{ editor }} dev-tools/doap/lucene.rdf"
         comment: Edit Lucene DOAP, add version {{ release_version }}
@@ -1288,16 +1288,16 @@ groups:
         cmd: git push origin
         logfile: push.log
         stdout: true
-        comment: Push the master branch
+        comment: Push the main branch
       - !Command
         cmd: "git checkout {{ stable_branch }} && git pull --ff-only"
         stdout: true
         comment: Checkout the stable branch
       - !Command
-        cmd: "git cherry-pick master"
+        cmd: "git cherry-pick main"
         logfile: commit.log
         stdout: true
-        comment: Cherrypick the DOAP changes from master onto the stable branch.
+        comment: Cherrypick the DOAP changes from main onto the stable branch.
       - !Command
         cmd: git show HEAD
         stdout: true
@@ -1407,24 +1407,24 @@ groups:
     commands: !Commands
       root_folder: '{{ git_checkout_folder }}'
       commands_text: |
-        Update versions on master and stable branch.
+        Update versions on main and stable branch.
         You may have to hand-edit some files before commit, so go slowly :)
       confirm_each_command: true
       commands:
       - !Command
-        cmd: git checkout master && git pull --ff-only && git clean -df && git checkout -- .
-        comment: Go to master branch
-        logfile: checkout-master.log
+        cmd: git checkout main && git pull --ff-only && git clean -df && git checkout -- .
+        comment: Go to main branch
+        logfile: checkout-main.log
       - !Command
         cmd: python3 -u dev-tools/scripts/addVersion.py {{ release_version }}
-        logfile: addversion-master.log
+        logfile: addversion-main.log
       - !Command
         cmd: git diff
-        logfile: diff-master.log
+        logfile: diff-main.log
         tee: true
       - !Command
         cmd: git add -u .  && git commit -m "Add bugfix version {{ release_version }}"  && git push
-        logfile: commit-master.log
+        logfile: commit-main.log
       - !Command
         cmd: git checkout {{ stable_branch }} && git pull --ff-only && git clean -df && git checkout -- .
         logfile: checkout-stable.log
@@ -1465,9 +1465,9 @@ groups:
         tee: true
         comment: Find version regexes
       - !Command
-        cmd: git checkout master && git pull --ff-only && git clean -df && git checkout -- .
-        comment: Go to master branch
-        logfile: checkout-master.log
+        cmd: git checkout main && git pull --ff-only && git clean -df && git checkout -- .
+        comment: Go to main branch
+        logfile: checkout-main.log
       - !Command
         cmd: "{{ editor }} solr/CHANGES.txt"
         comment: Edit Solr CHANGES, do necessary changes
@@ -1478,7 +1478,7 @@ groups:
         stdout: true
       - !Command
         cmd: git add -u .  && git commit -m "Sync CHANGES for {{ release_version }}"  && git push
-        logfile: commit-master.log
+        logfile: commit-main.log
       - !Command
         cmd: git checkout {{ stable_branch }} && git pull --ff-only && git clean -df && git checkout -- .
         comment: Go to stable branch

dev-tools/scripts/scriptutil.py

@@ -124,7 +124,7 @@ def find_branch_type():
   else:
     raise Exception('git status missing branch name')
 
-  if branchName == b'master':
+  if branchName == b'main':
     return BranchType.unstable
   if re.match(r'branch_(\d+)x', branchName.decode('UTF-8')):
     return BranchType.stable

solr/CHANGES.txt

@@ -143,6 +143,11 @@ when told to. The admin UI now tells it to. (Nazerke Seidan, David Smiley)
   This was already working for XML & "javabin"/SolrJ.  Previously, omitting the ID would be confused
   for a partial/atomic update.  (David Smiley)
 
+* SOLR-10887: Migrate "managed-schema" file naming to "managed-schema.xml" file name, with a fallback to the legacy "managed-schema".  (Eric Pugh, David Smiley)
+
+* SOLR-15630: Logging MDC values no longer include a hardcoded prefix, allowing custom logging configurations access to the plain values.
+  The default log4j2.xml PatternLayout has been updated to ensure the values are formatted with the existing prefixes. (hossman)
+
 Build
 ---------------------
 

solr/benchmark/log4j2-bench.xml

@@ -23,19 +23,19 @@
     <Console name="STDOUT" target="SYSTEM_OUT">
       <PatternLayout>
         <Pattern>
-          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%X{collection} %X{shard} %X{replica} %X{core}] %c{1.}
+          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.}
           %m%notEmpty{ =>%ex{short}}}{10240}%n
         </Pattern>
       </PatternLayout>
     </Console>
 
     <RollingRandomAccessFile
         name="MainLogFile"
-        fileName="${sys:solr.log.dir:-work/solr-logs}/${sys:solr.log.name:-solr}.log"
+        fileName="${sys:solr.log.dir:-build/work/solr-logs}/${sys:solr.log.name:-solr}.log"
         filePattern="${sys:solr.log.dir:-work/solr-logs}/${sys:solr.log.name:-solr}.log.%i">
       <PatternLayout>
         <Pattern>
-          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%X{collection} %X{shard} %X{replica} %X{core}] %c{1.}
+          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.}
           %m%notEmpty{ =>%ex{short}}}{10240}%n
         </Pattern>
       </PatternLayout>
@@ -48,11 +48,11 @@
 
     <RollingRandomAccessFile
         name="SlowLogFile"
-        fileName="${sys:solr.log.dir}/${sys:solr.log.name}_slow_requests.log"
+        fileName="${sys:solr.log.dir:-build/work/solr-logs}/${sys:solr.log.name:-solr}_slow_requests.log"
         filePattern="${sys:solr.log.dir}/${sys:solr.log.name}_slow_requests.log.%i">
       <PatternLayout>
         <Pattern>
-          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%X{collection} %X{shard} %X{replica} %X{core}] %c{1.}
+          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.}
           %m%notEmpty{ =>%ex{short}}}{10240}%n
         </Pattern>
       </PatternLayout>
@@ -69,14 +69,11 @@
         fileName="${sys:solr.log.dir:-build/work/solr-logs}/${sys:solr.log.name:-solr}_random_counts.log">
       <PatternLayout>
         <Pattern>
-          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%X{collection} %X{shard} %X{replica} %X{core}] %c{1.}
+          %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.}
           %m%notEmpty{ =>%ex{short}}}{10240}%n
         </Pattern>
       </PatternLayout>
     </RandomAccessFile>
-    <AsyncLogger name="org.apache.solr.bench.BaseBenchState.RandomCounts" level="info" additivity="false">
-      <AppenderRef ref="RandomCountsFile"/>
-    </AsyncLogger>
 
   </Appenders>
   <Loggers>
@@ -92,6 +89,9 @@
     <AsyncLogger name="org.apache.solr.core.SolrCore.SlowRequest" level="info" additivity="false">
       <AppenderRef ref="SlowLogFile"/>
     </AsyncLogger>
+    <AsyncLogger name="org.apache.solr.bench.BaseBenchState.RandomCounts" level="info" additivity="false">
+      <AppenderRef ref="RandomCountsFile"/>
+    </AsyncLogger>
 
     <AsyncRoot level="info">
       <AppenderRef ref="MainLogFile"/>

solr/benchmark/src/java/org/apache/solr/bench/generators/StringsDSL.java

@@ -19,13 +19,14 @@
 import static org.apache.solr.bench.generators.SourceDSL.checkArguments;
 import static org.apache.solr.bench.generators.SourceDSL.integers;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
+import java.util.ListIterator;
 import java.util.Objects;
-import java.util.Random;
+import java.util.RandomAccess;
 import java.util.Scanner;
 import java.util.SplittableRandom;
 import org.apache.solr.bench.BaseBenchState;
@@ -51,14 +52,18 @@ public class StringsDSL {
     // english word list via https://github.com/dwyl/english-words
 
     words = new ArrayList<>(1000);
-    InputStream inputStream = StringsDSL.class.getClassLoader().getResourceAsStream("words.txt");
-    try (Scanner scanner =
-        new Scanner(Objects.requireNonNull(inputStream), StandardCharsets.UTF_8.name())) {
+
+    try (InputStream inputStream =
+            StringsDSL.class.getClassLoader().getResourceAsStream("words.txt");
+        Scanner scanner =
+            new Scanner(Objects.requireNonNull(inputStream), StandardCharsets.UTF_8.name())) {
       while (scanner.hasNextLine()) {
         words.add(scanner.nextLine());
       }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
     }
-    Collections.shuffle(words, new Random(BaseBenchState.getRandomSeed()));
+    shuffle(words, new SplittableRandom(BaseBenchState.getRandomSeed()));
     WORD_SIZE = words.size();
   }
 
@@ -439,6 +444,33 @@ public String generate(RandomnessSource in) {
     }
   }
 
+  private static void shuffle(List<?> list, SplittableRandom random) {
+    @SuppressWarnings("unchecked") // we won't put foreign objects in
+    final List<Object> objectList = (List<Object>) list;
+
+    if (list instanceof RandomAccess) {
+      for (int i = objectList.size() - 1; i > 0; i--) {
+        int index = random.nextInt(i + 1);
+        objectList.set(index, objectList.set(i, objectList.get(index)));
+      }
+    } else {
+      Object[] array = objectList.toArray();
+      for (int i = array.length - 1; i > 0; i--) {
+        int index = random.nextInt(i + 1);
+        Object temp = array[i];
+        array[i] = array[index];
+        array[index] = temp;
+      }
+
+      int i = 0;
+      ListIterator<Object> it = objectList.listIterator();
+      while (it.hasNext()) {
+        it.next();
+        it.set(array[i++]);
+      }
+    }
+  }
+
   private static final int[] blockStarts = {
     0x0000, 0x0080, 0x0100, 0x0180, 0x0250, 0x02B0, 0x0300, 0x0370, 0x0400, 0x0500, 0x0530, 0x0590,
     0x0600, 0x0700, 0x0750, 0x0780, 0x07C0, 0x0800, 0x0900, 0x0980, 0x0A00, 0x0A80, 0x0B00, 0x0B80,