Require Java 21 and upgrade to Lucene 10.6.0 and Derby 10.17.1.0

.github/workflows/codeql-analysis.yml

@@ -45,10 +45,10 @@ jobs:
         # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
 
     steps:
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v6
         with:
-          java-version: 17
+          java-version: 21
           distribution: 'zulu'
           problem-matcher: false
 

.github/workflows/main.yml

@@ -32,7 +32,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [ '17', '21', '25' ]
+        java: [ '21', '25' ]
 
     steps:
       - name: Set up JDK ${{ matrix.java }}
@@ -63,7 +63,7 @@ jobs:
 
       # only on integration and only once in this matrix
       - name: Upload Dev Build on Integration
-        if: ${{ (matrix.java == '17') && (github.event_name == 'push') }}
+        if: ${{ (matrix.java == '21') && (github.event_name == 'push') }}
         uses: actions/upload-artifact@v7
         with:
           name: dev-build
@@ -79,10 +79,10 @@ jobs:
     timeout-minutes: 10
 
     steps:
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v6
         with:
-          java-version: '17'
+          java-version: '21'
           distribution: 'zulu'
 
       - name: Checkout Project
@@ -105,10 +105,10 @@ jobs:
     timeout-minutes: 30
 
     steps:
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v6
         with:
-          java-version: '17'
+          java-version: '21'
           distribution: 'zulu'
 
       - name: Checkout Project
@@ -167,10 +167,10 @@ jobs:
         auth: [ 'oidc', 'db-oidc' ]
 
     steps:
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v6
         with:
-          java-version: '17'
+          java-version: '21'
           distribution: 'zulu'
 
       - name: Checkout Project

CHANGES.md

@@ -114,9 +114,16 @@
   error and continues startup.
 
 - **Roller now runs on Jakarta EE**
-  ([ROL-2183](https://issues.apache.org/jira/browse/ROL-2183)). It needs Java 17
+  ([ROL-2183](https://issues.apache.org/jira/browse/ROL-2183)). It needs Java 21
   or later and a Servlet 6.1 container such as Tomcat 11. Tomcat 9 and other
   `javax.servlet` containers can no longer run Roller.
+- **The search index is rebuilt when upgrading from Roller 6.1.6 or
+  earlier.** Roller 7 uses Lucene 10, which reads the index Roller 6.1.7
+  writes. An older index either holds fields Roller no longer indexes or,
+  like the Lucene 8 index of Roller 6.0, can't be read at all. Roller
+  replaces it at first startup and rebuilds it in the background, so no
+  action is needed. Search results are complete again once the log shows
+  "Completed rebuilding index".
 - **OAuth 1.0a is removed** ([ROL-2183](https://issues.apache.org/jira/browse/ROL-2183)).
   AtomPub accepts only BASIC authentication, and the OAuth credentials page and
   consumer keys are gone. If `webservices.atomPubAuth` is still set to `oauth`,

Dockerfile

@@ -21,7 +21,7 @@
 
 # STAGE 1 - BUILD ------------------------------------------------
 
-FROM maven:3-eclipse-temurin-17 AS builder
+FROM maven:3.9.16-eclipse-temurin-21 AS builder
 
 COPY . /project
 WORKDIR /project
@@ -30,7 +30,7 @@ RUN mvn -Duser.home=/builder/home -DskipTests=true -B clean install
 
 # STAGE 2 - PACKAGE ------------------------------------------------
 
-FROM tomcat:11.0-jdk17-temurin
+FROM tomcat:11.0.29-jdk21-temurin
 
 # Remove existing Tomcat webapps
 

README.md

@@ -35,7 +35,7 @@ If you want to run Roller in production, then you should down load the latest of
 ## Quick start: Running via Maven
 
 You probably should not run Roller in production using this technique, but it's a relatively easy way to try Roller for yourself. 
-Assuming you've got a UNIX shell, Java, Maven and Git:
+Assuming you've got a UNIX shell, Java 21 or later, Maven and Git:
 
 Get the code:
 

app/pom.xml

@@ -48,7 +48,7 @@ limitations under the License.
         <eclipse-link.version>5.0.2</eclipse-link.version>
         <guice.version>7.0.0</guice.version>
         <log4j2.version>2.26.1</log4j2.version>
-        <lucene.version>9.12.3</lucene.version> <!-- lucene 10 requires JDK 21 -->
+        <lucene.version>10.6.0</lucene.version>
         <maven-war.version>3.5.1</maven-war.version>
         <maven-surefire.version>3.6.0</maven-surefire.version>
         <maven-antrun.version>1.0b3</maven-antrun.version>
@@ -737,6 +737,11 @@ limitations under the License.
 
                 <configuration>
                     <includeEmptyDirs>true</includeEmptyDirs>
+                    <!-- test resources are filtered; the search index
+                         fixtures are binary -->
+                    <nonFilteredFileExtensions>
+                        <nonFilteredFileExtension>zip</nonFilteredFileExtension>
+                    </nonFilteredFileExtensions>
                 </configuration>
 
             </plugin>

app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java

@@ -40,9 +40,12 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFormatTooNewException;
+import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.store.Directory;
@@ -162,9 +165,16 @@ public void initialize() throws InitializationException {
                         }
                     }
                 } catch (IOException | IllegalArgumentException ex) {  // IAE for incompatible codecs
-                    logger.warn("Failed to open search index, scheduling rebuild.", ex);
+                    if (ex instanceof IndexFormatTooOldException || ex instanceof IndexFormatTooNewException) {
+                        // for example the Lucene 8 index of Roller 6.0
+                        logger.warn("Search index uses an unsupported format, rebuilding it: " + ex.getMessage());
+                    } else {
+                        logger.warn("Failed to open search index, scheduling rebuild.", ex);
+                    }
                     inconsistentAtStartup = true;
                     deleteIndex();
+                    // until the rebuild is done, searches find nothing rather than fail
+                    createIndex(getIndexDirectory());
                 }
             } else {
                 logger.debug("Creating index");
@@ -474,11 +484,14 @@ static SearchResultList convertHitsToEntryList(
             List<VisibleHit> visible = new ArrayList<>();
             Timestamp now = new Timestamp(new Date().getTime());
             long wanted = (long) (pageNum + 1) * entryCount + 1;
+            // a StoredFields may only be used on the thread that got it, and
+            // the search ran on another one
+            StoredFields storedFields = search.getSearcher().storedFields();
             for (ScoreDoc hit : hits) {
                 if (visible.size() >= wanted) {
                     break;
                 }
-                Document doc = search.getSearcher().doc(hit.doc);
+                Document doc = storedFields.document(hit.doc);
                 // maybe null if search result returned inactive user
                 // or entry's user is not the requested user
                 WeblogEntry entry = weblogMgr.getWeblogEntry(doc.getField(FieldConstants.ID).stringValue());

app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java

@@ -212,7 +212,7 @@ public int getResultsCount() {
         if (searchresults == null) {
             return -1;
         }
-        return (int) searchresults.totalHits.value;
+        return (int) searchresults.totalHits.value();
     }
 
     /**

app/src/test/java/org/apache/roller/weblogger/business/search/lucene/IndexUpgradeTest.java

@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.roller.weblogger.business.search.lucene;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Timestamp;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexFormatTooOldException;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+import org.apache.roller.weblogger.business.WeblogEntryManager;
+import org.apache.roller.weblogger.business.Weblogger;
+import org.apache.roller.weblogger.business.runnable.ThreadManager;
+import org.apache.roller.weblogger.config.WebloggerConfig;
+import org.apache.roller.weblogger.pojos.Weblog;
+import org.apache.roller.weblogger.pojos.WeblogEntry;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+import org.mockito.MockedStatic;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Starting on a search index that an earlier Roller release left behind.
+ * Each fixture holds two entries, written with the Lucene version and the
+ * index fields of the release it is named after.
+ */
+class IndexUpgradeTest {
+
+    @TempDir
+    Path indexDir;
+
+    @Test
+    void indexFromRoller60IsReplacedAndRebuilt() throws Exception {
+        // Roller 6.0 shipped Lucene 8.3, which this Lucene cannot read
+        unzip("roller-6.0-lucene-8.3.0.zip");
+        try (Directory dir = FSDirectory.open(indexDir)) {
+            assertThrows(IndexFormatTooOldException.class, () -> DirectoryReader.open(dir));
+        }
+
+        ThreadManager threads = mock(ThreadManager.class);
+        WeblogEntryManager entries = mock(WeblogEntryManager.class);
+        LuceneIndexManager manager = manager(threads, entries);
+        try {
+            manager.initialize();
+
+            assertTrue(manager.isInconsistentAtStartup());
+            ArgumentCaptor<Runnable> rebuild = ArgumentCaptor.forClass(Runnable.class);
+            verify(threads).executeInBackground(rebuild.capture());
+            assertInstanceOf(RebuildWebsiteIndexOperation.class, rebuild.getValue());
+            try (Directory dir = FSDirectory.open(indexDir)) {
+                assertEquals(Version.LATEST.major,
+                        SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor());
+            }
+            // until the rebuild fills it, searches find nothing rather than fail
+            assertEquals(0, search(manager));
+
+            List<WeblogEntry> published = List.of(
+                    entry("entry-1", "Roller rebuilds its search index after an upgrade."),
+                    entry("entry-2", "Another Roller entry about search."));
+            when(entries.getWeblogEntries(any())).thenReturn(published);
+            rebuild.getValue().run();
+            assertEquals(2, search(manager));
+        } finally {
+            manager.shutdown();
+        }
+    }
+
+    @Test
+    void indexFromRoller617IsKept() throws Exception {
+        // Roller 6.1 shipped Lucene 9, which this Lucene reads through
+        // lucene-backward-codecs, and 6.1.7 indexes the fields this one does
+        unzip("roller-6.1.7-lucene-9.12.3.zip");
+        try (Directory dir = FSDirectory.open(indexDir)) {
+            assertEquals(9, SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor());
+        }
+
+        ThreadManager threads = mock(ThreadManager.class);
+        LuceneIndexManager manager = manager(threads, mock(WeblogEntryManager.class));
+        try {
+            manager.initialize();
+
+            assertFalse(manager.isInconsistentAtStartup());
+            verify(threads, never()).executeInBackground(any());
+            assertEquals(2, search(manager));
+
+            // new entries go into the same index
+            new WriteToIndexOperation(manager) {
+                @Override
+                protected void doRun() {
+                    Document doc = new Document();
+                    doc.add(new StringField(FieldConstants.ID, "new-entry", Field.Store.YES));
+                    doc.add(new StringField(FieldConstants.WEBSITE_HANDLE, "upgrade", Field.Store.YES));
+                    doc.add(new TextField(FieldConstants.CONTENT, "Written by Roller 7", Field.Store.NO));
+                    try {
+                        beginWriting().addDocument(doc);
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
+                    } finally {
+                        endWriting();
+                    }
+                }
+            }.run();
+            assertEquals(3, search(manager));
+        } finally {
+            manager.shutdown();
+        }
+    }
+
+    private void unzip(String fixture) throws IOException {
+        try (InputStream in = getClass().getResourceAsStream("/search-index/" + fixture);
+             ZipInputStream zip = new ZipInputStream(in)) {
+            for (ZipEntry entry = zip.getNextEntry(); entry != null; entry = zip.getNextEntry()) {
+                Files.copy(zip, indexDir.resolve(Path.of(entry.getName()).getFileName()));
+            }
+        }
+    }
+
+    private LuceneIndexManager manager(ThreadManager threads, WeblogEntryManager entries) {
+        Weblogger roller = mock(Weblogger.class);
+        when(roller.getThreadManager()).thenReturn(threads);
+        when(roller.getWeblogEntryManager()).thenReturn(entries);
+        WebloggerConfig.init();
+        try (MockedStatic<WebloggerConfig> config = mockStatic(WebloggerConfig.class, CALLS_REAL_METHODS)) {
+            config.when(() -> WebloggerConfig.getProperty("search.index.dir"))
+                    .thenReturn(indexDir.toString());
+            return new LuceneIndexManager(roller);
+        }
+    }
+
+    private static WeblogEntry entry(String id, String text) {
+        Weblog weblog = mock(Weblog.class);
+        when(weblog.getHandle()).thenReturn("upgrade");
+        Timestamp time = new Timestamp(System.currentTimeMillis());
+        WeblogEntry entry = mock(WeblogEntry.class);
+        when(entry.getId()).thenReturn(id);
+        when(entry.getWebsite()).thenReturn(weblog);
+        when(entry.getTitle()).thenReturn(id);
+        when(entry.getLocale()).thenReturn("en_US");
+        when(entry.getText()).thenReturn(text);
+        when(entry.getUpdateTime()).thenReturn(time);
+        when(entry.getPubTime()).thenReturn(time);
+        return entry;
+    }
+
+    /** Number of entries matching "roller", or -1 if the search failed. */
+    private static int search(LuceneIndexManager manager) {
+        SearchOperation search = new SearchOperation(manager);
+        search.setTerm("roller");
+        search.run();
+        return search.getResultsCount();
+    }
+}

app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java

@@ -47,7 +47,7 @@ void plainAndSearchFieldQueriesParse() throws Exception {
 
     @Test
     void matchAllParses() throws Exception {
-        assertEquals(new MatchAllDocsQuery(), parser().parse("*:*"));
+        assertEquals(MatchAllDocsQuery.INSTANCE, parser().parse("*:*"));
     }
 
     @Test