diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 07264ef246..d38f2c4de6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -45,10 +45,10 @@ jobs: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v6 with: - java-version: 17 + java-version: 21 distribution: 'zulu' problem-matcher: false diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6eb0ad6c21..39165fbacb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ '17', '21', '25' ] + java: [ '21', '25' ] steps: - name: Set up JDK ${{ matrix.java }} @@ -63,7 +63,7 @@ jobs: # only on integration and only once in this matrix - name: Upload Dev Build on Integration - if: ${{ (matrix.java == '17') && (github.event_name == 'push') }} + if: ${{ (matrix.java == '21') && (github.event_name == 'push') }} uses: actions/upload-artifact@v7 with: name: dev-build @@ -79,10 +79,10 @@ jobs: timeout-minutes: 10 steps: - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v6 with: - java-version: '17' + java-version: '21' distribution: 'zulu' - name: Checkout Project @@ -105,10 +105,10 @@ jobs: timeout-minutes: 30 steps: - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v6 with: - java-version: '17' + java-version: '21' distribution: 'zulu' - name: Checkout Project @@ -167,10 +167,10 @@ jobs: auth: [ 'oidc', 'db-oidc' ] steps: - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v6 with: - java-version: '17' + java-version: '21' distribution: 'zulu' - name: Checkout Project diff --git a/CHANGES.md b/CHANGES.md index 3144129598..e089c61a7b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -114,9 +114,16 @@ error and continues startup. - **Roller now runs on Jakarta EE** - ([ROL-2183](https://issues.apache.org/jira/browse/ROL-2183)). It needs Java 17 + ([ROL-2183](https://issues.apache.org/jira/browse/ROL-2183)). It needs Java 21 or later and a Servlet 6.1 container such as Tomcat 11. Tomcat 9 and other `javax.servlet` containers can no longer run Roller. +- **The search index is rebuilt when upgrading from Roller 6.1.6 or + earlier.** Roller 7 uses Lucene 10, which reads the index Roller 6.1.7 + writes. An older index either holds fields Roller no longer indexes or, + like the Lucene 8 index of Roller 6.0, can't be read at all. Roller + replaces it at first startup and rebuilds it in the background, so no + action is needed. Search results are complete again once the log shows + "Completed rebuilding index". - **OAuth 1.0a is removed** ([ROL-2183](https://issues.apache.org/jira/browse/ROL-2183)). AtomPub accepts only BASIC authentication, and the OAuth credentials page and consumer keys are gone. If `webservices.atomPubAuth` is still set to `oauth`, diff --git a/Dockerfile b/Dockerfile index 09e0a5fe2a..51b68bff8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ # STAGE 1 - BUILD ------------------------------------------------ -FROM maven:3-eclipse-temurin-17 AS builder +FROM maven:3.9.16-eclipse-temurin-21 AS builder COPY . /project WORKDIR /project @@ -30,7 +30,7 @@ RUN mvn -Duser.home=/builder/home -DskipTests=true -B clean install # STAGE 2 - PACKAGE ------------------------------------------------ -FROM tomcat:11.0-jdk17-temurin +FROM tomcat:11.0.29-jdk21-temurin # Remove existing Tomcat webapps diff --git a/README.md b/README.md index 4f258c5406..f7ad178472 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ If you want to run Roller in production, then you should down load the latest of ## Quick start: Running via Maven You probably should not run Roller in production using this technique, but it's a relatively easy way to try Roller for yourself. -Assuming you've got a UNIX shell, Java, Maven and Git: +Assuming you've got a UNIX shell, Java 21 or later, Maven and Git: Get the code: diff --git a/app/pom.xml b/app/pom.xml index e959000802..e508f25b85 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -48,7 +48,7 @@ limitations under the License. 5.0.2 7.0.0 2.26.1 - 9.12.3 + 10.6.0 3.5.1 3.6.0 1.0b3 @@ -737,6 +737,11 @@ limitations under the License. true + + + zip + diff --git a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java index fc7132a65a..0accc1e058 100644 --- a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java +++ b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java @@ -40,9 +40,12 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; @@ -162,9 +165,16 @@ public void initialize() throws InitializationException { } } } catch (IOException | IllegalArgumentException ex) { // IAE for incompatible codecs - logger.warn("Failed to open search index, scheduling rebuild.", ex); + if (ex instanceof IndexFormatTooOldException || ex instanceof IndexFormatTooNewException) { + // for example the Lucene 8 index of Roller 6.0 + logger.warn("Search index uses an unsupported format, rebuilding it: " + ex.getMessage()); + } else { + logger.warn("Failed to open search index, scheduling rebuild.", ex); + } inconsistentAtStartup = true; deleteIndex(); + // until the rebuild is done, searches find nothing rather than fail + createIndex(getIndexDirectory()); } } else { logger.debug("Creating index"); @@ -474,11 +484,14 @@ static SearchResultList convertHitsToEntryList( List visible = new ArrayList<>(); Timestamp now = new Timestamp(new Date().getTime()); long wanted = (long) (pageNum + 1) * entryCount + 1; + // a StoredFields may only be used on the thread that got it, and + // the search ran on another one + StoredFields storedFields = search.getSearcher().storedFields(); for (ScoreDoc hit : hits) { if (visible.size() >= wanted) { break; } - Document doc = search.getSearcher().doc(hit.doc); + Document doc = storedFields.document(hit.doc); // maybe null if search result returned inactive user // or entry's user is not the requested user WeblogEntry entry = weblogMgr.getWeblogEntry(doc.getField(FieldConstants.ID).stringValue()); diff --git a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java index 8a651ec26d..145686ceda 100644 --- a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java +++ b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java @@ -212,7 +212,7 @@ public int getResultsCount() { if (searchresults == null) { return -1; } - return (int) searchresults.totalHits.value; + return (int) searchresults.totalHits.value(); } /** diff --git a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/IndexUpgradeTest.java b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/IndexUpgradeTest.java new file mode 100644 index 0000000000..9141535c83 --- /dev/null +++ b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/IndexUpgradeTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.roller.weblogger.business.search.lucene; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Timestamp; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.Version; +import org.apache.roller.weblogger.business.WeblogEntryManager; +import org.apache.roller.weblogger.business.Weblogger; +import org.apache.roller.weblogger.business.runnable.ThreadManager; +import org.apache.roller.weblogger.config.WebloggerConfig; +import org.apache.roller.weblogger.pojos.Weblog; +import org.apache.roller.weblogger.pojos.WeblogEntry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Starting on a search index that an earlier Roller release left behind. + * Each fixture holds two entries, written with the Lucene version and the + * index fields of the release it is named after. + */ +class IndexUpgradeTest { + + @TempDir + Path indexDir; + + @Test + void indexFromRoller60IsReplacedAndRebuilt() throws Exception { + // Roller 6.0 shipped Lucene 8.3, which this Lucene cannot read + unzip("roller-6.0-lucene-8.3.0.zip"); + try (Directory dir = FSDirectory.open(indexDir)) { + assertThrows(IndexFormatTooOldException.class, () -> DirectoryReader.open(dir)); + } + + ThreadManager threads = mock(ThreadManager.class); + WeblogEntryManager entries = mock(WeblogEntryManager.class); + LuceneIndexManager manager = manager(threads, entries); + try { + manager.initialize(); + + assertTrue(manager.isInconsistentAtStartup()); + ArgumentCaptor rebuild = ArgumentCaptor.forClass(Runnable.class); + verify(threads).executeInBackground(rebuild.capture()); + assertInstanceOf(RebuildWebsiteIndexOperation.class, rebuild.getValue()); + try (Directory dir = FSDirectory.open(indexDir)) { + assertEquals(Version.LATEST.major, + SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor()); + } + // until the rebuild fills it, searches find nothing rather than fail + assertEquals(0, search(manager)); + + List published = List.of( + entry("entry-1", "Roller rebuilds its search index after an upgrade."), + entry("entry-2", "Another Roller entry about search.")); + when(entries.getWeblogEntries(any())).thenReturn(published); + rebuild.getValue().run(); + assertEquals(2, search(manager)); + } finally { + manager.shutdown(); + } + } + + @Test + void indexFromRoller617IsKept() throws Exception { + // Roller 6.1 shipped Lucene 9, which this Lucene reads through + // lucene-backward-codecs, and 6.1.7 indexes the fields this one does + unzip("roller-6.1.7-lucene-9.12.3.zip"); + try (Directory dir = FSDirectory.open(indexDir)) { + assertEquals(9, SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor()); + } + + ThreadManager threads = mock(ThreadManager.class); + LuceneIndexManager manager = manager(threads, mock(WeblogEntryManager.class)); + try { + manager.initialize(); + + assertFalse(manager.isInconsistentAtStartup()); + verify(threads, never()).executeInBackground(any()); + assertEquals(2, search(manager)); + + // new entries go into the same index + new WriteToIndexOperation(manager) { + @Override + protected void doRun() { + Document doc = new Document(); + doc.add(new StringField(FieldConstants.ID, "new-entry", Field.Store.YES)); + doc.add(new StringField(FieldConstants.WEBSITE_HANDLE, "upgrade", Field.Store.YES)); + doc.add(new TextField(FieldConstants.CONTENT, "Written by Roller 7", Field.Store.NO)); + try { + beginWriting().addDocument(doc); + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + endWriting(); + } + } + }.run(); + assertEquals(3, search(manager)); + } finally { + manager.shutdown(); + } + } + + private void unzip(String fixture) throws IOException { + try (InputStream in = getClass().getResourceAsStream("/search-index/" + fixture); + ZipInputStream zip = new ZipInputStream(in)) { + for (ZipEntry entry = zip.getNextEntry(); entry != null; entry = zip.getNextEntry()) { + Files.copy(zip, indexDir.resolve(Path.of(entry.getName()).getFileName())); + } + } + } + + private LuceneIndexManager manager(ThreadManager threads, WeblogEntryManager entries) { + Weblogger roller = mock(Weblogger.class); + when(roller.getThreadManager()).thenReturn(threads); + when(roller.getWeblogEntryManager()).thenReturn(entries); + WebloggerConfig.init(); + try (MockedStatic config = mockStatic(WebloggerConfig.class, CALLS_REAL_METHODS)) { + config.when(() -> WebloggerConfig.getProperty("search.index.dir")) + .thenReturn(indexDir.toString()); + return new LuceneIndexManager(roller); + } + } + + private static WeblogEntry entry(String id, String text) { + Weblog weblog = mock(Weblog.class); + when(weblog.getHandle()).thenReturn("upgrade"); + Timestamp time = new Timestamp(System.currentTimeMillis()); + WeblogEntry entry = mock(WeblogEntry.class); + when(entry.getId()).thenReturn(id); + when(entry.getWebsite()).thenReturn(weblog); + when(entry.getTitle()).thenReturn(id); + when(entry.getLocale()).thenReturn("en_US"); + when(entry.getText()).thenReturn(text); + when(entry.getUpdateTime()).thenReturn(time); + when(entry.getPubTime()).thenReturn(time); + return entry; + } + + /** Number of entries matching "roller", or -1 if the search failed. */ + private static int search(LuceneIndexManager manager) { + SearchOperation search = new SearchOperation(manager); + search.setTerm("roller"); + search.run(); + return search.getResultsCount(); + } +} diff --git a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java index 047921cb5f..c12eee08f4 100644 --- a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java +++ b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java @@ -47,7 +47,7 @@ void plainAndSearchFieldQueriesParse() throws Exception { @Test void matchAllParses() throws Exception { - assertEquals(new MatchAllDocsQuery(), parser().parse("*:*")); + assertEquals(MatchAllDocsQuery.INSTANCE, parser().parse("*:*")); } @Test diff --git a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchHitPagingTest.java b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchHitPagingTest.java index 1c3cf3851d..a7edc423c0 100644 --- a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchHitPagingTest.java +++ b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchHitPagingTest.java @@ -21,6 +21,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.roller.weblogger.business.URLStrategy; @@ -119,13 +120,15 @@ private static WeblogEntryManager entries(int draftIndex) throws Exception { private static SearchResultList convert(WeblogEntryManager entries, int page, int size) throws Exception { IndexSearcher searcher = mock(IndexSearcher.class); + StoredFields storedFields = mock(StoredFields.class); + when(searcher.storedFields()).thenReturn(storedFields); ScoreDoc[] hits = new ScoreDoc[HITS]; for (int i = 0; i < HITS; i++) { hits[i] = new ScoreDoc(i, 1f); Document doc = new Document(); doc.add(new StringField(FieldConstants.ID, "id" + i, Field.Store.YES)); doc.add(new StringField(FieldConstants.WEBSITE_HANDLE, "team", Field.Store.YES)); - when(searcher.doc(i)).thenReturn(doc); + when(storedFields.document(i)).thenReturn(doc); } SearchOperation search = mock(SearchOperation.class); when(search.getSearcher()).thenReturn(searcher); diff --git a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchResultLimitTest.java b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchResultLimitTest.java index 699238fb53..cb51fdb6f5 100644 --- a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchResultLimitTest.java +++ b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchResultLimitTest.java @@ -67,7 +67,7 @@ void countsEveryMatchPastFiveHundredAndPastOneThousand() throws Exception { SearchOperation search = search(reader); search.run(); assertEquals(MATCHES, search.getResultsCount()); - assertEquals(TotalHits.Relation.EQUAL_TO, search.getResults().totalHits.relation); + assertEquals(TotalHits.Relation.EQUAL_TO, search.getResults().totalHits.relation()); assertEquals(MATCHES, search.getResults().scoreDocs.length); } } @@ -105,7 +105,7 @@ void pagingStopsAtTheLimitButTheTotalDoesNot() throws Exception { // an exact count, not the lower bound Lucene gives by default // when there are more than 1,000 matches and fewer are kept assertEquals(MATCHES, search.getResultsCount()); - assertEquals(TotalHits.Relation.EQUAL_TO, search.getResults().totalHits.relation); + assertEquals(TotalHits.Relation.EQUAL_TO, search.getResults().totalHits.relation()); assertEquals(max, search.getResults().scoreDocs.length); // the page before the limit has a next page diff --git a/app/src/test/resources/search-index/roller-6.0-lucene-8.3.0.zip b/app/src/test/resources/search-index/roller-6.0-lucene-8.3.0.zip new file mode 100644 index 0000000000..05277f4985 Binary files /dev/null and b/app/src/test/resources/search-index/roller-6.0-lucene-8.3.0.zip differ diff --git a/app/src/test/resources/search-index/roller-6.1.7-lucene-9.12.3.zip b/app/src/test/resources/search-index/roller-6.1.7-lucene-9.12.3.zip new file mode 100644 index 0000000000..ddc40b4015 Binary files /dev/null and b/app/src/test/resources/search-index/roller-6.1.7-lucene-9.12.3.zip differ diff --git a/docs/roller-install-guide.adoc b/docs/roller-install-guide.adoc index 7cf2c81a40..f8558427df 100644 --- a/docs/roller-install-guide.adoc +++ b/docs/roller-install-guide.adoc @@ -87,18 +87,16 @@ Java, a Java Servlet container such as Tomcat, a connection to a database such as MySQL and optionally a connect to a mail server. More specifically, here’s what you need to install and run Roller: -* *Java Development Kit*, specifically the Java 2 SE 1.6 JDK or more -recent. The computer on which you install Roller should already have the -JDK installed and configured. -* *Java EE 6 Application Server*, or more specifically a Servlet -container that supports at least the Servlet 2.4 API. Hereinafter, we’ll -just call this your _server_. Roller has traditionally worked best on -Tomcat and Tomcat only, but Roller 6.0.0 is known to run on: -** Tomcat 8 -** Jetty 9 +* *Java Development Kit*, specifically Java 21 or later. The computer on +which you install Roller should already have the JDK installed and +configured. +* *Jakarta EE Servlet container* that supports the Servlet 6.1 API. +Hereinafter, we’ll just call this your _server_. Roller is tested on: +** Tomcat 11 +** Jetty 12 (EE 11) * *Relational database* such as MySQL or Apache Derby. Roller stores blog entries, comments, bookmarks and almost all other data in a -relational database accessed via the Java Persistence API 2.0. +relational database accessed via Jakarta Persistence. PostgresSQL, MySQL and Derby are best supported but Roller. Roller also includes database creation scripts for DB2, HSQL-DB, Microsoft SQL Server and Oracle but these have not been tested recently. @@ -108,8 +106,8 @@ for comments and other events via the JavaMail and Activation APIs. Roller WAR file, ready to deploy to your server, plus Roller license files, README and documentation. Unpack using either file based on a compression method your operating system supports: -** roller-release-6.0.0-standard.zip -** roller-release-6.0.0-standard.tar.gz +** apache-roller--binary.zip +** apache-roller--binary.tar.gz This step can also be done after deployment and determination that you would like themes other than the defaults available available, just @@ -1118,3 +1116,14 @@ Roller. *NOTE*: On Tomcat, before startup you should delete the contents of the Tomcat work directory (located under the webapps folder.) + +*NOTE*: Roller 7 keeps a search index written by Roller 6.1.7. It replaces +an older index at first startup and rebuilds it in the background, with no +action needed: an index from Roller 6.1.6 or earlier holds fields Roller no +longer indexes, and Roller 7's Lucene 10 can't read the Lucene 8 index of +Roller 6.0. Search results are complete again once the log shows +"Completed rebuilding index". To force a rebuild at any time, stop Roller +and delete the directory that `search.index.dir` names. +If you go back to Roller 6.1.x afterwards, it can't read the index Roller 7 +wrote. It deletes the index and rebuilds it in the background, and searches +return an error until that rebuild finishes. diff --git a/it-playwright/pom.xml b/it-playwright/pom.xml index 212b54f0f7..54f6912a26 100644 --- a/it-playwright/pom.xml +++ b/it-playwright/pom.xml @@ -48,7 +48,7 @@ --> - 17 + 21 UTF-8 1.63.0 6.1.3 diff --git a/pom.xml b/pom.xml index c58c942fe5..fbb8f939ee 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ limitations under the License. - 10.16.1.1 + 10.17.1.0 12.1.15 UTF-8 UTF-8 @@ -64,7 +64,7 @@ limitations under the License. maven-compiler-plugin 3.16.0 - 17 + 21 true -Xlint:all