[GR-75717] Implement PyREPL support

CHANGELOG.md

@@ -4,11 +4,12 @@ This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
 ## Version 25.6
-- New Java-based `decimal` module with significanlty better performance
+* Add support for PyREPL and make it the default. It can be turned off by setting `PYTHON_BASIC_REPL=1` environment variable
+* New Java-based `decimal` module with significantly better performance
 
 ## Version 25.5
-- Added `select.poll()` on Linux and Mac OS
-- Better Windows console support and Windows version reporting
+* Added `select.poll()` on Linux and Mac OS
+* Better Windows console support and Windows version reporting
 
 ## Version 25.4
 * Remove BouncyCastle entirely. BouncyCastle was only needed for legacy RSA, DSA, and EC private keys versions 0 and 1. To support these from Python embeddings, BouncyCastle must now be explicitly enabled by adding the `org.graalvm.python:python-bouncycastle-support` Maven _and_ adding `bcprov-jdk18on`, `bcpkix-jdk18on`, and `bcutil-jdk18on` dependencies.

ci/python-bench.libsonnet

@@ -260,7 +260,8 @@
             ruby: "==3.1.2",
             papi: "==5.5.1",
         },
-        environment +: environment(self.os, self.arch) + {
+        // Benchmarks may download build dependencies, such as Rust for pydantic-core.
+        environment +: environment(self.os, self.arch) + $.overlay_imports.INTERNET_ACCESS_ENV + {
             BISECT_BENCHMARK_CONFIG: "bisect-benchmark.ini",
             ENABLE_POLYBENCH_HPC: "yes",
             POLYBENCH_HPC_EXTRA_HEADERS: "/cm/shared/apps/papi/papi-5.5.1/include",

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java

@@ -113,6 +113,7 @@ public enum Flags {
         INTERACTIVE_TERMINAL,
         ASYNC_HACKS,
         ALLOW_INCOMPLETE_INPUT,
+        DONT_IMPLY_DEDENT,
     }
 
     private static final String BARRY_AS_BDFL = "with Barry as BDFL, use '<>' instead of '!='";
@@ -358,7 +359,7 @@ public Token getAndInitializeToken() {
         if (startRule == InputType.SINGLE && token.type == Token.Kind.ENDMARKER && parsingStarted) {
             token.type = Token.Kind.NEWLINE;
             parsingStarted = false;
-            if (tokenizer.getCurrentIndentIndex() > 0) {
+            if (tokenizer.getCurrentIndentIndex() > 0 && !flags.contains(Flags.DONT_IMPLY_DEDENT)) {
                 tokenizer.setPendingIndents(-tokenizer.getCurrentIndentIndex());
                 tokenizer.setCurrentIndentIndex(0);
             }

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -162,6 +162,7 @@ private static void appendDecodedChars(StringBuilder builder, CharBuffer out) {
     private int repeatedRuns = 1;
     private boolean snaptshotStartup = false;
     private boolean warnDefaultEncoding = false;
+    private final List<String> xOptions = new ArrayList<>();
     private int intMaxStrDigits = -1;
     private int cpuCount = -1;
     private boolean cpuCountXOptionSet = false;
@@ -462,6 +463,7 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
                         case 'X':
                             // CPython ignores unknown/unsupported -X options, so we can do that too
                             String xOption = getShortOptionParameter(argumentIterator, remainder, 'X');
+                            xOptions.add(xOption);
                             if ("warn_default_encoding".equals(xOption)) {
                                 warnDefaultEncoding = true;
                             } else if (xOption.startsWith("int_max_str_digits")) {
@@ -857,6 +859,11 @@ protected void launch(Builder contextBuilder) {
 
         origArgs.add(0, toAbsolutePath(executable));
         contextBuilder.option("python.OrigArgv", String.join(STRING_LIST_DELIMITER, origArgs));
+        StringBuilder serializedXOptions = new StringBuilder();
+        for (String xOption : xOptions) {
+            serializedXOptions.append(xOption.length()).append(':').append(xOption);
+        }
+        contextBuilder.option("python.XOptions", serializedXOptions.toString());
 
         // setting this to make sure our TopLevelExceptionHandler calls the excepthook
         // to print Python exceptions
@@ -929,8 +936,9 @@ protected void launch(Builder contextBuilder) {
                     }
                 }
 
+                boolean basicReplRequested = !tty || (!ignoreEnv && getBoolEnv("PYTHON_BASIC_REPL"));
                 sysModule.putMember("_console_handler", consoleHandler);
-                if (tty && !isolateFlag && (inspectFlag || (commandString == null && inputFile == null))) {
+                if (basicReplRequested && tty && !isolateFlag && (inspectFlag || (commandString == null && inputFile == null))) {
                     evalInternal(context, "import readline, rlcompleter");
                 }
 
@@ -949,8 +957,18 @@ protected void launch(Builder contextBuilder) {
                     }
                 }
                 if ((tty && commandString == null && inputFile == null) || inspectFlag) {
+                    boolean inspecting = commandString != null || inputFile != null;
                     inspectFlag = false;
-                    rc = readEvalPrint(context, consoleHandler, sysModule);
+                    runInteractiveHook(sysModule);
+                    if (inspecting) {
+                        // The inspected program may have changed os.environ since the launcher started.
+                        basicReplRequested = !tty || (!ignoreEnv && evalInternal(context, "import os; bool(os.environ.get('PYTHON_BASIC_REPL'))").asBoolean());
+                    }
+                    if (!basicReplRequested && canUsePyRepl(context)) {
+                        rc = readEvalPrintPyRepl(context, inspecting);
+                    } else {
+                        rc = readEvalPrint(context, consoleHandler, sysModule);
+                    }
                 }
             } catch (RuntimeException e) {
                 if (e.getMessage() != null && e.getMessage().contains("did not complete all polyglot threads")) {
@@ -1343,6 +1361,55 @@ private static ConsoleHandler createConsoleHandler(InputStream inStream, OutputS
         }
     }
 
+    private boolean canUsePyRepl(Context context) {
+        try {
+            return evalInternal(context, """
+                            from _pyrepl.main import CAN_USE_PYREPL, FAIL_REASON
+                            if not CAN_USE_PYREPL and FAIL_REASON:
+                                import sys
+                                from _pyrepl.trace import trace
+                                trace(FAIL_REASON)
+                                print(FAIL_REASON, file=sys.stderr)
+                            CAN_USE_PYREPL""").asBoolean();
+        } catch (PolyglotException e) {
+            return false;
+        }
+    }
+
+    private int readEvalPrintPyRepl(Context context, boolean inspecting) {
+        String source;
+        if (inspecting) {
+            source = """
+                            from _pyrepl.main import interactive_console
+                            interactive_console()""";
+        } else {
+            source = """
+                            import runpy
+                            runpy._run_module_as_main('_pyrepl')""";
+        }
+        try {
+            evalInternal(context, source);
+            return 0;
+        } catch (PolyglotException e) {
+            if (e.isExit()) {
+                return e.getExitStatus();
+            }
+            throw e;
+        }
+    }
+
+    private static void runInteractiveHook(Value sysModule) {
+        Value hook = null;
+        try {
+            hook = sysModule.getMember("__interactivehook__");
+        } catch (PolyglotException e) {
+            // ignore
+        }
+        if (hook != null) {
+            hook.execute();
+        }
+    }
+
     /**
      * The read-eval-print loop, which can take input from a console, command line expression or a
      * file. There are two ways the repl can terminate:
@@ -1355,15 +1422,6 @@ private static ConsoleHandler createConsoleHandler(InputStream inStream, OutputS
      */
     private int readEvalPrint(Context context, ConsoleHandler consoleHandler, Value sysModule) {
         try {
-            Value hook = null;
-            try {
-                hook = sysModule.getMember("__interactivehook__");
-            } catch (PolyglotException e) {
-                // ignore
-            }
-            if (hook != null) {
-                hook.execute();
-            }
             while (true) { // processing inputs
                 boolean doEcho = doEcho(context);
                 String ps1 = doEcho ? sysModule.getMember("ps1").asString() : null;
@@ -1498,7 +1556,7 @@ private static boolean canSkipFromEval(String input) {
     }
 
     private Value evalInternal(Context context, String code) {
-        return context.eval(Source.newBuilder(getLanguageId(), code, "<internal>").internal(true).buildLiteral());
+        return context.eval(Source.newBuilder(getLanguageId(), code, "<internal>").internal(true).option("python.NewGlobals", "true").buildLiteral());
     }
 
     /**

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/JLineConsoleHandler.java

@@ -100,7 +100,8 @@ public void initializeReadline(Value readlineModule) {
         try {
             // The grapheme capability probe consumes pending input, including commands sent through
             // a pseudo-terminal before readline is initialized (GR-80639).
-            terminal = TerminalBuilder.builder().streams(inputStream, outputStream).system(true).graphemeCluster(false).build();
+            // PyREPL also imports readline. Do not consume terminal input until readLine is used.
+            terminal = TerminalBuilder.builder().streams(inputStream, outputStream).system(true).graphemeCluster(false).paused(true).build();
         } catch (IOException ex) {
             throw new RuntimeException("unexpected error opening console reader", ex);
         } finally {
@@ -236,7 +237,14 @@ public String readLine(String prompt) {
         if (lineBuffer.isEmpty()) {
             Object[] signalHandlers = stashSignalHandlers();
             try {
-                String lines = reader.readLine(prompt);
+                Terminal terminal = reader.getTerminal();
+                String lines;
+                terminal.resume();
+                try {
+                    lines = reader.readLine(prompt);
+                } finally {
+                    terminal.pause(true);
+                }
                 Collections.addAll(lineBuffer, lines.split("\n"));
             } catch (UserInterruptException e) {
                 throw e;

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/compiler/UnparserTests.java

@@ -42,10 +42,14 @@
 
 import static org.junit.Assert.assertEquals;
 
+import java.util.EnumSet;
+
 import org.junit.Test;
 
+import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.compiler.Unparser;
 import com.oracle.graal.python.compiler.bytecode_dsl.BytecodeDSLCompiler;
+import com.oracle.graal.python.pegparser.AbstractParser;
 import com.oracle.graal.python.pegparser.InputType;
 import com.oracle.graal.python.pegparser.Parser;
 import com.oracle.graal.python.pegparser.ParserCallbacks;
@@ -93,7 +97,7 @@ public void testUnparseConstant() {
 
     private static void checkRoundTrip(String source) {
         ParserCallbacks parserCallbacks = new TestParserCallbacksImpl();
-        Parser parser = BytecodeDSLCompiler.createParser(source, parserCallbacks, InputType.EVAL, false, false);
+        Parser parser = BytecodeDSLCompiler.createParser(source, parserCallbacks, InputType.EVAL, EnumSet.noneOf(AbstractParser.Flags.class), PythonLanguage.MINOR);
         ModTy.Expression result = (ModTy.Expression) parser.parse();
         assertEquals(source, unparse(result.body));
     }

graalpython/com.oracle.graal.python.test/src/tests/test_cmd_line.py

@@ -98,6 +98,16 @@ def test_cpu_count_invalid(self):
                 result = self.run_cpu_count(cpu_count_env=value)
                 self.assertNotEqual(0, result.returncode)
 
+    def test_xoptions(self):
+        result = subprocess.run(
+            [sys.executable, '-Xa', '-Xb=c,d=e', '-Xc', '-Xa=overridden', '-X', '', '-c',
+             "import sys; print(tuple(sys._xoptions[key] for key in ('a', 'b', 'c', '')))"],
+            capture_output=True,
+            text=True,
+        )
+        self.assertEqual(0, result.returncode, result)
+        self.assertEqual("('overridden', 'c,d=e', True, True)", result.stdout.strip())
+
     def test_stdin_script_exit_code(self):
         code = "import sys\nsys.exit(42)\n"
         result = subprocess.run([sys.executable], input=code, text=True)

graalpython/com.oracle.graal.python.test/src/tests/test_parser.py

@@ -82,6 +82,20 @@ def test_compile_unrecognised_flags_message():
         assert False, "ValueError was not raised"
 
 
+def test_dont_imply_dedent():
+    flags = 0x4000 | 0x200  # PyCF_ALLOW_INCOMPLETE_INPUT | PyCF_DONT_IMPLY_DEDENT
+    source = "def foo():\n    return 42"
+
+    try:
+        compile(source, "<input>", "single", flags)
+    except _IncompleteInputError:
+        pass
+    else:
+        assert False, "_IncompleteInputError was not raised"
+
+    assert compile(source + "\n", "<input>", "single", flags)
+
+
 def test_lambdas_as_function_default_argument_values():
     globs = {}
     exec("""