Document Python stdout/stderr buffering behavior (#26624)

Document the behavior of IO with Chapel and Python interspersed, and
tell users how to deal with it.

- [x] `start_test test/library/packages/Python`
- [x] `start_test test/library/packages/Python` with GASNet

[Reviewed by @DanilaFe]

modules/packages/Python.chpl

@@ -23,8 +23,6 @@
 
 // TODO: implement operators as dunder methods
 
-// TODO: make python use chapel stdout/stderr
-
 /* Library for interfacing with Python from Chapel code.
 
   This module provides a Chapel interface to a Python interpreter.
@@ -297,6 +295,46 @@
   To translate custom Chapel types to Python objects, users should define and
   register custom :type:`TypeConverter` classes.
 
+  Notes on Python + Chapel I/O
+  ----------------------------
+
+  When interspersing Python and Chapel I/O, it is important to flush the output
+  buffers to ensure that the output is displayed in the correct order. This is
+  needed at the point where the output changes from Python to Chapel or
+  vice-versa. For example:
+
+  ..
+     START_TEST
+     FILENAME: Printing.chpl
+     START_GOOD
+     Hello from Chapel
+     Let's call some Python!
+     Hello, World!
+     Goodbye, World!
+     Back to Chapel
+     END_GOOD
+
+  .. code-block:: chapel
+
+     use Python, IO;
+
+     var interp = new Interpreter();
+     var func = new Function(interp, "lambda x,: print(x)");
+
+     writeln("Hello from Chapel");
+     writeln("Let's call some Python!");
+     IO.stdout.flush(); // flush the Chapel output buffer before calling Python
+
+     func(NoneType, "Hello, World!");
+     func(NoneType, "Goodbye, World!");
+     interp.flush(); // flush the Python output buffer before calling Chapel again
+
+     writeln("Back to Chapel");
+
+  ..
+     END_TEST
+
+
   More Examples:
   --------------
 
@@ -319,7 +357,7 @@ module Python {
     Use 'objgraph' to detect memory leaks in the Python code. Care should be
     taken when interpreting the output of this flag, not all memory leaks are
     under Chapel's control. For example, printing a Python list leaks memory
-    according to 'objgraph'. Furthermore, some memory is still held when until
+    according to 'objgraph'. Furthermore, some memory is still held until
     the interpreter is closed, like the module import cache.
   */
   config const pyMemLeaks = false;
@@ -461,10 +499,6 @@ module Python {
         PyList_Insert(path, 0, Py_BuildValue("s", "."));
         this.checkException();
       }
-      // TODO: reset stdout and stderr to Chapel's handles
-      // I think we can do this by setting sys.stdout and sys.stderr to a python
-      // object that looks like a python file but forwards calls like write to
-      // Chapel's write
 
       if !ArrayTypes.createArrayTypes() {
         throwChapelException("Failed to create Python array types for Chapel arrays");
@@ -687,6 +721,26 @@ module Python {
       }
     }
 
+    /*
+      Flush the standard output buffers of the Python interpreter. This is
+      useful when mixing Python and Chapel I/O to ensure that the output is
+      displayed in the correct order.
+    */
+    inline proc flush(flushStderr: bool = false) throws {
+      var stdout = PySys_GetObject("stdout");
+      if stdout == nil then throw new ChapelException("stdout not found");
+
+      var flushStr = this.toPython("flush");
+      defer Py_DECREF(flushStr);
+
+      PyObject_CallMethodNoArgs(stdout, flushStr);
+      if flushStderr {
+        var stderr = PySys_GetObject("stderr");
+        if stderr == nil then throw new ChapelException("stderr not found");
+        PyObject_CallMethodNoArgs(stderr, flushStr);
+      }
+    }
+
     @chpldoc.nodoc
     inline proc importModule(in modName: string): PyObjectPtr throws {
       var mod = PyImport_ImportModule(modName.c_str());
@@ -2003,6 +2057,8 @@ module Python {
     extern "chpl_PY_MINOR_VERSION" const PY_MINOR_VERSION: c_ulong;
     extern "chpl_PY_MICRO_VERSION" const PY_MICRO_VERSION: c_ulong;
 
+    extern proc PySys_GetObject(name: c_ptrConst(c_char)): PyObjectPtr;
+
 
     /*
       Sub Interpreters

test/library/packages/Python/correctness/argPassingTest.chpl

@@ -1,5 +1,6 @@
 use Python;
 import Reflection;
+use IO;
 
 proc test_no_args(mod: borrowed Module) {
   const funcName = "no_args";
@@ -9,13 +10,13 @@ proc test_no_args(mod: borrowed Module) {
 
   // error: wrong return type
   try { func(int); }
-  catch e: PythonException { writeln("Caught PythonException: ", e.message()); }
-  catch                    { writeln("Caught unknown exception"); }
+  catch e: PythonException { mod.interpreter.flush(); writeln("Caught PythonException: ", e.message()); IO.stdout.flush(); }
+  catch                    { mod.interpreter.flush(); writeln("Caught unknown exception"); IO.stdout.flush(); }
 
   // error: too many args
   try { func(NoneType, 2); }
-  catch e: PythonException { writeln("Caught PythonException: ", e.message()); }
-  catch                    { writeln("Caught unknown exception"); }
+  catch e: PythonException { mod.interpreter.flush(); writeln("Caught PythonException: ", e.message()); IO.stdout.flush(); }
+  catch                    { mod.interpreter.flush(); writeln("Caught unknown exception"); IO.stdout.flush(); }
 }
 proc test_one_arg(mod: borrowed Module) {
   const funcName = "one_arg";
@@ -25,13 +26,13 @@ proc test_one_arg(mod: borrowed Module) {
 
   // error: not enough args
   try { func(NoneType); }
-  catch e: PythonException { writeln("Caught PythonException: ", e.message()); }
-  catch                    { writeln("Caught unknown exception"); }
+  catch e: PythonException { mod.interpreter.flush(); writeln("Caught PythonException: ", e.message()); IO.stdout.flush(); }
+  catch                    { mod.interpreter.flush(); writeln("Caught unknown exception"); IO.stdout.flush(); }
 
   // error: too many args
   try { func(NoneType, 2, 3); }
-  catch e: PythonException { writeln("Caught PythonException: ", e.message()); }
-  catch                    { writeln("Caught unknown exception"); }
+  catch e: PythonException { mod.interpreter.flush(); writeln("Caught PythonException: ", e.message()); IO.stdout.flush(); }
+  catch                    { mod.interpreter.flush(); writeln("Caught unknown exception"); IO.stdout.flush(); }
 }
 proc test_two_args(mod: borrowed Module) {
   const funcName = "two_args";
@@ -43,8 +44,8 @@ proc test_two_args(mod: borrowed Module) {
 
   // error: not enough args
   try { func(NoneType, 3); }
-  catch e: PythonException { writeln("Caught PythonException: ", e.message()); }
-  catch                    { writeln("Caught unknown exception"); }
+  catch e: PythonException { mod.interpreter.flush(); writeln("Caught PythonException: ", e.message()); IO.stdout.flush(); }
+  catch                    { mod.interpreter.flush(); writeln("Caught unknown exception"); IO.stdout.flush(); }
 }
 proc test_three_args(mod: borrowed Module) {
   const funcName = "three_args";

test/library/packages/Python/correctness/argPassingTest.good

@@ -1,14 +1,14 @@
+called no_args
+called no_args
 Caught PythonException: 'NoneType' object cannot be interpreted as an integer
 Caught PythonException: no_args() takes 0 positional arguments but 1 was given
+called one_arg with 1
 Caught PythonException: one_arg() missing 1 required positional argument: 'a'
 Caught PythonException: one_arg() takes 1 positional argument but 2 were given
-Caught PythonException: two_args() missing 1 required positional argument: 'b'
-called no_args
-called no_args
-called one_arg with 1
 called two_args with 1 and 2
 called two_args with hello and world
 called two_args with None and None
+Caught PythonException: two_args() missing 1 required positional argument: 'b'
 called three_args with 1, 2, and 3
 called varargs
   args:

test/library/packages/Python/correctness/customType.chpl

@@ -1,6 +1,7 @@
 use Python;
 import Reflection;
 use CTypes;
+use IO;
 
 record myRec {
   var x: int;
@@ -35,6 +36,7 @@ proc main() {
   var pyClsType = new Class(m, "MyRec");
   interp.registerConverter(new myRecConverter(pyClsType));
 
+  IO.stdout.flush();
   var printAndReturn = new Function(m, "printAndReturn");
   var fromPy = printAndReturn(myRec, new myRec(42, "hello"));
   writeln(fromPy);

test/library/packages/Python/correctness/customType.good

@@ -1,2 +1,2 @@
-(x = 42, y = hello)
 MyRec(x=42, y=hello)
+(x = 42, y = hello)

test/library/packages/Python/correctness/customType.py

@@ -1,4 +1,5 @@
 
+import sys
 
 class MyRec:
     def __init__(self, x, y):
@@ -12,4 +13,5 @@ def __str__(self):
 def printAndReturn(rec):
     assert isinstance(rec, MyRec)
     print(rec)
+    sys.stdout.flush()
     return rec