From 076452f020da716d25abf1e3e6d222cf2dcfc2d7 Mon Sep 17 00:00:00 2001 From: esendjer Date: Wed, 5 Jan 2022 20:23:34 +0500 Subject: [PATCH 1/7] added goroutines example --- examples/goroutines/foo.py | 17 ++++++++ examples/goroutines/main.go | 77 +++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 examples/goroutines/foo.py create mode 100644 examples/goroutines/main.go diff --git a/examples/goroutines/foo.py b/examples/goroutines/foo.py new file mode 100644 index 0000000..9f87896 --- /dev/null +++ b/examples/goroutines/foo.py @@ -0,0 +1,17 @@ +import sys + +def print_odds(limit=10): + """ + Print odds numbers < limit + """ + for i in range(limit): + if i%2: + sys.stderr.write("odds: {}\n".format(i)) + +def print_even(limit=10): + """ + Print even numbers < limit + """ + for i in range(limit): + if i%2 == 0: + sys.stderr.write("even: {}\n".format(i)) diff --git a/examples/goroutines/main.go b/examples/goroutines/main.go new file mode 100644 index 0000000..954b166 --- /dev/null +++ b/examples/goroutines/main.go @@ -0,0 +1,77 @@ +package main + +import ( + "fmt" + "os" + "sync" + + python3 "github.com/go-python/cpy3" +) + +func main() { + // The following will also create the GIL explicitly + // by calling PyEval_InitThreads(), without waiting + // for the interpreter to do that + python3.Py_Initialize() + + if !python3.Py_IsInitialized() { + fmt.Println("Error initializing the python interpreter") + os.Exit(1) + } + + var wg sync.WaitGroup + wg.Add(2) + + fooModule := python3.PyImport_ImportModule("foo") + if fooModule == nil { + fmt.Println("The module fooModule doesn't exist") + } + oddsAttr := "print_odds" + evenAttr := "print_even" + + if !fooModule.HasAttrString(oddsAttr) { + fmt.Printf("The module doesn't have attribute %s", oddsAttr) + os.Exit(1) + } + + if !fooModule.HasAttrString(evenAttr) { + fmt.Printf("The module doesn't have attribute %s", evenAttr) + os.Exit(1) + } + + odds := fooModule.GetAttrString(oddsAttr) + even := fooModule.GetAttrString(evenAttr) + + if odds == nil || even == nil { + panic("Error importing function") + } + + // Py_Initialize() has locked the the GIL but at this point we don't need it + // anymore. We save the current state and release the lock + // so that goroutines can acquire it + state := python3.PyEval_SaveThread() + + go func() { + _gstate := python3.PyGILState_Ensure() + odds.Call(python3.PyTuple_New(0), python3.PyDict_New()) + python3.PyGILState_Release(_gstate) + + wg.Done() + }() + + go func() { + _gstate := python3.PyGILState_Ensure() + even.Call(python3.PyTuple_New(0), python3.PyDict_New()) + python3.PyGILState_Release(_gstate) + + wg.Done() + }() + + wg.Wait() + + // At this point we know we won't need python anymore in this + // program, we can restore the state and lock the GIL to perform + // the final operations before exiting. + python3.PyEval_RestoreThread(state) + python3.Py_Finalize() +} From c871e091cf4442948d660b84658c5f7186a680b6 Mon Sep 17 00:00:00 2001 From: esendjer Date: Thu, 6 Jan 2022 09:06:30 +0500 Subject: [PATCH 2/7] a little fix: Moved the name of foo python module to a variable. Added line breaks in Printf --- examples/goroutines/main.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/goroutines/main.go b/examples/goroutines/main.go index 954b166..1e813e5 100644 --- a/examples/goroutines/main.go +++ b/examples/goroutines/main.go @@ -22,20 +22,21 @@ func main() { var wg sync.WaitGroup wg.Add(2) - fooModule := python3.PyImport_ImportModule("foo") + nameModule := "foo" + fooModule := python3.PyImport_ImportModule(nameModule) if fooModule == nil { - fmt.Println("The module fooModule doesn't exist") + fmt.Printf("The module %s doesn't exist\n", nameModule) } oddsAttr := "print_odds" evenAttr := "print_even" if !fooModule.HasAttrString(oddsAttr) { - fmt.Printf("The module doesn't have attribute %s", oddsAttr) + fmt.Printf("The module doesn't have attribute %s\n", oddsAttr) os.Exit(1) } if !fooModule.HasAttrString(evenAttr) { - fmt.Printf("The module doesn't have attribute %s", evenAttr) + fmt.Printf("The module doesn't have attribute %s\n", evenAttr) os.Exit(1) } From e6a8b562ad1a53a535cb20e60ee3c878f808867a Mon Sep 17 00:00:00 2001 From: esendjer Date: Thu, 6 Jan 2022 17:04:58 +0500 Subject: [PATCH 3/7] Updated according to the comment - https://github.com/go-python/cpy3/pull/5#issuecomment-1006287080 --- examples/goroutines/main.go | 62 ++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/examples/goroutines/main.go b/examples/goroutines/main.go index 1e813e5..94f6486 100644 --- a/examples/goroutines/main.go +++ b/examples/goroutines/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "os" "sync" @@ -9,14 +10,30 @@ import ( ) func main() { + var err error + + // At the end of the main function execution + // it will check, print and exit with non-zero code if any error is here + defer func() { + if err != nil { + fmt.Println(err) + os.Exit(1) + } + }() + // it will undo all initializations made by Py_Initialize() + defer python3.Py_Finalize() + // it will print any python error if it is here + // no needs to call it after each single check of PyErr_Occurred() + defer python3.PyErr_Print() + // The following will also create the GIL explicitly // by calling PyEval_InitThreads(), without waiting // for the interpreter to do that python3.Py_Initialize() if !python3.Py_IsInitialized() { - fmt.Println("Error initializing the python interpreter") - os.Exit(1) + err = errors.New("Error initializing the python interpreter") + return } var wg sync.WaitGroup @@ -24,27 +41,37 @@ func main() { nameModule := "foo" fooModule := python3.PyImport_ImportModule(nameModule) - if fooModule == nil { - fmt.Printf("The module %s doesn't exist\n", nameModule) + defer fooModule.DecRef() + if python3.PyErr_Occurred() != nil { + err = errors.New("Error importing the python module") + return } + oddsAttr := "print_odds" evenAttr := "print_even" - if !fooModule.HasAttrString(oddsAttr) { - fmt.Printf("The module doesn't have attribute %s\n", oddsAttr) - os.Exit(1) - } - - if !fooModule.HasAttrString(evenAttr) { - fmt.Printf("The module doesn't have attribute %s\n", evenAttr) - os.Exit(1) + odds := fooModule.GetAttrString(oddsAttr) + defer odds.DecRef() + if python3.PyErr_Occurred() != nil { + err = errors.New("Error getting the attribute print_odds") + return } - odds := fooModule.GetAttrString(oddsAttr) even := fooModule.GetAttrString(evenAttr) + defer even.DecRef() + if python3.PyErr_Occurred() != nil { + err = errors.New("Error getting the attribute print_even") + return + } - if odds == nil || even == nil { - panic("Error importing function") + limit := python3.PyLong_FromGoInt(50) + args := python3.PyTuple_New(1) + ret := python3.PyTuple_SetItem(args, 0, limit) + if ret != 0 || python3.PyErr_Occurred() != nil { + args.DecRef() + limit.DecRef() + err = errors.New("Error setting a tuple item") + return } // Py_Initialize() has locked the the GIL but at this point we don't need it @@ -54,7 +81,7 @@ func main() { go func() { _gstate := python3.PyGILState_Ensure() - odds.Call(python3.PyTuple_New(0), python3.PyDict_New()) + odds.Call(args, python3.PyDict_New()) python3.PyGILState_Release(_gstate) wg.Done() @@ -62,7 +89,7 @@ func main() { go func() { _gstate := python3.PyGILState_Ensure() - even.Call(python3.PyTuple_New(0), python3.PyDict_New()) + even.Call(args, python3.PyDict_New()) python3.PyGILState_Release(_gstate) wg.Done() @@ -74,5 +101,4 @@ func main() { // program, we can restore the state and lock the GIL to perform // the final operations before exiting. python3.PyEval_RestoreThread(state) - python3.Py_Finalize() } From 896499bf23bf6025a23433a39e81d195464ba917 Mon Sep 17 00:00:00 2001 From: esendjer Date: Thu, 6 Jan 2022 21:53:33 +0500 Subject: [PATCH 4/7] fixed using the DecRef() of python objects in defer statements --- examples/goroutines/main.go | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/examples/goroutines/main.go b/examples/goroutines/main.go index 94f6486..a19f0b4 100644 --- a/examples/goroutines/main.go +++ b/examples/goroutines/main.go @@ -41,9 +41,9 @@ func main() { nameModule := "foo" fooModule := python3.PyImport_ImportModule(nameModule) - defer fooModule.DecRef() - if python3.PyErr_Occurred() != nil { + if fooModule == nil && python3.PyErr_Occurred() != nil { err = errors.New("Error importing the python module") + fooModule.DecRef() return } @@ -51,23 +51,35 @@ func main() { evenAttr := "print_even" odds := fooModule.GetAttrString(oddsAttr) - defer odds.DecRef() - if python3.PyErr_Occurred() != nil { + if odds == nil && python3.PyErr_Occurred() != nil { err = errors.New("Error getting the attribute print_odds") + odds.DecRef() return } even := fooModule.GetAttrString(evenAttr) - defer even.DecRef() - if python3.PyErr_Occurred() != nil { + if even == nil && python3.PyErr_Occurred() != nil { err = errors.New("Error getting the attribute print_even") + even.DecRef() return } limit := python3.PyLong_FromGoInt(50) + if limit == nil && python3.PyErr_Occurred() != nil { + err = errors.New("Error creating python long object") + limit.DecRef() + return + } + args := python3.PyTuple_New(1) + if args == nil && python3.PyErr_Occurred() != nil { + err = errors.New("Error creating python tuple object") + args.DecRef() + return + } + ret := python3.PyTuple_SetItem(args, 0, limit) - if ret != 0 || python3.PyErr_Occurred() != nil { + if ret != 0 { args.DecRef() limit.DecRef() err = errors.New("Error setting a tuple item") From a9c69338af9e4894005061ee2f38939636985b47 Mon Sep 17 00:00:00 2001 From: esendjer Date: Fri, 7 Jan 2022 13:17:24 +0500 Subject: [PATCH 5/7] fixed the problem with random panic on a call Py_Finalize due to not right working with references and its decrement --- examples/goroutines/main.go | 62 +++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/examples/goroutines/main.go b/examples/goroutines/main.go index a19f0b4..ff66765 100644 --- a/examples/goroutines/main.go +++ b/examples/goroutines/main.go @@ -12,27 +12,25 @@ import ( func main() { var err error - // At the end of the main function execution - // it will check, print and exit with non-zero code if any error is here + // At the end of the main function execution + // it will check, print and exit with non-zero code if any error is here defer func() { if err != nil { fmt.Println(err) os.Exit(1) } }() - // it will undo all initializations made by Py_Initialize() + // it will undo all initializations made by Py_Initialize() defer python3.Py_Finalize() - // it will print any python error if it is here - // no needs to call it after each single check of PyErr_Occurred() + // it will print any python error if it is here + // no needs to call it after each single check of PyErr_Occurred() defer python3.PyErr_Print() - // The following will also create the GIL explicitly - // by calling PyEval_InitThreads(), without waiting - // for the interpreter to do that + // Initialize the Python interpreter and create the GIL explicitly python3.Py_Initialize() if !python3.Py_IsInitialized() { - err = errors.New("Error initializing the python interpreter") + err = errors.New("error initializing the python interpreter") return } @@ -40,55 +38,53 @@ func main() { wg.Add(2) nameModule := "foo" - fooModule := python3.PyImport_ImportModule(nameModule) + fooModule := python3.PyImport_ImportModule(nameModule) // New reference if fooModule == nil && python3.PyErr_Occurred() != nil { - err = errors.New("Error importing the python module") - fooModule.DecRef() + err = errors.New("error importing the python module") return } + defer fooModule.DecRef() oddsAttr := "print_odds" evenAttr := "print_even" - - odds := fooModule.GetAttrString(oddsAttr) + odds := fooModule.GetAttrString(oddsAttr) // New reference if odds == nil && python3.PyErr_Occurred() != nil { - err = errors.New("Error getting the attribute print_odds") - odds.DecRef() + err = errors.New("error getting the attribute print_odds") return } + defer odds.DecRef() - even := fooModule.GetAttrString(evenAttr) + even := fooModule.GetAttrString(evenAttr) // New reference if even == nil && python3.PyErr_Occurred() != nil { - err = errors.New("Error getting the attribute print_even") - even.DecRef() + err = errors.New("error getting the attribute print_even") return } + defer even.DecRef() - limit := python3.PyLong_FromGoInt(50) + limit := python3.PyLong_FromGoInt(50) // New reference, will stolen later if limit == nil && python3.PyErr_Occurred() != nil { - err = errors.New("Error creating python long object") - limit.DecRef() + err = errors.New("error creating python long object") return } - args := python3.PyTuple_New(1) + args := python3.PyTuple_New(1) // New reference if args == nil && python3.PyErr_Occurred() != nil { - err = errors.New("Error creating python tuple object") - args.DecRef() + err = errors.New("error creating python tuple object") return } + defer args.DecRef() - ret := python3.PyTuple_SetItem(args, 0, limit) + ret := python3.PyTuple_SetItem(args, 0, limit) // Steals reference to limit + limit = nil if ret != 0 { - args.DecRef() + err = errors.New("error setting a tuple item") limit.DecRef() - err = errors.New("Error setting a tuple item") + limit = nil return } - // Py_Initialize() has locked the the GIL but at this point we don't need it - // anymore. We save the current state and release the lock - // so that goroutines can acquire it + // Save the current state and release the lock + // so that goroutines can acquire it state := python3.PyEval_SaveThread() go func() { @@ -109,8 +105,6 @@ func main() { wg.Wait() - // At this point we know we won't need python anymore in this - // program, we can restore the state and lock the GIL to perform - // the final operations before exiting. + // Restore the state and lock the GIL python3.PyEval_RestoreThread(state) } From 2f094fc65f804b2a940a0d4418d3bf1daa6f6d7f Mon Sep 17 00:00:00 2001 From: esendjer Date: Thu, 13 Jan 2022 16:23:57 +0500 Subject: [PATCH 6/7] Fixed comment. Reached to idiomatic as possible. Applied linters. --- examples/goroutines/main.go | 55 ++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/examples/goroutines/main.go b/examples/goroutines/main.go index ff66765..4c2de24 100644 --- a/examples/goroutines/main.go +++ b/examples/goroutines/main.go @@ -12,22 +12,26 @@ import ( func main() { var err error - // At the end of the main function execution - // it will check, print and exit with non-zero code if any error is here + // At the end of all, if there was an error + // prints the error and exit with the non-zero code defer func() { if err != nil { fmt.Println(err) os.Exit(1) } }() - // it will undo all initializations made by Py_Initialize() + + // Undo all initializations made by Py_Initialize() and subsequent defer python3.Py_Finalize() - // it will print any python error if it is here - // no needs to call it after each single check of PyErr_Occurred() + + // Prints any python error if it was here + // no needs to call it after each single check of PyErr_Occurred() defer python3.PyErr_Print() - // Initialize the Python interpreter and create the GIL explicitly - python3.Py_Initialize() + // Initialize the Python interpreter and + // since version 3.7 it also create the GIL explicitly by calling PyEval_InitThreads() + // so you don’t have to call PyEval_InitThreads() yourself anymore + python3.Py_Initialize() // create the GIL, the GIL is locked by the main thread if !python3.Py_IsInitialized() { err = errors.New("error initializing the python interpreter") @@ -37,45 +41,46 @@ func main() { var wg sync.WaitGroup wg.Add(2) - nameModule := "foo" - fooModule := python3.PyImport_ImportModule(nameModule) // New reference + fooModule := python3.PyImport_ImportModule("foo") // new reference, a call DecRef() is needed if fooModule == nil && python3.PyErr_Occurred() != nil { err = errors.New("error importing the python module") return } defer fooModule.DecRef() - oddsAttr := "print_odds" - evenAttr := "print_even" - odds := fooModule.GetAttrString(oddsAttr) // New reference + odds := fooModule.GetAttrString("print_odds") // new reference, a call DecRef() is needed if odds == nil && python3.PyErr_Occurred() != nil { err = errors.New("error getting the attribute print_odds") return } defer odds.DecRef() - even := fooModule.GetAttrString(evenAttr) // New reference + even := fooModule.GetAttrString("print_even") // new reference, a call DecRef() is needed if even == nil && python3.PyErr_Occurred() != nil { err = errors.New("error getting the attribute print_even") return } defer even.DecRef() - limit := python3.PyLong_FromGoInt(50) // New reference, will stolen later + limit := python3.PyLong_FromGoInt(50) // new reference, will stolen later, a call DecRef() is NOT needed if limit == nil && python3.PyErr_Occurred() != nil { err = errors.New("error creating python long object") return } - args := python3.PyTuple_New(1) // New reference + args := python3.PyTuple_New(1) // new reference, a call DecRef() is needed if args == nil && python3.PyErr_Occurred() != nil { err = errors.New("error creating python tuple object") return } defer args.DecRef() - ret := python3.PyTuple_SetItem(args, 0, limit) // Steals reference to limit + ret := python3.PyTuple_SetItem(args, 0, limit) // steals reference to limit + + // Cleans the Go variable, because now a new owner is caring about related PyObject + // no action, such as a call DecRef(), is needed here limit = nil + if ret != 0 { err = errors.New("error setting a tuple item") limit.DecRef() @@ -83,28 +88,28 @@ func main() { return } - // Save the current state and release the lock - // so that goroutines can acquire it - state := python3.PyEval_SaveThread() + // Save the current state and release the GIL + // so that goroutines can acquire it + state := python3.PyEval_SaveThread() // release the GIL, the GIL is unlocked for using by goroutines go func() { - _gstate := python3.PyGILState_Ensure() + _gstate := python3.PyGILState_Ensure() // acquire the GIL, the GIL is locked by the 1st goroutine odds.Call(args, python3.PyDict_New()) - python3.PyGILState_Release(_gstate) + python3.PyGILState_Release(_gstate) // release the GIL, the GIL is unlocked for using by others wg.Done() }() go func() { - _gstate := python3.PyGILState_Ensure() + _gstate := python3.PyGILState_Ensure() // acquire the GIL, the GIL is locked by the 2nd goroutine even.Call(args, python3.PyDict_New()) - python3.PyGILState_Release(_gstate) + python3.PyGILState_Release(_gstate) // release the GIL, the GIL is unlocked for using by others wg.Done() }() wg.Wait() - // Restore the state and lock the GIL - python3.PyEval_RestoreThread(state) + // Restore the state and lock the GIL + python3.PyEval_RestoreThread(state) // acquire the GIL, the GIL is locked by the main thread } From 191bf332d98968fbfec673c46a1393b7d4acb043 Mon Sep 17 00:00:00 2001 From: esendjer Date: Thu, 13 Jan 2022 16:46:07 +0500 Subject: [PATCH 7/7] Used log instead fmt for printing error messages --- examples/goroutines/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/goroutines/main.go b/examples/goroutines/main.go index 4c2de24..cab91be 100644 --- a/examples/goroutines/main.go +++ b/examples/goroutines/main.go @@ -2,7 +2,7 @@ package main import ( "errors" - "fmt" + "log" "os" "sync" @@ -16,7 +16,7 @@ func main() { // prints the error and exit with the non-zero code defer func() { if err != nil { - fmt.Println(err) + log.Printf("%+v", err) os.Exit(1) } }()