Support multithreaded execution of Python components with the HTTPD event MPM.

git-svn-id: http://svn.us.apache.org/repos/asf/tuscany@1165452 13f79535-47bb-0310-9956-ffa450edef68

1 files changed, 219 insertions, 63 deletions

diff --git a/sca-cpp/trunk/modules/python/eval.hpp b/sca-cpp/trunk/modules/python/eval.hpp
index f106ff1659..a1685d11ff 100644
--- a/sca-cpp/trunk/modules/python/eval.hpp
+++ b/sca-cpp/trunk/modules/python/eval.hpp
@@ -37,23 +37,53 @@
 namespace tuscany {
 namespace python {
 
+class PythonThreadIn;
+class PythonThreadOut;
+class PythonRuntimeLock;
+
 /**
  * Represent a Python runtime.
  */
+
 class PythonRuntime {
 public:
     PythonRuntime() {
         debug("python::pythonruntime");
-        if (Py_IsInitialized())
-            return;
-        Py_InitializeEx(0);
-        const char* arg0 = "";
-        PySys_SetArgv(0, const_cast<char**>(&arg0));
+        pthread_mutex_init(&modulemx, NULL);
+
+        // Initialize the Python interpreter
+        if (!Py_IsInitialized()) {
+            debug("python::pythonruntime::initialize");
+            Py_InitializeEx(0);
+
+            // Set default interpreter args
+            const char* arg0 = "";
+            PySys_SetArgv(0, const_cast<char**>(&arg0));
+
+#ifdef WANT_THREADS
+            // Initialize the Python thread support
+            PyEval_InitThreads();
+
+            // Release Python lock
+            PyEval_ReleaseLock();
+#endif
+        }
+
     }
 
     ~PythonRuntime() {
         debug("python::~pythonruntime");
     }
+
+private:
+
+#ifdef WANT_THREADS
+    pthread_mutex_t modulemx;
+#endif
+
+    friend class PythonThreadIn;
+    friend class PythonThreadOut;
+    friend class PythonRuntimeLock;
 };
 
 /**
@@ -83,22 +113,90 @@ const string lastError() {
 }
 
 /**
+ * Represent a lock on the Python runtime.
+ */
+class PythonRuntimeLock {
+public:
+    PythonRuntimeLock(PythonRuntime* py) : py(py) {
+#ifdef WANT_THREADS
+        pthread_mutex_lock(&py->modulemx);
+#endif
+    }
+
+    ~PythonRuntimeLock() {
+#ifdef WANT_THREADS
+        pthread_mutex_unlock(&py->modulemx);
+#endif
+    }
+
+private:
+    PythonRuntime* py;
+};
+
+/**
+ * Represent a thread calling into the Python interpreter.
+ */
+class PythonThreadIn {
+public:
+    PythonThreadIn(PythonRuntime* py) : py(py) {
+#ifdef WANT_THREADS
+        //debug("python::gil::ensure");
+        gstate = PyGILState_Ensure();
+#endif
+    }
+
+    ~PythonThreadIn() {
+#ifdef WANT_THREADS
+        //debug("python::gil::release");
+        PyGILState_Release(gstate);
+#endif
+    }
+
+private:
+    PythonRuntime* py;
+    PyGILState_STATE gstate;
+};
+
+/**
+ * Represent a thread calling out of the Python interpreter.
+ */
+class PythonThreadOut {
+public:
+    PythonThreadOut(PythonRuntime* py) : py(py) {
+#ifdef WANT_THREADS
+        //tstate = PyEval_SaveThread();
+#endif
+    }
+
+    ~PythonThreadOut() {
+#ifdef WANT_THREADS
+        //PyEval_RestoreThread(tstate);
+#endif
+    }
+
+private:
+    PythonRuntime* py;
+    PyThreadState* tstate;
+};
+
+/**
  * Declare conversion functions.
  */
-PyObject* valueToPyObject(const value& v);
-const value pyObjectToValue(PyObject *o);
-PyObject* valuesToPyTuple(const list<value>& v);
-const list<value> pyTupleToValues(PyObject* o);
+PyObject* valueToPyObject(const value& v, PythonRuntime* py);
+const value pyObjectToValue(PyObject *o, PythonRuntime* py);
+PyObject* valuesToPyTuple(const list<value>& v, PythonRuntime* py);
+const list<value> pyTupleToValues(PyObject* o, PythonRuntime* py);
 
 /**
  * Callable python type used to represent a lambda expression.
  */
 typedef struct {
-  PyObject_HEAD
-  lambda<value(const list<value>&)> func;
+    PyObject_HEAD
+    lambda<value(const list<value>&)>* func;
+    PythonRuntime* py;
 } pyLambda;
 
-PyObject *mkPyLambda(const lambda<value(const list<value>&)>& l);
+PyObject *mkPyLambda(const lambda<value(const list<value>&)>& l, PythonRuntime* py);
 
 void pyLambda_dealloc(PyObject* self) {
     debug(self, "python::pylambda_dealloc");
@@ -112,12 +210,17 @@ const string pyRepr(PyObject* o) {
     return s;
 }
 
+const value pyLambda_callout(const pyLambda* pyl, const list<value>& args, PythonRuntime* py) {
+    PythonThreadOut pyout(py);
+    return (*(pyl->func))(args);
+}
+
 PyObject* pyLambda_call(PyObject* self, PyObject* args, unused PyObject* kwds) {
     debug("python::call");
-    const pyLambda* pyl = (pyLambda*)self;
-    const value result = pyl->func(pyTupleToValues(args));
+    const pyLambda* pyl = (const pyLambda*)self;
+    const value result =  pyLambda_callout(pyl, pyTupleToValues(args, pyl->py), pyl->py);
     debug(result, "python::call::result");
-    PyObject *pyr = valueToPyObject(result);
+    PyObject *pyr = valueToPyObject(result, pyl->py);
     return pyr;
 }
 
@@ -148,53 +251,89 @@ PyObject* pyLambda_getattr(PyObject *self, PyObject *attrname) {
 
     const pyLambda* pyl = (pyLambda*)self;
     debug(name, "python::getattr::name");
-    PyObject* pyr = mkPyLambda(pyProxy(name, pyl->func));
+    PyObject* pyr = mkPyLambda(pyProxy(name, *(pyl->func)), pyl->py);
     return pyr;
 }
 
-PyTypeObject pyLambda_type = {
-    PyObject_HEAD_INIT(0)
-    0,
-    "lambda",
-    sizeof(pyLambda),
-    0,
-    (destructor)pyLambda_dealloc,
-    0, 0, 0, 0, 0, 0, 0, 0, 0,
-    (ternaryfunc)pyLambda_call,
-    0, 
-    (binaryfunc)pyLambda_getattr,
-    0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0
+ PyTypeObject pyLambda_type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "lambda",                     /*tp_name*/
+    sizeof(pyLambda),             /*tp_basicsize*/
+    0,                            /*tp_itemsize*/
+    /* methods */
+    (destructor)pyLambda_dealloc, /*tp_dealloc*/
+    0,                            /*tp_print*/
+    0,                            /*tp_getattr*/
+    0,                            /*tp_setattr*/
+    0,                            /*tp_compare*/
+    0,                            /*tp_repr*/
+    0,                            /*tp_as_number*/
+    0,                            /*tp_as_sequence*/
+    0,                            /*tp_as_mapping*/
+    0,                            /*tp_hash*/
+    (ternaryfunc)pyLambda_call,   /*tp_call*/
+    0,                            /*tp_str*/
+    (binaryfunc)pyLambda_getattr, /*tp_getattro*/
+    0,                            /*tp_setattro*/
+    0,                            /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT,           /*tp_flags*/
+    0,                            /*tp_doc*/
+    0,                            /*tp_traverse*/
+    0,                            /*tp_clear*/
+    0,                            /*tp_richcompare*/
+    0,                            /*tp_weaklistoffset*/
+    0,                            /*tp_iter*/
+    0,                            /*tp_iternext*/
+    0,                            /*tp_methods*/
+    0,                            /*tp_members*/
+    0,                            /*tp_getset*/
+    0,                            /*tp_base*/
+    0,                            /*tp_dict*/
+    0,                            /*tp_descr_get*/
+    0,                            /*tp_descr_set*/
+    0,                            /*tp_dictoffset*/
+    0,                            /*tp_init*/
+    0,                            /*tp_alloc*/
+    0,                            /*tp_new*/
+    0,                            /*tp_free*/
+    0,                            /*tp_is_gc*/
+    0,                            /*tp_bases*/
+    0,                            /*tp_mro*/
+    0,                            /*tp_cache*/
+    0,                            /*tp_subclasses*/
+    0,                            /*tp_weaklist*/
+    0,                            /*tp_del*/
+    0                             /*tp_version_tag*/
 };
 
+
 /**
  * Create a new python object representing a lambda expression.
  */
-PyObject *mkPyLambda(const lambda<value(const list<value>&)>& l) {
-    pyLambda* pyl = NULL;
-    pyl = PyObject_New(pyLambda, &pyLambda_type);
-    if (pyl != NULL)
-      pyl->func = l;
+PyObject *mkPyLambda(const lambda<value(const list<value>&)>& l, PythonRuntime* py) {
+    pyLambda* pyl = PyObject_New(pyLambda, &pyLambda_type);
+    if (pyl != NULL) {
+        pyl->func = new (gc_new<lambda<value(const list<value>&)> >()) lambda<value(const list<value>&)>(l);
+        pyl->py = py;
+    }
     debug(pyl, "python::mkpylambda");
-    return (PyObject *)pyl;
+    return (PyObject*)pyl;
 }
 
 /**
  * Convert a list of values to a python list.
  */
-PyObject* valuesToPyListHelper(PyObject* l, const list<value>& v) {
+PyObject* valuesToPyListHelper(PyObject* l, const list<value>& v, PythonRuntime* py) {
     if (isNil(v))
         return l;
-    PyObject* pyv = valueToPyObject(car(v));
+    PyObject* pyv = valueToPyObject(car(v), py);
     PyList_Append(l, pyv);
     Py_DECREF(pyv);
-    return valuesToPyListHelper(l, cdr(v));
+    return valuesToPyListHelper(l, cdr(v), py);
 }
 
-PyObject* valuesToPyTuple(const list<value>& v) {
-    PyObject* pyl = valuesToPyListHelper(PyList_New(0), v);
+PyObject* valuesToPyTuple(const list<value>& v, PythonRuntime* py) {
+    PyObject* pyl = valuesToPyListHelper(PyList_New(0), v, py);
     PyObject* pyt = PyList_AsTuple(pyl);
     Py_DECREF(pyl);
     return pyt;
@@ -203,12 +342,12 @@ PyObject* valuesToPyTuple(const list<value>& v) {
 /**
  * Convert a value to a python object.
  */
-PyObject* valueToPyObject(const value& v) {
+PyObject* valueToPyObject(const value& v, PythonRuntime* py) {
     switch (type(v)) {
     case value::List:
-        return valuesToPyTuple(v);
+        return valuesToPyTuple(v, py);
     case value::Lambda:
-        return mkPyLambda(v);
+        return mkPyLambda(v, py);
     case value::Symbol:
         return PyString_FromString(c_str(string("'") + v));
     case value::String: {
@@ -228,14 +367,14 @@ PyObject* valueToPyObject(const value& v) {
  * Convert a python tuple to a list of values.
  */
 
-const list<value> pyTupleToValuesHelper(PyObject* o, const size_t i, const size_t size) {
+const list<value> pyTupleToValuesHelper(PyObject* o, const size_t i, const size_t size, PythonRuntime* py) {
     if (i == size)
         return list<value>();
-    return cons(pyObjectToValue(PyTuple_GetItem(o, i)), pyTupleToValuesHelper(o, i + 1, size));
+    return cons(pyObjectToValue(PyTuple_GetItem(o, i), py), pyTupleToValuesHelper(o, i + 1, size, py));
 }
 
-const list<value> pyTupleToValues(PyObject* o) {
-    return pyTupleToValuesHelper(o, 0, PyTuple_Size(o));
+const list<value> pyTupleToValues(PyObject* o, PythonRuntime* py) {
+    return pyTupleToValuesHelper(o, 0, PyTuple_Size(o), py);
 }
 
 /**
@@ -243,8 +382,9 @@ const list<value> pyTupleToValues(PyObject* o) {
  */
 struct pyCallable {
     PyObject* func;
+    PythonRuntime* py;
 
-    pyCallable(PyObject* func) : func(func) {
+    pyCallable(PyObject* func, PythonRuntime* py) : func(func), py(py) {
         Py_INCREF(func);
     }
 
@@ -253,9 +393,10 @@ struct pyCallable {
     }
 
     const value operator()(const list<value>& args) const {
-        PyObject* pyargs = valuesToPyTuple(args);
+        PythonThreadIn pyin(py);
+        PyObject* pyargs = valuesToPyTuple(args, py);
         PyObject* result = PyObject_CallObject(func, pyargs);
-        const value v = pyObjectToValue(result);
+        const value v = pyObjectToValue(result, py);
         Py_DECREF(pyargs);
         Py_DECREF(result);
         return v;
@@ -265,7 +406,7 @@ struct pyCallable {
 /**
  * Convert a python object to a value.
  */
-const value pyObjectToValue(PyObject *o) {
+const value pyObjectToValue(PyObject *o, PythonRuntime* py) {
     if (PyString_Check(o)) {
         char* s = NULL;
         Py_ssize_t l = 0;
@@ -283,9 +424,9 @@ const value pyObjectToValue(PyObject *o) {
     if (PyFloat_Check(o))
         return value((double)PyFloat_AsDouble(o));
     if (PyTuple_Check(o))
-        return pyTupleToValues(o);
+        return pyTupleToValues(o, py);
     if (PyCallable_Check(o))
-        return lambda<value(const list<value>&)>(pyCallable(o));
+        return lambda<value(const list<value>&)>(pyCallable(o, py));
     return value();
 }
 
@@ -299,7 +440,8 @@ const string moduleName(const string& path) {
 /**
  * Evaluate an expression against a script provided as a python object.
  */
-const failable<value> evalScript(const value& expr, PyObject* script) {
+const failable<value> evalScript(const value& expr, PyObject* script, PythonRuntime& py) {
+    PythonThreadIn pyin(&py);
 
     // Get the requested function
     PyObject* func = PyObject_GetAttrString(script, c_str(car<value>(expr)));
@@ -320,7 +462,7 @@ const failable<value> evalScript(const value& expr, PyObject* script) {
     }
 
     // Convert args to python objects
-    PyObject* args = valuesToPyTuple(cdr<value>(expr));
+    PyObject* args = valuesToPyTuple(cdr<value>(expr), &py);
 
     // Call the function
     PyObject* result = PyObject_CallObject(func, args);
@@ -330,7 +472,7 @@ const failable<value> evalScript(const value& expr, PyObject* script) {
         return mkfailure<value>(string("Function call failed: ") + car<value>(expr) + " : " + lastError());
 
     // Convert python result to a value
-    const value v = pyObjectToValue(result);
+    const value v = pyObjectToValue(result, &py);
     Py_DECREF(result);
     return v;
 }
@@ -338,7 +480,9 @@ const failable<value> evalScript(const value& expr, PyObject* script) {
 /**
  * Read a python script from an input stream.
  */
-const failable<PyObject*> readScript(const string& name, const string& path, istream& is) {
+const failable<PyObject*> readScript(const string& name, const string& path, istream& is, PythonRuntime& py) {
+    PythonThreadIn pyin(&py);
+
     const list<string> ls = streamList(is);
     ostringstream os;
     write(ls, os);
@@ -346,19 +490,31 @@ const failable<PyObject*> readScript(const string& name, const string& path, ist
     if (code == NULL)
         return mkfailure<PyObject*>(string("Couldn't compile script: ") + path + " : " + lastError());
     PyObject* mod = PyImport_ExecCodeModuleEx(const_cast<char*>(c_str(name)), code, const_cast<char*>(c_str(path)));
-    if (mod == NULL)
+    if (mod == NULL) {
+        Py_DECREF(code);
         return mkfailure<PyObject*>(string("Couldn't import module: ") + path + " : " + lastError());
+    }
+    Py_DECREF(code);
     return mod;
 }
 
 /**
+ * Release a python script.
+ */
+const failable<bool> releaseScript(PyObject* script, PythonRuntime& py) {
+    PythonThreadIn pyin(&py);
+    Py_DECREF(script);
+    return true;
+}
+
+/**
  * Evaluate an expression against a script provided as an input stream.
  */
-const failable<value> evalScript(const value& expr, istream& is) {
-    failable<PyObject*> script = readScript("script", "script.py", is);
+const failable<value> evalScript(const value& expr, istream& is, PythonRuntime& py) {
+    failable<PyObject*> script = readScript("script", "script.py", is, py);
     if (!hasContent(script))
         return mkfailure<value>(reason(script));
-    return evalScript(expr, content(script));
+    return evalScript(expr, content(script), py);
 }
 
 }