MINIFICPP-2746 Allow python processors to run in parallel

extensions/python/ExecutePythonProcessor.h

@@ -42,7 +42,8 @@ class ExecutePythonProcessor : public core::ProcessorImpl {
  public:
   explicit ExecutePythonProcessor(core::ProcessorMetadata metadata)
       : ProcessorImpl(metadata),
-        python_dynamic_(false) {
+        python_dynamic_(false),
+        python_single_threaded_(false) {
     python_logger_ = core::logging::LoggerFactory<ExecutePythonProcessor>::getAliasedLogger(getName(), metadata.uuid);
   }
 
@@ -59,12 +60,12 @@ class ExecutePythonProcessor : public core::ProcessorImpl {
   EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false;
   EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false;
   EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_ALLOWED;
-  EXTENSIONAPI static constexpr bool IsSingleThreaded = true;
+  EXTENSIONAPI static constexpr bool IsSingleThreaded = false;
 
   bool supportsDynamicProperties() const override { return python_dynamic_; }
   bool supportsDynamicRelationships() const override { return SupportsDynamicRelationships; }
   minifi::core::annotation::Input getInputRequirement() const override { return InputRequirement; }
-  bool isSingleThreaded() const override { return IsSingleThreaded; }
+  bool isSingleThreaded() const override { return python_single_threaded_; }
   ADD_GET_PROCESSOR_NAME
 
   void initialize() override;
@@ -75,6 +76,10 @@ class ExecutePythonProcessor : public core::ProcessorImpl {
     python_dynamic_ = true;
   }
 
+  void setSingleThreaded() {
+    python_single_threaded_ = true;
+  }
+
   void addProperty(const std::string &name, const std::string &description, const std::optional<std::string> &defaultvalue, bool required, bool el, bool sensitive,
       const std::optional<int64_t>& property_type_code, gsl::span<const std::string_view> allowable_values, const std::optional<std::string>& controller_service_type_name);
 
@@ -126,6 +131,7 @@ class ExecutePythonProcessor : public core::ProcessorImpl {
   std::optional<std::string> version_;
 
   bool python_dynamic_;
+  bool python_single_threaded_;
 
   std::string script_to_exec_;
   std::optional<std::chrono::file_clock::time_point> last_script_write_time_;

extensions/python/PYTHON.md

@@ -222,6 +222,13 @@ class VaderSentiment(object):
     return len(self.content)
 ```
 
+By default the MiNiFi C++ style native python processors are executed with multiple threads in parallel, but it is possible to set the processor to be single threaded by calling the `setSingleThreaded()` method in the processor while initializing. When a processor is set to be single threaded, only one thread will execute the processor, and setting the max concurrent tasks to more than 1 in the flow configuration will not have any effect.
+
+```python
+def onInitialize(processor):
+  processor.setSingleThreaded()
+```
+
 ## Using NiFi Python Processors
 
 MiNiFi C++ supports the use of NiFi Python processors, that are inherited from the FlowFileTransform, RecordTransform or the FlowFileSource base class. To use these processors, copy the Python processor module to the nifi_python_processors subdirectory of the python directory. By default, the python directory is ${minifi_root}/minifi-python. To see how to write NiFi Python processors, please refer to the Python Developer Guide under the [Apache NiFi documentation](https://nifi.apache.org/nifi-docs/python-developer-guide.html).
@@ -243,6 +250,27 @@ Due to some differences between the NiFi and MiNiFi C++ processors and implement
 - MiNiFi C++ uses a single embedded Python interpreter for all Python processors, so the Python processors share the same Python interpreter. This means that the Python processors cannot have different Python versions or use different Python packages. The Python packages are installed on the system or in a single virtualenv that is shared by all Python processors.
 - State manager API is available with the same interface as in NiFi, but MiNiFi C++ uses transactional state management, due to this when a state is changed in a processor trigger the state cannot be read due to the dirty read protection. The state can be read in the next trigger after the state is committed at the end of a session. Also MiNiFi C++ does not use clustering, so the scope parameter for the state operations is ignored as it is always local.
 
+### Setting processor to be single threaded
+
+One feature that is currently only available in MiNiFi C++ is the ability to set a python processor to be single threaded. This is not yet available in NiFi's API, but similarly to Nifi's `@TriggerSerially` annotation, in MiNiFi C++ the `@trigger_serially` decorator can be used to make a processor single threaded. Setting this will make sure that only one thread executes the processor, and setting the max concurrent tasks to more than 1 in the flow configuration will not have any effect.
+
+```python
+from nifiapi.flowfilesource import FlowFileSource, FlowFileSourceResult
+from nifiapi.processorutils import trigger_serially
+
+
+@trigger_serially
+class SingleThreadedProcessor(FlowFileSource):
+    class Java:
+        implements = ['org.apache.nifi.python.processor.FlowFileSource']
+
+    def __init__(self, **kwargs):
+        pass
+
+    def create(self, context):
+        return FlowFileSourceResult(relationship='success', contents="result")
+```
+
 ## Use Python processors from virtualenv
 
 It is possible to set a virtualenv to be used by the Python processors in Apache MiNiFi C++. This is the default behavior. If the virtualenv directory is set, the Python processors will be executed using the packages installed in the virtualenv or if dependencies are not found there MiNiFi C++ will try to find the required packages installed on the system. If the virtualenv directory is not set, the Python processors will be executed using only the packages installed on the system.

extensions/python/PythonProcessor.cpp

@@ -31,6 +31,10 @@ void PythonProcessor::setSupportsDynamicProperties() {
   processor_->setSupportsDynamicProperties();
 }
 
+void PythonProcessor::setSingleThreaded() {
+  processor_->setSingleThreaded();
+}
+
 void PythonProcessor::setDescription(const std::string& desc) {
   processor_->setDescription(desc);
 }

extensions/python/PythonProcessor.h

@@ -38,6 +38,8 @@ class PythonProcessor {
 
   void setSupportsDynamicProperties();
 
+  void setSingleThreaded();
+
   void setDescription(const std::string& desc);
 
   void setVersion(const std::string& version);

extensions/python/pythonprocessors/nifiapi/processorbase.py

@@ -53,6 +53,9 @@ def onInitialize(self, processor: Processor):
         else:
             self.supports_dynamic_properties = False
 
+        if hasattr(self, '_trigger_serially') and self._trigger_serially:
+            processor.setSingleThreaded()
+
         for property in self.getPropertyDescriptors():
             expression_language_supported = True if property.expressionLanguageScope != ExpressionLanguageScope.NONE else False
             property_type_code = translateStandardValidatorToMiNiFiPropertype(property.validators)

extensions/python/pythonprocessors/nifiapi/processorutils.py

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def trigger_serially(cls):
+    cls._trigger_serially = True
+    return cls

extensions/python/tests/PythonManifestTests.cpp

@@ -66,6 +66,7 @@ TEST_CASE("Python processor's description is part of the manifest") {
     "def describe(proc):\n"
     "  proc.setDescription('Another amazing processor')\n"
     "  proc.setSupportsDynamicProperties()\n"
+    "  proc.setSingleThreaded()\n"
     "  proc.addProperty('Prop1', 'A great property', 'banana', True, False, False, None, ['apple', 'orange', 'banana', 'durian'], None)\n";
 
   const auto executable_dir = minifi::utils::file::FileUtils::get_executable_dir();
@@ -77,7 +78,9 @@ TEST_CASE("Python processor's description is part of the manifest") {
   std::ofstream{python_dir / "nifi_python_processors" / "MyPyProc3.py"} << R"(
 from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult
 from nifiapi.properties import ExpressionLanguageScope, PropertyDescriptor, StandardValidators
+from nifiapi.processorutils import trigger_serially
 
+@trigger_serially
 class MyPyProc3(FlowFileTransform):
 
     class Java:
@@ -203,7 +206,7 @@ class MyPyProc5(FlowFileTransform):
     auto MyPyProc = getProcessorNode(python_bundle);
 
     CHECK(getNode(MyPyProc->children, "inputRequirement").value == "INPUT_ALLOWED");
-    CHECK(getNode(MyPyProc->children, "isSingleThreaded").value == true);
+    CHECK(getNode(MyPyProc->children, "isSingleThreaded").value == false);
     CHECK(getNode(MyPyProc->children, "typeDescription").value == "An amazing processor");
     CHECK(getNode(MyPyProc->children, "supportsDynamicRelationships").value == false);
     CHECK(getNode(MyPyProc->children, "supportsDynamicProperties").value == false);
@@ -302,6 +305,8 @@ class MyPyProc5(FlowFileTransform):
   {
     auto python_bundle = findPythonBundle("MyPyProc4");
     CHECK(getNode(python_bundle->children, "version").value == minifi::AgentBuild::VERSION);
+    auto MyPyProc4 = getProcessorNode(python_bundle);
+    CHECK(getNode(MyPyProc4->children, "isSingleThreaded").value == false);
   }
 
   {

extensions/python/tests/features/environment.py

@@ -60,6 +60,10 @@ def before_all(context):
 COPY LogDynamicProperties.py {minifi_python_dir}/LogDynamicProperties.py
 COPY ExpressionLanguagePropertyWithValidator.py {minifi_python_dir}/nifi_python_processors/ExpressionLanguagePropertyWithValidator.py
 COPY EvaluateExpressionLanguageChecker.py {minifi_python_dir}/nifi_python_processors/EvaluateExpressionLanguageChecker.py
+COPY SleepForever.py {minifi_python_dir}/nifi_python_processors/SleepForever.py
+COPY SingleThreadedSleepForever.py {minifi_python_dir}/nifi_python_processors/SingleThreadedSleepForever.py
+COPY MinifiSleepForever.py {minifi_python_dir}/MinifiSleepForever.py
+COPY SingleThreadedMinifiSleepForever.py {minifi_python_dir}/SingleThreadedMinifiSleepForever.py
 RUN python3 -m venv {minifi_python_venv_parent}/venv
     """.format(base_image=context.minifi_container_image,
                pip3_install_command=pip3_install_command,

extensions/python/tests/features/python.feature

@@ -353,3 +353,53 @@ Feature: MiNiFi can use python processors in its flows
     And the Minifi logs contain the following message: "Non-existent property value is empty" in less than 1 seconds
     And the Minifi logs contain the following message: "My Dynamic Property value is: Dynamic ${my.attribute}" in less than 1 seconds
     And the Minifi logs contain the following message: "My Dynamic Property evaluated value is: Dynamic my.value" in less than 1 seconds
+
+  Scenario Outline: NiFi python processors are allowed to run in parallel
+    Given a org.apache.nifi.minifi.processors.nifi_python_processors.SleepForever processor with the name "SleepForever"
+    And the max concurrent tasks attribute of the SleepForever processor is set to <Max concurrent tasks>
+    And the scheduling period of the SleepForever processor is set to "100 milliseconds"
+    And SleepForever's success relationship is auto-terminated
+
+    When the MiNiFi instance starts up
+
+    Then the Minifi logs contain the following message: "Sleeping forever" <Max concurrent tasks> times after 5 seconds
+
+    Examples:
+    | Max concurrent tasks |
+    | 3                    |
+    | 1                    |
+
+  Scenario: NiFi python processors can be set to be single threaded
+    Given a org.apache.nifi.minifi.processors.nifi_python_processors.SingleThreadedSleepForever processor with the name "SingleThreadedSleepForever"
+    And the max concurrent tasks attribute of the SingleThreadedSleepForever processor is set to 5
+    And the scheduling period of the SingleThreadedSleepForever processor is set to "100 milliseconds"
+    And SingleThreadedSleepForever's success relationship is auto-terminated
+
+    When the MiNiFi instance starts up
+
+    Then the Minifi logs contain the following message: "Sleeping forever" 1 times after 5 seconds
+
+  Scenario Outline: MiNiFi python processors are allowed to run in parallel
+    Given a MinifiSleepForever processor
+    And the max concurrent tasks attribute of the MinifiSleepForever processor is set to <Max concurrent tasks>
+    And the scheduling period of the MinifiSleepForever processor is set to "100 milliseconds"
+    And MinifiSleepForever's success relationship is auto-terminated
+
+    When the MiNiFi instance starts up
+
+    Then the Minifi logs contain the following message: "Sleeping forever" <Max concurrent tasks> times after 5 seconds
+
+    Examples:
+    | Max concurrent tasks |
+    | 3                    |
+    | 1                    |
+
+  Scenario: MiNiFi python processors can be set to be single threaded
+    Given a SingleThreadedMinifiSleepForever processor
+    And the max concurrent tasks attribute of the SingleThreadedMinifiSleepForever processor is set to 5
+    And the scheduling period of the SingleThreadedMinifiSleepForever processor is set to "100 milliseconds"
+    And SingleThreadedMinifiSleepForever's success relationship is auto-terminated
+
+    When the MiNiFi instance starts up
+
+    Then the Minifi logs contain the following message: "Sleeping forever" 1 times after 5 seconds

extensions/python/tests/features/resources/MinifiSleepForever.py

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import time
+
+
+def describe(processor):
+    processor.setDescription("Sleep forever processor.")
+
+
+def onTrigger(context, session):
+    log.info("Sleeping forever")
+    while True:
+        time.sleep(1)

extensions/python/tests/features/resources/SingleThreadedMinifiSleepForever.py

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import time
+
+
+def describe(processor):
+    processor.setDescription("Sleep forever processor.")
+
+
+def onInitialize(processor):
+    processor.setSingleThreaded()
+
+
+def onTrigger(context, session):
+    log.info("Sleeping forever")
+    while True:
+        time.sleep(1)

extensions/python/tests/features/resources/SingleThreadedSleepForever.py

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from nifiapi.flowfilesource import FlowFileSource
+from nifiapi.processorutils import trigger_serially
+
+
+@trigger_serially
+class SingleThreadedSleepForever(FlowFileSource):
+    class Java:
+        implements = ['org.apache.nifi.python.processor.FlowFileSource']
+
+    class ProcessorDetails:
+        version = '2.0.0-snapshot'
+        description = '''Test Python source processor.'''
+        tags = ['text', 'test', 'python', 'source']
+
+    def __init__(self, **kwargs):
+        pass
+
+    def create(self, context):
+        self.logger.info("Sleeping forever")
+        while True:
+            time.sleep(1)

extensions/python/tests/features/resources/SleepForever.py

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from nifiapi.flowfilesource import FlowFileSource
+
+
+class SleepForever(FlowFileSource):
+    class Java:
+        implements = ['org.apache.nifi.python.processor.FlowFileSource']
+
+    class ProcessorDetails:
+        version = '2.0.0-snapshot'
+        description = '''Test Python source processor.'''
+        tags = ['text', 'test', 'python', 'source']
+
+    def __init__(self, **kwargs):
+        pass
+
+    def create(self, context):
+        self.logger.info("Sleeping forever")
+        while True:
+            time.sleep(1)

extensions/python/types/PyProcessor.cpp

@@ -43,6 +43,7 @@ extern "C" {
 
 static PyMethodDef PyProcessor_methods[] = {  // NOLINT(cppcoreguidelines-avoid-c-arrays)
     {"setSupportsDynamicProperties", (PyCFunction) PyProcessor::setSupportsDynamicProperties, METH_VARARGS, nullptr},
+    {"setSingleThreaded", (PyCFunction) PyProcessor::setSingleThreaded, METH_VARARGS, nullptr},
     {"setDescription", (PyCFunction) PyProcessor::setDescription, METH_VARARGS, nullptr},
     {"setVersion", (PyCFunction) PyProcessor::setVersion, METH_VARARGS, nullptr},
     {"addProperty", (PyCFunction) PyProcessor::addProperty, METH_VARARGS, nullptr},
@@ -89,6 +90,17 @@ PyObject* PyProcessor::setSupportsDynamicProperties(PyProcessor* self, PyObject*
   Py_RETURN_NONE;
 }
 
+PyObject* PyProcessor::setSingleThreaded(PyProcessor* self, PyObject*) {
+  auto processor = self->processor_.lock();
+  if (!processor) {
+    PyErr_SetString(PyExc_AttributeError, "tried reading processor outside 'on_trigger'");
+    return nullptr;
+  }
+
+  processor->setSingleThreaded();
+  Py_RETURN_NONE;
+}
+
 PyObject* PyProcessor::setDescription(PyProcessor* self, PyObject* args) {
   auto processor = self->processor_.lock();
   if (!processor) {

extensions/python/types/PyProcessor.h

@@ -32,6 +32,7 @@ struct PyProcessor {
   static int init(PyProcessor* self, PyObject* args, PyObject* kwds);
 
   static PyObject* setSupportsDynamicProperties(PyProcessor* self, PyObject* args);
+  static PyObject* setSingleThreaded(PyProcessor* self, PyObject* args);
   static PyObject* setDescription(PyProcessor* self, PyObject* args);
   static PyObject* setVersion(PyProcessor* self, PyObject* args);
   static PyObject* addProperty(PyProcessor* self, PyObject* args);