diff --git a/extensions/python/ExecutePythonProcessor.h b/extensions/python/ExecutePythonProcessor.h index a237e19bda..a4c1a66b84 100644 --- a/extensions/python/ExecutePythonProcessor.h +++ b/extensions/python/ExecutePythonProcessor.h @@ -42,7 +42,8 @@ class ExecutePythonProcessor : public core::ProcessorImpl { public: explicit ExecutePythonProcessor(core::ProcessorMetadata metadata) : ProcessorImpl(metadata), - python_dynamic_(false) { + python_dynamic_(false), + python_single_threaded_(false) { python_logger_ = core::logging::LoggerFactory::getAliasedLogger(getName(), metadata.uuid); } @@ -59,12 +60,12 @@ class ExecutePythonProcessor : public core::ProcessorImpl { EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_ALLOWED; - EXTENSIONAPI static constexpr bool IsSingleThreaded = true; + EXTENSIONAPI static constexpr bool IsSingleThreaded = false; bool supportsDynamicProperties() const override { return python_dynamic_; } bool supportsDynamicRelationships() const override { return SupportsDynamicRelationships; } minifi::core::annotation::Input getInputRequirement() const override { return InputRequirement; } - bool isSingleThreaded() const override { return IsSingleThreaded; } + bool isSingleThreaded() const override { return python_single_threaded_; } ADD_GET_PROCESSOR_NAME void initialize() override; @@ -75,6 +76,10 @@ class ExecutePythonProcessor : public core::ProcessorImpl { python_dynamic_ = true; } + void setSingleThreaded() { + python_single_threaded_ = true; + } + void addProperty(const std::string &name, const std::string &description, const std::optional &defaultvalue, bool required, bool el, bool sensitive, const std::optional& property_type_code, gsl::span allowable_values, const std::optional& controller_service_type_name); @@ -126,6 +131,7 @@ class ExecutePythonProcessor : public core::ProcessorImpl { std::optional version_; bool python_dynamic_; + bool python_single_threaded_; std::string script_to_exec_; std::optional last_script_write_time_; diff --git a/extensions/python/PYTHON.md b/extensions/python/PYTHON.md index 13044261db..28382a5da0 100644 --- a/extensions/python/PYTHON.md +++ b/extensions/python/PYTHON.md @@ -222,6 +222,13 @@ class VaderSentiment(object): return len(self.content) ``` +By default the MiNiFi C++ style native python processors are executed with multiple threads in parallel, but it is possible to set the processor to be single threaded by calling the `setSingleThreaded()` method in the processor while initializing. When a processor is set to be single threaded, only one thread will execute the processor, and setting the max concurrent tasks to more than 1 in the flow configuration will not have any effect. + +```python +def onInitialize(processor): + processor.setSingleThreaded() +``` + ## Using NiFi Python Processors MiNiFi C++ supports the use of NiFi Python processors, that are inherited from the FlowFileTransform, RecordTransform or the FlowFileSource base class. To use these processors, copy the Python processor module to the nifi_python_processors subdirectory of the python directory. By default, the python directory is ${minifi_root}/minifi-python. To see how to write NiFi Python processors, please refer to the Python Developer Guide under the [Apache NiFi documentation](https://nifi.apache.org/nifi-docs/python-developer-guide.html). @@ -243,6 +250,27 @@ Due to some differences between the NiFi and MiNiFi C++ processors and implement - MiNiFi C++ uses a single embedded Python interpreter for all Python processors, so the Python processors share the same Python interpreter. This means that the Python processors cannot have different Python versions or use different Python packages. The Python packages are installed on the system or in a single virtualenv that is shared by all Python processors. - State manager API is available with the same interface as in NiFi, but MiNiFi C++ uses transactional state management, due to this when a state is changed in a processor trigger the state cannot be read due to the dirty read protection. The state can be read in the next trigger after the state is committed at the end of a session. Also MiNiFi C++ does not use clustering, so the scope parameter for the state operations is ignored as it is always local. +### Setting processor to be single threaded + +One feature that is currently only available in MiNiFi C++ is the ability to set a python processor to be single threaded. This is not yet available in NiFi's API, but similarly to Nifi's `@TriggerSerially` annotation, in MiNiFi C++ the `@trigger_serially` decorator can be used to make a processor single threaded. Setting this will make sure that only one thread executes the processor, and setting the max concurrent tasks to more than 1 in the flow configuration will not have any effect. + +```python +from nifiapi.flowfilesource import FlowFileSource, FlowFileSourceResult +from nifiapi.processorutils import trigger_serially + + +@trigger_serially +class SingleThreadedProcessor(FlowFileSource): + class Java: + implements = ['org.apache.nifi.python.processor.FlowFileSource'] + + def __init__(self, **kwargs): + pass + + def create(self, context): + return FlowFileSourceResult(relationship='success', contents="result") +``` + ## Use Python processors from virtualenv It is possible to set a virtualenv to be used by the Python processors in Apache MiNiFi C++. This is the default behavior. If the virtualenv directory is set, the Python processors will be executed using the packages installed in the virtualenv or if dependencies are not found there MiNiFi C++ will try to find the required packages installed on the system. If the virtualenv directory is not set, the Python processors will be executed using only the packages installed on the system. diff --git a/extensions/python/PythonProcessor.cpp b/extensions/python/PythonProcessor.cpp index c7a231f216..282a8cf9fa 100644 --- a/extensions/python/PythonProcessor.cpp +++ b/extensions/python/PythonProcessor.cpp @@ -31,6 +31,10 @@ void PythonProcessor::setSupportsDynamicProperties() { processor_->setSupportsDynamicProperties(); } +void PythonProcessor::setSingleThreaded() { + processor_->setSingleThreaded(); +} + void PythonProcessor::setDescription(const std::string& desc) { processor_->setDescription(desc); } diff --git a/extensions/python/PythonProcessor.h b/extensions/python/PythonProcessor.h index 0abe2be279..2f048863f3 100644 --- a/extensions/python/PythonProcessor.h +++ b/extensions/python/PythonProcessor.h @@ -38,6 +38,8 @@ class PythonProcessor { void setSupportsDynamicProperties(); + void setSingleThreaded(); + void setDescription(const std::string& desc); void setVersion(const std::string& version); diff --git a/extensions/python/pythonprocessors/nifiapi/processorbase.py b/extensions/python/pythonprocessors/nifiapi/processorbase.py index 1ef376c4ad..2088b03ff9 100644 --- a/extensions/python/pythonprocessors/nifiapi/processorbase.py +++ b/extensions/python/pythonprocessors/nifiapi/processorbase.py @@ -53,6 +53,9 @@ def onInitialize(self, processor: Processor): else: self.supports_dynamic_properties = False + if hasattr(self, '_trigger_serially') and self._trigger_serially: + processor.setSingleThreaded() + for property in self.getPropertyDescriptors(): expression_language_supported = True if property.expressionLanguageScope != ExpressionLanguageScope.NONE else False property_type_code = translateStandardValidatorToMiNiFiPropertype(property.validators) diff --git a/extensions/python/pythonprocessors/nifiapi/processorutils.py b/extensions/python/pythonprocessors/nifiapi/processorutils.py new file mode 100644 index 0000000000..502817f621 --- /dev/null +++ b/extensions/python/pythonprocessors/nifiapi/processorutils.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def trigger_serially(cls): + cls._trigger_serially = True + return cls diff --git a/extensions/python/tests/PythonManifestTests.cpp b/extensions/python/tests/PythonManifestTests.cpp index b4a3945df2..700673005a 100644 --- a/extensions/python/tests/PythonManifestTests.cpp +++ b/extensions/python/tests/PythonManifestTests.cpp @@ -66,6 +66,7 @@ TEST_CASE("Python processor's description is part of the manifest") { "def describe(proc):\n" " proc.setDescription('Another amazing processor')\n" " proc.setSupportsDynamicProperties()\n" + " proc.setSingleThreaded()\n" " proc.addProperty('Prop1', 'A great property', 'banana', True, False, False, None, ['apple', 'orange', 'banana', 'durian'], None)\n"; const auto executable_dir = minifi::utils::file::FileUtils::get_executable_dir(); @@ -77,7 +78,9 @@ TEST_CASE("Python processor's description is part of the manifest") { std::ofstream{python_dir / "nifi_python_processors" / "MyPyProc3.py"} << R"( from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult from nifiapi.properties import ExpressionLanguageScope, PropertyDescriptor, StandardValidators +from nifiapi.processorutils import trigger_serially +@trigger_serially class MyPyProc3(FlowFileTransform): class Java: @@ -203,7 +206,7 @@ class MyPyProc5(FlowFileTransform): auto MyPyProc = getProcessorNode(python_bundle); CHECK(getNode(MyPyProc->children, "inputRequirement").value == "INPUT_ALLOWED"); - CHECK(getNode(MyPyProc->children, "isSingleThreaded").value == true); + CHECK(getNode(MyPyProc->children, "isSingleThreaded").value == false); CHECK(getNode(MyPyProc->children, "typeDescription").value == "An amazing processor"); CHECK(getNode(MyPyProc->children, "supportsDynamicRelationships").value == false); CHECK(getNode(MyPyProc->children, "supportsDynamicProperties").value == false); @@ -302,6 +305,8 @@ class MyPyProc5(FlowFileTransform): { auto python_bundle = findPythonBundle("MyPyProc4"); CHECK(getNode(python_bundle->children, "version").value == minifi::AgentBuild::VERSION); + auto MyPyProc4 = getProcessorNode(python_bundle); + CHECK(getNode(MyPyProc4->children, "isSingleThreaded").value == false); } { diff --git a/extensions/python/tests/features/environment.py b/extensions/python/tests/features/environment.py index d22adcf271..0aaa3c3327 100644 --- a/extensions/python/tests/features/environment.py +++ b/extensions/python/tests/features/environment.py @@ -60,6 +60,10 @@ def before_all(context): COPY LogDynamicProperties.py {minifi_python_dir}/LogDynamicProperties.py COPY ExpressionLanguagePropertyWithValidator.py {minifi_python_dir}/nifi_python_processors/ExpressionLanguagePropertyWithValidator.py COPY EvaluateExpressionLanguageChecker.py {minifi_python_dir}/nifi_python_processors/EvaluateExpressionLanguageChecker.py +COPY SleepForever.py {minifi_python_dir}/nifi_python_processors/SleepForever.py +COPY SingleThreadedSleepForever.py {minifi_python_dir}/nifi_python_processors/SingleThreadedSleepForever.py +COPY MinifiSleepForever.py {minifi_python_dir}/MinifiSleepForever.py +COPY SingleThreadedMinifiSleepForever.py {minifi_python_dir}/SingleThreadedMinifiSleepForever.py RUN python3 -m venv {minifi_python_venv_parent}/venv """.format(base_image=context.minifi_container_image, pip3_install_command=pip3_install_command, diff --git a/extensions/python/tests/features/python.feature b/extensions/python/tests/features/python.feature index 657137c990..9fcb15efee 100644 --- a/extensions/python/tests/features/python.feature +++ b/extensions/python/tests/features/python.feature @@ -353,3 +353,53 @@ Feature: MiNiFi can use python processors in its flows And the Minifi logs contain the following message: "Non-existent property value is empty" in less than 1 seconds And the Minifi logs contain the following message: "My Dynamic Property value is: Dynamic ${my.attribute}" in less than 1 seconds And the Minifi logs contain the following message: "My Dynamic Property evaluated value is: Dynamic my.value" in less than 1 seconds + + Scenario Outline: NiFi python processors are allowed to run in parallel + Given a org.apache.nifi.minifi.processors.nifi_python_processors.SleepForever processor with the name "SleepForever" + And the max concurrent tasks attribute of the SleepForever processor is set to + And the scheduling period of the SleepForever processor is set to "100 milliseconds" + And SleepForever's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "Sleeping forever" times after 5 seconds + + Examples: + | Max concurrent tasks | + | 3 | + | 1 | + + Scenario: NiFi python processors can be set to be single threaded + Given a org.apache.nifi.minifi.processors.nifi_python_processors.SingleThreadedSleepForever processor with the name "SingleThreadedSleepForever" + And the max concurrent tasks attribute of the SingleThreadedSleepForever processor is set to 5 + And the scheduling period of the SingleThreadedSleepForever processor is set to "100 milliseconds" + And SingleThreadedSleepForever's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "Sleeping forever" 1 times after 5 seconds + + Scenario Outline: MiNiFi python processors are allowed to run in parallel + Given a MinifiSleepForever processor + And the max concurrent tasks attribute of the MinifiSleepForever processor is set to + And the scheduling period of the MinifiSleepForever processor is set to "100 milliseconds" + And MinifiSleepForever's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "Sleeping forever" times after 5 seconds + + Examples: + | Max concurrent tasks | + | 3 | + | 1 | + + Scenario: MiNiFi python processors can be set to be single threaded + Given a SingleThreadedMinifiSleepForever processor + And the max concurrent tasks attribute of the SingleThreadedMinifiSleepForever processor is set to 5 + And the scheduling period of the SingleThreadedMinifiSleepForever processor is set to "100 milliseconds" + And SingleThreadedMinifiSleepForever's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "Sleeping forever" 1 times after 5 seconds diff --git a/extensions/python/tests/features/resources/MinifiSleepForever.py b/extensions/python/tests/features/resources/MinifiSleepForever.py new file mode 100644 index 0000000000..a28b13fe5f --- /dev/null +++ b/extensions/python/tests/features/resources/MinifiSleepForever.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import time + + +def describe(processor): + processor.setDescription("Sleep forever processor.") + + +def onTrigger(context, session): + log.info("Sleeping forever") + while True: + time.sleep(1) diff --git a/extensions/python/tests/features/resources/SingleThreadedMinifiSleepForever.py b/extensions/python/tests/features/resources/SingleThreadedMinifiSleepForever.py new file mode 100644 index 0000000000..db08f03905 --- /dev/null +++ b/extensions/python/tests/features/resources/SingleThreadedMinifiSleepForever.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import time + + +def describe(processor): + processor.setDescription("Sleep forever processor.") + + +def onInitialize(processor): + processor.setSingleThreaded() + + +def onTrigger(context, session): + log.info("Sleeping forever") + while True: + time.sleep(1) diff --git a/extensions/python/tests/features/resources/SingleThreadedSleepForever.py b/extensions/python/tests/features/resources/SingleThreadedSleepForever.py new file mode 100644 index 0000000000..6f8bc2cc4e --- /dev/null +++ b/extensions/python/tests/features/resources/SingleThreadedSleepForever.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from nifiapi.flowfilesource import FlowFileSource +from nifiapi.processorutils import trigger_serially + + +@trigger_serially +class SingleThreadedSleepForever(FlowFileSource): + class Java: + implements = ['org.apache.nifi.python.processor.FlowFileSource'] + + class ProcessorDetails: + version = '2.0.0-snapshot' + description = '''Test Python source processor.''' + tags = ['text', 'test', 'python', 'source'] + + def __init__(self, **kwargs): + pass + + def create(self, context): + self.logger.info("Sleeping forever") + while True: + time.sleep(1) diff --git a/extensions/python/tests/features/resources/SleepForever.py b/extensions/python/tests/features/resources/SleepForever.py new file mode 100644 index 0000000000..4deef78782 --- /dev/null +++ b/extensions/python/tests/features/resources/SleepForever.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from nifiapi.flowfilesource import FlowFileSource + + +class SleepForever(FlowFileSource): + class Java: + implements = ['org.apache.nifi.python.processor.FlowFileSource'] + + class ProcessorDetails: + version = '2.0.0-snapshot' + description = '''Test Python source processor.''' + tags = ['text', 'test', 'python', 'source'] + + def __init__(self, **kwargs): + pass + + def create(self, context): + self.logger.info("Sleeping forever") + while True: + time.sleep(1) diff --git a/extensions/python/types/PyProcessor.cpp b/extensions/python/types/PyProcessor.cpp index d39b9c3ed5..723f430733 100644 --- a/extensions/python/types/PyProcessor.cpp +++ b/extensions/python/types/PyProcessor.cpp @@ -43,6 +43,7 @@ extern "C" { static PyMethodDef PyProcessor_methods[] = { // NOLINT(cppcoreguidelines-avoid-c-arrays) {"setSupportsDynamicProperties", (PyCFunction) PyProcessor::setSupportsDynamicProperties, METH_VARARGS, nullptr}, + {"setSingleThreaded", (PyCFunction) PyProcessor::setSingleThreaded, METH_VARARGS, nullptr}, {"setDescription", (PyCFunction) PyProcessor::setDescription, METH_VARARGS, nullptr}, {"setVersion", (PyCFunction) PyProcessor::setVersion, METH_VARARGS, nullptr}, {"addProperty", (PyCFunction) PyProcessor::addProperty, METH_VARARGS, nullptr}, @@ -89,6 +90,17 @@ PyObject* PyProcessor::setSupportsDynamicProperties(PyProcessor* self, PyObject* Py_RETURN_NONE; } +PyObject* PyProcessor::setSingleThreaded(PyProcessor* self, PyObject*) { + auto processor = self->processor_.lock(); + if (!processor) { + PyErr_SetString(PyExc_AttributeError, "tried reading processor outside 'on_trigger'"); + return nullptr; + } + + processor->setSingleThreaded(); + Py_RETURN_NONE; +} + PyObject* PyProcessor::setDescription(PyProcessor* self, PyObject* args) { auto processor = self->processor_.lock(); if (!processor) { diff --git a/extensions/python/types/PyProcessor.h b/extensions/python/types/PyProcessor.h index ff9e804a88..891b4714f0 100644 --- a/extensions/python/types/PyProcessor.h +++ b/extensions/python/types/PyProcessor.h @@ -32,6 +32,7 @@ struct PyProcessor { static int init(PyProcessor* self, PyObject* args, PyObject* kwds); static PyObject* setSupportsDynamicProperties(PyProcessor* self, PyObject* args); + static PyObject* setSingleThreaded(PyProcessor* self, PyObject* args); static PyObject* setDescription(PyProcessor* self, PyObject* args); static PyObject* setVersion(PyProcessor* self, PyObject* args); static PyObject* addProperty(PyProcessor* self, PyObject* args); diff --git a/packaging/msi/WixWin.wsi.in b/packaging/msi/WixWin.wsi.in index 9c912f604e..9b8060ba8e 100644 --- a/packaging/msi/WixWin.wsi.in +++ b/packaging/msi/WixWin.wsi.in @@ -405,6 +405,7 @@ ${WIX_EXTRA_COMPONENTS} + diff --git a/packaging/rpm/expected-rpm-contents.in b/packaging/rpm/expected-rpm-contents.in index b4eb8a37f1..36812b4ea1 100644 --- a/packaging/rpm/expected-rpm-contents.in +++ b/packaging/rpm/expected-rpm-contents.in @@ -61,6 +61,7 @@ /var/lib/nifi-minifi-cpp/minifi-python/nifiapi/flowfilesource.py /var/lib/nifi-minifi-cpp/minifi-python/nifiapi/flowfiletransform.py /var/lib/nifi-minifi-cpp/minifi-python/nifiapi/processorbase.py +/var/lib/nifi-minifi-cpp/minifi-python/nifiapi/processorutils.py /var/lib/nifi-minifi-cpp/minifi-python/nifiapi/properties.py /var/lib/nifi-minifi-cpp/minifi-python/nifiapi/recordtransform.py /var/lib/nifi-minifi-cpp/minifi-python/nifiapi/relationship.py