diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3b78abb9..25238614 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,8 +3,7 @@ name: Tests on: push: branches: [ main ] - pull_request: - branches: [ main ] + pull_request: ~ # Allow job to be triggered manually. workflow_dispatch: @@ -36,6 +35,8 @@ jobs: - name: Acquire sources uses: actions/checkout@v3 + with: + fetch-depth: 0 # https://github.com/docker-practice/actions-setup-docker # - name: Install Docker diff --git a/CHANGES.rst b/CHANGES.rst index 16f20fa5..6999f75c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,7 @@ mqttwarn changelog in progress =========== - [udf] Unlock JavaScript for user-defined functions. Thanks, @extremeheat. +- [udf] Unlock Lua for user-defined functions. Thanks, @scoder. 2023-10-15 0.35.0 diff --git a/Dockerfile b/Dockerfile index f92a1e23..5b6114e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ RUN --mount=type=cache,id=pip,target=/root/.cache/pip \ true \ && pip install --upgrade pip \ && pip install --prefer-binary versioningit wheel \ - && pip install --use-pep517 --prefer-binary '/src[javascript]' + && pip install --use-pep517 --prefer-binary '/src[javascript,lua]' # Uninstall build prerequisites again. RUN apt-get --yes remove --purge git && apt-get --yes autoremove diff --git a/docs/configure/transformation.md b/docs/configure/transformation.md index 6a3344d0..e441cd98 100644 --- a/docs/configure/transformation.md +++ b/docs/configure/transformation.md @@ -447,6 +447,19 @@ export its main entry point symbol, configure mqttwarn to use `functions = myclo and adjust its settings to use your MQTT broker endpoint at the beginning of the data pipeline, invoke mqttwarn, and turn off Kafka. It works! +On the next day, after investigating if you need to migrate any other system components, +you realize that there is an Nginx instance, which receives a certain share of telemetry +traffic using HTTP, and processes it using Lua. One quick `mosquitto_pub` later, you are +sure those telemetry messages are _also_ available on the MQTT bus already. Another set +of transformation rules written in Lua was quickly identified, and, after applying the +same procedure of inlining it into a single-file version, and configuring another mqttwarn +instance with `functions = mycloud.lua`, you are ready to turn off your whole cloud +infrastructure, and save valuable resources. + +After a while, you are able to hire back half of your previous engineering team, and, +based on the new architecture, you will happily start contributing back to mqttwarn, +both in terms of maintenance, and by adding new features. + :::{note} Rest assured we are overexaggerating a bit, and [Kafka] can only be compared to [MQTT] if you are also willing to compare apples with oranges, but you will get the point that @@ -467,6 +480,15 @@ available [OCI images](#using-oci-image). You can find an example implementation for a `filter` function written in JavaScript at the [OwnTracks-to-ntfy example tutorial](#owntracks-ntfy-variants-udf). +#### Lua + +For running user-defined functions code written in Lua, mqttwarn uses the excellent +[lupa] package. For adding JavaScript support to mqttwarn, install it using pip like +`pip install --upgrade 'mqttwarn[lua]'`, or use one of the available +[OCI images](#using-oci-image). + +You can find an example implementation for a `filter` function written in Lua +at the [OwnTracks-to-ntfy example tutorial](#owntracks-ntfy-variants-udf). ## User-defined function examples @@ -707,6 +729,7 @@ weather,topic=tasmota/temp/ds/1 temperature=19.7 1517525319000 [Jinja2 templates]: https://jinja.palletsprojects.com/templates/ [JSPyBridge]: https://pypi.org/project/javascript/ [Kafka]: https://en.wikipedia.org/wiki/Apache_Kafka +[lupa]: https://github.com/scoder/lupa [MQTT]: https://en.wikipedia.org/wiki/MQTT [Node.js]: https://en.wikipedia.org/wiki/Node.js [OwnTracks]: https://owntracks.org diff --git a/docs/usage/pip.md b/docs/usage/pip.md index 31770753..fef6b7b0 100644 --- a/docs/usage/pip.md +++ b/docs/usage/pip.md @@ -11,9 +11,9 @@ that. pip install --upgrade mqttwarn ``` -Add JavaScript support for user-defined functions. +Add JavaScript and Lua support for user-defined functions. ```bash -pip install --upgrade 'mqttwarn[javascript]' +pip install --upgrade 'mqttwarn[javascript,lua]' ``` You can also add support for a specific service plugin. diff --git a/examples/owntracks-ntfy/mqttwarn-owntracks.lua b/examples/owntracks-ntfy/mqttwarn-owntracks.lua new file mode 100644 index 00000000..b731cdad --- /dev/null +++ b/examples/owntracks-ntfy/mqttwarn-owntracks.lua @@ -0,0 +1,28 @@ +--[[ +Forward OwnTracks low-battery warnings to ntfy. +https://mqttwarn.readthedocs.io/en/latest/examples/owntracks-battery/readme.html +--]] + +-- mqttwarn filter function, returning true if the message should be ignored. +-- In this case, ignore all battery level telemetry values above a certain threshold. +function owntracks_batteryfilter(topic, message) + local ignore = true + + -- Decode inbound message. + local data = json.decode(message) + + -- Evaluate filtering rule. + if data ~= nil and data.batt ~= nil then + ignore = tonumber(data.batt) > 20 + end + + return ignore +end + +-- Status message. +print("Loaded Lua module.") + +-- Export symbols. +return { + owntracks_batteryfilter = owntracks_batteryfilter, +} diff --git a/examples/owntracks-ntfy/readme-variants.md b/examples/owntracks-ntfy/readme-variants.md index 8f91d485..aae5f7c3 100644 --- a/examples/owntracks-ntfy/readme-variants.md +++ b/examples/owntracks-ntfy/readme-variants.md @@ -50,9 +50,9 @@ targets = {'testdrive': 'http://localhost:5555/testdrive'} ### JavaScript -In order to try that on the OwnTracks-to-ntfy example, use the alternative -`mqttwarn-owntracks.js` implementation by adjusting the `functions` setting within the -`[defaults]` section of your configuration file, and restart mqttwarn. +In order to explore JavaScript user-defined functions using the OwnTracks-to-ntfy recipe, +use the alternative `mqttwarn-owntracks.js` implementation by adjusting the `functions` +setting within the `[defaults]` section of your configuration file, and restart mqttwarn. ```ini [defaults] functions = mqttwarn-owntracks.js @@ -69,3 +69,25 @@ previous one, which was written in Python. The feature to run JavaScript code is currently considered to be experimental. Please use it responsibly. ::: + +### Lua + +In order to explore Lua user-defined functions using the OwnTracks-to-ntfy recipe, +use the alternative `mqttwarn-owntracks.lua` implementation by adjusting the `functions` +setting within the `[defaults]` section of your configuration file, and restart mqttwarn. +```ini +[defaults] +functions = mqttwarn-owntracks.lua +``` + +The Lua function `owntracks_batteryfilter()` implements the same rule as the +previous ones, which was written in Python and JavaScript. + +:::{literalinclude} mqttwarn-owntracks.lua +:language: lua +::: + +:::{attention} +The feature to run Lua code is currently considered to be experimental. +Please use it responsibly. +::: diff --git a/mqttwarn/util.py b/mqttwarn/util.py index c8b59b8d..4d3cc570 100644 --- a/mqttwarn/util.py +++ b/mqttwarn/util.py @@ -13,6 +13,7 @@ import os import re import string +import sys import threading import types import typing as t @@ -150,6 +151,8 @@ def load_module_from_file(path: t.Union[str, Path]) -> types.ModuleType: loader = importlib.machinery.SourcelessFileLoader(fullname=name, path=str(path)) elif path.suffix in [".js", ".javascript"]: return load_source_js(name, str(path)) + elif path.suffix == ".lua": + return load_source_lua(name, str(path)) else: raise ImportError(f"Loading file type failed (only .py, .pyc, .js, .javascript): {path}") spec = importlib.util.spec_from_loader(loader.name, loader) @@ -317,3 +320,41 @@ def load_source_js(mod_name, filepath): javascript.eval_js(js_code) threading.Event().wait(0.01) return module_factory(mod_name, module["exports"]) + + +class LuaJsonAdapter: + """ + Support Lua as if it had its `json` module. + + Wasn't able to make Lua's `json` module work, so this provides minimal functionality + instead. It will be injected into the Lua context's global `json` symbol. + """ + + @staticmethod + def decode(data): + if data is None: + return None + return json.loads(data) + + +def load_source_lua(mod_name, filepath): + """ + Load a Lua module, and import its exported symbols into a synthetic Python module. + """ + import lupa + + lua = lupa.LuaRuntime(unpack_returned_tuples=True) + + # Lua modules want to be loaded without suffix, but the interpreter would like to know about their path. + modfile = Path(filepath).with_suffix("").name + modpath = Path(filepath).parent + # Yeah, Windows. + if sys.platform == "win32": + modpath = str(modpath).replace("\\", "\\\\") + lua.execute(rf'package.path = package.path .. ";{str(modpath)}/?.lua"') + + logger.info(f"Loading Lua module {modfile} from path {modpath}") + module, filepath = lua.require(modfile) + # FIXME: Add support for common modules, as long as they are not available natively. + lua.globals()["json"] = LuaJsonAdapter + return module_factory(mod_name, module) diff --git a/setup.py b/setup.py index 072bc408..ff3667fe 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,9 @@ "javascript": [ "javascript==1!1.0.1; python_version>='3.7'", ], + "lua": [ + "lupa<3", + ], "mysql": [ "mysql", ], @@ -205,6 +208,9 @@ "Operating System :: MacOS", "Operating System :: Microsoft :: Windows", "Programming Language :: JavaScript", + "Programming Language :: Lua", + "Programming Language :: Other", + "Programming Language :: Other Scripting Engines", "Programming Language :: Python", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", diff --git a/tests/test_util.py b/tests/test_util.py index d8c8fb5b..91eb89ed 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -202,6 +202,41 @@ def test_load_functions_javascript_runtime_failure(tmp_path): assert ex.match("ReferenceError: bar is not defined") +def test_load_functions_lua_success(tmp_path): + """ + Verify that Lua module loading, including symbol exporting and invocation, works well. + """ + luafile = tmp_path / "test.lua" + luafile.write_text("return { forty_two = function() return 42 end }") + luamod = load_functions(luafile) + assert luamod.forty_two() == 42 + + +def test_load_functions_lua_compile_failure(tmp_path): + """ + Verify that Lua module loading, including symbol exporting and invocation, works well. + """ + luafile = tmp_path / "test.lua" + luafile.write_text("Hotzenplotz") + with pytest.raises(Exception) as ex: + load_functions(luafile) + assert ex.typename == "LuaError" + assert ex.match("syntax error near ") + + +def test_load_functions_lua_runtime_failure(tmp_path): + """ + Verify that Lua module loading, including symbol exporting and invocation, works well. + """ + luafile = tmp_path / "test.lua" + luafile.write_text("return { foo = function() bar() end }") + luamod = load_functions(luafile) + with pytest.raises(Exception) as ex: + luamod.foo() + assert ex.typename == "LuaError" + assert ex.match(re.escape("attempt to call a nil value (global 'bar')")) + + def test_load_function(): # Load valid functions file