From c17b77e537ec9e23ccfeef2e9d4a80ea1827e94a Mon Sep 17 00:00:00 2001 From: Jean Abou Samra Date: Sun, 29 May 2022 22:47:47 +0200 Subject: [PATCH 1/6] Switch from pkg_resources to importlib.metadata importlib.metadata is a standard library replacement for pkg_resources, available starting with Python 3.8. It is faster than pkg_resources. This lets the plugin interface use it when available, i.e. on Python 3.8 and later. On earlier Python, it uses the importlib_metadata PyPI backport if available, and finally falls back on pkg_resources. setup.cfg gains an extra called plugins, which can allow a project to install "pygments[plugins]" in order to ensure that plugins support is present even if installed with older Python versions. Timings for lexing an empty file with a lexer from a random plugin: Before: real 0m0,238s user 0m0,210s sys 0m0,029s After: real 0m0,141s user 0m0,125s sys 0m0,017s Fixes #2116, #2126 --- doc/docs/plugins.rst | 39 ++++++++++++++++++++++++++++++++++----- pygments/plugin.py | 28 ++++++++++++++++++++-------- setup.cfg | 4 ++++ 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/doc/docs/plugins.rst b/doc/docs/plugins.rst index 1008013aa..67388607a 100644 --- a/doc/docs/plugins.rst +++ b/doc/docs/plugins.rst @@ -1,6 +1,6 @@ -================ -Register Plugins -================ +======= +Plugins +======= If you want to extend Pygments without hacking the sources, but want to use the lexer/formatter/style/filter lookup functions (`lexers.get_lexer_by_name` @@ -13,8 +13,37 @@ That means you can use your highlighter modules with the `pygmentize` script, which relies on the mentioned functions. -Entrypoints -=========== +Plugin discovery +================ + +At runtime, discovering plugins is preferentially done using Python's +standard library module `importlib.metadata`_, available in Python 3.8 +and higher. In earlier Python versions, Pygments attempts to use the +`importlib_metadata`_ backport, if available. If not available, a +fallback is attempted on the older `pkg_resources`_ module. Finally, if +``pkg_resources`` is not available, no plugins will be loaded at +all. Note that ``pkg_resources`` is distributed with `setuptools`_, and +thus available on most Python environments. However, ``pkg_resources`` +is considerably slower than ``importlib.metadata`` or its +``importlib_metadata`` backport. For this reason, if you run Pygments +under Python older than 3.8, it is recommended to install +``importlib-metadata``. Pygments defines a ``plugins`` packaging extra, +so you can ensure it is installed with best plugin support (i.e., that +``importlib-metadata`` is also installed in case you are running Python +earlier than 3.8) by specifying ``pygments[plugins]`` as the +requirement, for example, with ``pip``: + +.. sourcecode:: shell + + $ python -m pip install --user pygments[plugins] + +.. _importlib.metadata: https://docs.python.org/3.10/library/importlib.metadata.html +.. _importlib_metadata: https://pypi.org/project/importlib-metadata +.. _pkg_resources: https://setuptools.pypa.io/en/latest/pkg_resources.html + + +Defining plugins through entrypoints +==================================== Here is a list of setuptools entrypoints that Pygments understands: diff --git a/pygments/plugin.py b/pygments/plugin.py index 247d5083b..d2afafebd 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -2,9 +2,11 @@ pygments.plugin ~~~~~~~~~~~~~~~ - Pygments setuptools plugin interface. The methods defined - here also work if setuptools isn't installed but they just - return nothing. + Pygments plugin interface. By default, this tries to use + ``importlib.metadata``, which is in the Python standard + library since Python 3.8. It falls back on ``pkg_resources`` + if not found. Finally, if ``pkg_resources`` is not found + either, no plugins are loaded at all. lexer plugins:: @@ -42,11 +44,21 @@ def iter_entry_points(group_name): try: - import pkg_resources - except (ImportError, OSError): - return [] - - return pkg_resources.iter_entry_points(group_name) + from importlib.metadata import entry_points + except ImportError: + try: + from importlib_metadata import entry_points + except ImportError: + try: + from pkg_resources import iter_entry_points + except (ImportError, OSError): + return [] + else: + return iter_entry_points(group_name) + else: + return entry_points(group=group_name) + else: + return entry_points(group=group_name) def find_plugin_lexers(): diff --git a/setup.cfg b/setup.cfg index 661914c2f..a59082268 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,10 @@ include = console_scripts = pygmentize = pygments.cmdline:main +[options.extras_require] +plugins = + importlib-metadata;python_version<'3.8' + [aliases] release = egg_info -Db '' upload = upload --sign --identity=36580288 From ab346aaf7dcd4766c15c4140c4e5dce2ee8c1adb Mon Sep 17 00:00:00 2001 From: Jean Abou Samra Date: Mon, 30 May 2022 00:01:12 +0200 Subject: [PATCH 2/6] fixup: don't use group kwarg It was apparently added in Python 3.10 or recent versions of the importlib_metadata backport. --- pygments/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/plugin.py b/pygments/plugin.py index d2afafebd..0deea439f 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -56,9 +56,9 @@ def iter_entry_points(group_name): else: return iter_entry_points(group_name) else: - return entry_points(group=group_name) + return entry_points().get(group_name, []) else: - return entry_points(group=group_name) + return entry_points().get(group_name, []) def find_plugin_lexers(): From d80c185090bb4dd6d41477a93d5466d775efb3c6 Mon Sep 17 00:00:00 2001 From: Jean Abou Samra Date: Mon, 30 May 2022 00:13:50 +0200 Subject: [PATCH 3/6] Silence warning (sigh) --- pygments/plugin.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pygments/plugin.py b/pygments/plugin.py index 0deea439f..a84d4ef27 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -36,6 +36,9 @@ :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ + +from warnings import filterwarnings + LEXER_ENTRY_POINT = 'pygments.lexers' FORMATTER_ENTRY_POINT = 'pygments.formatters' STYLE_ENTRY_POINT = 'pygments.styles' @@ -43,6 +46,8 @@ def iter_entry_points(group_name): + # We can use .select() when we no longer support Python 3.9. + filterwarnings('ignore', 'SelectableGroups dict interface is deprecated. Use select.', DeprecationWarning) try: from importlib.metadata import entry_points except ImportError: From 31efed2e86d4be2e88212e9751fa748fc47e50b0 Mon Sep 17 00:00:00 2001 From: Jean Abou Samra Date: Mon, 30 May 2022 10:33:31 +0200 Subject: [PATCH 4/6] Likely better to use the non-deprecated interface in Python 3.10 & newer importlib_metadata --- pygments/plugin.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pygments/plugin.py b/pygments/plugin.py index a84d4ef27..135edd05c 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -46,8 +46,6 @@ def iter_entry_points(group_name): - # We can use .select() when we no longer support Python 3.9. - filterwarnings('ignore', 'SelectableGroups dict interface is deprecated. Use select.', DeprecationWarning) try: from importlib.metadata import entry_points except ImportError: @@ -60,10 +58,15 @@ def iter_entry_points(group_name): return [] else: return iter_entry_points(group_name) - else: - return entry_points().get(group_name, []) + groups = entry_points() + if hasattr(groups, 'select'): + # New interface in Python 3.10 and newer versions of the + # importlib_metadata backport. + return groups.select(group=group_name) else: - return entry_points().get(group_name, []) + # Older interface, deprecated in Python 3.10 and recent + # importlib_metadata, but we need it in Python 3.8 and 3.9. + return groups.get(group_name, []) def find_plugin_lexers(): From 25ec5d94dd2b4b7b2128d07c3ca818d2700f2be3 Mon Sep 17 00:00:00 2001 From: Jean Abou Samra Date: Mon, 30 May 2022 13:50:50 +0200 Subject: [PATCH 5/6] Docstring fix --- pygments/plugin.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pygments/plugin.py b/pygments/plugin.py index 135edd05c..ceef8d079 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -4,9 +4,10 @@ Pygments plugin interface. By default, this tries to use ``importlib.metadata``, which is in the Python standard - library since Python 3.8. It falls back on ``pkg_resources`` - if not found. Finally, if ``pkg_resources`` is not found - either, no plugins are loaded at all. + library since Python 3.8, or its ``importlib_metadata`` + backport for earlier versions of Python. It falls back on + ``pkg_resources`` if not found. Finally, if ``pkg_resources`` + is not found either, no plugins are loaded at all. lexer plugins:: From 9b83858be633c26b4309857115294f9123b78de6 Mon Sep 17 00:00:00 2001 From: Jean Abou Samra Date: Mon, 30 May 2022 13:51:07 +0200 Subject: [PATCH 6/6] Forgot to remove import --- pygments/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pygments/plugin.py b/pygments/plugin.py index ceef8d079..0ffef47ee 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -38,8 +38,6 @@ :license: BSD, see LICENSE for details. """ -from warnings import filterwarnings - LEXER_ENTRY_POINT = 'pygments.lexers' FORMATTER_ENTRY_POINT = 'pygments.formatters' STYLE_ENTRY_POINT = 'pygments.styles'