From 68a1563c14546e43bc607c1b15ece29d7323dbe3 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 24 Sep 2025 12:34:08 +0200 Subject: [PATCH 01/12] Rework unicode support for old bugs and Python2 Signed-off-by: Jose Luis Rivero --- bloom/generators/debian/generator.py | 34 +++++----------------------- bloom/generators/rpm/generator.py | 18 ++++----------- 2 files changed, 11 insertions(+), 41 deletions(-) diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index c1abe493..5da90252 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -108,17 +108,6 @@ debug(traceback.format_exc()) error("empy was not detected, please install it.", exit=True) -# Fix unicode bug in empy -# This should be removed once upstream empy is fixed -# See: https://github.com/ros-infrastructure/bloom/issues/196 -try: - em.str = unicode - em.Stream.write_old = em.Stream.write - em.Stream.write = lambda self, data: em.Stream.write_old(self, data.encode('utf8')) -except NameError: - pass -# End fix - # Drop the first log prefix for this command enable_drop_first_log_prefix(True) @@ -151,10 +140,7 @@ def __place_template_folder(group, src, dst, gbp=False): debug("Not overwriting existing file '{0}'".format(template_dst)) else: with io.open(template_dst, 'w', encoding='utf-8') as f: - if not isinstance(template, str): - template = template.decode('utf-8') - # Python 2 API needs a `unicode` not a utf-8 string. - elif sys.version_info.major == 2: + if isinstance(template, bytes): template = template.decode('utf-8') f.write(template) shutil.copystat(template_abs_path, template_dst) @@ -483,17 +469,11 @@ def generate_substitutions_from_package( data['Licenses'] = licenses def convertToUnicode(obj): - if sys.version_info.major == 2: - if isinstance(obj, str): - return unicode(obj.decode('utf8')) - elif isinstance(obj, unicode): - return obj - else: - if isinstance(obj, bytes): - return str(obj.decode('utf8')) - elif isinstance(obj, str): - return obj - if isinstance(obj, list): + if isinstance(obj, bytes): + return str(obj.decode('utf8')) + elif isinstance(obj, str): + return obj + elif isinstance(obj, list): for i, val in enumerate(obj): obj[i] = convertToUnicode(val) return obj @@ -542,8 +522,6 @@ def __process_template_folder(path, subs): continue # Write the result with io.open(template_path, 'w', encoding='utf-8') as f: - if sys.version_info.major == 2: - result = result.decode('utf-8') f.write(result) # Copy the permissions shutil.copymode(item, template_path) diff --git a/bloom/generators/rpm/generator.py b/bloom/generators/rpm/generator.py index 66c8f3fe..dc7752b4 100644 --- a/bloom/generators/rpm/generator.py +++ b/bloom/generators/rpm/generator.py @@ -325,17 +325,11 @@ def generate_substitutions_from_package( summarize_dependency_mapping(data, depends, build_depends, resolved_deps) def convertToUnicode(obj): - if sys.version_info.major == 2: - if isinstance(obj, str): - return unicode(obj.decode('utf8')) - elif isinstance(obj, unicode): - return obj - else: - if isinstance(obj, bytes): - return str(obj.decode('utf8')) - elif isinstance(obj, str): - return obj - if isinstance(obj, list): + if isinstance(obj, bytes): + return str(obj.decode('utf8')) + elif isinstance(obj, str): + return obj + elif isinstance(obj, list): for i, val in enumerate(obj): obj[i] = convertToUnicode(val) return obj @@ -381,8 +375,6 @@ def __process_template_folder(path, subs): result = em.expand(template, **subs) # Write the result with io.open(template_path, 'w', encoding='utf-8') as f: - if sys.version_info.major == 2: - result = result.decode('utf-8') f.write(result) # Copy the permissions shutil.copymode(item, template_path) From 057af7389dbec4f4d61f3061030b203aea8a8e00 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 24 Sep 2025 15:52:45 +0200 Subject: [PATCH 02/12] Support empy4 together with empy3 Use a wrapper function to make the em.expand calls. Signed-off-by: Jose Luis Rivero --- bloom/generators/debian/generator.py | 10 +++------- bloom/generators/rpm/generator.py | 9 ++------- bloom/util.py | 19 +++++++++++++++++++ setup.py | 3 +-- .../test_debian/test_generator.py | 4 ++-- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index 5da90252..4439e814 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -78,6 +78,8 @@ from bloom.logging import is_debug from bloom.logging import warning +from bloom.util import expand_template_em + from bloom.commands.git.patch.common import get_patch_config from bloom.commands.git.patch.common import set_patch_config @@ -102,12 +104,6 @@ debug(traceback.format_exc()) error("rosdistro was not detected, please install it.", exit=True) -try: - import em -except ImportError: - debug(traceback.format_exc()) - error("empy was not detected, please install it.", exit=True) - # Drop the first log prefix for this command enable_drop_first_log_prefix(True) @@ -514,7 +510,7 @@ def __process_template_folder(path, subs): info("Expanding '{0}' -> '{1}'".format( os.path.relpath(item), os.path.relpath(template_path))) - result = em.expand(template, **subs) + result = expand_template_em(template, subs) # Don't write an empty file if len(result) == 0 and \ os.path.basename(template_path) in ['copyright']: diff --git a/bloom/generators/rpm/generator.py b/bloom/generators/rpm/generator.py index dc7752b4..2d8bc62d 100644 --- a/bloom/generators/rpm/generator.py +++ b/bloom/generators/rpm/generator.py @@ -81,6 +81,7 @@ from bloom.util import code from bloom.util import execute_command +from bloom.util import expand_template_em from bloom.util import maybe_continue try: @@ -89,12 +90,6 @@ debug(traceback.format_exc()) error("rosdistro was not detected, please install it.", exit=True) -try: - import em -except ImportError: - debug(traceback.format_exc()) - error("empy was not detected, please install it.", exit=True) - # Drop the first log prefix for this command enable_drop_first_log_prefix(True) @@ -372,7 +367,7 @@ def __process_template_folder(path, subs): info("Expanding '{0}' -> '{1}'".format( os.path.relpath(item), os.path.relpath(template_path))) - result = em.expand(template, **subs) + result = expand_template_em(template, subs) # Write the result with io.open(template_path, 'w', encoding='utf-8') as f: f.write(result) diff --git a/bloom/util.py b/bloom/util.py index 7ef46093..a39c84a5 100755 --- a/bloom/util.py +++ b/bloom/util.py @@ -84,6 +84,25 @@ to_unicode = str +def expand_template_em(template, subs): + """ + Compatibility function for EmPy 3 and 4. + EmPy 3: em.expand(template, **kwargs) + EmPy 4: em.expand(template, locals=dict) + """ + try: + import em + except ImportError: + error("empy was not detected, please install it.", exit=True) + + try: + # Try EmPy 4 API first + return em.expand(template, locals=subs) + except (TypeError, NameError): + # Fall back to EmPy 3 API + return em.expand(template, **subs) + + def flush_stdin(): try: from termios import tcflush, TCIFLUSH diff --git a/setup.py b/setup.py index 476911b0..f07c3b9c 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import sys from setuptools import find_packages, setup @@ -20,7 +19,7 @@ install_requires=[ 'catkin_pkg >= 0.4.3', 'setuptools', - 'empy < 4', + 'empy', 'packaging', 'python-dateutil', 'PyYAML', diff --git a/test/unit_tests/test_generators/test_debian/test_generator.py b/test/unit_tests/test_generators/test_debian/test_generator.py index d0eb2723..466fcfb0 100644 --- a/test/unit_tests/test_generators/test_debian/test_generator.py +++ b/test/unit_tests/test_generators/test_debian/test_generator.py @@ -2,9 +2,9 @@ from ....utils.common import redirected_stdio -from bloom.generators.debian.generator import em from bloom.generators.debian.generator import get_changelogs from bloom.generators.debian.generator import format_description +from bloom.util import expand_template_em from catkin_pkg.packages import find_packages @@ -24,7 +24,7 @@ def test_unicode_templating(): assert 'bad_changelog_pkg' in packages chlogs = get_changelogs(packages['bad_changelog_pkg']) template = "@(changelog)" - em.expand(template, {'changelog': chlogs[0][2]}) + expand_template_em(template, {'changelog': chlogs[0][2]}) def test_format_description(): From 78ff8bbd121cbf88e205d777c36abca6e64b1af8 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 24 Sep 2025 16:10:01 +0200 Subject: [PATCH 03/12] Add a custom test for empy legacy Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9a4501ee..2cc1a534 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,6 +12,28 @@ jobs: uses: ros-infrastructure/ci/.github/workflows/pytest.yaml@main with: matrix-filter: del(.matrix.os[] | select(contains("windows"))) + + # Dedicated job for testing with empy < 4 + pytest-empy-legacy: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.11', '3.12', '3.13'] + name: Test empy < 4 compatibility (Python ${{ matrix.python-version }}) + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies with empy < 4 + run: | + python -m pip install --upgrade pip + pip install 'empy<4' + pip install -e .[test] + - name: Run tests + run: | + python -m pytest test/ -v yamllint: runs-on: ubuntu-latest steps: From abc6a11086fd7bc145981fad02a6de00985827d0 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 26 Jan 2026 18:59:16 +0100 Subject: [PATCH 04/12] Refactor EmPy version handling in template expansion --- bloom/util.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bloom/util.py b/bloom/util.py index a39c84a5..78c0a180 100755 --- a/bloom/util.py +++ b/bloom/util.py @@ -95,12 +95,10 @@ def expand_template_em(template, subs): except ImportError: error("empy was not detected, please install it.", exit=True) - try: - # Try EmPy 4 API first - return em.expand(template, locals=subs) - except (TypeError, NameError): - # Fall back to EmPy 3 API + if em.__version__.startswith('3'): return em.expand(template, **subs) + else: + return em.expand(template, locals=subs) def flush_stdin(): From 1faa8de120fadfe9eb63377a8791d83d614021ff Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 9 Feb 2026 16:39:49 +0100 Subject: [PATCH 05/12] Update .github/workflows/ci.yaml Co-authored-by: Scott K Logan --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2cc1a534..b68b377c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,8 +29,7 @@ jobs: - name: Install dependencies with empy < 4 run: | python -m pip install --upgrade pip - pip install 'empy<4' - pip install -e .[test] + pip install -e .[test] 'empy<4' - name: Run tests run: | python -m pytest test/ -v From 37c2c90bc9ef29699bcb47d397dbcb152111d907 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 9 Feb 2026 16:46:27 +0100 Subject: [PATCH 06/12] Reduce the testing matrix to just one Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b68b377c..53e0cac6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.11', '3.12', '3.13'] + python-version: ['3.13'] name: Test empy < 4 compatibility (Python ${{ matrix.python-version }}) steps: - uses: actions/checkout@v4 From 7f1db432813f7c05bcaecf62d87d257ad3ccc022 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 9 Feb 2026 17:11:38 +0100 Subject: [PATCH 07/12] Revert changes not related to empy Signed-off-by: Jose Luis Rivero --- bloom/generators/debian/generator.py | 40 +++++++++++++++++++++++----- bloom/generators/rpm/generator.py | 18 +++++++++---- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index 4439e814..5a11a9f9 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -104,6 +104,23 @@ debug(traceback.format_exc()) error("rosdistro was not detected, please install it.", exit=True) +try: + import em +except ImportError: + debug(traceback.format_exc()) + error("empy was not detected, please install it.", exit=True) + +# Fix unicode bug in empy +# This should be removed once upstream empy is fixed +# See: https://github.com/ros-infrastructure/bloom/issues/196 +try: + em.str = unicode + em.Stream.write_old = em.Stream.write + em.Stream.write = lambda self, data: em.Stream.write_old(self, data.encode('utf8')) +except NameError: + pass +# End fix + # Drop the first log prefix for this command enable_drop_first_log_prefix(True) @@ -136,7 +153,10 @@ def __place_template_folder(group, src, dst, gbp=False): debug("Not overwriting existing file '{0}'".format(template_dst)) else: with io.open(template_dst, 'w', encoding='utf-8') as f: - if isinstance(template, bytes): + if not isinstance(template, str): + template = template.decode('utf-8') + # Python 2 API needs a `unicode` not a utf-8 string. + elif sys.version_info.major == 2: template = template.decode('utf-8') f.write(template) shutil.copystat(template_abs_path, template_dst) @@ -465,11 +485,17 @@ def generate_substitutions_from_package( data['Licenses'] = licenses def convertToUnicode(obj): - if isinstance(obj, bytes): - return str(obj.decode('utf8')) - elif isinstance(obj, str): - return obj - elif isinstance(obj, list): + if sys.version_info.major == 2: + if isinstance(obj, str): + return unicode(obj.decode('utf8')) + elif isinstance(obj, unicode): + return obj + else: + if isinstance(obj, bytes): + return str(obj.decode('utf8')) + elif isinstance(obj, str): + return obj + if isinstance(obj, list): for i, val in enumerate(obj): obj[i] = convertToUnicode(val) return obj @@ -518,6 +544,8 @@ def __process_template_folder(path, subs): continue # Write the result with io.open(template_path, 'w', encoding='utf-8') as f: + if sys.version_info.major == 2: + result = result.decode('utf-8') f.write(result) # Copy the permissions shutil.copymode(item, template_path) diff --git a/bloom/generators/rpm/generator.py b/bloom/generators/rpm/generator.py index 2d8bc62d..00359080 100644 --- a/bloom/generators/rpm/generator.py +++ b/bloom/generators/rpm/generator.py @@ -320,11 +320,17 @@ def generate_substitutions_from_package( summarize_dependency_mapping(data, depends, build_depends, resolved_deps) def convertToUnicode(obj): - if isinstance(obj, bytes): - return str(obj.decode('utf8')) - elif isinstance(obj, str): - return obj - elif isinstance(obj, list): + if sys.version_info.major == 2: + if isinstance(obj, str): + return unicode(obj.decode('utf8')) + elif isinstance(obj, unicode): + return obj + else: + if isinstance(obj, bytes): + return str(obj.decode('utf8')) + elif isinstance(obj, str): + return obj + if isinstance(obj, list): for i, val in enumerate(obj): obj[i] = convertToUnicode(val) return obj @@ -370,6 +376,8 @@ def __process_template_folder(path, subs): result = expand_template_em(template, subs) # Write the result with io.open(template_path, 'w', encoding='utf-8') as f: + if sys.version_info.major == 2: + result = result.decode('utf-8') f.write(result) # Copy the permissions shutil.copymode(item, template_path) From 9772887a00a319cd3ddaafb9055226bf10c67aeb Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 10 Feb 2026 12:40:28 +0100 Subject: [PATCH 08/12] Update .github/workflows/ci.yaml Co-authored-by: Scott K Logan --- .github/workflows/ci.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53e0cac6..8bc5e48a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,16 +16,11 @@ jobs: # Dedicated job for testing with empy < 4 pytest-empy-legacy: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.13'] - name: Test empy < 4 compatibility (Python ${{ matrix.python-version }}) + name: Test empy < 4 compatibility steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - name: Install dependencies with empy < 4 run: | python -m pip install --upgrade pip From 6f6a571c8cd157fd399f90ee1c000662df5c1830 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 10 Feb 2026 19:00:15 +0100 Subject: [PATCH 09/12] Update install_requires in setup.py Removed 'setuptools' and 'empy < 4' from install_requires. --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index b695e076..1b850d83 100755 --- a/setup.py +++ b/setup.py @@ -18,10 +18,8 @@ }, install_requires=[ 'catkin_pkg >= 0.4.3', - 'setuptools', 'empy', 'setuptools < 82', - 'empy < 4', 'packaging', 'python-dateutil', 'PyYAML', From a9153dfb62fc84eab364e70444f1fab9d812224d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 10 Feb 2026 19:25:57 +0100 Subject: [PATCH 10/12] Revert "Update install_requires in setup.py" This reverts commit 6f6a571c8cd157fd399f90ee1c000662df5c1830. --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 1b850d83..b695e076 100755 --- a/setup.py +++ b/setup.py @@ -18,8 +18,10 @@ }, install_requires=[ 'catkin_pkg >= 0.4.3', + 'setuptools', 'empy', 'setuptools < 82', + 'empy < 4', 'packaging', 'python-dateutil', 'PyYAML', From cb30d533c476d9a5f011ee1f0aadda09e51e99be Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 10 Feb 2026 12:38:03 -0600 Subject: [PATCH 11/12] Drop extra setuptools dep --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index b695e076..93a3e7b6 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ }, install_requires=[ 'catkin_pkg >= 0.4.3', - 'setuptools', 'empy', 'setuptools < 82', 'empy < 4', From 23ffdbfa9d36dbaef1058cca577688236815146f Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 10 Feb 2026 12:47:49 -0600 Subject: [PATCH 12/12] Un-pin empy --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 93a3e7b6..f4f99065 100755 --- a/setup.py +++ b/setup.py @@ -18,9 +18,8 @@ }, install_requires=[ 'catkin_pkg >= 0.4.3', - 'empy', 'setuptools < 82', - 'empy < 4', + 'empy', 'packaging', 'python-dateutil', 'PyYAML',