From 1c3b070d342df2bf4f005f0713fee8b0d249db04 Mon Sep 17 00:00:00 2001 From: Evan Rusackas Date: Mon, 6 Jul 2026 19:31:41 -0700 Subject: [PATCH] feat(i18n): do-not-translate registry + '#. do-not-translate' marker standard (#41651) Co-authored-by: Amin Ghadersohi Co-authored-by: Claude Opus 4.8 --- UPDATING.md | 2 + docs/developer_docs/contributing/howtos.md | 11 ++ .../translations/apply_do_not_translate.py | 128 ++++++++++++++++++ scripts/translations/babel_update.sh | 8 ++ scripts/translations/backfill_po.py | 63 +++++++++ superset/translations/do-not-translate.txt | 59 ++++++++ superset/translations/messages.pot | 27 ++++ .../scripts/translations/backfill_po_test.py | 98 ++++++++++++++ 8 files changed, 396 insertions(+) create mode 100644 scripts/translations/apply_do_not_translate.py create mode 100644 superset/translations/do-not-translate.txt diff --git a/UPDATING.md b/UPDATING.md index ec0a06275cc..65598d0025a 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -42,6 +42,8 @@ assists people when migrating to a new version. - **New config flag `EMBEDDED_DISABLE_PERMALINK_ORIGIN_REWRITE` (default `False`).** Share/permalink URLs now substitute `window.location.origin` for the backend-supplied origin so a proxied or subdirectory-deployed Superset never hands the user an unreachable internal hostname. Operators whose reverse proxy correctly forwards `X-Forwarded-Host` *and* who want permalinks to carry the backend's literal origin can opt out by setting `EMBEDDED_DISABLE_PERMALINK_ORIGIN_REWRITE = True` in `superset_config.py`. Default `False` (rewrite is on); flipping the default would regress the dominant proxied/subdir deployment to an unreachable host. +- [41651](https://github.com/apache/superset/pull/41651): **New do-not-translate standard for translation catalogs.** Strings that must stay identical to the source — icon names (e.g. `bolt`), enum/option values (`step-after`), SQL keywords, API field names (`error_message`), code constants, and example placeholders — are now marked with a `#. do-not-translate` extracted comment. The list lives in the `superset/translations/do-not-translate.txt` registry; `scripts/translations/apply_do_not_translate.py` stamps the marker onto `messages.pot` during `babel_update.sh`, and `pybabel update` propagates it to every `.po`, so the status is consistent across all languages. The AI backfill (`backfill_po.py`) and translators leave these entries untranslated (source fallback). The legacy per-catalog convention (a `# Не переводить` translator comment in the `ru` catalog) is still honored for back-compat but is superseded by this standard; contributors adding new machine-read strings should add the msgid to the registry rather than annotating individual catalogs. + ### SQL Lab denies large-object and information_schema access by default `DISALLOWED_SQL_FUNCTIONS` and `DISALLOWED_SQL_TABLES` now ship with additional default entries, so SQL Lab and chart-data queries that reference them are rejected where they were previously allowed: diff --git a/docs/developer_docs/contributing/howtos.md b/docs/developer_docs/contributing/howtos.md index 452f705ae57..3fc44b50a15 100644 --- a/docs/developer_docs/contributing/howtos.md +++ b/docs/developer_docs/contributing/howtos.md @@ -355,6 +355,17 @@ translations](#applying-translations) above), so an AI-generated string is shown in the UI as soon as it is built and deployed. Reviewers should verify each entry and remove the `#, fuzzy` flag to promote it to a confirmed translation. +The script never touches entries that must stay literal — icon names, enum +values, SQL keywords, API field names, and example placeholders. These are +registered in `superset/translations/do-not-translate.txt`; +`scripts/translations/apply_do_not_translate.py` stamps them in `messages.pot` +with a `#. do-not-translate` extracted comment (run automatically +from `babel_update.sh`), which `pybabel update` then propagates to every +catalog. To mark a new string do-not-translate, add its msgid to the registry. +The backfill also honors that marker and any legacy do-not-translate translator +comment (e.g. the `ru` catalog's `# Не переводить`), leaving such entries +untranslated so they fall back to the source token. + #### Prerequisites ```bash diff --git a/scripts/translations/apply_do_not_translate.py b/scripts/translations/apply_do_not_translate.py new file mode 100644 index 00000000000..fbd9dd71d81 --- /dev/null +++ b/scripts/translations/apply_do_not_translate.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Stamp do-not-translate msgids in a .pot with an extracted-comment marker. + +For every msgid listed in ``superset/translations/do-not-translate.txt`` that is +present in the target .pot, add a ``#. do-not-translate`` extracted +comment. gettext extracted comments (``#.``) propagate from the .pot into every +language .po on ``pybabel update``, so the do-not-translate status stays +consistent across all catalogs from a single registry. + +Run from ``babel_update.sh`` after the .pot is extracted and normalized (and +before ``pybabel update``). Idempotent: re-running makes no further changes. + +Usage: + python scripts/translations/apply_do_not_translate.py [POT_PATH] + # POT_PATH defaults to superset/translations/messages.pot +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +# The standardized extracted-comment marker. Kept in sync with backfill_po.py. +MARKER: str = "do-not-translate" +_MARKER_LINE: str = f"#. {MARKER}" + +TRANSLATIONS_DIR: Path = ( + Path(__file__).parent.parent.parent / "superset" / "translations" +) +DEFAULT_POT: Path = TRANSLATIONS_DIR / "messages.pot" +REGISTRY: Path = TRANSLATIONS_DIR / "do-not-translate.txt" + + +def load_registry(path: Path = REGISTRY) -> set[str]: + """Return the set of do-not-translate msgids (skips comments/blank lines). + + Each line is stripped before the blank/comment check, so trailing + whitespace or an indented comment never yields a msgid that fails to match + the .pot. + """ + if not path.exists(): + return set() + entries: set[str] = set() + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line: str = raw_line.strip() + if line and not line.startswith("#"): + entries.add(line) + return entries + + +def _escape(msgid: str) -> str: + """Escape a msgid the way gettext writes it on a `msgid "..."` line.""" + return msgid.replace("\\", "\\\\").replace('"', '\\"') + + +def apply_markers(pot_path: Path, registry: set[str]) -> int: + """Insert the marker comment above each registry msgid via text edit. + + Text manipulation (rather than a polib round-trip) preserves the .pot's + exact wrapping/layout, so the only change is the added marker lines. + Idempotent. Returns the number of entries newly marked. + """ + lines: list[str] = pot_path.read_text(encoding="utf-8").split("\n") + targets: set[str] = {f'msgid "{_escape(m)}"' for m in registry} + out: list[str] = [] + changed: int = 0 + for line in lines: + if line in targets and (not out or out[-1] != _MARKER_LINE): + # `#.` extracted comments precede `msgid`; these registry entries are + # bare single-line msgids, so inserting directly above is correct. + out.append(_MARKER_LINE) + changed += 1 + out.append(line) + if changed: + pot_path.write_text("\n".join(out), encoding="utf-8") + return changed + + +def main() -> None: + """Stamp the marker onto the target .pot from the registry.""" + pot_path: Path = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_POT + if not pot_path.exists(): + print(f"POT file not found: {pot_path}", file=sys.stderr) + sys.exit(1) + # Fail fast if the registry file is absent: babel_update.sh depends on this + # step to stamp the .pot, and continuing would silently publish catalogs + # without any do-not-translate markers. An existing-but-empty registry is a + # valid state (nothing to mark), so only a missing file is an error. + if not REGISTRY.exists(): + print( + f"do-not-translate registry not found at {REGISTRY}; refusing to " + "produce unmarked translation artifacts.", + file=sys.stderr, + ) + sys.exit(1) + registry: set[str] = load_registry() + if not registry: + print( + f"do-not-translate registry {REGISTRY} is empty; nothing to mark.", + file=sys.stderr, + ) + return + changed: int = apply_markers(pot_path, registry) + print( + f"do-not-translate: marked {changed} new entr(y/ies) with {MARKER} " + f"in {pot_path.name} ({len(registry)} msgids in registry).", + file=sys.stderr, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/translations/babel_update.sh b/scripts/translations/babel_update.sh index 78689d38a5a..988c67c2021 100755 --- a/scripts/translations/babel_update.sh +++ b/scripts/translations/babel_update.sh @@ -55,6 +55,14 @@ msgcat --sort-by-msgid --no-wrap --no-location superset/translations/messages.po cat $LICENSE_TMP superset/translations/messages.pot > messages.pot.tmp \ && mv messages.pot.tmp superset/translations/messages.pot +# Stamp do-not-translate msgids (superset/translations/do-not-translate.txt) with +# a `#. do-not-translate` extracted comment. Extracted comments +# propagate from the .pot into every catalog on the `pybabel update` below, so +# the do-not-translate status stays consistent across all languages. +# Fail fast: without this guard the script would continue past a marker-stamping +# failure and `pybabel update` would publish catalogs missing the markers. +python scripts/translations/apply_do_not_translate.py superset/translations/messages.pot || exit 1 + # --no-fuzzy-matching: when a *new* source string is added, Babel's fuzzy # matcher otherwise guesses a "close" existing translation and marks it # `#, fuzzy` in every language catalog. Those guesses are (a) usually wrong diff --git a/scripts/translations/backfill_po.py b/scripts/translations/backfill_po.py index 25eed9ed444..76f863f8955 100644 --- a/scripts/translations/backfill_po.py +++ b/scripts/translations/backfill_po.py @@ -152,6 +152,60 @@ def _is_missing(entry: polib.POEntry) -> bool: return not entry.msgstr +# Canonical registry of msgids that must never be machine-translated: literal +# tokens compared against source (SQL keywords, confirmation words), enum values +# (d3 interpolation modes), icon names (e.g. "bolt" -> the ⚡ Explore control +# icon), API field names, code constants, and example placeholders. Translating +# them can break icon lookups, enum matching, or API contracts, or is simply +# meaningless (proper nouns, example values). apply_do_not_translate.py stamps +# these msgids in messages.pot with a `#. do-not-translate` +# extracted comment that propagates to every catalog on `pybabel update`. +DO_NOT_TRANSLATE_REGISTRY: Path = TRANSLATIONS_DIR / "do-not-translate.txt" + + +def _load_do_not_translate(path: Path = DO_NOT_TRANSLATE_REGISTRY) -> frozenset[str]: + """Load the do-not-translate msgids (skips comment/blank lines). + + Lines are stripped before the blank/comment checks, matching the parsing in + apply_do_not_translate.py, so trailing whitespace or an indented comment + never yields a msgid that fails to match a catalog entry. + """ + if not path.exists(): + return frozenset() + return frozenset( + stripped + for line in path.read_text(encoding="utf-8").splitlines() + if (stripped := line.strip()) and not stripped.startswith("#") + ) + + +DO_NOT_TRANSLATE: frozenset[str] = _load_do_not_translate() + +# An explicit do-not-translate marker on an entry, matched in either the +# extracted comment (`#. do-not-translate`, the standard propagated +# from the .pot) or a translator comment (e.g. the ru catalog's legacy +# "# Не переводить"). Honored so a human's deliberate decision is never +# overridden even if a msgid is missing from the registry. +_DO_NOT_TRANSLATE_COMMENT: re.Pattern[str] = re.compile( + r"не\s+переводить|do[\s-]?not[\s-]?translate|don'?t\s+translate", + re.IGNORECASE, +) + + +def _is_do_not_translate(entry: polib.POEntry) -> bool: + """Return True if an entry must be left for a human (never machine-filled). + + Either its msgid is in the do-not-translate registry, or the entry carries + an explicit do-not-translate marker in its extracted or translator comment. + """ + if entry.msgid in DO_NOT_TRANSLATE: + return True + return any( + comment and _DO_NOT_TRANSLATE_COMMENT.search(comment) + for comment in (entry.comment, entry.tcomment) + ) + + def _context_langs( item: dict[str, Any], index: dict[str, Any], target_lang: str ) -> list[str]: @@ -647,6 +701,15 @@ def backfill( missing: list[polib.POEntry] = [e for e in cat if e.msgid and _is_missing(e)] print(f"Found {len(missing)} untranslated entries for '{lang}'.", file=sys.stderr) + skipped_dnt: list[polib.POEntry] = [e for e in missing if _is_do_not_translate(e)] + if skipped_dnt: + missing = [e for e in missing if not _is_do_not_translate(e)] + print( + f"Skipping {len(skipped_dnt)} do-not-translate entries (literal " + f"tokens / translator-marked); they are left untranslated.", + file=sys.stderr, + ) + if min_context > 0: before = len(missing) missing = [ diff --git a/superset/translations/do-not-translate.txt b/superset/translations/do-not-translate.txt new file mode 100644 index 00000000000..a2ba3f5bf3b --- /dev/null +++ b/superset/translations/do-not-translate.txt @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Do-not-translate registry. +# +# One msgid per line. These strings must never be translated because they are +# machine-consumed or otherwise must stay identical to the source: icon names, +# enum/option values, SQL keywords, API field names, code constants, and example +# placeholders. Translating them can break icon lookups, enum matching, or API +# contracts, or is simply meaningless (proper nouns, example values). +# +# scripts/translations/apply_do_not_translate.py stamps each of these msgids in +# messages.pot with a `#. MACHINE_READ-DO_NOT_TRANSLATE` extracted comment +# (run from babel_update.sh after extraction). pybabel update then propagates +# that marker into every language catalog, so the do-not-translate status is +# consistent across languages. scripts/translations/backfill_po.py reads this +# file (and honors the marker / legacy translator comments) to leave these +# entries untranslated. Lines starting with '#' and blank lines are ignored. +10000 +DELETE +ECharts +EMAIL_REPORTS_CTA +GROUP BY +NOT GROUPED BY +OVERWRITE +TEMPORAL_RANGE +WFS +WMS +XYZ +bolt +crontab +error_message +pivoted_xlsx +schema1,schema2 +sql +step-after +step-before +superset.example.com +your-project-1234-a1 +deck.gl Geojson +dttm +p1 +p5 +p95 +p99 diff --git a/superset/translations/messages.pot b/superset/translations/messages.pot index 5b93b3bdefc..f4b7f922d3e 100644 --- a/superset/translations/messages.pot +++ b/superset/translations/messages.pot @@ -564,6 +564,7 @@ msgstr "" msgid "10/90 percentiles" msgstr "" +#. do-not-translate msgid "10000" msgstr "" @@ -3957,6 +3958,7 @@ msgstr "" msgid "DEC" msgstr "" +#. do-not-translate msgid "DELETE" msgstr "" @@ -5195,12 +5197,14 @@ msgstr "" msgid "Dynamically select grouping columns from a dataset" msgstr "" +#. do-not-translate msgid "ECharts" msgstr "" msgid "ECharts Options (JS object literals)" msgstr "" +#. do-not-translate msgid "EMAIL_REPORTS_CTA" msgstr "" @@ -6472,6 +6476,7 @@ msgstr "" msgid "Further customize how to display each metric" msgstr "" +#. do-not-translate msgid "GROUP BY" msgstr "" @@ -8247,6 +8252,7 @@ msgstr "" msgid "N/A" msgstr "" +#. do-not-translate msgid "NOT GROUPED BY" msgstr "" @@ -8738,6 +8744,7 @@ msgstr "" msgid "OR" msgstr "" +#. do-not-translate msgid "OVERWRITE" msgstr "" @@ -12071,6 +12078,7 @@ msgstr "" msgid "TABLES" msgstr "" +#. do-not-translate msgid "TEMPORAL_RANGE" msgstr "" @@ -14737,9 +14745,11 @@ msgstr "" msgid "WED" msgstr "" +#. do-not-translate msgid "WFS" msgstr "" +#. do-not-translate msgid "WMS" msgstr "" @@ -15208,6 +15218,7 @@ msgstr "" msgid "X-scale interval" msgstr "" +#. do-not-translate msgid "XYZ" msgstr "" @@ -15713,6 +15724,7 @@ msgstr "" msgid "bfill" msgstr "" +#. do-not-translate msgid "bolt" msgstr "" @@ -15798,6 +15810,7 @@ msgstr "" msgid "create dataset from SQL query" msgstr "" +#. do-not-translate msgid "crontab" msgstr "" @@ -15864,6 +15877,7 @@ msgstr "" msgid "deck.gl Contour" msgstr "" +#. do-not-translate msgid "deck.gl Geojson" msgstr "" @@ -15912,6 +15926,7 @@ msgstr "" msgid "documentation" msgstr "" +#. do-not-translate msgid "dttm" msgstr "" @@ -15975,6 +15990,7 @@ msgstr "" msgid "error" msgstr "" +#. do-not-translate msgid "error_message" msgstr "" @@ -16251,15 +16267,19 @@ msgstr "" msgid "p-value precision" msgstr "" +#. do-not-translate msgid "p1" msgstr "" +#. do-not-translate msgid "p5" msgstr "" +#. do-not-translate msgid "p95" msgstr "" +#. do-not-translate msgid "p99" msgstr "" @@ -16274,6 +16294,7 @@ msgstr "" msgid "permalink state not found" msgstr "" +#. do-not-translate msgid "pivoted_xlsx" msgstr "" @@ -16334,6 +16355,7 @@ msgstr "" msgid "save" msgstr "" +#. do-not-translate msgid "schema1,schema2" msgstr "" @@ -16349,6 +16371,7 @@ msgid "" "series" msgstr "" +#. do-not-translate msgid "sql" msgstr "" @@ -16364,9 +16387,11 @@ msgstr "" msgid "std" msgstr "" +#. do-not-translate msgid "step-after" msgstr "" +#. do-not-translate msgid "step-before" msgstr "" @@ -16388,6 +16413,7 @@ msgstr "" msgid "sum" msgstr "" +#. do-not-translate msgid "superset.example.com" msgstr "" @@ -16495,6 +16521,7 @@ msgstr "" msgid "year" msgstr "" +#. do-not-translate msgid "your-project-1234-a1" msgstr "" diff --git a/tests/unit_tests/scripts/translations/backfill_po_test.py b/tests/unit_tests/scripts/translations/backfill_po_test.py index 6734362b0ae..253c8335b3a 100644 --- a/tests/unit_tests/scripts/translations/backfill_po_test.py +++ b/tests/unit_tests/scripts/translations/backfill_po_test.py @@ -436,3 +436,101 @@ def test_resilient_translate_propagates_runtime_error( monkeypatch.setattr(backfill_po, "translate_batch", _boom) with pytest.raises(RuntimeError): backfill_po._resilient_translate("m", "fr", [_qitem("Alpha")], {}) + + +# --- _is_do_not_translate: never machine-fill literal tokens ------------------- + + +def test_is_do_not_translate_registry_msgid() -> None: + """A msgid in the do-not-translate registry is protected (icon names, + enum values, SQL keywords, API field names, placeholders).""" + for msgid in ("bolt", "error_message", "step-after", "GROUP BY"): + assert backfill_po._is_do_not_translate(polib.POEntry(msgid=msgid, msgstr="")) + + +def test_load_do_not_translate_strips_whitespace(tmp_path: Path) -> None: + """Registry lines are stripped before the blank/comment checks (matching + apply_do_not_translate.py), so trailing spaces or indented comments never + yield msgids that fail to match catalog entries.""" + registry = tmp_path / "do-not-translate.txt" + registry.write_text( + "error_message \n # indented comment\n\t\nbolt\n", encoding="utf-8" + ) + assert backfill_po._load_do_not_translate(registry) == frozenset( + {"error_message", "bolt"} + ) + + +def test_is_do_not_translate_honors_extracted_marker() -> None: + """The standardized `#. do-not-translate` extracted comment + (propagated from the .pot) is honored even for a msgid not in the registry.""" + entry = polib.POEntry(msgid="not-in-registry-token", msgstr="") + entry.comment = "do-not-translate" # polib .comment == `#.` + assert backfill_po._is_do_not_translate(entry) + + +def test_is_do_not_translate_honors_translator_comment() -> None: + """An explicit do-not-translate translator comment is honored, in any + language (e.g. the ru catalog's Cyrillic marker) and phrasing.""" + for comment in ("Не переводить", "do not translate", "DO-NOT-TRANSLATE"): + entry = polib.POEntry(msgid="Some label", msgstr="") + entry.tcomment = comment + assert backfill_po._is_do_not_translate(entry) + + +def test_is_do_not_translate_allows_normal_entry() -> None: + """An ordinary translatable string is not flagged.""" + entry = polib.POEntry(msgid="Save dashboard", msgstr="") + entry.tcomment = "Machine-translated via backfill_po.py (claude-x) [no refs]" + assert not backfill_po._is_do_not_translate(entry) + + +def test_backfill_skips_do_not_translate_entries_end_to_end( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """End-to-end: ``backfill`` must never hand a do-not-translate entry to the + translator, and must leave it untranslated in the written .po, while normal + entries are filled. Guards against the filter being applied at the wrong + stage or dropped entirely.""" + lang = "es" + po_dir = tmp_path / lang / "LC_MESSAGES" + po_dir.mkdir(parents=True) + po_path = po_dir / "messages.po" + # One curated DNT msgid, one translator-marked DNT entry, one normal entry. + po_path.write_text( + 'msgid ""\nmsgstr ""\n\n' + 'msgid "bolt"\nmsgstr ""\n\n' + '# Не переводить\nmsgid "Keep me literal"\nmsgstr ""\n\n' + 'msgid "Save dashboard"\nmsgstr ""\n', + encoding="utf-8", + ) + index_path = tmp_path / "translation_index.json" + index_path.write_text("{}", encoding="utf-8") + + monkeypatch.setattr(backfill_po, "TRANSLATIONS_DIR", tmp_path) + + seen_msgids: list[str] = [] + + def _fake_translate_batch( + model: str, + target_lang: str, + batch: list[dict[str, str]], + index: dict[str, object], + ) -> dict[int, str]: + seen_msgids.extend(it["msgid"] for it in batch) + return {i: f"T:{it['msgid']}" for i, it in enumerate(batch)} + + monkeypatch.setattr(backfill_po, "translate_batch", _fake_translate_batch) + + backfill_po.backfill(lang, index_path=index_path, mark_fuzzy=False) + + # DNT entries never reached the translator … + assert "bolt" not in seen_msgids + assert "Keep me literal" not in seen_msgids + assert seen_msgids == ["Save dashboard"] + + # … and stay untranslated in the written file, while the normal one is filled. + written = polib.pofile(str(po_path)) + assert written.find("bolt").msgstr == "" + assert written.find("Keep me literal").msgstr == "" + assert written.find("Save dashboard").msgstr == "T:Save dashboard"