feat(i18n): do-not-translate registry + '#. do-not-translate' marker standard (#41651)

Co-authored-by: Amin Ghadersohi <amin.ghadersohi@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

UPDATING.md

@@ -42,6 +42,8 @@ assists people when migrating to a new version.
 
 - **New config flag `EMBEDDED_DISABLE_PERMALINK_ORIGIN_REWRITE` (default `False`).** Share/permalink URLs now substitute `window.location.origin` for the backend-supplied origin so a proxied or subdirectory-deployed Superset never hands the user an unreachable internal hostname. Operators whose reverse proxy correctly forwards `X-Forwarded-Host` *and* who want permalinks to carry the backend's literal origin can opt out by setting `EMBEDDED_DISABLE_PERMALINK_ORIGIN_REWRITE = True` in `superset_config.py`. Default `False` (rewrite is on); flipping the default would regress the dominant proxied/subdir deployment to an unreachable host.
 
+- [41651](https://github.com/apache/superset/pull/41651): **New do-not-translate standard for translation catalogs.** Strings that must stay identical to the source — icon names (e.g. `bolt`), enum/option values (`step-after`), SQL keywords, API field names (`error_message`), code constants, and example placeholders — are now marked with a `#. do-not-translate` extracted comment. The list lives in the `superset/translations/do-not-translate.txt` registry; `scripts/translations/apply_do_not_translate.py` stamps the marker onto `messages.pot` during `babel_update.sh`, and `pybabel update` propagates it to every `.po`, so the status is consistent across all languages. The AI backfill (`backfill_po.py`) and translators leave these entries untranslated (source fallback). The legacy per-catalog convention (a `# Не переводить` translator comment in the `ru` catalog) is still honored for back-compat but is superseded by this standard; contributors adding new machine-read strings should add the msgid to the registry rather than annotating individual catalogs.
+
 ### SQL Lab denies large-object and information_schema access by default
 
 `DISALLOWED_SQL_FUNCTIONS` and `DISALLOWED_SQL_TABLES` now ship with additional default entries, so SQL Lab and chart-data queries that reference them are rejected where they were previously allowed:

docs/developer_docs/contributing/howtos.md

@@ -355,6 +355,17 @@ translations](#applying-translations) above), so an AI-generated string is shown
 in the UI as soon as it is built and deployed. Reviewers should verify each
 entry and remove the `#, fuzzy` flag to promote it to a confirmed translation.
 
+The script never touches entries that must stay literal — icon names, enum
+values, SQL keywords, API field names, and example placeholders. These are
+registered in `superset/translations/do-not-translate.txt`;
+`scripts/translations/apply_do_not_translate.py` stamps them in `messages.pot`
+with a `#. do-not-translate` extracted comment (run automatically
+from `babel_update.sh`), which `pybabel update` then propagates to every
+catalog. To mark a new string do-not-translate, add its msgid to the registry.
+The backfill also honors that marker and any legacy do-not-translate translator
+comment (e.g. the `ru` catalog's `# Не переводить`), leaving such entries
+untranslated so they fall back to the source token.
+
 #### Prerequisites
 
 ```bash

scripts/translations/apply_do_not_translate.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Stamp do-not-translate msgids in a .pot with an extracted-comment marker.
+
+For every msgid listed in ``superset/translations/do-not-translate.txt`` that is
+present in the target .pot, add a ``#. do-not-translate`` extracted
+comment. gettext extracted comments (``#.``) propagate from the .pot into every
+language .po on ``pybabel update``, so the do-not-translate status stays
+consistent across all catalogs from a single registry.
+
+Run from ``babel_update.sh`` after the .pot is extracted and normalized (and
+before ``pybabel update``). Idempotent: re-running makes no further changes.
+
+Usage:
+  python scripts/translations/apply_do_not_translate.py [POT_PATH]
+  # POT_PATH defaults to superset/translations/messages.pot
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+# The standardized extracted-comment marker. Kept in sync with backfill_po.py.
+MARKER: str = "do-not-translate"
+_MARKER_LINE: str = f"#. {MARKER}"
+
+TRANSLATIONS_DIR: Path = (
+    Path(__file__).parent.parent.parent / "superset" / "translations"
+)
+DEFAULT_POT: Path = TRANSLATIONS_DIR / "messages.pot"
+REGISTRY: Path = TRANSLATIONS_DIR / "do-not-translate.txt"
+
+
+def load_registry(path: Path = REGISTRY) -> set[str]:
+    """Return the set of do-not-translate msgids (skips comments/blank lines).
+
+    Each line is stripped before the blank/comment check, so trailing
+    whitespace or an indented comment never yields a msgid that fails to match
+    the .pot.
+    """
+    if not path.exists():
+        return set()
+    entries: set[str] = set()
+    for raw_line in path.read_text(encoding="utf-8").splitlines():
+        line: str = raw_line.strip()
+        if line and not line.startswith("#"):
+            entries.add(line)
+    return entries
+
+
+def _escape(msgid: str) -> str:
+    """Escape a msgid the way gettext writes it on a `msgid "..."` line."""
+    return msgid.replace("\\", "\\\\").replace('"', '\\"')
+
+
+def apply_markers(pot_path: Path, registry: set[str]) -> int:
+    """Insert the marker comment above each registry msgid via text edit.
+
+    Text manipulation (rather than a polib round-trip) preserves the .pot's
+    exact wrapping/layout, so the only change is the added marker lines.
+    Idempotent. Returns the number of entries newly marked.
+    """
+    lines: list[str] = pot_path.read_text(encoding="utf-8").split("\n")
+    targets: set[str] = {f'msgid "{_escape(m)}"' for m in registry}
+    out: list[str] = []
+    changed: int = 0
+    for line in lines:
+        if line in targets and (not out or out[-1] != _MARKER_LINE):
+            # `#.` extracted comments precede `msgid`; these registry entries are
+            # bare single-line msgids, so inserting directly above is correct.
+            out.append(_MARKER_LINE)
+            changed += 1
+        out.append(line)
+    if changed:
+        pot_path.write_text("\n".join(out), encoding="utf-8")
+    return changed
+
+
+def main() -> None:
+    """Stamp the marker onto the target .pot from the registry."""
+    pot_path: Path = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_POT
+    if not pot_path.exists():
+        print(f"POT file not found: {pot_path}", file=sys.stderr)
+        sys.exit(1)
+    # Fail fast if the registry file is absent: babel_update.sh depends on this
+    # step to stamp the .pot, and continuing would silently publish catalogs
+    # without any do-not-translate markers. An existing-but-empty registry is a
+    # valid state (nothing to mark), so only a missing file is an error.
+    if not REGISTRY.exists():
+        print(
+            f"do-not-translate registry not found at {REGISTRY}; refusing to "
+            "produce unmarked translation artifacts.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    registry: set[str] = load_registry()
+    if not registry:
+        print(
+            f"do-not-translate registry {REGISTRY} is empty; nothing to mark.",
+            file=sys.stderr,
+        )
+        return
+    changed: int = apply_markers(pot_path, registry)
+    print(
+        f"do-not-translate: marked {changed} new entr(y/ies) with {MARKER} "
+        f"in {pot_path.name} ({len(registry)} msgids in registry).",
+        file=sys.stderr,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/translations/babel_update.sh

@@ -55,6 +55,14 @@ msgcat --sort-by-msgid --no-wrap --no-location superset/translations/messages.po
 cat $LICENSE_TMP superset/translations/messages.pot > messages.pot.tmp \
   && mv messages.pot.tmp superset/translations/messages.pot
 
+# Stamp do-not-translate msgids (superset/translations/do-not-translate.txt) with
+# a `#. do-not-translate` extracted comment. Extracted comments
+# propagate from the .pot into every catalog on the `pybabel update` below, so
+# the do-not-translate status stays consistent across all languages.
+# Fail fast: without this guard the script would continue past a marker-stamping
+# failure and `pybabel update` would publish catalogs missing the markers.
+python scripts/translations/apply_do_not_translate.py superset/translations/messages.pot || exit 1
+
 # --no-fuzzy-matching: when a *new* source string is added, Babel's fuzzy
 # matcher otherwise guesses a "close" existing translation and marks it
 # `#, fuzzy` in every language catalog. Those guesses are (a) usually wrong

scripts/translations/backfill_po.py

@@ -152,6 +152,60 @@ def _is_missing(entry: polib.POEntry) -> bool:
     return not entry.msgstr
 
 
+# Canonical registry of msgids that must never be machine-translated: literal
+# tokens compared against source (SQL keywords, confirmation words), enum values
+# (d3 interpolation modes), icon names (e.g. "bolt" -> the ⚡ Explore control
+# icon), API field names, code constants, and example placeholders. Translating
+# them can break icon lookups, enum matching, or API contracts, or is simply
+# meaningless (proper nouns, example values). apply_do_not_translate.py stamps
+# these msgids in messages.pot with a `#. do-not-translate`
+# extracted comment that propagates to every catalog on `pybabel update`.
+DO_NOT_TRANSLATE_REGISTRY: Path = TRANSLATIONS_DIR / "do-not-translate.txt"
+
+
+def _load_do_not_translate(path: Path = DO_NOT_TRANSLATE_REGISTRY) -> frozenset[str]:
+    """Load the do-not-translate msgids (skips comment/blank lines).
+
+    Lines are stripped before the blank/comment checks, matching the parsing in
+    apply_do_not_translate.py, so trailing whitespace or an indented comment
+    never yields a msgid that fails to match a catalog entry.
+    """
+    if not path.exists():
+        return frozenset()
+    return frozenset(
+        stripped
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if (stripped := line.strip()) and not stripped.startswith("#")
+    )
+
+
+DO_NOT_TRANSLATE: frozenset[str] = _load_do_not_translate()
+
+# An explicit do-not-translate marker on an entry, matched in either the
+# extracted comment (`#. do-not-translate`, the standard propagated
+# from the .pot) or a translator comment (e.g. the ru catalog's legacy
+# "# Не переводить"). Honored so a human's deliberate decision is never
+# overridden even if a msgid is missing from the registry.
+_DO_NOT_TRANSLATE_COMMENT: re.Pattern[str] = re.compile(
+    r"не\s+переводить|do[\s-]?not[\s-]?translate|don'?t\s+translate",
+    re.IGNORECASE,
+)
+
+
+def _is_do_not_translate(entry: polib.POEntry) -> bool:
+    """Return True if an entry must be left for a human (never machine-filled).
+
+    Either its msgid is in the do-not-translate registry, or the entry carries
+    an explicit do-not-translate marker in its extracted or translator comment.
+    """
+    if entry.msgid in DO_NOT_TRANSLATE:
+        return True
+    return any(
+        comment and _DO_NOT_TRANSLATE_COMMENT.search(comment)
+        for comment in (entry.comment, entry.tcomment)
+    )
+
+
 def _context_langs(
     item: dict[str, Any], index: dict[str, Any], target_lang: str
 ) -> list[str]:
@@ -647,6 +701,15 @@ def backfill(
     missing: list[polib.POEntry] = [e for e in cat if e.msgid and _is_missing(e)]
     print(f"Found {len(missing)} untranslated entries for '{lang}'.", file=sys.stderr)
 
+    skipped_dnt: list[polib.POEntry] = [e for e in missing if _is_do_not_translate(e)]
+    if skipped_dnt:
+        missing = [e for e in missing if not _is_do_not_translate(e)]
+        print(
+            f"Skipping {len(skipped_dnt)} do-not-translate entries (literal "
+            f"tokens / translator-marked); they are left untranslated.",
+            file=sys.stderr,
+        )
+
     if min_context > 0:
         before = len(missing)
         missing = [

superset/translations/do-not-translate.txt

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Do-not-translate registry.
+#
+# One msgid per line. These strings must never be translated because they are
+# machine-consumed or otherwise must stay identical to the source: icon names,
+# enum/option values, SQL keywords, API field names, code constants, and example
+# placeholders. Translating them can break icon lookups, enum matching, or API
+# contracts, or is simply meaningless (proper nouns, example values).
+#
+# scripts/translations/apply_do_not_translate.py stamps each of these msgids in
+# messages.pot with a `#. MACHINE_READ-DO_NOT_TRANSLATE` extracted comment
+# (run from babel_update.sh after extraction). pybabel update then propagates
+# that marker into every language catalog, so the do-not-translate status is
+# consistent across languages. scripts/translations/backfill_po.py reads this
+# file (and honors the marker / legacy translator comments) to leave these
+# entries untranslated. Lines starting with '#' and blank lines are ignored.
+10000
+DELETE
+ECharts
+EMAIL_REPORTS_CTA
+GROUP BY
+NOT GROUPED BY
+OVERWRITE
+TEMPORAL_RANGE
+WFS
+WMS
+XYZ
+bolt
+crontab
+error_message
+pivoted_xlsx
+schema1,schema2
+sql
+step-after
+step-before
+superset.example.com
+your-project-1234-a1
+deck.gl Geojson
+dttm
+p1
+p5
+p95
+p99

superset/translations/messages.pot

@@ -564,6 +564,7 @@ msgstr ""
 msgid "10/90 percentiles"
 msgstr ""
 
+#. do-not-translate
 msgid "10000"
 msgstr ""
 
@@ -3957,6 +3958,7 @@ msgstr ""
 msgid "DEC"
 msgstr ""
 
+#. do-not-translate
 msgid "DELETE"
 msgstr ""
 
@@ -5195,12 +5197,14 @@ msgstr ""
 msgid "Dynamically select grouping columns from a dataset"
 msgstr ""
 
+#. do-not-translate
 msgid "ECharts"
 msgstr ""
 
 msgid "ECharts Options (JS object literals)"
 msgstr ""
 
+#. do-not-translate
 msgid "EMAIL_REPORTS_CTA"
 msgstr ""
 
@@ -6472,6 +6476,7 @@ msgstr ""
 msgid "Further customize how to display each metric"
 msgstr ""
 
+#. do-not-translate
 msgid "GROUP BY"
 msgstr ""
 
@@ -8247,6 +8252,7 @@ msgstr ""
 msgid "N/A"
 msgstr ""
 
+#. do-not-translate
 msgid "NOT GROUPED BY"
 msgstr ""
 
@@ -8738,6 +8744,7 @@ msgstr ""
 msgid "OR"
 msgstr ""
 
+#. do-not-translate
 msgid "OVERWRITE"
 msgstr ""
 
@@ -12071,6 +12078,7 @@ msgstr ""
 msgid "TABLES"
 msgstr ""
 
+#. do-not-translate
 msgid "TEMPORAL_RANGE"
 msgstr ""
 
@@ -14737,9 +14745,11 @@ msgstr ""
 msgid "WED"
 msgstr ""
 
+#. do-not-translate
 msgid "WFS"
 msgstr ""
 
+#. do-not-translate
 msgid "WMS"
 msgstr ""
 
@@ -15208,6 +15218,7 @@ msgstr ""
 msgid "X-scale interval"
 msgstr ""
 
+#. do-not-translate
 msgid "XYZ"
 msgstr ""
 
@@ -15713,6 +15724,7 @@ msgstr ""
 msgid "bfill"
 msgstr ""
 
+#. do-not-translate
 msgid "bolt"
 msgstr ""
 
@@ -15798,6 +15810,7 @@ msgstr ""
 msgid "create dataset from SQL query"
 msgstr ""
 
+#. do-not-translate
 msgid "crontab"
 msgstr ""
 
@@ -15864,6 +15877,7 @@ msgstr ""
 msgid "deck.gl Contour"
 msgstr ""
 
+#. do-not-translate
 msgid "deck.gl Geojson"
 msgstr ""
 
@@ -15912,6 +15926,7 @@ msgstr ""
 msgid "documentation"
 msgstr ""
 
+#. do-not-translate
 msgid "dttm"
 msgstr ""
 
@@ -15975,6 +15990,7 @@ msgstr ""
 msgid "error"
 msgstr ""
 
+#. do-not-translate
 msgid "error_message"
 msgstr ""
 
@@ -16251,15 +16267,19 @@ msgstr ""
 msgid "p-value precision"
 msgstr ""
 
+#. do-not-translate
 msgid "p1"
 msgstr ""
 
+#. do-not-translate
 msgid "p5"
 msgstr ""
 
+#. do-not-translate
 msgid "p95"
 msgstr ""
 
+#. do-not-translate
 msgid "p99"
 msgstr ""
 
@@ -16274,6 +16294,7 @@ msgstr ""
 msgid "permalink state not found"
 msgstr ""
 
+#. do-not-translate
 msgid "pivoted_xlsx"
 msgstr ""
 
@@ -16334,6 +16355,7 @@ msgstr ""
 msgid "save"
 msgstr ""
 
+#. do-not-translate
 msgid "schema1,schema2"
 msgstr ""
 
@@ -16349,6 +16371,7 @@ msgid ""
 "series"
 msgstr ""
 
+#. do-not-translate
 msgid "sql"
 msgstr ""
 
@@ -16364,9 +16387,11 @@ msgstr ""
 msgid "std"
 msgstr ""
 
+#. do-not-translate
 msgid "step-after"
 msgstr ""
 
+#. do-not-translate
 msgid "step-before"
 msgstr ""
 
@@ -16388,6 +16413,7 @@ msgstr ""
 msgid "sum"
 msgstr ""
 
+#. do-not-translate
 msgid "superset.example.com"
 msgstr ""
 
@@ -16495,6 +16521,7 @@ msgstr ""
 msgid "year"
 msgstr ""
 
+#. do-not-translate
 msgid "your-project-1234-a1"
 msgstr ""
 

tests/unit_tests/scripts/translations/backfill_po_test.py

@@ -436,3 +436,101 @@ def test_resilient_translate_propagates_runtime_error(
     monkeypatch.setattr(backfill_po, "translate_batch", _boom)
     with pytest.raises(RuntimeError):
         backfill_po._resilient_translate("m", "fr", [_qitem("Alpha")], {})
+
+
+# --- _is_do_not_translate: never machine-fill literal tokens -------------------
+
+
+def test_is_do_not_translate_registry_msgid() -> None:
+    """A msgid in the do-not-translate registry is protected (icon names,
+    enum values, SQL keywords, API field names, placeholders)."""
+    for msgid in ("bolt", "error_message", "step-after", "GROUP BY"):
+        assert backfill_po._is_do_not_translate(polib.POEntry(msgid=msgid, msgstr=""))
+
+
+def test_load_do_not_translate_strips_whitespace(tmp_path: Path) -> None:
+    """Registry lines are stripped before the blank/comment checks (matching
+    apply_do_not_translate.py), so trailing spaces or indented comments never
+    yield msgids that fail to match catalog entries."""
+    registry = tmp_path / "do-not-translate.txt"
+    registry.write_text(
+        "error_message \n  # indented comment\n\t\nbolt\n", encoding="utf-8"
+    )
+    assert backfill_po._load_do_not_translate(registry) == frozenset(
+        {"error_message", "bolt"}
+    )
+
+
+def test_is_do_not_translate_honors_extracted_marker() -> None:
+    """The standardized `#. do-not-translate` extracted comment
+    (propagated from the .pot) is honored even for a msgid not in the registry."""
+    entry = polib.POEntry(msgid="not-in-registry-token", msgstr="")
+    entry.comment = "do-not-translate"  # polib .comment == `#.`
+    assert backfill_po._is_do_not_translate(entry)
+
+
+def test_is_do_not_translate_honors_translator_comment() -> None:
+    """An explicit do-not-translate translator comment is honored, in any
+    language (e.g. the ru catalog's Cyrillic marker) and phrasing."""
+    for comment in ("Не переводить", "do not translate", "DO-NOT-TRANSLATE"):
+        entry = polib.POEntry(msgid="Some label", msgstr="")
+        entry.tcomment = comment
+        assert backfill_po._is_do_not_translate(entry)
+
+
+def test_is_do_not_translate_allows_normal_entry() -> None:
+    """An ordinary translatable string is not flagged."""
+    entry = polib.POEntry(msgid="Save dashboard", msgstr="")
+    entry.tcomment = "Machine-translated via backfill_po.py (claude-x) [no refs]"
+    assert not backfill_po._is_do_not_translate(entry)
+
+
+def test_backfill_skips_do_not_translate_entries_end_to_end(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """End-to-end: ``backfill`` must never hand a do-not-translate entry to the
+    translator, and must leave it untranslated in the written .po, while normal
+    entries are filled. Guards against the filter being applied at the wrong
+    stage or dropped entirely."""
+    lang = "es"
+    po_dir = tmp_path / lang / "LC_MESSAGES"
+    po_dir.mkdir(parents=True)
+    po_path = po_dir / "messages.po"
+    # One curated DNT msgid, one translator-marked DNT entry, one normal entry.
+    po_path.write_text(
+        'msgid ""\nmsgstr ""\n\n'
+        'msgid "bolt"\nmsgstr ""\n\n'
+        '# Не переводить\nmsgid "Keep me literal"\nmsgstr ""\n\n'
+        'msgid "Save dashboard"\nmsgstr ""\n',
+        encoding="utf-8",
+    )
+    index_path = tmp_path / "translation_index.json"
+    index_path.write_text("{}", encoding="utf-8")
+
+    monkeypatch.setattr(backfill_po, "TRANSLATIONS_DIR", tmp_path)
+
+    seen_msgids: list[str] = []
+
+    def _fake_translate_batch(
+        model: str,
+        target_lang: str,
+        batch: list[dict[str, str]],
+        index: dict[str, object],
+    ) -> dict[int, str]:
+        seen_msgids.extend(it["msgid"] for it in batch)
+        return {i: f"T:{it['msgid']}" for i, it in enumerate(batch)}
+
+    monkeypatch.setattr(backfill_po, "translate_batch", _fake_translate_batch)
+
+    backfill_po.backfill(lang, index_path=index_path, mark_fuzzy=False)
+
+    # DNT entries never reached the translator …
+    assert "bolt" not in seen_msgids
+    assert "Keep me literal" not in seen_msgids
+    assert seen_msgids == ["Save dashboard"]
+
+    # … and stay untranslated in the written file, while the normal one is filled.
+    written = polib.pofile(str(po_path))
+    assert written.find("bolt").msgstr == ""
+    assert written.find("Keep me literal").msgstr == ""
+    assert written.find("Save dashboard").msgstr == "T:Save dashboard"