Add multilingual audit CI pipeline + extract mandelblog_content_guard

2026-03-29 20:49:42 +02:00
parent 2a51989fa4
commit 1f05011a63
104 changed files with 3372 additions and 6 deletions
--- a/mandelblog_content_guard/tests.py
+++ b/mandelblog_content_guard/tests.py
@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import json
+
+from django.test import SimpleTestCase
+
+from mandelblog_content_guard.agents import get_language_agent
+from mandelblog_content_guard.extractors.visible_text import extract_visible_rendered_text
+from mandelblog_content_guard.system_strings import build_system_rewrite_candidates, build_system_vocabulary
+from mandelblog_content_guard.validators.multilingual import validate_text_nodes
+
+
+class PackageLevelContentGuardTests(SimpleTestCase):
+    def test_system_string_replacement_catalog(self):
+        self.assertEqual(build_system_vocabulary("fr")["PLAN"], "FORFAIT")
+        self.assertEqual(build_system_vocabulary("pt")["Unverbindlich"], "Sem compromisso")
+        self.assertEqual(build_system_rewrite_candidates()["PLAN"], "generic_badge_label")
+
+    def test_canonical_source_suppression(self):
+        nl_issues = validate_text_nodes("nl", [("body.badge", "PLAN")])
+        it_issues = validate_text_nodes("it", [("body.badge", "PIANO")])
+        self.assertFalse(any(issue.bad_value == "PLAN" for issue in nl_issues))
+        self.assertFalse(any(issue.bad_value == "PIANO" for issue in it_issues))
+
+    def test_visible_text_extraction(self):
+        html = """
+        <html><body>
+        <script>var x = 1;</script>
+        <style>.hidden{display:none}</style>
+        <h1>Visible heading</h1>
+        <p aria-hidden="true">Invisible text</p>
+        <a href="#">Visible link</a>
+        </body></html>
+        """
+        extracted = extract_visible_rendered_text(html)
+        self.assertIn("Visible heading", extracted)
+        self.assertIn("Visible link", extracted)
+        self.assertNotIn("Invisible text", extracted)
+        self.assertNotIn("var x", extracted)
+
+    def test_locale_normalizers(self):
+        de_agent = get_language_agent("de")
+        en_agent = get_language_agent("en")
+        self.assertEqual(de_agent.rewrite("Was du bekommst", "body.heading"), "Was Sie erhalten")
+        self.assertEqual(en_agent.rewrite("PLAN", "body.badge"), "Package")
+
+    def test_audit_json_contract_shape(self):
+        payload = {
+            "run_id": 1,
+            "summary": {"en": {"total_urls_checked": 1, "issues_found": 0, "issues_fixed": 0, "remaining_issues": 0, "by_severity": {"block": 0, "warn": 0, "log": 0}}},
+            "issues": {"en": []},
+        }
+        rendered = json.dumps(payload)
+        parsed = json.loads(rendered)
+        self.assertEqual(sorted(parsed.keys()), ["issues", "run_id", "summary"])
+        self.assertIn("by_severity", parsed["summary"]["en"])