manage-traffic-jam: wip check lang

Lib/gftools/push/items.py

@@ -6,6 +6,7 @@
 from fontTools.ttLib import TTFont  # type: ignore
 from gftools.designers_pb2 import DesignerInfoProto
 from gftools.fonts_public_pb2 import FamilyProto
+from gftools.languages_public_pb2 import LanguageProto
 from gftools.push.utils import google_path_to_repo_path
 from gftools.util.google_fonts import ReadProto
 from gftools.utils import (
@@ -20,6 +21,7 @@
 from google.protobuf.json_format import MessageToDict  # type: ignore
 from typing import Optional
 import re
+import os
 
 
 log = logging.getLogger("gftools.push")
@@ -236,4 +238,61 @@ def from_fp(cls, fp):
             return cls(name, parse_html(bio))
 
 
-Items = "Axis | Designer | Family | FamilyMeta"
+@dataclass
+class SampleText(Itemer):
+    language: str
+    masthead_full: str = ""
+    masthead_partial: str = ""
+    styles: str = ""
+    tester: str = ""
+    poster_sm: str = ""
+    poster_md: str = ""
+    poster_lg: str = ""
+    specimen_48: str = ""
+    specimen_36: str = ""
+    specimen_32: str = ""
+    specimen_21: str = ""
+    specimen_16: str = ""
+
+    @classmethod
+    def from_gf_json(cls, lang, data):
+        if not data:
+            return cls(language=lang)
+        return cls(
+            language=lang,
+            masthead_full=data["mastheadFull"],
+            masthead_partial=data["mastheadPartial"],
+            styles=data["styles"],
+            tester=data["tester"],
+            poster_sm=data["posterSm"],
+            poster_md=data["posterMd"],
+            poster_lg=data["posterLg"],
+            specimen_48=data["specimen48"],
+            specimen_36=data["specimen36"],
+            specimen_32=data["specimen32"],
+            specimen_21=data["specimen21"],
+            specimen_16=data["specimen16"],
+        )
+
+    @classmethod
+    def from_fp(cls, fp):
+        meta = ReadProto(LanguageProto(), fp)
+        meta = meta.sample_text
+        return cls(
+            language=os.path.basename(fp).replace(".textproto", ""),
+            masthead_full=meta.masthead_full,
+            masthead_partial=meta.masthead_partial,
+            styles=meta.styles,
+            tester=meta.tester,
+            poster_sm=meta.poster_sm,
+            poster_md=meta.poster_md,
+            poster_lg=meta.poster_lg,
+            specimen_48=meta.specimen_48,
+            specimen_36=meta.specimen_36,
+            specimen_32=meta.specimen_32,
+            specimen_21=meta.specimen_21,
+            specimen_16=meta.specimen_16,
+        )
+
+
+Items = "Axis | Designer | Family | FamilyMeta | SampleText"

Lib/gftools/push/servers.py

@@ -14,6 +14,7 @@
     Designer,
     Family,
     FamilyMeta,
+    SampleText,
     Itemer,
     Items,
 )
@@ -34,21 +35,33 @@
     DEV_FAMILY_DOWNLOAD = config["urls"]["dev_family_download"]
     DEV_META_URL = config["urls"]["dev_meta"]
     DEV_VERSIONS_URL = config["urls"]["dev_versions"]
+    DEV_LANG_URL = config["urls"]["dev_lang"]
+    DEV_SAMPLE_TEXT_URL = config["urls"]["dev_sample_text"]
     SANDBOX_FAMILY_DOWNLOAD = config["urls"]["sandbox_family_download"]
     SANDBOX_META_URL = config["urls"]["sandbox_meta"]
     SANDBOX_VERSIONS_URL = config["urls"]["sandbox_versions"]
+    SANDBOX_LANG_URL = config["urls"]["sandbox_lang"]
+    SANDBOX_SAMPLE_TEXT_URL = config["urls"]["sandbox_sample_text"]
     PRODUCTION_META_URL = config["urls"]["production_meta"]
     PRODUCTION_VERSIONS_URL = config["urls"]["production_versions"]
+    PRODUCTION_LANG_URL = config["urls"]["production_lang"]
+    PRODUCTION_SAMPLE_TEXT_URL = config["urls"]["production_sample_text"]
 
 else:
     DEV_FAMILY_DOWNLOAD = os.environ.get("DEV_FAMILY_DOWNLOAD")
     DEV_META_URL = os.environ.get("DEV_META_URL")
     DEV_VERSIONS_URL = os.environ.get("DEV_VERSIONS_URL")
+    DEV_LANG_URL = os.environ.get("DEV_LANG_URL")
+    DEV_SAMPLE_TEXT_URL = os.environ.get("DEV_SAMPLE_TEXT_URL")
     SANDBOX_FAMILY_DOWNLOAD = os.environ.get("SANDBOX_FAMILY_DOWNLOAD")
     SANDBOX_META_URL = os.environ.get("SANDBOX_META_URL")
     SANDBOX_VERSIONS_URL = os.environ.get("SANDBOX_VERSIONS_URL")
+    SANDBOX_LANG_URL = os.environ.get("SANDBOX_LANG_URL")
+    SANDBOX_SAMPLE_TEXT_URL = os.environ.get("SANDBOX_SAMPLE_TEXT_URL")
     PRODUCTION_META_URL = os.environ.get("PRODUCTION_META_URL")
     PRODUCTION_VERSIONS_URL = os.environ.get("PRODUCTION_VERSIONS_URL")
+    PRODUCTION_LANG_URL = os.environ.get("PRODUCTION_LANG_URL")
+    PRODUCTION_SAMPLE_TEXT_URL = os.environ.get("PRODUCTION_SAMPLE_TEXT_URL")
 
 
 @lru_cache
@@ -73,22 +86,52 @@ def gf_server_family_metadata(url: str, family: str):
     return info
 
 
+def sample_text(lang: str, url: str = PRODUCTION_SAMPLE_TEXT_URL):
+    def sample(family, lang):
+        resp = requests.get(url.format(family=family, lang=lang))
+        if resp.status_code != 200:
+            raise ValueError(
+                f"Language '{lang}' is either mispelled or not supported. "
+                "Also check if the endpoint still exists."
+            )
+        data = json.loads(resp.text[5:])
+        if lang in data["sampleText"]["languages"]:
+            return data["sampleText"]
+        return None
+
+    # This approach only seems to work for characters inside the Basic
+    # Multi-lingual Plane. We use this approach first since Noto doesn't
+    # fully cover the BMP yet and it lacks Latin African fonts.
+    res = sample("Adobe+Blank", lang)
+    if res:
+        return res
+    noto = requests.get(PRODUCTION_LANG_URL).json()
+    supported_families = noto["langToNotoFamilies"].get(lang)
+    if not supported_families:
+        return None
+    family = supported_families[0].replace(" ", "+")
+    return sample(family, lang)
+
+
 class GFServer(Itemer):
     def __init__(
         self,
         name: str,
         url: str = PRODUCTION_META_URL,
         dl_url: str = PROD_FAMILY_DOWNLOAD,
         version_url: str = PRODUCTION_VERSIONS_URL,
+        sample_text_url: str = PRODUCTION_SAMPLE_TEXT_URL,
     ):
         self.name = name
         self.url = url
         self.dl_url = dl_url
         self.version_url = version_url
+        self.sample_text_url = sample_text_url
         self.families: dict[str, Family] = {}
         self.designers: dict[str, Designer] = {}
         self.metadata: dict[str, FamilyMeta] = {}
         self.axisregistry: dict[str, Axis] = {}
+        self.sample_text: dict[str, SampleText] = {}
         self.family_versions = json.loads(requests.get(self.version_url).text[5:])
         self.family_versions = {
             i["name"]: i["fontVersions"][0]["version"]
@@ -115,6 +158,10 @@ def find_item(self, item):
     def update_axis_registry(self, axis_data):
         for axis in axis_data:
             self.axisregistry[axis["tag"]] = Axis.from_gf_json(axis)
+
+    def update_sample_text(self, lang: str):
+        data = sample_text(lang, self.sample_text_url)
+        self.sample_text[lang] = SampleText.from_gf_json(lang, data)
 
     def update_family(self, name: str):
         family_version = self.family_versions.get(name)
@@ -172,19 +219,21 @@ class GFServers(Itemer):
     def __init__(self):
         self.last_checked = datetime.fromordinal(1).isoformat().split("T")[0]
         self.dev = GFServer(
-            GFServers.DEV, DEV_META_URL, DEV_FAMILY_DOWNLOAD, DEV_VERSIONS_URL
+            GFServers.DEV, DEV_META_URL, DEV_FAMILY_DOWNLOAD, DEV_VERSIONS_URL, DEV_SAMPLE_TEXT_URL
         )
         self.sandbox = GFServer(
             GFServers.SANDBOX,
             SANDBOX_META_URL,
             SANDBOX_FAMILY_DOWNLOAD,
             SANDBOX_VERSIONS_URL,
+            SANDBOX_SAMPLE_TEXT_URL,
         )
         self.production = GFServer(
             GFServers.PRODUCTION,
             PRODUCTION_META_URL,
             PROD_FAMILY_DOWNLOAD,
             PRODUCTION_VERSIONS_URL,
+            PRODUCTION_SAMPLE_TEXT_URL
         )
         self.fp = None
 
@@ -248,6 +297,8 @@ def from_dict(cls, data):
                         server.axisregistry[k].fallback = [
                             AxisFallback(**a) for a in v.fallback
                         ]
+                elif item_type == "sample_text":
+                    server.sample_text = {k: SampleText(**v) for k, v in item_value.items()}
                 else:
                     setattr(server, item_type, item_value)
         return inst

Lib/gftools/push/trafficjam.py

@@ -10,7 +10,7 @@
 from typing import Optional, Any
 from functools import cached_property
 
-from gftools.push.items import Axis, Designer, Family, FamilyMeta
+from gftools.push.items import Axis, Designer, Family, FamilyMeta, SampleText
 from gftools.push.utils import google_path_to_repo_path, repo_path_to_google_path
 import json
 
@@ -214,6 +214,8 @@ def item(self):
             return FamilyMeta.from_fp(self.path)
         elif self.category == PushCategory.AXIS_REGISTRY:
             return Axis.from_fp(self.path)
+        elif self.category == PushCategory.SAMPLE_TEXTS:
+            return SampleText.from_fp(self.path)
         return None
 
     def set_server(self, server: STATUS_OPTION_IDS):

Lib/gftools/scripts/manage_traffic_jam.py

@@ -194,7 +194,9 @@ def run(self):
             if push_item.category == PushCategory.OTHER:
                 print("no push category defined. Skipping")
                 continue
-
+            if push_item.category == PushCategory.SAMPLE_TEXTS:
+                import pdb
+                pdb.set_trace()
             self.git_checkout_item(push_item)
             self.update_server(push_item, self.servers)
             self.display_item(push_item)