From 5d39ef084049deac6907380d16564a27a838cbda Mon Sep 17 00:00:00 2001 From: Marc Foley Date: Mon, 18 Mar 2024 15:31:38 +0000 Subject: [PATCH] wip check lang --- Lib/gftools/push/items.py | 61 ++++++++++++++++++++++- Lib/gftools/push/servers.py | 53 +++++++++++++++++++- Lib/gftools/push/trafficjam.py | 4 +- Lib/gftools/scripts/manage_traffic_jam.py | 4 +- 4 files changed, 118 insertions(+), 4 deletions(-) diff --git a/Lib/gftools/push/items.py b/Lib/gftools/push/items.py index 449a288d8..11b651118 100644 --- a/Lib/gftools/push/items.py +++ b/Lib/gftools/push/items.py @@ -6,6 +6,7 @@ from fontTools.ttLib import TTFont # type: ignore from gftools.designers_pb2 import DesignerInfoProto from gftools.fonts_public_pb2 import FamilyProto +from gftools.languages_public_pb2 import LanguageProto from gftools.push.utils import google_path_to_repo_path from gftools.util.google_fonts import ReadProto from gftools.utils import ( @@ -20,6 +21,7 @@ from google.protobuf.json_format import MessageToDict # type: ignore from typing import Optional import re +import os log = logging.getLogger("gftools.push") @@ -236,4 +238,61 @@ def from_fp(cls, fp): return cls(name, parse_html(bio)) -Items = "Axis | Designer | Family | FamilyMeta" +@dataclass +class SampleText(Itemer): + language: str + masthead_full: str = "" + masthead_partial: str = "" + styles: str = "" + tester: str = "" + poster_sm: str = "" + poster_md: str = "" + poster_lg: str = "" + specimen_48: str = "" + specimen_36: str = "" + specimen_32: str = "" + specimen_21: str = "" + specimen_16: str = "" + + @classmethod + def from_gf_json(cls, lang, data): + if not data: + return cls(language=lang) + return cls( + language=lang, + masthead_full=data["mastheadFull"], + masthead_partial=data["mastheadPartial"], + styles=data["styles"], + tester=data["tester"], + poster_sm=data["posterSm"], + poster_md=data["posterMd"], + poster_lg=data["posterLg"], + specimen_48=data["specimen48"], + specimen_36=data["specimen36"], + specimen_32=data["specimen32"], + specimen_21=data["specimen21"], + specimen_16=data["specimen16"], + ) + + @classmethod + def from_fp(cls, fp): + meta = ReadProto(LanguageProto(), fp) + meta = meta.sample_text + return cls( + language=os.path.basename(fp).replace(".textproto", ""), + masthead_full=meta.masthead_full, + masthead_partial=meta.masthead_partial, + styles=meta.styles, + tester=meta.tester, + poster_sm=meta.poster_sm, + poster_md=meta.poster_md, + poster_lg=meta.poster_lg, + specimen_48=meta.specimen_48, + specimen_36=meta.specimen_36, + specimen_32=meta.specimen_32, + specimen_21=meta.specimen_21, + specimen_16=meta.specimen_16, + ) + + +Items = "Axis | Designer | Family | FamilyMeta | SampleText" diff --git a/Lib/gftools/push/servers.py b/Lib/gftools/push/servers.py index 6e8b8c1d2..599e118c0 100644 --- a/Lib/gftools/push/servers.py +++ b/Lib/gftools/push/servers.py @@ -14,6 +14,7 @@ Designer, Family, FamilyMeta, + SampleText, Itemer, Items, ) @@ -34,21 +35,33 @@ DEV_FAMILY_DOWNLOAD = config["urls"]["dev_family_download"] DEV_META_URL = config["urls"]["dev_meta"] DEV_VERSIONS_URL = config["urls"]["dev_versions"] + DEV_LANG_URL = config["urls"]["dev_lang"] + DEV_SAMPLE_TEXT_URL = config["urls"]["dev_sample_text"] SANDBOX_FAMILY_DOWNLOAD = config["urls"]["sandbox_family_download"] SANDBOX_META_URL = config["urls"]["sandbox_meta"] SANDBOX_VERSIONS_URL = config["urls"]["sandbox_versions"] + SANDBOX_LANG_URL = config["urls"]["sandbox_lang"] + SANDBOX_SAMPLE_TEXT_URL = config["urls"]["sandbox_sample_text"] PRODUCTION_META_URL = config["urls"]["production_meta"] PRODUCTION_VERSIONS_URL = config["urls"]["production_versions"] + PRODUCTION_LANG_URL = config["urls"]["production_lang"] + PRODUCTION_SAMPLE_TEXT_URL = config["urls"]["production_sample_text"] else: DEV_FAMILY_DOWNLOAD = os.environ.get("DEV_FAMILY_DOWNLOAD") DEV_META_URL = os.environ.get("DEV_META_URL") DEV_VERSIONS_URL = os.environ.get("DEV_VERSIONS_URL") + DEV_LANG_URL = os.environ.get("DEV_LANG_URL") + DEV_SAMPLE_TEXT_URL = os.environ.get("DEV_SAMPLE_TEXT_URL") SANDBOX_FAMILY_DOWNLOAD = os.environ.get("SANDBOX_FAMILY_DOWNLOAD") SANDBOX_META_URL = os.environ.get("SANDBOX_META_URL") SANDBOX_VERSIONS_URL = os.environ.get("SANDBOX_VERSIONS_URL") + SANDBOX_LANG_URL = os.environ.get("SANDBOX_LANG_URL") + SANDBOX_SAMPLE_TEXT_URL = os.environ.get("SANDBOX_SAMPLE_TEXT_URL") PRODUCTION_META_URL = os.environ.get("PRODUCTION_META_URL") PRODUCTION_VERSIONS_URL = os.environ.get("PRODUCTION_VERSIONS_URL") + PRODUCTION_LANG_URL = os.environ.get("PRODUCTION_LANG_URL") + PRODUCTION_SAMPLE_TEXT_URL = os.environ.get("PRODUCTION_SAMPLE_TEXT_URL") @lru_cache @@ -73,6 +86,33 @@ def gf_server_family_metadata(url: str, family: str): return info +def sample_text(lang: str, url: str = PRODUCTION_SAMPLE_TEXT_URL): + def sample(family, lang): + resp = requests.get(url.format(family=family, lang=lang)) + if resp.status_code != 200: + raise ValueError( + f"Language '{lang}' is either mispelled or not supported. " + "Also check if the endpoint still exists." + ) + data = json.loads(resp.text[5:]) + if lang in data["sampleText"]["languages"]: + return data["sampleText"] + return None + + # This approach only seems to work for characters inside the Basic + # Multi-lingual Plane. We use this approach first since Noto doesn't + # fully cover the BMP yet and it lacks Latin African fonts. + res = sample("Adobe+Blank", lang) + if res: + return res + noto = requests.get(PRODUCTION_LANG_URL).json() + supported_families = noto["langToNotoFamilies"].get(lang) + if not supported_families: + return None + family = supported_families[0].replace(" ", "+") + return sample(family, lang) + + class GFServer(Itemer): def __init__( self, @@ -80,15 +120,18 @@ def __init__( url: str = PRODUCTION_META_URL, dl_url: str = PROD_FAMILY_DOWNLOAD, version_url: str = PRODUCTION_VERSIONS_URL, + sample_text_url: str = PRODUCTION_SAMPLE_TEXT_URL, ): self.name = name self.url = url self.dl_url = dl_url self.version_url = version_url + self.sample_text_url = sample_text_url self.families: dict[str, Family] = {} self.designers: dict[str, Designer] = {} self.metadata: dict[str, FamilyMeta] = {} self.axisregistry: dict[str, Axis] = {} + self.sample_text: dict[str, SampleText] = {} self.family_versions = json.loads(requests.get(self.version_url).text[5:]) self.family_versions = { i["name"]: i["fontVersions"][0]["version"] @@ -115,6 +158,10 @@ def find_item(self, item): def update_axis_registry(self, axis_data): for axis in axis_data: self.axisregistry[axis["tag"]] = Axis.from_gf_json(axis) + + def update_sample_text(self, lang: str): + data = sample_text(lang, self.sample_text_url) + self.sample_text[lang] = SampleText.from_gf_json(lang, data) def update_family(self, name: str): family_version = self.family_versions.get(name) @@ -172,19 +219,21 @@ class GFServers(Itemer): def __init__(self): self.last_checked = datetime.fromordinal(1).isoformat().split("T")[0] self.dev = GFServer( - GFServers.DEV, DEV_META_URL, DEV_FAMILY_DOWNLOAD, DEV_VERSIONS_URL + GFServers.DEV, DEV_META_URL, DEV_FAMILY_DOWNLOAD, DEV_VERSIONS_URL, DEV_SAMPLE_TEXT_URL ) self.sandbox = GFServer( GFServers.SANDBOX, SANDBOX_META_URL, SANDBOX_FAMILY_DOWNLOAD, SANDBOX_VERSIONS_URL, + SANDBOX_SAMPLE_TEXT_URL, ) self.production = GFServer( GFServers.PRODUCTION, PRODUCTION_META_URL, PROD_FAMILY_DOWNLOAD, PRODUCTION_VERSIONS_URL, + PRODUCTION_SAMPLE_TEXT_URL ) self.fp = None @@ -248,6 +297,8 @@ def from_dict(cls, data): server.axisregistry[k].fallback = [ AxisFallback(**a) for a in v.fallback ] + elif item_type == "sample_text": + server.sample_text = {k: SampleText(**v) for k, v in item_value.items()} else: setattr(server, item_type, item_value) return inst diff --git a/Lib/gftools/push/trafficjam.py b/Lib/gftools/push/trafficjam.py index f08278b1a..aab2117c2 100644 --- a/Lib/gftools/push/trafficjam.py +++ b/Lib/gftools/push/trafficjam.py @@ -10,7 +10,7 @@ from typing import Optional, Any from functools import cached_property -from gftools.push.items import Axis, Designer, Family, FamilyMeta +from gftools.push.items import Axis, Designer, Family, FamilyMeta, SampleText from gftools.push.utils import google_path_to_repo_path, repo_path_to_google_path import json @@ -214,6 +214,8 @@ def item(self): return FamilyMeta.from_fp(self.path) elif self.category == PushCategory.AXIS_REGISTRY: return Axis.from_fp(self.path) + elif self.category == PushCategory.SAMPLE_TEXTS: + return SampleText.from_fp(self.path) return None def set_server(self, server: STATUS_OPTION_IDS): diff --git a/Lib/gftools/scripts/manage_traffic_jam.py b/Lib/gftools/scripts/manage_traffic_jam.py index be9985cff..f6e42531e 100644 --- a/Lib/gftools/scripts/manage_traffic_jam.py +++ b/Lib/gftools/scripts/manage_traffic_jam.py @@ -194,7 +194,9 @@ def run(self): if push_item.category == PushCategory.OTHER: print("no push category defined. Skipping") continue - + if push_item.category == PushCategory.SAMPLE_TEXTS: + import pdb + pdb.set_trace() self.git_checkout_item(push_item) self.update_server(push_item, self.servers) self.display_item(push_item)