From 00278da29a58bad8685ee430d014021369c3c415 Mon Sep 17 00:00:00 2001 From: Smaarn Date: Sat, 22 Jul 2023 18:36:10 +0200 Subject: [PATCH 01/11] Add mustache to dev env --- Dockerfile | 1 + README.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5c9fd88c741..fbb3018c1c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,6 +61,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ python3-distutils \ rename \ rsync \ + ruby-mustache \ scons \ subversion \ sudo \ diff --git a/README.md b/README.md index fcf2d0838fa..961da5e7487 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ sudo apt install autoconf-archive autogen automake autopoint bash bc bison \ g++-multilib gawk gettext git gperf imagemagick intltool jq libbz2-dev libc6-i386 \ libcppunit-dev libffi-dev libgc-dev libgmp3-dev libltdl-dev libmount-dev libncurses-dev \ libpcre3-dev libssl-dev libtool libunistring-dev lzip mercurial moreutils ninja-build \ - patchelf php pkg-config python2 python3 python3-distutils rename rsync scons subversion \ + patchelf php pkg-config python2 python3 python3-distutils rename ruby-mustache rsync scons subversion \ swig texinfo unzip xmlto zlib1g-dev wget https://bootstrap.pypa.io/pip/2.7/get-pip.py -O - | sudo python2 sudo pip2 install wheel httpie @@ -87,7 +87,7 @@ lxc exec spksrc -- /usr/bin/apt install autoconf-archive autogen automake autopo g++-multilib gawk gettext git gperf imagemagick intltool jq libbz2-dev libc6-i386 \ libcppunit-dev libffi-dev libgc-dev libgmp3-dev libltdl-dev libmount-dev libncurses-dev \ libpcre3-dev libssl-dev libtool libunistring-dev lzip mercurial moreutils ninja-build \ - patchelf php pkg-config python2 python3 python3-distutils rename rsync scons subversion \ + patchelf php pkg-config python2 python3 python3-distutils rename rsync ruby-mustache scons subversion \ swig texinfo unzip xmlto zlib1g-dev ``` 5. Install `python2` wheels: From b644c25a69fad5abf9782aeb4af2419ff844a81b Mon Sep 17 00:00:00 2001 From: Smaarn Date: Sat, 22 Jul 2023 19:14:28 +0200 Subject: [PATCH 02/11] new framework for wizards --- mk/spksrc.spk.mk | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index f9787aab614..a669be783b9 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -282,6 +282,9 @@ endif # Wizard DSM_WIZARDS_DIR = $(WORK_DIR)/WIZARD_UIFILES +ifneq ($(strip $(WIZARDS_TEMPLATES_DIR)),) +WIZARDS_DIR = $(WORK_DIR)/generated-wizards +endif ifneq ($(WIZARDS_DIR),) # export working wizards dir to the shell for use later at compile-time export SPKSRC_WIZARDS_DIR=$(WIZARDS_DIR) @@ -390,6 +393,24 @@ ifeq ($(strip $(WIZARDS_DIR)),) $(eval SPK_CONTENT += WIZARD_UIFILES) endif endif +ifneq ($(strip $(WIZARDS_TEMPLATES_DIR)),) + @$(MSG) "Generate DSM Wizards from templates" + @mkdir -p $(WIZARDS_DIR) + @for template in `find $(WIZARDS_TEMPLATES_DIR) -maxdepth 1 -type f -and \( $(WIZARD_FILE_NAMES) \) -print`; do \ + template_filename="$$(basename $${template})"; \ + template_name="$${template_filename%.*}"; \ + if [ "$${template_name}" = "$${template_filename}" ]; then \ + template_suffix=; \ + else \ + template_suffix=".$${template_filename:$${#template_filename} + 1}"; \ + fi; \ + for suffix in '' $(patsubst %,_%,$(SUPPORTED_LANGUAGES)) ; do \ + mustache -e \ + "$(WIZARDS_TEMPLATES_DIR)/$${template_name}$${suffix}.yml" \ + "$(WIZARDS_TEMPLATES_DIR)/$${template_filename}" >"$(WIZARDS_DIR)/$${template_name}$${suffix}$${template_suffix}"; \ + done; \ + done +endif ifneq ($(strip $(WIZARDS_DIR)),) @$(MSG) "Create DSM Wizards" $(eval SPK_CONTENT += WIZARD_UIFILES) From bcf125c8e40b0cead29dac1f23f8e98f7f949422 Mon Sep 17 00:00:00 2001 From: Smaarn Date: Sat, 22 Jul 2023 19:57:18 +0200 Subject: [PATCH 03/11] Fix indentings --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fbb3018c1c8..b5f4db2d8fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,7 +61,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ python3-distutils \ rename \ rsync \ - ruby-mustache \ + ruby-mustache \ scons \ subversion \ sudo \ From 26b63a2abad79339532e92110f221bc5662a8ce4 Mon Sep 17 00:00:00 2001 From: Smaarn Date: Sun, 6 Aug 2023 13:04:13 +0200 Subject: [PATCH 04/11] Rework mustache logic to use the LANGUAGES variable instead of the SUPPORTED LANGUAGES (and make it easier) --- mk/spksrc.spk.mk | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index a669be783b9..f11ffb0ee58 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -404,10 +404,14 @@ ifneq ($(strip $(WIZARDS_TEMPLATES_DIR)),) else \ template_suffix=".$${template_filename:$${#template_filename} + 1}"; \ fi; \ - for suffix in '' $(patsubst %,_%,$(SUPPORTED_LANGUAGES)) ; do \ - mustache -e \ - "$(WIZARDS_TEMPLATES_DIR)/$${template_name}$${suffix}.yml" \ - "$(WIZARDS_TEMPLATES_DIR)/$${template_filename}" >"$(WIZARDS_DIR)/$${template_name}$${suffix}$${template_suffix}"; \ + template_file_path="$(WIZARDS_TEMPLATES_DIR)/$${template_filename}"; \ + for suffix in '' $(patsubst %,_%,$(LANGUAGES)) ; do \ + template_file_localization_data_path="$(WIZARDS_TEMPLATES_DIR)/$${template_name}$${suffix}.yml"; \ + if [ -f "$${template_file_localization_data_path}" ]; then \ + mustache -e \ + "$${template_file_localization_data_path}" \ + "$${template_file_path}" >"$(WIZARDS_DIR)/$${template_name}$${suffix}$${template_suffix}"; \ + fi; \ done; \ done endif From 49559e0371021bafda28513c9447fd7adcc902f8 Mon Sep 17 00:00:00 2001 From: hgy59 Date: Sun, 6 Aug 2023 17:31:11 +0200 Subject: [PATCH 05/11] fix Dockerfile - avoid too long RUN command (max length is 1024 chars) - add zip package to dev env. --- Dockerfile | 120 +++++++++++++++++++++++++++-------------------------- README.md | 4 +- 2 files changed, 63 insertions(+), 61 deletions(-) diff --git a/Dockerfile b/Dockerfile index b5f4db2d8fe..71f4c853a30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,66 +10,68 @@ ENV LANG C.UTF-8 RUN dpkg --add-architecture i386 # Install required packages (in sync with README.rst instructions) +# ATTENTION: the total length of the following RUN command must not exceed 1024 characters RUN apt-get update && apt-get install --no-install-recommends -y \ - autoconf-archive \ - autogen \ - automake \ - autopoint \ - bash \ - bc \ - bison \ - build-essential \ - check \ - cmake \ - curl \ - cython3 \ - debootstrap \ - ed \ - expect \ - fakeroot \ - flex \ - g++-multilib \ - gawk \ - gettext \ - git \ - gperf \ - imagemagick \ - intltool \ - jq \ - libbz2-dev \ - libc6-i386 \ - libcppunit-dev \ - libffi-dev \ - libgc-dev \ - libgmp3-dev \ - libltdl-dev \ - libmount-dev \ - libncurses-dev \ - libpcre3-dev \ - libssl-dev \ - libtool \ - libunistring-dev \ - lzip \ - mercurial \ - moreutils \ - ninja-build \ - patchelf \ - php \ - pkg-config \ - python2 \ - python3 \ - python3-distutils \ - rename \ - rsync \ - ruby-mustache \ - scons \ - subversion \ - sudo \ - swig \ - texinfo \ - unzip \ - xmlto \ - zlib1g-dev && \ + autoconf-archive \ + autogen \ + automake \ + autopoint \ + bash \ + bc \ + bison \ + build-essential \ + check \ + cmake \ + curl \ + cython3 \ + debootstrap \ + ed \ + expect \ + fakeroot \ + flex \ + g++-multilib \ + gawk \ + gettext \ + git \ + gperf \ + imagemagick \ + intltool \ + jq \ + libbz2-dev \ + libc6-i386 \ + libcppunit-dev \ + libffi-dev \ + libgc-dev \ + libgmp3-dev \ + libltdl-dev \ + libmount-dev \ + libncurses-dev \ + libpcre3-dev \ + libssl-dev \ + libtool \ + libunistring-dev \ + lzip \ + mercurial \ + moreutils \ + ninja-build \ + patchelf \ + php \ + pkg-config \ + python2 \ + python3 \ + python3-distutils \ + rename \ + rsync \ + ruby-mustache \ + scons \ + subversion \ + sudo \ + swig \ + texinfo \ + unzip \ + xmlto \ + zip \ + zlib1g-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ adduser --disabled-password --gecos '' user && \ diff --git a/README.md b/README.md index 961da5e7487..29b7be0e3b2 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ sudo apt install autoconf-archive autogen automake autopoint bash bc bison \ libcppunit-dev libffi-dev libgc-dev libgmp3-dev libltdl-dev libmount-dev libncurses-dev \ libpcre3-dev libssl-dev libtool libunistring-dev lzip mercurial moreutils ninja-build \ patchelf php pkg-config python2 python3 python3-distutils rename ruby-mustache rsync scons subversion \ - swig texinfo unzip xmlto zlib1g-dev + swig texinfo unzip xmlto zip zlib1g-dev wget https://bootstrap.pypa.io/pip/2.7/get-pip.py -O - | sudo python2 sudo pip2 install wheel httpie wget https://bootstrap.pypa.io/get-pip.py -O - | sudo python3 @@ -88,7 +88,7 @@ lxc exec spksrc -- /usr/bin/apt install autoconf-archive autogen automake autopo libcppunit-dev libffi-dev libgc-dev libgmp3-dev libltdl-dev libmount-dev libncurses-dev \ libpcre3-dev libssl-dev libtool libunistring-dev lzip mercurial moreutils ninja-build \ patchelf php pkg-config python2 python3 python3-distutils rename rsync ruby-mustache scons subversion \ - swig texinfo unzip xmlto zlib1g-dev + swig texinfo unzip xmlto zip zlib1g-dev ``` 5. Install `python2` wheels: ```bash From 1d6dd0ff1ba3fa852916188684df2b0e955c3390 Mon Sep 17 00:00:00 2001 From: hgy59 Date: Sun, 6 Aug 2023 19:57:00 +0200 Subject: [PATCH 06/11] fix sudoers - group is user (not users) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 71f4c853a30..db92d7f9a91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ adduser --disabled-password --gecos '' user && \ adduser user sudo && \ - echo "%users ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/users + echo "%user ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/users # Install setuptools, wheel and pip for Python2 RUN wget https://bootstrap.pypa.io/pip/2.7/get-pip.py -O - | python2 From c4e411a622a41ade80e20c5d77f5125f84c6a099 Mon Sep 17 00:00:00 2001 From: Smaarn Date: Sun, 6 Aug 2023 21:13:16 +0200 Subject: [PATCH 07/11] Add error detection when generating wizards --- mk/spksrc.spk.mk | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index f11ffb0ee58..4254fb354f1 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -407,10 +407,17 @@ ifneq ($(strip $(WIZARDS_TEMPLATES_DIR)),) template_file_path="$(WIZARDS_TEMPLATES_DIR)/$${template_filename}"; \ for suffix in '' $(patsubst %,_%,$(LANGUAGES)) ; do \ template_file_localization_data_path="$(WIZARDS_TEMPLATES_DIR)/$${template_name}$${suffix}.yml"; \ + output_file="$(WIZARDS_DIR)/$${template_name}$${suffix}$${template_suffix}"; \ if [ -f "$${template_file_localization_data_path}" ]; then \ mustache -e \ "$${template_file_localization_data_path}" \ - "$${template_file_path}" >"$(WIZARDS_DIR)/$${template_name}$${suffix}$${template_suffix}"; \ + "$${template_file_path}" >"$${output_file}"; \ + errors=$$(jq . "$${output_file}" 2>&1); \ + if [ "$$?" != "0" ]; then \ + echo "Invalid wizard file generated $${output_file}:"; \ + echo "$${errors}" \ + exit 1; \ + fi; \ fi; \ done; \ done From e8f08a056a247e3cbb4cb0d34ae2d4849a3ef5d6 Mon Sep 17 00:00:00 2001 From: Smaarn Date: Mon, 7 Aug 2023 11:52:58 +0200 Subject: [PATCH 08/11] mustache: fixed handling of install shell script --- mk/spksrc.spk.mk | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index 4254fb354f1..0179e124486 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -402,7 +402,7 @@ ifneq ($(strip $(WIZARDS_TEMPLATES_DIR)),) if [ "$${template_name}" = "$${template_filename}" ]; then \ template_suffix=; \ else \ - template_suffix=".$${template_filename:$${#template_filename} + 1}"; \ + template_suffix=".$${template_filename##*.}"; \ fi; \ template_file_path="$(WIZARDS_TEMPLATES_DIR)/$${template_filename}"; \ for suffix in '' $(patsubst %,_%,$(LANGUAGES)) ; do \ @@ -412,11 +412,13 @@ ifneq ($(strip $(WIZARDS_TEMPLATES_DIR)),) mustache -e \ "$${template_file_localization_data_path}" \ "$${template_file_path}" >"$${output_file}"; \ - errors=$$(jq . "$${output_file}" 2>&1); \ - if [ "$$?" != "0" ]; then \ - echo "Invalid wizard file generated $${output_file}:"; \ - echo "$${errors}" \ - exit 1; \ + if [ "$${template_suffix}" = "" ]; then \ + errors=$$(jq . "$${output_file}" 2>&1); \ + if [ "$$?" != "0" ]; then \ + echo "Invalid wizard file generated $${output_file}:"; \ + echo "$${errors}" \ + exit 1; \ + fi; \ fi; \ fi; \ done; \ From 0df7c85d3529e2216fa699998d6fa4e44c26a542 Mon Sep 17 00:00:00 2001 From: Smaarn Date: Mon, 7 Aug 2023 17:20:13 +0200 Subject: [PATCH 09/11] Mustache: inject `IS_DSM_6_OR_GREATER` and `IS_DSM_7_OR_GREATER` --- mk/spksrc.spk.mk | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index 0179e124486..41b278f4d4f 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -409,9 +409,11 @@ ifneq ($(strip $(WIZARDS_TEMPLATES_DIR)),) template_file_localization_data_path="$(WIZARDS_TEMPLATES_DIR)/$${template_name}$${suffix}.yml"; \ output_file="$(WIZARDS_DIR)/$${template_name}$${suffix}$${template_suffix}"; \ if [ -f "$${template_file_localization_data_path}" ]; then \ - mustache -e \ - "$${template_file_localization_data_path}" \ - "$${template_file_path}" >"$${output_file}"; \ + { \ + echo "IS_DSM_6_OR_GREATER: $(if $(filter 1,$(call version_gt, $(TCVERSION), 6.0)),true,false)"; \ + echo "IS_DSM_7_OR_GREATER: $(if $(filter 1,$(call version_gt, $(TCVERSION), 7.0)),true,false)"; \ + cat "$${template_file_localization_data_path}"; \ + } | mustache - "$${template_file_path}" >"$${output_file}"; \ if [ "$${template_suffix}" = "" ]; then \ errors=$$(jq . "$${output_file}" 2>&1); \ if [ "$$?" != "0" ]; then \ From 1e16da0e4a43c94e70267a6070c4c6aa634ec741 Mon Sep 17 00:00:00 2001 From: Smaarn Date: Mon, 7 Aug 2023 17:40:12 +0200 Subject: [PATCH 10/11] Add `IS_DSM_6` and `IS_DSM_7` --- mk/spksrc.spk.mk | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index 41b278f4d4f..ad4249d353c 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -396,6 +396,10 @@ endif ifneq ($(strip $(WIZARDS_TEMPLATES_DIR)),) @$(MSG) "Generate DSM Wizards from templates" @mkdir -p $(WIZARDS_DIR) + $(eval IS_DSM_6_OR_GREATER = $(if $(filter 1,$(call version_gt, $(TCVERSION), 6.0)),true,false)) + $(eval IS_DSM_7_OR_GREATER = $(if $(filter 1,$(call version_gt, $(TCVERSION), 7.0)),true,false)) + $(eval IS_DSM_7 = $(IS_DSM_7_OR_GREATER)) + $(eval IS_DSM_6 = $(if $(filter true,$(IS_DSM_6_OR_GREATER)),$(if $(filter true,$(IS_DSM_7)),false,true),false)) @for template in `find $(WIZARDS_TEMPLATES_DIR) -maxdepth 1 -type f -and \( $(WIZARD_FILE_NAMES) \) -print`; do \ template_filename="$$(basename $${template})"; \ template_name="$${template_filename%.*}"; \ @@ -410,8 +414,10 @@ ifneq ($(strip $(WIZARDS_TEMPLATES_DIR)),) output_file="$(WIZARDS_DIR)/$${template_name}$${suffix}$${template_suffix}"; \ if [ -f "$${template_file_localization_data_path}" ]; then \ { \ - echo "IS_DSM_6_OR_GREATER: $(if $(filter 1,$(call version_gt, $(TCVERSION), 6.0)),true,false)"; \ - echo "IS_DSM_7_OR_GREATER: $(if $(filter 1,$(call version_gt, $(TCVERSION), 7.0)),true,false)"; \ + echo "IS_DSM_6_OR_GREATER: $(IS_DSM_6_OR_GREATER)"; \ + echo "IS_DSM_6: $(IS_DSM_6)"; \ + echo "IS_DSM_7_OR_GREATER: $(IS_DSM_7_OR_GREATER)"; \ + echo "IS_DSM_7: $(IS_DSM_7)"; \ cat "$${template_file_localization_data_path}"; \ } | mustache - "$${template_file_path}" >"$${output_file}"; \ if [ "$${template_suffix}" = "" ]; then \ From 2c8ec7e5a5a0f25fea3ed88bedc8cbe87049c395 Mon Sep 17 00:00:00 2001 From: Smaarn Date: Mon, 7 Aug 2023 17:51:19 +0200 Subject: [PATCH 11/11] Mustache: fixed error handling --- mk/spksrc.spk.mk | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index ad4249d353c..28a9668cfdc 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -421,10 +421,11 @@ ifneq ($(strip $(WIZARDS_TEMPLATES_DIR)),) cat "$${template_file_localization_data_path}"; \ } | mustache - "$${template_file_path}" >"$${output_file}"; \ if [ "$${template_suffix}" = "" ]; then \ - errors=$$(jq . "$${output_file}" 2>&1); \ - if [ "$$?" != "0" ]; then \ + jq_failed=0; \ + errors=$$(jq . "$${output_file}" 2>&1) || jq_failed=1; \ + if [ "$${jq_failed}" != "0" ]; then \ echo "Invalid wizard file generated $${output_file}:"; \ - echo "$${errors}" \ + echo "$${errors}"; \ exit 1; \ fi; \ fi; \