[FL-2859,2838] fbt: improvements for FAPs (flipperdevices#1813)

* fbt: assets builder for apps WIP
* fbt: automatically building private fap assets
* docs: details on how to use image assets
* fbt: renamed fap_assets -> fap_icons
* fbt: support for fap_extbuild field
* docs: info on fap_extbuild
* fbt: added --proxy-env parame ter
* fbt: made firmware_cdb & updater_cdb targets always available
* fbt: renamed fap_icons -> fap_icon_assets
* fbt: deprecated firmware_* target names for faps; new alias is "fap_APPID"
* fbt: changed intermediate file locations for external apps
* fbt: support for fap_private_libs; docs: updates
* restored mbedtls as global lib
* scripts: lint.py: skip "lib" subfolder
* fbt: Sanity checks for building advanced faps as part of fw
* docs: info on fap_private_libs; fbt: optimized *.fam indexing
* fbt: cleanup; samples: added sample_icons app
* fbt: moved example app to applications/examples
* linter fix
* docs: readme fixes
* added applications/examples/application.fam stub
* docs: more info on private libs

Co-authored-by: あく <[email protected]>

applications/examples/application.fam

@@ -0,0 +1,5 @@
+App(
+    appid="sample_apps",
+    name="Sample apps bundle",
+    apptype=FlipperAppType.METAPACKAGE,
+)

applications/examples/example_images/application.fam

@@ -0,0 +1,10 @@
+App(
+    appid="example_images",
+    name="Example: Images",
+    apptype=FlipperAppType.EXTERNAL,
+    entry_point="example_images_main",
+    requires=["gui"],
+    stack_size=1 * 1024,
+    fap_category="Examples",
+    fap_icon_assets="images",
+)

applications/examples/example_images/example_images.c

@@ -0,0 +1,79 @@
+#include <furi.h>
+#include <furi_hal.h>
+
+#include <gui/gui.h>
+#include <input/input.h>
+
+#include "example_images_icons.h"
+
+typedef struct {
+    uint8_t x, y;
+} ImagePosition;
+
+static ImagePosition image_position = {.x = 0, .y = 0};
+
+// Screen is 128x64 px
+static void app_draw_callback(Canvas* canvas, void* ctx) {
+    UNUSED(ctx);
+
+    canvas_clear(canvas);
+    canvas_draw_icon(canvas, image_position.x % 128, image_position.y % 64, &I_dolphin_71x25);
+}
+
+static void app_input_callback(InputEvent* input_event, void* ctx) {
+    furi_assert(ctx);
+
+    FuriMessageQueue* event_queue = ctx;
+    furi_message_queue_put(event_queue, input_event, FuriWaitForever);
+}
+
+int32_t example_images_main(void* p) {
+    UNUSED(p);
+    FuriMessageQueue* event_queue = furi_message_queue_alloc(8, sizeof(InputEvent));
+
+    // Configure view port
+    ViewPort* view_port = view_port_alloc();
+    view_port_draw_callback_set(view_port, app_draw_callback, view_port);
+    view_port_input_callback_set(view_port, app_input_callback, event_queue);
+
+    // Register view port in GUI
+    Gui* gui = furi_record_open(RECORD_GUI);
+    gui_add_view_port(gui, view_port, GuiLayerFullscreen);
+
+    InputEvent event;
+
+    bool running = true;
+    while(running) {
+        if(furi_message_queue_get(event_queue, &event, 100) == FuriStatusOk) {
+            if((event.type == InputTypePress) || (event.type == InputTypeRepeat)) {
+                switch(event.key) {
+                case InputKeyLeft:
+                    image_position.x -= 2;
+                    break;
+                case InputKeyRight:
+                    image_position.x += 2;
+                    break;
+                case InputKeyUp:
+                    image_position.y -= 2;
+                    break;
+                case InputKeyDown:
+                    image_position.y += 2;
+                    break;
+                default:
+                    running = false;
+                    break;
+                }
+            }
+        }
+        view_port_update(view_port);
+    }
+
+    view_port_enabled_set(view_port, false);
+    gui_remove_view_port(gui, view_port);
+    view_port_free(view_port);
+    furi_message_queue_free(event_queue);
+
+    furi_record_close(RECORD_GUI);
+
+    return 0;
+}

applications/plugins/picopass/application.fam

@@ -10,8 +10,11 @@ App(
     stack_size=4 * 1024,
     order=30,
     fap_icon="../../../assets/icons/Archive/125_10px.png",
-    fap_libs=[
-        "mbedtls",
-    ],
     fap_category="Tools",
+    fap_libs=["mbedtls"],
+    fap_private_libs=[
+        Lib(
+            name="loclass",
+        ),
+    ],
 )

applications/plugins/picopass/picopass_device.h

@@ -7,8 +7,8 @@
 #include <mbedtls/des.h>
 
 #include "rfal_picopass.h"
-#include "loclass/optimized_ikeys.h"
-#include "loclass/optimized_cipher.h"
+#include <optimized_ikeys.h>
+#include <optimized_cipher.h>
 
 #define PICOPASS_DEV_NAME_MAX_LEN 22
 #define PICOPASS_READER_DATA_MAX_SIZE 64

assets/SConscript

@@ -9,28 +9,13 @@ assetsenv = env.Clone(
 )
 assetsenv.ApplyLibFlags()
 
-if not assetsenv["VERBOSE"]:
-    assetsenv.SetDefault(
-        ICONSCOMSTR="\tICONS\t${TARGET}",
-        PROTOCOMSTR="\tPROTO\t${SOURCE}",
-        DOLPHINCOMSTR="\tDOLPHIN\t${DOLPHIN_RES_TYPE}",
-        RESMANIFESTCOMSTR="\tMANIFEST\t${TARGET}",
-        PBVERCOMSTR="\tPBVER\t${TARGET}",
-    )
-
-# Gathering icons sources
-icons_src = assetsenv.GlobRecursive("*.png", "icons")
-icons_src += assetsenv.GlobRecursive("frame_rate", "icons")
-
-icons = assetsenv.IconBuilder(
-    assetsenv.Dir("compiled"), ICON_SRC_DIR=assetsenv.Dir("#/assets/icons")
+icons = assetsenv.CompileIcons(
+    assetsenv.Dir("compiled"), assetsenv.Dir("#/assets/icons")
 )
-assetsenv.Depends(icons, icons_src)
 assetsenv.Alias("icons", icons)
 
 
 # Protobuf .proto -> .c + .h
-
 proto_src = assetsenv.Glob("protobuf/*.proto", source=True)
 proto_options = assetsenv.Glob("protobuf/*.options", source=True)
 proto = assetsenv.ProtoBuilder(assetsenv.Dir("compiled"), proto_src)

documentation/AppManifests.md

@@ -41,16 +41,71 @@ Only 2 parameters are mandatory: ***appid*** and ***apptype***, others are optio
 * **order**: Order of an application within its group when sorting entries in it. The lower the order is, the closer to the start of the list the item is placed. *Used for ordering startup hooks and menu entries.* 
 * **sdk_headers**: List of C header files from this app's code to include in API definitions for external applications.
 
+
+#### Parameters for external applications
+
 The following parameters are used only for [FAPs](./AppsOnSDCard.md):
 
-* **sources**: list of strings, file name masks, used for gathering sources within app folder. Default value of `["*.c*"]` includes C and CPP source files.
+* **sources**: list of strings, file name masks, used for gathering sources within app folder. Default value of `["*.c*"]` includes C and C++ source files. Application cannot use `"lib"` folder for their own source code, as it is reserved for **fap_private_libs**.
 * **fap_version**: tuple, 2 numbers in form of (x,y): application version to be embedded within .fap file. Default value is (0,1), meanig version "0.1".
 * **fap_icon**: name of a .png file, 1-bit color depth, 10x10px, to be embedded within .fap file.
 * **fap_libs**: list of extra libraries to link application against. Provides access to extra functions that are not exported as a part of main firmware at expense of increased .fap file size and RAM consumption.
 * **fap_category**: string, may be empty. App subcategory, also works as path of FAP within apps folder in the file system.
 * **fap_description**: string, may be empty. Short application description.
 * **fap_author**: string, may be empty. Application's author.
 * **fap_weburl**: string, may be empty. Application's homepage.
+* **fap_icon_assets**: string. If present, defines a folder name to be used for gathering image assets for this application. These images will be preprocessed and built alongside the application. See [FAP assets](./AppsOnSDCard.md#fap-assets) for details.
+* **fap_extbuild**: provides support for parts of application sources to be build by external tools. Contains a list of `ExtFile(path="file name", command="shell command")` definitions. **`fbt`** will run the specified command for each file in the list.
+Note that commands are executed at the firmware root folder's root, and all intermediate files must be placed in a application's temporary build folder. For that, you can use pattern expansion by **`fbt`**: `${FAP_WORK_DIR}` will be replaced with the path to the application's temporary build folder, and `${FAP_SRC_DIR}` will be replaced with the path to the application's source folder. You can also use other variables defined internally by **`fbt`**. 
+
+Example for building an app from Rust sources:
+
+```python
+    sources=["target/thumbv7em-none-eabihf/release/libhello_rust.a"],
+    fap_extbuild=(
+        ExtFile(
+            path="${FAP_WORK_DIR}/target/thumbv7em-none-eabihf/release/libhello_rust.a",
+            command="cargo build --release --verbose --target thumbv7em-none-eabihf --target-dir ${FAP_WORK_DIR}/target --manifest-path ${FAP_SRC_DIR}/Cargo.toml",
+        ),
+    ),
+```
+
+* **fap_private_libs**: list of additional libraries that are distributed as sources alongside the application. These libraries will be built as a part of the application build process. 
+Library sources must be placed in a subfolder of "`lib`" folder within the application's source folder.
+Each library is defined as a call to `Lib()` function, accepting the following parameters:
+
+    - **name**: name of library's folder. Required.
+    - **fap_include_paths**: list of library's relative paths to add to parent fap's include path list. Default value is `["."]` meaning  library's source root.
+    - **sources**: list of filename masks to be used for gathering include files for this library. Default value is `["*.c*"]`.
+    - **cflags**: list of additional compiler flags to be used for building this library. Default value is `[]`.
+    - **cdefines**: list of additional preprocessor definitions to be used for building this library. Default value is `[]`.
+    - **cincludes**: list of additional include paths to be used for building this library. Can be used for providing external search paths for this library's code - for configuration headers. Default value is `[]`.
+
+Example for building an app with a private library:
+
+```python
+    fap_private_libs=[
+            Lib(
+                name="mbedtls",
+                fap_include_paths=["include"],
+                sources=[
+                    "library/des.c",
+                    "library/sha1.c",
+                    "library/platform_util.c",
+                ],
+                cdefines=["MBEDTLS_ERROR_C"],
+            ),
+            Lib(
+                name="loclass",
+                cflags=["-Wno-error"],
+            ),
+        ],
+```
+
+For that snippet, **`fbt`** will build 2 libraries: one from sources in `lib/mbedtls` folder, and another from sources in `lib/loclass` folder. For `mbedtls` library, **`fbt`** will add `lib/mbedtls/include` to the list of include paths for the application and compile only the files specified in `sources` list. Additionally, **`fbt`** will enable `MBEDTLS_ERROR_C` preprocessor definition for `mbedtls` sources. 
+For `loclass` library, **`fbt`** will add `lib/loclass` to the list of include paths for the application and build all sources in that folder. Also **`fbt`** will disable treating compiler warnings as errors for `loclass` library specifically - that can be useful when compiling large 3rd-party codebases.
+
+Both libraries will be linked into the application.
 
 
 ## .fam file contents

documentation/AppsOnSDCard.md

@@ -2,7 +2,7 @@
 
 [fbt](./fbt.md) has support for building applications as FAP files. FAP are essentially .elf executables with extra metadata and resources bundled in.
 
-FAPs are built with `firmware_extapps` (or `plugin_dist`) **`fbt`** targets.
+FAPs are built with `faps` **`fbt`** target. They can also be deployed to `dist` folder with `plugin_dist` **`fbt`** target.
 
 FAPs do not depend on being run on a specific firmware version. Compatibility is determined by the FAP's metadata, which includes the required [API version](#api-versioning).
 
@@ -18,6 +18,17 @@ To build your application as a FAP, just create a folder with your app's source
  * To build all FAPs, run `./fbt plugin_dist`.
 
 
+## FAP assets
+
+FAPs can include static and animated images as private assets. They will be automatically compiled alongside application sources and can be referenced the same way as assets from the main firmware.
+
+To use that feature, put your images in a subfolder inside your application's folder, then reference that folder in your application's manifest in `fap_icon_assets` field. See [Application Manifests](./AppManifests.md#application-definition) for more details.
+
+To use these assets in your application, put `#include "{APPID}_icons.h"` in your application's source code, where `{APPID}` is the `appid` value field from your application's manifest. Then you can use all icons from your application's assets the same way as if they were a part of `assets_icons.h` of the main firmware.
+
+Images and animated icons must follow the same [naming convention](../assets/ReadMe.md#asset-naming-rules) as those from the main firmware.
+
+
 ## Debugging FAPs
 
 **`fbt`** includes a script for gdb-py to provide debugging support for FAPs, `debug/flipperapps.py`. It is loaded in default debugging configurations by **`fbt`** and stock VSCode configurations.
@@ -53,13 +64,13 @@ App loader allocates memory for the application and copies it to RAM, processing
 
 Not all parts of firmware are available for external applications. A subset of available functions and variables is defined in "api_symbols.csv" file, which is a part of firmware target definition in `firmware/targets/` directory. 
 
-**`fbt`** uses semantic versioning for API versioning. Major version is incremented when there are breaking changes in the API, minor version is incremented when there are new features added. 
+**`fbt`** uses semantic versioning for API. Major version is incremented when there are breaking changes in the API, minor version is incremented when new features are added. 
 
 Breaking changes include:
 - removal of a function or a global variable;
 - changing the signature of a function.
 
-API versioning is mostly automated by **`fbt`**. When rebuilding the firmware, **`fbt`** checks if there are any changes in the API exposed by headers gathered from `SDK_HEADERS`. If there are, it stops the build, adjusts the API version and asks the user to go through the changes in .csv file. New entries are marked with "`?`" mark, and the user is supposed to change the mark to "`+`" for the entry to be exposed for FAPs, "`-`" for it to be unavailable.
+API versioning is mostly automated by **`fbt`**. When rebuilding the firmware, **`fbt`** checks if there are any changes in the API exposed by headers gathered from `SDK_HEADERS`. If so, it stops the build, adjusts the API version and asks the user to go through the changes in .csv file. New entries are marked with "`?`" mark, and the user is supposed to change the mark to "`+`" for the entry to be exposed for FAPs, "`-`" for it to be unavailable.
 
 **`fbt`** will not allow building a firmware until all "`?`" entries are changed to "`+`" or "`-`".
 

documentation/fbt.md

@@ -59,10 +59,11 @@ To run cleanup (think of `make clean`) for specified targets, add `-c` option.
 
 ### Firmware targets
 
-- `firmware_extapps` - build all plug-ins as separate .elf files
-    - `firmware_snake_game`, etc - build single plug-in as .elf by its name
-    - Check out `--extra-ext-apps` for force adding extra apps to external build 
-    - `firmware_snake_game_list`, etc - generate source + assembler listing for app's .elf
+- `faps` - build all external & plugin apps as [.faps](./AppsOnSDCard.md#fap-flipper-application-package). 
+- **`fbt`** also defines per-app targets. For example, for an app with `appid=snake_game` target names are:
+    - `fap_snake_game`, etc - build single app as .fap by its application ID.
+    - Check out [`--extra-ext-apps`](#command-line-parameters) for force adding extra apps to external build
+    - `fap_snake_game_list`, etc - generate source + assembler listing for app's .fap
 - `flash`, `firmware_flash` - flash current version to attached device with OpenOCD over ST-Link
 - `jflash` - flash current version to attached device with JFlash using J-Link probe. JFlash executable must be on your $PATH
 - `flash_blackmagic` - flash current version to attached device with Blackmagic probe
@@ -83,9 +84,9 @@ To run cleanup (think of `make clean`) for specified targets, add `-c` option.
 ## Command-line parameters
 
 - `--options optionfile.py` (default value `fbt_options.py`) - load file with multiple configuration values
-- `--with-updater` - enables updater-related targets and dependency tracking. Enabling this option introduces extra startup time costs, so use it when bundling update packages. _Explicily enabling this should no longer be required, **`fbt`** now has specific handling for updater-related targets_
 - `--extra-int-apps=app1,app2,appN` - forces listed apps to be built as internal with `firmware` target
 - `--extra-ext-apps=app1,app2,appN` - forces listed apps to be built as external with `firmware_extapps` target
+- `--proxy-env=VAR1,VAR2` - additional environment variables to expose to subprocesses spawned by `fbt`. By default, `fbt` sanitizes execution environment and doesn't forward all inherited environment variables. You can find list of variables that are always forwarded in `environ.scons` file. 
 
 
 ## Configuration 

firmware.scons

@@ -1,5 +1,6 @@
 Import("ENV", "fw_build_meta")
 
+from SCons.Errors import UserError
 import itertools
 
 from fbt.util import (
@@ -164,13 +165,25 @@ apps_c = fwenv.ApplicationsC(
 # Adding dependency on manifest files so apps.c is rebuilt when any manifest is changed
 for app_dir, _ in env["APPDIRS"]:
     app_dir_node = env.Dir("#").Dir(app_dir)
-    fwenv.Depends(apps_c, fwenv.GlobRecursive("*.fam", app_dir_node))
+    fwenv.Depends(apps_c, app_dir_node.glob("*/application.fam"))
+
+# Sanity check - certain external apps are using features that are not available in base firmware
+if advanced_faps := list(
+    filter(
+        lambda app: app.fap_extbuild or app.fap_private_libs or app.fap_icon_assets,
+        fwenv["APPBUILD"].get_builtin_apps(),
+    )
+):
+    raise UserError(
+        "An Application that is using fap-specific features cannot be built into base firmware."
+        f" Offending app(s): {', '.join(app.appid for app in advanced_faps)}"
+    )
 
 sources = [apps_c]
 # Gather sources only from app folders in current configuration
 sources.extend(
     itertools.chain.from_iterable(
-        fwenv.GlobRecursive(source_type, appdir.relpath)
+        fwenv.GlobRecursive(source_type, appdir.relpath, exclude="lib")
         for appdir, source_type in fwenv["APPBUILD"].get_builtin_app_folders()
     )
 )
@@ -259,18 +272,18 @@ fw_artifacts = fwenv["FW_ARTIFACTS"] = [
     fwenv["FW_VERSION_JSON"],
 ]
 
+
+fwcdb = fwenv.CompilationDatabase()
+# without filtering, both updater & firmware commands would be generated in same file
+fwenv.Replace(COMPILATIONDB_PATH_FILTER=fwenv.subst("*${FW_FLAVOR}*"))
+AlwaysBuild(fwcdb)
+Precious(fwcdb)
+NoClean(fwcdb)
+Alias(fwenv["FIRMWARE_BUILD_CFG"] + "_cdb", fwcdb)
+
 # If current configuration was explicitly requested, generate compilation database
 # and link its directory as build/latest
 if should_gen_cdb_and_link_dir(fwenv, BUILD_TARGETS):
-    fwcdb = fwenv.CompilationDatabase()
-    # without filtering, both updater & firmware commands would be generated
-    fwenv.Replace(COMPILATIONDB_PATH_FILTER=fwenv.subst("*${FW_FLAVOR}*"))
-    AlwaysBuild(fwcdb)
-    Precious(fwcdb)
-    NoClean(fwcdb)
-    Alias(fwenv["FIRMWARE_BUILD_CFG"] + "_cdb", fwcdb)
-    AlwaysBuild(fwenv["FIRMWARE_BUILD_CFG"] + "_cdb", fwcdb)
-    Alias(fwcdb, "")
     fw_artifacts.append(fwcdb)
 
     # Adding as a phony target, so folder link is updated even if elf didn't change