[FL-2832] fbt: more fixes & improvements (flipperdevices#1854)

* github: bundling debug folder with scripts; docs: fixes & updates; fbt: added FAP_EXAMPLES variable to enable building example apps. Disabled by default. fbt: added TERM to list of proxied environment variables * fbt: better help output; disabled implicit_deps_unchanged; added color to import validator reports * fbt: moved debug configuration to separate tool * fbt: proper dependency tracker for SDK source file; renamed linker script for external apps * fbt: fixed debug elf path * fbt: packaging sdk archive * scripts: fixed sconsdist.py * fbt: reworked sdk packing; docs: updates * docs: info on cli target; linter fixes * fbt: moved main code to scripts folder * scripts: packing update into .tgz * fbt, scripts: reworked copro_dist to build .tgz * scripts: fixed naming for archived updater package * Scripts: fix ぐるぐる回る Co-authored-by: Aleksandr Kutuzov <[email protected]>
blackvault88 · Oct 12, 2022 · eb4ff3c · eb4ff3c
1 parent afff1ad
commit eb4ff3c
Show file tree

Hide file tree

Showing 41 changed files with 413 additions and 272 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -56,14 +56,14 @@ jobs:
       - name: 'Bundle scripts'
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          tar czpf artifacts/flipper-z-any-scripts-${SUFFIX}.tgz scripts
+          tar czpf artifacts/flipper-z-any-scripts-${SUFFIX}.tgz scripts debug
 
       - name: 'Build the firmware'
         run: |
           set -e
           for TARGET in ${TARGETS}; do
             FBT_TOOLCHAIN_PATH=/runner/_work ./fbt TARGET_HW="$(echo "${TARGET}" | sed 's/f//')" \
-                updater_package ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }}
+                copro_dist updater_package ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }}
           done
 
       - name: 'Move upload files'
@@ -74,17 +74,6 @@ jobs:
             mv dist/${TARGET}-*/* artifacts/
           done
 
-      - name: 'Bundle self-update package'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
-        run: |
-          set -e
-          for UPDATEBUNDLE in artifacts/*/; do
-            BUNDLE_NAME="$(echo "$UPDATEBUNDLE" | cut -d'/' -f2)"
-            echo Packaging "${BUNDLE_NAME}"
-            tar czpf "artifacts/flipper-z-${BUNDLE_NAME}.tgz" -C artifacts "${BUNDLE_NAME}"
-            rm -rf "artifacts/${BUNDLE_NAME}"
-          done
-
       - name: "Check for uncommitted changes"
         run: |
           git diff --exit-code
@@ -97,8 +86,7 @@ jobs:
       - name: 'Bundle core2 firmware'
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          FBT_TOOLCHAIN_PATH=/runner/_work ./fbt copro_dist
-          tar czpf "artifacts/flipper-z-any-core2_firmware-${SUFFIX}.tgz" -C assets core2_firmware
+          cp build/core2_firmware.tgz "artifacts/flipper-z-any-core2_firmware-${SUFFIX}.tgz"
 
       - name: 'Copy .map file'
         run: |

diff --git a/SConstruct b/SConstruct
@@ -7,25 +7,29 @@
 # construction of certain targets behind command-line options.
 
 import os
-import subprocess
 
 DefaultEnvironment(tools=[])
 
 EnsurePythonVersion(3, 8)
 
 # Progress(["OwO\r", "owo\r", "uwu\r", "owo\r"], interval=15)
 
-
 # This environment is created only for loading options & validating file/dir existence
 fbt_variables = SConscript("site_scons/commandline.scons")
-cmd_environment = Environment(tools=[], variables=fbt_variables)
-Help(fbt_variables.GenerateHelpText(cmd_environment))
+cmd_environment = Environment(
+    toolpath=["#/scripts/fbt_tools"],
+    tools=[
+        ("fbt_help", {"vars": fbt_variables}),
+    ],
+    variables=fbt_variables,
+)
 
 # Building basic environment - tools, utility methods, cross-compilation
 # settings, gcc flags for Cortex-M4, basic builders and more
 coreenv = SConscript(
     "site_scons/environ.scons",
     exports={"VAR_ENV": cmd_environment},
+    toolpath=["#/scripts/fbt_tools"],
 )
 SConscript("site_scons/cc.scons", exports={"ENV": coreenv})
 
@@ -35,41 +39,13 @@ coreenv["ROOT_DIR"] = Dir(".")
 
 # Create a separate "dist" environment and add construction envs to it
 distenv = coreenv.Clone(
-    tools=["fbt_dist", "openocd", "blackmagic", "jflash"],
-    OPENOCD_GDB_PIPE=[
-        "|openocd -c 'gdb_port pipe; log_output debug/openocd.log' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}"
-    ],
-    GDBOPTS_BASE=[
-        "-ex",
-        "target extended-remote ${GDBREMOTE}",
-        "-ex",
-        "set confirm off",
-        "-ex",
-        "set pagination off",
-    ],
-    GDBOPTS_BLACKMAGIC=[
-        "-ex",
-        "monitor swdp_scan",
-        "-ex",
-        "monitor debug_bmp enable",
-        "-ex",
-        "attach 1",
-        "-ex",
-        "set mem inaccessible-by-default off",
-    ],
-    GDBPYOPTS=[
-        "-ex",
-        "source debug/FreeRTOS/FreeRTOS.py",
-        "-ex",
-        "source debug/flipperapps.py",
-        "-ex",
-        "source debug/PyCortexMDebug/PyCortexMDebug.py",
-        "-ex",
-        "svd_load ${SVD_FILE}",
-        "-ex",
-        "compare-sections",
+    tools=[
+        "fbt_dist",
+        "fbt_debugopts",
+        "openocd",
+        "blackmagic",
+        "jflash",
     ],
-    JFLASHPROJECT="${ROOT_DIR.abspath}/debug/fw.jflash",
     ENV=os.environ,
 )
 
@@ -166,7 +142,7 @@ basic_dist = distenv.DistCommand("fw_dist", distenv["DIST_DEPENDS"])
 distenv.Default(basic_dist)
 
 dist_dir = distenv.GetProjetDirName()
-plugin_dist = [
+fap_dist = [
     distenv.Install(
         f"#/dist/{dist_dir}/apps/debug_elf",
         firmware_env["FW_EXTAPPS"]["debug"].values(),
@@ -176,9 +152,9 @@ plugin_dist = [
         for dist_entry in firmware_env["FW_EXTAPPS"]["dist"].values()
     ),
 ]
-Depends(plugin_dist, firmware_env["FW_EXTAPPS"]["validators"].values())
-Alias("plugin_dist", plugin_dist)
-# distenv.Default(plugin_dist)
+Depends(fap_dist, firmware_env["FW_EXTAPPS"]["validators"].values())
+Alias("fap_dist", fap_dist)
+# distenv.Default(fap_dist)
 
 plugin_resources_dist = list(
     distenv.Install(f"#/assets/resources/apps/{dist_entry[0]}", dist_entry[1])
@@ -189,9 +165,10 @@ distenv.Depends(firmware_env["FW_RESOURCES"], plugin_resources_dist)
 
 # Target for bundling core2 package for qFlipper
 copro_dist = distenv.CoproBuilder(
-    distenv.Dir("assets/core2_firmware"),
+    "#/build/core2_firmware.tgz",
     [],
 )
+distenv.AlwaysBuild(copro_dist)
 distenv.Alias("copro_dist", copro_dist)
 
 firmware_flash = distenv.AddOpenOCDFlashTarget(firmware_env)

diff --git a/applications/examples/example_images/example_images.c b/applications/examples/example_images/example_images.c
@@ -4,6 +4,8 @@
 #include <gui/gui.h>
 #include <input/input.h>
 
+/* Magic happens here -- this file is generated by fbt.
+ * Just set fap_icon_assets in application.fam and #include {APPID}_icons.h */
 #include "example_images_icons.h"
 
 typedef struct {

diff --git a/documentation/AppManifests.md b/documentation/AppManifests.md
@@ -30,7 +30,7 @@ Only 2 parameters are mandatory: ***appid*** and ***apptype***, others are optio
 | METAPACKAGE  | Does not define any code to be run, used for declaring dependencies and application bundles |
 
 * **name**: Name that is displayed in menus.
-* **entry_point**: C function to be used as application's entry point.
+* **entry_point**: C function to be used as application's entry point. Note that C++ function names are mangled, so you need to wrap them in `extern "C"` in order to use them as entry points.
 * **flags**: Internal flags for system apps. Do not use.
 * **cdefines**: C preprocessor definitions to declare globally for other apps when current application is included in active build configuration.
 * **requires**: List of application IDs to also include in build configuration, when current application is referenced in list of applications to build.
@@ -55,7 +55,7 @@ The following parameters are used only for [FAPs](./AppsOnSDCard.md):
 * **fap_author**: string, may be empty. Application's author.
 * **fap_weburl**: string, may be empty. Application's homepage.
 * **fap_icon_assets**: string. If present, defines a folder name to be used for gathering image assets for this application. These images will be preprocessed and built alongside the application. See [FAP assets](./AppsOnSDCard.md#fap-assets) for details.
-* **fap_extbuild**: provides support for parts of application sources to be build by external tools. Contains a list of `ExtFile(path="file name", command="shell command")` definitions. **`fbt`** will run the specified command for each file in the list.
+* **fap_extbuild**: provides support for parts of application sources to be built by external tools. Contains a list of `ExtFile(path="file name", command="shell command")` definitions. **`fbt`** will run the specified command for each file in the list.
 Note that commands are executed at the firmware root folder's root, and all intermediate files must be placed in a application's temporary build folder. For that, you can use pattern expansion by **`fbt`**: `${FAP_WORK_DIR}` will be replaced with the path to the application's temporary build folder, and `${FAP_SRC_DIR}` will be replaced with the path to the application's source folder. You can also use other variables defined internally by **`fbt`**. 
 
 Example for building an app from Rust sources:

diff --git a/documentation/AppsOnSDCard.md b/documentation/AppsOnSDCard.md
@@ -2,7 +2,7 @@
 
 [fbt](./fbt.md) has support for building applications as FAP files. FAP are essentially .elf executables with extra metadata and resources bundled in.
 
-FAPs are built with `faps` **`fbt`** target. They can also be deployed to `dist` folder with `plugin_dist` **`fbt`** target.
+FAPs are built with `faps` target. They can also be deployed to `dist` folder with `fap_dist` target.
 
 FAPs do not depend on being run on a specific firmware version. Compatibility is determined by the FAP's metadata, which includes the required [API version](#api-versioning).
 
@@ -15,7 +15,7 @@ To build your application as a FAP, just create a folder with your app's source
 
  * To build your application, run `./fbt fap_{APPID}`, where APPID is your application's ID in its manifest.
  * To build your app, then upload it over USB & run it on Flipper, use `./fbt launch_app APPSRC=applications/path/to/app`. This command is configured in default [VSCode profile](../.vscode/ReadMe.md) as "Launch App on Flipper" build action (Ctrl+Shift+B menu).
- * To build all FAPs, run `./fbt plugin_dist`.
+ * To build all FAPs, run `./fbt faps` or `./fbt fap_dist`.
 
 
 ## FAP assets

diff --git a/documentation/fbt.md b/documentation/fbt.md
@@ -43,7 +43,7 @@ To run cleanup (think of `make clean`) for specified targets, add `-c` option.
 ### High-level (what you most likely need)
 
 - `fw_dist` - build & publish firmware to `dist` folder. This is a default target, when no other are specified
-- `plugin_dist` - build external plugins & publish to `dist` folder  
+- `fap_dist` - build external plugins & publish to `dist` folder  
 - `updater_package`, `updater_minpackage` - build self-update package. Minimal version only inclues firmware's DFU file; full version also includes radio stack & resources for SD card
 - `copro_dist` - bundle Core2 FUS+stack binaries for qFlipper
 - `flash` - flash attached device with OpenOCD over ST-Link
@@ -56,6 +56,7 @@ To run cleanup (think of `make clean`) for specified targets, add `-c` option.
 - `get_blackmagic` - output blackmagic address in gdb remote format. Useful for IDE integration
 - `lint`, `format` - run clang-format on C source code to check and reformat it according to `.clang-format` specs
 - `lint_py`, `format_py` - run [black](https://black.readthedocs.io/en/stable/index.html) on Python source code, build system files & application manifests 
+- `cli` - start Flipper CLI session over USB
 
 ### Firmware targets
 

diff --git a/firmware.scons b/firmware.scons
@@ -3,7 +3,7 @@ Import("ENV", "fw_build_meta")
 from SCons.Errors import UserError
 import itertools
 
-from fbt.util import (
+from fbt_extra.util import (
     should_gen_cdb_and_link_dir,
     link_elf_dir_as_latest,
 )
@@ -141,6 +141,10 @@ else:
 if extra_int_apps := GetOption("extra_int_apps"):
     fwenv.Append(APPS=extra_int_apps.split(","))
 
+
+if fwenv["FAP_EXAMPLES"]:
+    fwenv.Append(APPDIRS=[("applications/examples", False)])
+
 fwenv.LoadApplicationManifests()
 fwenv.PrepareApplicationsBuild()
 
@@ -316,10 +320,13 @@ if fwenv["IS_BASE_FIRMWARE"]:
             "-D__inline__=inline",
         ],
     )
-    Depends(sdk_source, (fwenv["SDK_HEADERS"], fwenv["FW_ASSETS_HEADERS"]))
+    # Depends(sdk_source, (fwenv["SDK_HEADERS"], fwenv["FW_ASSETS_HEADERS"]))
+    Depends(sdk_source, fwenv.ProcessSdkDepends("sdk_origin.d"))
 
-    sdk_tree = fwenv.SDKTree("sdk/sdk.opts", "sdk_origin")
-    AlwaysBuild(sdk_tree)
+    fwenv["SDK_DIR"] = fwenv.Dir("sdk")
+    sdk_tree = fwenv.SDKTree(fwenv["SDK_DIR"], "sdk_origin")
+    fw_artifacts.append(sdk_tree)
+    # AlwaysBuild(sdk_tree)
     Alias("sdk_tree", sdk_tree)
 
     sdk_apicheck = fwenv.SDKSymUpdater(fwenv.subst("$SDK_DEFINITION"), "sdk_origin")
@@ -329,7 +336,7 @@ if fwenv["IS_BASE_FIRMWARE"]:
     Alias("sdk_check", sdk_apicheck)
 
     sdk_apisyms = fwenv.SDKSymGenerator(
-        "assets/compiled/symbols.h", fwenv.subst("$SDK_DEFINITION")
+        "assets/compiled/symbols.h", fwenv["SDK_DEFINITION"]
     )
     Alias("api_syms", sdk_apisyms)
 

diff --git a/firmware/targets/f7/application-ext.ld → firmware/targets/f7/application_ext.ld b/firmware/targets/f7/application-ext.ld → firmware/targets/f7/application_ext.ld
diff --git a/site_scons/fbt/__init__.py → scripts/fbt/__init__.py b/site_scons/fbt/__init__.py → scripts/fbt/__init__.py
diff --git a/site_scons/fbt/appmanifest.py → scripts/fbt/appmanifest.py b/site_scons/fbt/appmanifest.py → scripts/fbt/appmanifest.py
diff --git a/site_scons/fbt/elfmanifest.py → scripts/fbt/elfmanifest.py b/site_scons/fbt/elfmanifest.py → scripts/fbt/elfmanifest.py
diff --git a/site_scons/fbt/sdk.py → scripts/fbt/sdk.py b/site_scons/fbt/sdk.py → scripts/fbt/sdk.py
diff --git a/site_scons/fbt/util.py → scripts/fbt/util.py b/site_scons/fbt/util.py → scripts/fbt/util.py
@@ -41,25 +41,3 @@ def link_dir(target_path, source_path, is_windows):
 
 def single_quote(arg_list):
     return " ".join(f"'{arg}'" if " " in arg else str(arg) for arg in arg_list)
-
-
-def link_elf_dir_as_latest(env, elf_node):
-    elf_dir = elf_node.Dir(".")
-    latest_dir = env.Dir("#build/latest")
-    print(f"Setting {elf_dir} as latest built dir (./build/latest/)")
-    return link_dir(latest_dir.abspath, elf_dir.abspath, env["PLATFORM"] == "win32")
-
-
-def should_gen_cdb_and_link_dir(env, requested_targets):
-    explicitly_building_updater = False
-    # Hacky way to check if updater-related targets were requested
-    for build_target in requested_targets:
-        if "updater" in str(build_target):
-            explicitly_building_updater = True
-
-    is_updater = not env["IS_BASE_FIRMWARE"]
-    # If updater is explicitly requested, link to the latest updater
-    # Otherwise, link to firmware
-    return (is_updater and explicitly_building_updater) or (
-        not is_updater and not explicitly_building_updater
-    )
diff --git a/site_scons/fbt/version.py → scripts/fbt/version.py b/site_scons/fbt/version.py → scripts/fbt/version.py
diff --git a/site_scons/site_tools/blackmagic.py → scripts/fbt_tools/blackmagic.py b/site_scons/site_tools/blackmagic.py → scripts/fbt_tools/blackmagic.py
diff --git a/site_scons/site_tools/ccache.py → scripts/fbt_tools/ccache.py b/site_scons/site_tools/ccache.py → scripts/fbt_tools/ccache.py
diff --git a/site_scons/site_tools/crosscc.py → scripts/fbt_tools/crosscc.py b/site_scons/site_tools/crosscc.py → scripts/fbt_tools/crosscc.py
diff --git a/site_scons/site_tools/fbt_apps.py → scripts/fbt_tools/fbt_apps.py b/site_scons/site_tools/fbt_apps.py → scripts/fbt_tools/fbt_apps.py
diff --git a/site_scons/site_tools/fbt_assets.py → scripts/fbt_tools/fbt_assets.py b/site_scons/site_tools/fbt_assets.py → scripts/fbt_tools/fbt_assets.py
diff --git a/scripts/fbt_tools/fbt_debugopts.py b/scripts/fbt_tools/fbt_debugopts.py
@@ -0,0 +1,41 @@
+def generate(env, **kw):
+    env.SetDefault(
+        OPENOCD_GDB_PIPE=[
+            "|openocd -c 'gdb_port pipe; log_output debug/openocd.log' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}"
+        ],
+        GDBOPTS_BASE=[
+            "-ex",
+            "target extended-remote ${GDBREMOTE}",
+            "-ex",
+            "set confirm off",
+            "-ex",
+            "set pagination off",
+        ],
+        GDBOPTS_BLACKMAGIC=[
+            "-ex",
+            "monitor swdp_scan",
+            "-ex",
+            "monitor debug_bmp enable",
+            "-ex",
+            "attach 1",
+            "-ex",
+            "set mem inaccessible-by-default off",
+        ],
+        GDBPYOPTS=[
+            "-ex",
+            "source debug/FreeRTOS/FreeRTOS.py",
+            "-ex",
+            "source debug/flipperapps.py",
+            "-ex",
+            "source debug/PyCortexMDebug/PyCortexMDebug.py",
+            "-ex",
+            "svd_load ${SVD_FILE}",
+            "-ex",
+            "compare-sections",
+        ],
+        JFLASHPROJECT="${ROOT_DIR.abspath}/debug/fw.jflash",
+    )
+
+
+def exists(env):
+    return True
diff --git a/site_scons/site_tools/fbt_dist.py → scripts/fbt_tools/fbt_dist.py b/site_scons/site_tools/fbt_dist.py → scripts/fbt_tools/fbt_dist.py
@@ -136,7 +136,6 @@ def generate(env):
             "CoproBuilder": Builder(
                 action=Action(
                     [
-                        Mkdir("$TARGET"),
                         '${PYTHON3} "${ROOT_DIR.abspath}/scripts/assets.py" '
                         "copro ${COPRO_CUBE_DIR} "
                         "${TARGET} ${COPRO_MCU_FAMILY} "
@@ -145,7 +144,7 @@ def generate(env):
                         '--stack_file="${COPRO_STACK_BIN}" '
                         "--stack_addr=${COPRO_STACK_ADDR} ",
                     ],
-                    "",
+                    "\tCOPRO\t${TARGET}",
                 )
             ),
         }

diff --git a/site_scons/site_tools/fbt_extapps.py → scripts/fbt_tools/fbt_extapps.py b/site_scons/site_tools/fbt_extapps.py → scripts/fbt_tools/fbt_extapps.py
@@ -6,12 +6,10 @@
 import os
 import pathlib
 from fbt.elfmanifest import assemble_manifest_data
-from fbt.appmanifest import FlipperManifestException
+from fbt.appmanifest import FlipperApplication, FlipperManifestException
 from fbt.sdk import SdkCache
 import itertools
 
-from site_scons.fbt.appmanifest import FlipperApplication
-
 
 def BuildAppElf(env, app):
     ext_apps_work_dir = env.subst("$EXT_APPS_WORK_DIR")
@@ -111,7 +109,7 @@ def legacy_app_build_stub(**kw):
     )
 
     app_elf_raw = app_env.Program(
-        os.path.join(app_work_dir, f"{app.appid}_d"),
+        os.path.join(ext_apps_work_dir, f"{app.appid}_d"),
         app_sources,
         APP_ENTRY=app.entry_point,
     )
@@ -180,7 +178,7 @@ def validate_app_imports(target, source, env):
     if unresolved_syms:
         SCons.Warnings.warn(
             SCons.Warnings.LinkWarning,
-            f"{source[0].path}: app won't run. Unresolved symbols: {unresolved_syms}",
+            f"\033[93m{source[0].path}: app won't run. Unresolved symbols: \033[95m{unresolved_syms}\033[0m",
         )