[FL-3600] Added fal_embedded parameter for PLUGIN apps (#3083)

* fbt, ufbt: added `fal_embedded` parameter for PLIGIN apps, to embed them into .fap
* fbt: fixed dependency settings for assets
* fbt: extapps: Removed unneeded casts
* fbt: extapps: code simplification
* fbt: fal_embedded: fixed dependency relations

Co-authored-by: あく <alleteam@gmail.com>

documentation/AppManifests.md

@@ -56,6 +56,7 @@ The following parameters are used only for [FAPs](./AppsOnSDCard.md):
 - **fap_weburl**: string, may be empty. Application's homepage.
 - **fap_icon_assets**: string. If present, it defines a folder name to be used for gathering image assets for this application. These images will be preprocessed and built alongside the application. See [FAP assets](./AppsOnSDCard.md#fap-assets) for details.
 - **fap_extbuild**: provides support for parts of application sources to be built by external tools. Contains a list of `ExtFile(path="file name", command="shell command")` definitions. **`fbt`** will run the specified command for each file in the list.
+- **fal_embedded**: boolean, default `False`. Applies only to PLUGIN type. If `True`, the plugin will be embedded into host application's .fap file as a resource and extracted to `apps_assets/APPID` folder on its start. This allows plugins to be distributed as a part of the host application.
 
 Note that commands are executed at the firmware root folder, and all intermediate files must be placed in an application's temporary build folder. For that, you can use pattern expansion by **`fbt`**: `${FAP_WORK_DIR}` will be replaced with the path to the application's temporary build folder, and `${FAP_SRC_DIR}` will be replaced with the path to the application's source folder. You can also use other variables defined internally by **`fbt`**.
 

firmware.scons

@@ -143,7 +143,7 @@ fwenv.PrepareApplicationsBuild()
 
 # Build external apps + configure SDK
 if env["IS_BASE_FIRMWARE"]:
-    fwenv.SetDefault(FBT_FAP_DEBUG_ELF_ROOT="${BUILD_DIR}/.extapps")
+    fwenv.SetDefault(FBT_FAP_DEBUG_ELF_ROOT=fwenv["BUILD_DIR"].Dir(".extapps"))
     fwenv["FW_EXTAPPS"] = SConscript(
         "site_scons/extapps.scons",
         exports={"ENV": fwenv},

scripts/fbt/appmanifest.py

@@ -79,11 +79,19 @@ class FlipperApplication:
     fap_extbuild: List[ExternallyBuiltFile] = field(default_factory=list)
     fap_private_libs: List[Library] = field(default_factory=list)
     fap_file_assets: Optional[str] = None
+    fal_embedded: bool = False
     # Internally used by fbt
     _appmanager: Optional["AppManager"] = None
     _appdir: Optional[object] = None
     _apppath: Optional[str] = None
     _plugins: List["FlipperApplication"] = field(default_factory=list)
+    _assets_dirs: List[object] = field(default_factory=list)
+    _section_fapmeta: Optional[object] = None
+    _section_fapfileassets: Optional[object] = None
+
+    @property
+    def embeds_plugins(self):
+        return any(plugin.fal_embedded for plugin in self._plugins)
 
     def supports_hardware_target(self, target: str):
         return target in self.targets or "all" in self.targets
@@ -137,6 +145,11 @@ class AppManager:
                 raise FlipperManifestException(
                     f"Plugin {kw.get('appid')} must have 'requires' in manifest"
                 )
+        else:
+            if kw.get("fal_embedded"):
+                raise FlipperManifestException(
+                    f"App {kw.get('appid')} cannot have fal_embedded set"
+                )
         # Harmless - cdefines for external apps are meaningless
         # if apptype == FlipperAppType.EXTERNAL and kw.get("cdefines"):
         #     raise FlipperManifestException(

scripts/fbt/fapassets.py

@@ -1,7 +1,7 @@
 import hashlib
 import os
 import struct
-from typing import TypedDict
+from typing import TypedDict, List
 
 
 class File(TypedDict):
@@ -32,20 +32,19 @@ class FileBundler:
       u8[] file_content
     """
 
-    def __init__(self, directory_path: str):
+    def __init__(self, assets_dirs: List[object]):
-        self.directory_path = directory_path
+        self.src_dirs = list(assets_dirs)
-        self.file_list: list[File] = []
-        self.directory_list: list[Dir] = []
-        self._gather()
 
-    def _gather(self):
+    def _gather(self, directory_path: str):
-        for root, dirs, files in os.walk(self.directory_path):
+        if not os.path.isdir(directory_path):
+            raise Exception(f"Assets directory {directory_path} does not exist")
+        for root, dirs, files in os.walk(directory_path):
             for file_info in files:
                 file_path = os.path.join(root, file_info)
                 file_size = os.path.getsize(file_path)
                 self.file_list.append(
                     {
-                        "path": os.path.relpath(file_path, self.directory_path),
+                        "path": os.path.relpath(file_path, directory_path),
                         "size": file_size,
                         "content_path": file_path,
                     }
@@ -57,15 +56,20 @@ class FileBundler:
                 #     os.path.getsize(os.path.join(dir_path, f)) for f in os.listdir(dir_path)
                 # )
                 self.directory_list.append(
-                    {
+                    {"path": os.path.relpath(dir_path, directory_path)}
-                        "path": os.path.relpath(dir_path, self.directory_path),
-                    }
                 )
 
         self.file_list.sort(key=lambda f: f["path"])
         self.directory_list.sort(key=lambda d: d["path"])
 
+    def _process_src_dirs(self):
+        self.file_list: list[File] = []
+        self.directory_list: list[Dir] = []
+        for directory_path in self.src_dirs:
+            self._gather(directory_path)
+
     def export(self, target_path: str):
+        self._process_src_dirs()
         self._md5_hash = hashlib.md5()
         with open(target_path, "wb") as f:
             # Write header magic and version

scripts/fbt_tools/fbt_extapps.py

@@ -3,7 +3,7 @@ import os
 import pathlib
 import shutil
 from dataclasses import dataclass, field
-from typing import Optional, Dict, List
+from typing import Dict, List, Optional
 
 import SCons.Warnings
 from ansi.color import fg
@@ -32,11 +32,15 @@ class FlipperExternalAppInfo:
 
 
 class AppBuilder:
+    @staticmethod
+    def get_app_work_dir(env, app):
+        return env["EXT_APPS_WORK_DIR"].Dir(app.appid)
+
     def __init__(self, env, app):
         self.fw_env = env
         self.app = app
-        self.ext_apps_work_dir = env.subst("$EXT_APPS_WORK_DIR")
+        self.ext_apps_work_dir = env["EXT_APPS_WORK_DIR"]
-        self.app_work_dir = os.path.join(self.ext_apps_work_dir, self.app.appid)
+        self.app_work_dir = self.get_app_work_dir(env, app)
         self.app_alias = f"fap_{self.app.appid}"
         self.externally_built_files = []
         self.private_libs = []
@@ -83,9 +87,9 @@ class AppBuilder:
             return
 
         fap_icons = self.app_env.CompileIcons(
-            self.app_env.Dir(self.app_work_dir),
+            self.app_work_dir,
             self.app._appdir.Dir(self.app.fap_icon_assets),
-            icon_bundle_name=f"{self.app.fap_icon_assets_symbol if self.app.fap_icon_assets_symbol else self.app.appid }_icons",
+            icon_bundle_name=f"{self.app.fap_icon_assets_symbol or self.app.appid }_icons",
         )
         self.app_env.Alias("_fap_icons", fap_icons)
         self.fw_env.Append(_APP_ICONS=[fap_icons])
@@ -95,7 +99,7 @@ class AppBuilder:
             self.private_libs.append(self._build_private_lib(lib_def))
 
     def _build_private_lib(self, lib_def):
-        lib_src_root_path = os.path.join(self.app_work_dir, "lib", lib_def.name)
+        lib_src_root_path = self.app_work_dir.Dir("lib").Dir(lib_def.name)
         self.app_env.AppendUnique(
             CPPPATH=list(
                 self.app_env.Dir(lib_src_root_path)
@@ -119,9 +123,7 @@ class AppBuilder:
 
         private_lib_env = self.app_env.Clone()
         private_lib_env.AppendUnique(
-            CCFLAGS=[
+            CCFLAGS=lib_def.cflags,
-                *lib_def.cflags,
-            ],
             CPPDEFINES=lib_def.cdefines,
             CPPPATH=list(
                 map(
@@ -132,14 +134,17 @@ class AppBuilder:
         )
 
         return private_lib_env.StaticLibrary(
-            os.path.join(self.app_work_dir, lib_def.name),
+            self.app_work_dir.File(lib_def.name),
             lib_sources,
         )
 
     def _build_app(self):
+        if self.app.fap_file_assets:
+            self.app._assets_dirs = [self.app._appdir.Dir(self.app.fap_file_assets)]
+
         self.app_env.Append(
             LIBS=[*self.app.fap_libs, *self.private_libs],
-            CPPPATH=[self.app_env.Dir(self.app_work_dir), self.app._appdir],
+            CPPPATH=[self.app_work_dir, self.app._appdir],
         )
 
         app_sources = list(
@@ -155,32 +160,46 @@ class AppBuilder:
 
         app_artifacts = FlipperExternalAppInfo(self.app)
         app_artifacts.debug = self.app_env.Program(
-            os.path.join(self.ext_apps_work_dir, f"{self.app.appid}_d"),
+            self.ext_apps_work_dir.File(f"{self.app.appid}_d.elf"),
             app_sources,
             APP_ENTRY=self.app.entry_point,
         )[0]
 
         app_artifacts.compact = self.app_env.EmbedAppMetadata(
-            os.path.join(self.ext_apps_work_dir, self.app.appid),
+            self.ext_apps_work_dir.File(f"{self.app.appid}.fap"),
             app_artifacts.debug,
             APP=self.app,
         )[0]
 
+        if self.app.embeds_plugins:
+            self.app._assets_dirs.append(self.app_work_dir.Dir("assets"))
+
         app_artifacts.validator = self.app_env.ValidateAppImports(
             app_artifacts.compact
         )[0]
 
         if self.app.apptype == FlipperAppType.PLUGIN:
             for parent_app_id in self.app.requires:
-                fal_path = (
+                if self.app.fal_embedded:
-                    f"apps_data/{parent_app_id}/plugins/{app_artifacts.compact.name}"
+                    parent_app = self.app._appmanager.get(parent_app_id)
-                )
+                    if not parent_app:
-                deployable = True
+                        raise UserError(
-                # If it's a plugin for a non-deployable app, don't include it in the resources
+                            f"Embedded plugin {self.app.appid} requires unknown app {parent_app_id}"
-                if parent_app := self.app._appmanager.get(parent_app_id):
+                        )
-                    if not parent_app.is_default_deployable:
+                    self.app_env.Install(
-                        deployable = False
+                        target=self.get_app_work_dir(self.app_env, parent_app)
-                app_artifacts.dist_entries.append((deployable, fal_path))
+                        .Dir("assets")
+                        .Dir("plugins"),
+                        source=app_artifacts.compact,
+                    )
+                else:
+                    fal_path = f"apps_data/{parent_app_id}/plugins/{app_artifacts.compact.name}"
+                    deployable = True
+                    # If it's a plugin for a non-deployable app, don't include it in the resources
+                    if parent_app := self.app._appmanager.get(parent_app_id):
+                        if not parent_app.is_default_deployable:
+                            deployable = False
+                    app_artifacts.dist_entries.append((deployable, fal_path))
         else:
             fap_path = f"apps/{self.app.fap_category}/{app_artifacts.compact.name}"
             app_artifacts.dist_entries.append(
@@ -194,7 +213,7 @@ class AppBuilder:
         # Extra things to clean up along with the app
         self.app_env.Clean(
             app_artifacts.debug,
-            [*self.externally_built_files, self.app_env.Dir(self.app_work_dir)],
+            [*self.externally_built_files, self.app_work_dir],
         )
 
         # Create listing of the app
@@ -219,13 +238,10 @@ class AppBuilder:
             )
 
         # Add dependencies on file assets
-        if self.app.fap_file_assets:
+        for assets_dir in self.app._assets_dirs:
             self.app_env.Depends(
                 app_artifacts.compact,
-                self.app_env.GlobRecursive(
+                (assets_dir, self.app_env.GlobRecursive("*", assets_dir)),
-                    "*",
-                    self.app._appdir.Dir(self.app.fap_file_assets),
-                ),
             )
 
         # Always run the validator for the app's binary when building the app
@@ -344,25 +360,26 @@ def embed_app_metadata_emitter(target, source, env):
     if app.apptype == FlipperAppType.PLUGIN:
         target[0].name = target[0].name.replace(".fap", ".fal")
 
-    target.append(env.File(source[0].abspath + _FAP_META_SECTION))
+    app_work_dir = AppBuilder.get_app_work_dir(env, app)
+    app._section_fapmeta = app_work_dir.File(_FAP_META_SECTION)
+    target.append(app._section_fapmeta)
 
-    if app.fap_file_assets:
+    # At this point, we haven't added dir with embedded plugins to _assets_dirs yet
-        target.append(env.File(source[0].abspath + _FAP_FILEASSETS_SECTION))
+    if app._assets_dirs or app.embeds_plugins:
+        app._section_fapfileassets = app_work_dir.File(_FAP_FILEASSETS_SECTION)
+        target.append(app._section_fapfileassets)
 
     return (target, source)
 
 
-def prepare_app_files(target, source, env):
+def prepare_app_file_assets(target, source, env):
     files_section_node = next(
         filter(lambda t: t.name.endswith(_FAP_FILEASSETS_SECTION), target)
     )
 
-    app = env["APP"]
+    bundler = FileBundler(
-    directory = env.Dir(app._apppath).Dir(app.fap_file_assets)
+        list(env.Dir(asset_dir).abspath for asset_dir in env["APP"]._assets_dirs)
-    if not directory.exists():
+    )
-        raise UserError(f"File asset directory {directory} does not exist")
-
-    bundler = FileBundler(directory.abspath)
     bundler.export(files_section_node.abspath)
 
 
@@ -376,12 +393,14 @@ def generate_embed_app_metadata_actions(source, target, env, for_signature):
     objcopy_str = (
         "${OBJCOPY} "
         "--remove-section .ARM.attributes "
-        "--add-section ${_FAP_META_SECTION}=${SOURCE}${_FAP_META_SECTION} "
+        "--add-section ${_FAP_META_SECTION}=${APP._section_fapmeta} "
     )
 
-    if app.fap_file_assets:
+    if app._section_fapfileassets:
-        actions.append(Action(prepare_app_files, "$APPFILE_COMSTR"))
+        actions.append(Action(prepare_app_file_assets, "$APPFILE_COMSTR"))
-        objcopy_str += "--add-section ${_FAP_FILEASSETS_SECTION}=${SOURCE}${_FAP_FILEASSETS_SECTION} "
+        objcopy_str += (
+            "--add-section ${_FAP_FILEASSETS_SECTION}=${APP._section_fapfileassets} "
+        )
 
     objcopy_str += (
         "--set-section-flags ${_FAP_META_SECTION}=contents,noload,readonly,data "
@@ -470,7 +489,7 @@ def AddAppBuildTarget(env, appname, build_target_name):
 
 def generate(env, **kw):
     env.SetDefault(
-        EXT_APPS_WORK_DIR="${FBT_FAP_DEBUG_ELF_ROOT}",
+        EXT_APPS_WORK_DIR=env.Dir(env["FBT_FAP_DEBUG_ELF_ROOT"]),
         APP_RUN_SCRIPT="${FBT_SCRIPT_DIR}/runfap.py",
     )
     if not env["VERBOSE"]: