[FL-3386] Fast FAP Loader (#2790)

* FBT: build and add FastFAP(tm) sections
* Elf file: fast loading fap files. Really fast, like x15 times faster.
* fastfap.py: cleanup unused imports
* Toolchain: 23 version
* Elf File: remove log messages
* Scripts: fix file permissions
* FBT: explicit interpreter for fastfap invocation

Co-authored-by: あく <alleteam@gmail.com>

firmware/targets/f18/api_symbols.csv

@@ -679,7 +679,8 @@ Function,+,elements_slightly_rounded_box,void,"Canvas*, uint8_t, uint8_t, uint8_
 Function,+,elements_slightly_rounded_frame,void,"Canvas*, uint8_t, uint8_t, uint8_t, uint8_t"
 Function,+,elements_string_fit_width,void,"Canvas*, FuriString*, uint8_t"
 Function,+,elements_text_box,void,"Canvas*, uint8_t, uint8_t, uint8_t, uint8_t, Align, Align, const char*, _Bool"
-Function,+,elf_resolve_from_hashtable,_Bool,"const ElfApiInterface*, const char*, Elf32_Addr*"
+Function,+,elf_resolve_from_hashtable,_Bool,"const ElfApiInterface*, uint32_t, Elf32_Addr*"
+Function,+,elf_symbolname_hash,uint32_t,const char*
 Function,+,empty_screen_alloc,EmptyScreen*,
 Function,+,empty_screen_free,void,EmptyScreen*
 Function,+,empty_screen_get_view,View*,EmptyScreen*

firmware/targets/f7/api_symbols.csv

@@ -808,7 +808,8 @@ Function,+,elements_slightly_rounded_box,void,"Canvas*, uint8_t, uint8_t, uint8_
 Function,+,elements_slightly_rounded_frame,void,"Canvas*, uint8_t, uint8_t, uint8_t, uint8_t"
 Function,+,elements_string_fit_width,void,"Canvas*, FuriString*, uint8_t"
 Function,+,elements_text_box,void,"Canvas*, uint8_t, uint8_t, uint8_t, uint8_t, Align, Align, const char*, _Bool"
-Function,+,elf_resolve_from_hashtable,_Bool,"const ElfApiInterface*, const char*, Elf32_Addr*"
+Function,+,elf_resolve_from_hashtable,_Bool,"const ElfApiInterface*, uint32_t, Elf32_Addr*"
+Function,+,elf_symbolname_hash,uint32_t,const char*
 Function,+,empty_screen_alloc,EmptyScreen*,
 Function,+,empty_screen_free,void,EmptyScreen*
 Function,+,empty_screen_get_view,View*,EmptyScreen*

lib/flipper_application/api_hashtable/api_hashtable.cpp

@@ -7,27 +7,22 @@
 
 bool elf_resolve_from_hashtable(
     const ElfApiInterface* interface,
-    const char* name,
+    uint32_t hash,
     Elf32_Addr* address) {
+    bool result = false;
     const HashtableApiInterface* hashtable_interface =
         static_cast<const HashtableApiInterface*>(interface);
-    bool result = false;
-    uint32_t gnu_sym_hash = elf_gnu_hash(name);
 
     sym_entry key = {
-        .hash = gnu_sym_hash,
+        .hash = hash,
         .address = 0,
     };
 
     auto find_res =
         std::lower_bound(hashtable_interface->table_cbegin, hashtable_interface->table_cend, key);
-    if((find_res == hashtable_interface->table_cend || (find_res->hash != gnu_sym_hash))) {
+    if((find_res == hashtable_interface->table_cend || (find_res->hash != hash))) {
         FURI_LOG_W(
-            TAG,
-            "Can't find symbol '%s' (hash %lx) @ %p!",
-            name,
-            gnu_sym_hash,
-            hashtable_interface->table_cbegin);
+            TAG, "Can't find symbol with hash %lx @ %p!", hash, hashtable_interface->table_cbegin);
         result = false;
     } else {
         result = true;
@@ -36,3 +31,7 @@ bool elf_resolve_from_hashtable(
 
     return result;
 }
+
+uint32_t elf_symbolname_hash(const char* s) {
+    return elf_gnu_hash(s);
+}

lib/flipper_application/api_hashtable/api_hashtable.h

@@ -19,15 +19,17 @@ struct sym_entry {
 /**
  * @brief Resolver for API entries using a pre-sorted table with hashes
  * @param interface pointer to HashtableApiInterface
- * @param name function name
+ * @param hash gnu hash of function name
  * @param address output for function address
  * @return true if the table contains a function
  */
 bool elf_resolve_from_hashtable(
     const ElfApiInterface* interface,
-    const char* name,
+    uint32_t hash,
     Elf32_Addr* address);
 
+uint32_t elf_symbolname_hash(const char* s);
+
 #ifdef __cplusplus
 }
 
@@ -48,8 +50,10 @@ struct HashtableApiInterface : public ElfApiInterface {
         .hash = elf_gnu_hash(#x), .address = (uint32_t)(static_cast<ret_type(*) args_type>(x)) \
     }
 
-#define API_VARIABLE(x, var_type) \
-    sym_entry { .hash = elf_gnu_hash(#x), .address = (uint32_t)(&(x)), }
+#define API_VARIABLE(x, var_type)                              \
+    sym_entry {                                                \
+        .hash = elf_gnu_hash(#x), .address = (uint32_t)(&(x)), \
+    }
 
 constexpr bool operator<(const sym_entry& k1, const sym_entry& k2) {
     return k1.hash < k2.hash;

lib/flipper_application/elf/elf_api_interface.h

@@ -11,6 +11,6 @@ typedef struct ElfApiInterface {
     uint16_t api_version_minor;
     bool (*resolver_callback)(
         const struct ElfApiInterface* interface,
-        const char* name,
+        uint32_t hash,
         Elf32_Addr* address);
 } ElfApiInterface;

lib/flipper_application/elf/elf_file.c

@@ -2,6 +2,7 @@
 #include "elf_file.h"
 #include "elf_file_i.h"
 #include "elf_api_interface.h"
+#include "../api_hashtable/api_hashtable.h"
 
 #define TAG "elf"
 
@@ -9,6 +10,7 @@
 #define SECTION_OFFSET(e, n) ((e)->section_table + (n) * sizeof(Elf32_Shdr))
 #define IS_FLAGS_SET(v, m) (((v) & (m)) == (m))
 #define RESOLVER_THREAD_YIELD_STEP 30
+#define FAST_RELOCATION_VERSION 1
 
 // #define ELF_DEBUG_LOG 1
 
@@ -71,6 +73,7 @@ static ELFSection* elf_file_get_or_put_section(ELFFile* elf, const char* name) {
                 .size = 0,
                 .rel_count = 0,
                 .rel_offset = 0,
+                .fast_rel = NULL,
             });
         section_p = elf_file_get_section(elf, name);
     }
@@ -168,7 +171,8 @@ static ELFSection* elf_section_of(ELFFile* elf, int index) {
 static Elf32_Addr elf_address_of(ELFFile* elf, Elf32_Sym* sym, const char* sName) {
     if(sym->st_shndx == SHN_UNDEF) {
         Elf32_Addr addr = 0;
-        if(elf->api_interface->resolver_callback(elf->api_interface, sName, &addr)) {
+        uint32_t hash = elf_symbolname_hash(sName);
+        if(elf->api_interface->resolver_callback(elf->api_interface, hash, &addr)) {
             return addr;
         }
     } else {
@@ -424,6 +428,7 @@ typedef enum {
     SectionTypeSymTab = 1 << 3,
     SectionTypeStrTab = 1 << 4,
     SectionTypeDebugLink = 1 << 5,
+    SectionTypeFastRelData = 1 << 6,
 
     SectionTypeValid = SectionTypeSymTab | SectionTypeStrTab,
 } SectionType;
@@ -505,7 +510,8 @@ static SectionType elf_preload_section(
     // TODO: how to do it not by name?
     // .ARM: type 0x70000001, flags SHF_ALLOC | SHF_LINK_ORDER
     // .rel.ARM: type 0x9, flags SHT_REL
-    if(str_prefix(name, ".ARM.") || str_prefix(name, ".rel.ARM.")) {
+    if(str_prefix(name, ".ARM.") || str_prefix(name, ".rel.ARM.") ||
+       str_prefix(name, ".fast.rel.ARM.")) {
         FURI_LOG_D(TAG, "Ignoring ARM section");
         return SectionTypeUnused;
     }
@@ -536,11 +542,31 @@ static SectionType elf_preload_section(
 
     // Load link info section
     if(section_header->sh_flags & SHF_INFO_LINK) {
-        name = name + strlen(".rel");
+        if(str_prefix(name, ".rel")) {
+            name = name + strlen(".rel");
+            ELFSection* section_p = elf_file_get_or_put_section(elf, name);
+            section_p->rel_count = section_header->sh_size / sizeof(Elf32_Rel);
+            section_p->rel_offset = section_header->sh_offset;
+            return SectionTypeRelData;
+        } else {
+            FURI_LOG_E(TAG, "Unknown link info section '%s'", name);
+            return SectionTypeERROR;
+        }
+    }
+
+    // Load fast rel section
+    if(str_prefix(name, ".fast.rel")) {
+        name = name + strlen(".fast.rel");
         ELFSection* section_p = elf_file_get_or_put_section(elf, name);
-        section_p->rel_count = section_header->sh_size / sizeof(Elf32_Rel);
-        section_p->rel_offset = section_header->sh_offset;
-        return SectionTypeRelData;
+        section_p->fast_rel = malloc(sizeof(ELFSection));
+
+        if(!elf_load_section_data(elf, section_p->fast_rel, section_header)) {
+            FURI_LOG_E(TAG, "Error loading section '%s'", name);
+            return SectionTypeERROR;
+        }
+
+        FURI_LOG_D(TAG, "Loaded fast rel section for '%s'", name);
+        return SectionTypeFastRelData;
     }
 
     // Load symbol table
@@ -571,8 +597,90 @@ static SectionType elf_preload_section(
     return SectionTypeUnused;
 }
 
+static Elf32_Addr elf_address_of_by_hash(ELFFile* elf, uint32_t hash) {
+    Elf32_Addr addr = 0;
+    if(elf->api_interface->resolver_callback(elf->api_interface, hash, &addr)) {
+        return addr;
+    }
+    return ELF_INVALID_ADDRESS;
+}
+
+static bool elf_relocate_fast(ELFFile* elf, ELFSection* s) {
+    UNUSED(elf);
+    const uint8_t* start = s->fast_rel->data;
+    const uint8_t version = *start;
+
+    if(version != FAST_RELOCATION_VERSION) {
+        FURI_LOG_E(TAG, "Unsupported fast relocation version %d", version);
+        return false;
+    }
+    start += 1;
+
+    const uint32_t records_count = *((uint32_t*)start);
+    start += 4;
+    FURI_LOG_D(TAG, "Fast relocation records count: %ld", records_count);
+
+    for(uint32_t i = 0; i < records_count; i++) {
+        bool is_section = (*start & (0x1 << 7)) ? true : false;
+        uint8_t type = *start & 0x7F;
+        start += 1;
+        uint32_t hash_or_section_index = *((uint32_t*)start);
+        start += 4;
+
+        uint32_t section_value = ELF_INVALID_ADDRESS;
+        if(is_section) {
+            section_value = *((uint32_t*)start);
+            start += 4;
+        }
+
+        const uint32_t offsets_count = *((uint32_t*)start);
+        start += 4;
+
+        FURI_LOG_D(
+            TAG,
+            "Fast relocation record %ld: is_section=%d, type=%d, hash_or_section_index=%lX, offsets_count=%ld",
+            i,
+            is_section,
+            type,
+            hash_or_section_index,
+            offsets_count);
+
+        Elf32_Addr address = 0;
+        if(is_section) {
+            ELFSection* symSec = elf_section_of(elf, hash_or_section_index);
+            if(symSec) {
+                address = ((Elf32_Addr)symSec->data) + section_value;
+            }
+        } else {
+            address = elf_address_of_by_hash(elf, hash_or_section_index);
+        }
+
+        if(address == ELF_INVALID_ADDRESS) {
+            FURI_LOG_E(TAG, "Failed to resolve address for hash %lX", hash_or_section_index);
+            return false;
+        }
+
+        for(uint32_t j = 0; j < offsets_count; j++) {
+            uint32_t offset = *((uint32_t*)start) & 0x00FFFFFF;
+            start += 3;
+            // FURI_LOG_I(TAG, "  Fast relocation offset %ld: %ld", j, offset);
+            Elf32_Addr relAddr = ((Elf32_Addr)s->data) + offset;
+            elf_relocate_symbol(elf, relAddr, type, address);
+        }
+    }
+
+    aligned_free(s->fast_rel->data);
+    free(s->fast_rel);
+    s->fast_rel = NULL;
+
+    return true;
+}
+
 static bool elf_relocate_section(ELFFile* elf, ELFSection* section) {
-    if(section->rel_count) {
+    if(section->fast_rel) {
+        FURI_LOG_D(TAG, "Fast relocating section");
+        return elf_relocate_fast(elf, section);
+    } else if(section->rel_count) {
         FURI_LOG_D(TAG, "Relocating section");
         return elf_relocate(elf, section);
     } else {
@@ -630,6 +738,10 @@ void elf_file_free(ELFFile* elf) {
             if(itref->value.data) {
                 aligned_free(itref->value.data);
             }
+            if(itref->value.fast_rel) {
+                aligned_free(itref->value.fast_rel->data);
+                free(itref->value.fast_rel);
+            }
             free((void*)itref->key);
         }
 

lib/flipper_application/elf/elf_file_i.h

@@ -13,14 +13,18 @@ DICT_DEF2(AddressCache, int, M_DEFAULT_OPLIST, Elf32_Addr, M_DEFAULT_OPLIST)
  */
 typedef int32_t(entry_t)(void*);
 
-typedef struct {
+typedef struct ELFSection ELFSection;
+
+struct ELFSection {
     void* data;
-    uint16_t sec_idx;
     Elf32_Word size;
 
     size_t rel_count;
     Elf32_Off rel_offset;
-} ELFSection;
+    ELFSection* fast_rel;
+
+    uint16_t sec_idx;
+};
 
 DICT_DEF2(ELFSectionDict, const char*, M_CSTR_OPLIST, ELFSection, M_POD_OPLIST)
 

lib/flipper_application/plugins/composite_resolver.c

@@ -13,12 +13,12 @@ struct CompositeApiResolver {
 
 static bool composite_api_resolver_callback(
     const ElfApiInterface* interface,
-    const char* name,
+    uint32_t hash,
     Elf32_Addr* address) {
     CompositeApiResolver* resolver = (CompositeApiResolver*)interface;
     for
         M_EACH(interface, resolver->interfaces, ElfApiInterfaceList_t) {
-            if((*interface)->resolver_callback(*interface, name, address)) {
+            if((*interface)->resolver_callback(*interface, hash, address)) {
                 return true;
             }
         }

scripts/fastfap.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+import hashlib
+import os
+import struct
+import subprocess
+import tempfile
+from collections import defaultdict
+from dataclasses import dataclass
+
+from elftools.elf.elffile import ELFFile
+from elftools.elf.relocation import RelocationSection
+from elftools.elf.sections import SymbolTableSection
+from fbt.sdk.hashes import gnu_sym_hash
+from flipper.app import App
+
+VERSION = 1
+
+
+@dataclass
+class RelData:
+    section: int
+    section_value: int
+    type: int
+    offset: int
+    name: str
+
+
+@dataclass(frozen=True)
+class UniqueRelData:
+    section: int
+    section_value: int
+    type: int
+    name: str
+
+
+@dataclass
+class RelSection:
+    name: str
+    oringinal_name: str
+    data: dict[UniqueRelData, list[int]]
+
+
+def serialize_relsection_data(data: dict[UniqueRelData, list[int]]) -> bytes:
+    result = struct.pack("<B", VERSION)
+    result += struct.pack("<I", len(data))
+    for unique, values in data.items():
+        if unique.section > 0:
+            result += struct.pack("<B", (1 << 7) | unique.type & 0x7F)
+            result += struct.pack("<I", unique.section)
+            result += struct.pack("<I", unique.section_value)
+        else:
+            result += struct.pack("<B", (0 << 7) | unique.type & 0x7F)
+            result += struct.pack("<I", gnu_sym_hash(unique.name))
+
+        result += struct.pack("<I", len(values))
+        for offset in values:
+            result += struct.pack(
+                "<BBB", offset & 0xFF, (offset >> 8) & 0xFF, (offset >> 16) & 0xFF
+            )
+
+    return result
+
+
+class Main(App):
+    def init(self):
+        self.parser.add_argument("fap_src_path", help="App file to upload")
+        self.parser.add_argument("objcopy_path", help="Objcopy path")
+        self.parser.set_defaults(func=self.process)
+
+    def process(self):
+        fap_path = self.args.fap_src_path
+        objcopy_path = self.args.objcopy_path
+
+        sections: list[RelSection] = []
+
+        with open(fap_path, "rb") as f:
+            elf_file = ELFFile(f)
+
+            relocation_sections: list[RelocationSection] = []
+            symtab_section: SymbolTableSection | None = None
+
+            for section in elf_file.iter_sections():
+                if isinstance(section, RelocationSection):
+                    relocation_sections.append(section)
+
+                if isinstance(section, SymbolTableSection):
+                    symtab_section = section
+
+            if not symtab_section:
+                self.logger.error("No symbol table found")
+                return 1
+
+            if not relocation_sections:
+                self.logger.info("No relocation sections found")
+                return 0
+
+            for section in relocation_sections:
+                section_relocations: list[RelData] = []
+
+                for relocation in section.iter_relocations():
+                    symbol_id: int = relocation.entry["r_info_sym"]
+                    offset: int = relocation.entry["r_offset"]
+                    type: int = relocation.entry["r_info_type"]
+                    symbol = symtab_section.get_symbol(symbol_id)
+                    section_index: int = symbol["st_shndx"]
+                    section_value: int = symbol["st_value"]
+                    if section_index == "SHN_UNDEF":
+                        section_index = 0
+
+                    section_relocations.append(
+                        RelData(section_index, section_value, type, offset, symbol.name)
+                    )
+
+                unique_relocations: dict[UniqueRelData, list[int]] = defaultdict(list)
+                for relocation in section_relocations:
+                    unique = UniqueRelData(
+                        relocation.section,
+                        relocation.section_value,
+                        relocation.type,
+                        relocation.name,
+                    )
+
+                    unique_relocations[unique].append(relocation.offset)
+
+                section_name = section.name
+                if section_name.startswith(".rel"):
+                    section_name = ".fast.rel" + section_name[4:]
+                else:
+                    self.logger.error(
+                        "Unknown relocation section name: %s", section_name
+                    )
+                    return 1
+
+                sections.append(
+                    RelSection(section_name, section.name, unique_relocations)
+                )
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            for section in sections:
+                data = serialize_relsection_data(section.data)
+                hash_name = hashlib.md5(section.name.encode()).hexdigest()
+                filename = f"{temp_dir}/{hash_name}.bin"
+
+                if os.path.isfile(filename):
+                    self.logger.error(f"File {filename} already exists")
+                    return 1
+
+                with open(filename, "wb") as f:
+                    f.write(data)
+
+                exit_code = subprocess.run(
+                    [
+                        objcopy_path,
+                        "--add-section",
+                        f"{section.name}={filename}",
+                        fap_path,
+                    ],
+                    check=True,
+                )
+
+                if exit_code.returncode != 0:
+                    self.logger.error("objcopy failed")
+                    return 1
+
+        return 0
+
+
+if __name__ == "__main__":
+    Main()()

scripts/fbt/sdk/collector.py

@@ -1,4 +1,5 @@
 from typing import List
+from .hashes import gnu_sym_hash
 
 from cxxheaderparser.parser import CxxParser
 from . import (
@@ -72,13 +73,6 @@ class SymbolManager:
         self.api.headers.add(ApiHeader(header))
 
 
-def gnu_sym_hash(name: str):
-    h = 0x1505
-    for c in name:
-        h = (h << 5) + h + ord(c)
-    return str(hex(h))[-8:]
-
-
 class SdkCollector:
     def __init__(self):
         self.symbol_manager = SymbolManager()

scripts/fbt/sdk/hashes.py

@@ -0,0 +1,5 @@
+def gnu_sym_hash(name: str) -> int:
+    h = 0x1505
+    for c in name:
+        h = ((h << 5) + h + ord(c)) & 0xFFFFFFFF
+    return h

scripts/fbt_tools/fbt_extapps.py

@@ -384,10 +384,16 @@ def generate_embed_app_metadata_actions(source, target, env, for_signature):
         "${SOURCES} ${TARGET}"
     )
 
-    actions.append(
-        Action(
-            objcopy_str,
-            "$APPMETAEMBED_COMSTR",
+    actions.extend(
+        (
+            Action(
+                objcopy_str,
+                "$APPMETAEMBED_COMSTR",
+            ),
+            Action(
+                "${PYTHON3} ${FBT_SCRIPT_DIR}/fastfap.py ${TARGET} ${OBJCOPY}",
+                "$FASTFAP_COMSTR",
+            ),
         )
     )
 
@@ -450,6 +456,7 @@ def generate(env, **kw):
             APPMETA_COMSTR="\tAPPMETA\t${TARGET}",
             APPFILE_COMSTR="\tAPPFILE\t${TARGET}",
             APPMETAEMBED_COMSTR="\tFAP\t${TARGET}",
+            FASTFAP_COMSTR="\tFASTFAP\t${TARGET}",
             APPCHECK_COMSTR="\tAPPCHK\t${SOURCE}",
         )
 

scripts/toolchain/fbtenv.cmd

@@ -13,7 +13,7 @@ if not ["%FBT_NOENV%"] == [""] (
     exit /b 0
 )
 
-set "FLIPPER_TOOLCHAIN_VERSION=21"
+set "FLIPPER_TOOLCHAIN_VERSION=22"
 
 if ["%FBT_TOOLCHAIN_PATH%"] == [""] (
     set "FBT_TOOLCHAIN_PATH=%FBT_ROOT%"

scripts/toolchain/fbtenv.sh

@@ -4,7 +4,7 @@
 
 # public variables
 DEFAULT_SCRIPT_PATH="$(pwd -P)";
-FBT_TOOLCHAIN_VERSION="${FBT_TOOLCHAIN_VERSION:-"21"}";
+FBT_TOOLCHAIN_VERSION="${FBT_TOOLCHAIN_VERSION:-"22"}";
 
 if [ -z ${FBT_TOOLCHAIN_PATH+x} ] ; then
     FBT_TOOLCHAIN_PATH_WAS_SET=0;