[FL-2554] Embedded arm-none-eabi toolchain (#1351)

.gitattributes

@@ -1 +1,4 @@
 * text=auto eol=lf
+*.bat eol=crlf
+*.ps1 eol=crlf
+*.cmd eol=crlf

.gitignore

@@ -46,4 +46,7 @@ dist
 build/
 
 # Toolchain
-toolchain*/
+/toolchain
+
+# openocd output file
+openocd.log

SConstruct

@@ -8,6 +8,8 @@
 
 import os
 
+EnsurePythonVersion(3, 8)
+
 DefaultEnvironment(tools=[])
 # Progress(["OwO\r", "owo\r", "uwu\r", "owo\r"], interval=15)
 
@@ -32,7 +34,7 @@ coreenv["ROOT_DIR"] = Dir(".")
 # Create a separate "dist" environment and add construction envs to it
 distenv = coreenv.Clone(
     tools=["fbt_dist", "openocd", "blackmagic"],
-    OPENOCD_GDB_PIPE=["|openocd -c 'gdb_port pipe' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}"],
+    OPENOCD_GDB_PIPE=["|openocd -c 'gdb_port pipe; log_output debug/openocd.log' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}"],
     GDBOPTS_BASE=[
         "-ex",
         "target extended-remote ${GDBREMOTE}",

applications/extapps.scons

@@ -3,7 +3,6 @@ Import("ENV")
 
 from fbt.appmanifest import FlipperAppType
 
-
 appenv = ENV.Clone(tools=["fbt_extapps"])
 
 appenv.Replace(

fbt

@@ -1,18 +1,22 @@
-#!/bin/bash
+#!/bin/sh
 
-set -e
+# shellcheck disable=SC2086 source=/dev/null
+# unofficial strict mode
+set -eu;
 
-SCRIPTDIR="$( dirname -- "$0"; )";
-SCONS_EP=${SCRIPTDIR}/lib/scons/scripts/scons.py
+SCONS_DEFAULT_FLAGS="-Q --warn=target-not-built";
+SCRIPT_PATH="$(cd "$(dirname "$0")" && pwd -P)";
 
-if [[ -z "${FBT_NO_SYNC:-}" ]] ; then
-    if [[ -d .git ]]; then
-        git submodule update --init
-    else
-        echo Not in a git repo, please clone with git clone --recursive
-        exit 1
-    fi
+if [ -z "${FBT_NOENV:-}" ]; then
+    . "$SCRIPT_PATH/scripts/toolchain/fbtenv.sh";
 fi
 
-SCONS_DEFAULT_FLAGS="-Q --warn=target-not-built"
-python3 ${SCONS_EP} ${SCONS_DEFAULT_FLAGS} "$@"
+if [ -z "${FBT_NO_SYNC:-}" ]; then
+    if [ ! -d "$SCRIPT_PATH/.git" ]; then
+        echo "\".git\" directory not found, please clone repo via \"git clone --recursive\"";
+        exit 1;
+    fi
+    git submodule update --init;
+fi
+
+python3 "$SCRIPT_PATH/lib/scons/scripts/scons.py" $SCONS_DEFAULT_FLAGS "$@"

fbt.cmd

@@ -1,15 +1,17 @@
 @echo off
+call %~dp0scripts\toolchain\fbtenv.cmd env
 
 set SCONS_EP=%~dp0\lib\scons\scripts\scons.py
 
 if [%FBT_NO_SYNC%] == [] (
-	if exist ".git" (
-		git submodule update --init
-	) else (
-		echo Not in a git repo, please clone with git clone --recursive
-		exit /b 1
-	)
+    if exist ".git" (
+        git submodule update --init
+    ) else (
+        echo Not in a git repo, please clone with git clone --recursive
+        exit /b 1
+    )
 )
+git submodule update --init
 
 set "SCONS_DEFAULT_FLAGS=-Q --warn=target-not-built"
-python %SCONS_EP% %SCONS_DEFAULT_FLAGS% %*
+python lib\scons\scripts\scons.py %SCONS_DEFAULT_FLAGS% %*

fbt_options.py

@@ -10,8 +10,8 @@ COMPACT = 0
 ## Optimize for debugging experience
 DEBUG = 1
 
-# Suffix to add to files when building distribution.
-# If OS environment has DIST_SUFFIX set, it will be used instead..
+# Suffix to add to files when building distribution
+# If OS environment has DIST_SUFFIX set, it will be used instead
 DIST_SUFFIX = "local"
 
 # Coprocessor firmware
@@ -27,7 +27,7 @@ COPRO_STACK_BIN = "stm32wb5x_BLE_Stack_light_fw.bin"
 # Firmware also supports "ble_full", but it might not fit into debug builds
 COPRO_STACK_TYPE = "ble_light"
 
-# Leave 0 to lets scripts automatically calculate it
+# Leave 0 to let scripts automatically calculate it
 COPRO_STACK_ADDR = "0x0"
 
 # If you override COPRO_CUBE_DIR on commandline, override this aswell
@@ -56,7 +56,7 @@ OPENOCD_OPTS = [
 
 SVD_FILE = "debug/STM32WB55_CM4.svd"
 
-# Look for blackmagic probe on serial ports
+# Look for blackmagic probe on serial ports and local network
 BLACKMAGIC = "auto"
 
 FIRMWARE_APPS = {

firmware.scons

@@ -201,7 +201,7 @@ fwelf = fwenv["FW_ELF"] = fwenv.Program(
     ],
 )
 
-
+# Make it depend on everything child builders returned
 # Firmware depends on everything child builders returned
 Depends(fwelf, lib_targets)
 # Output extra details after building firmware

scripts/toolchain/fbtenv.cmd

@@ -0,0 +1,45 @@
+@echo off
+
+if not [%FBT_ROOT%] == [] (
+    goto already_set
+)
+
+set "FBT_ROOT=%~dp0\..\..\"
+pushd %FBT_ROOT%
+set "FBT_ROOT=%cd%"
+popd
+
+if not [%FBT_NOENV%] == [] (
+    exit /b 0
+)
+
+set "FLIPPER_TOOLCHAIN_VERSION=3"
+set "FBT_TOOLCHAIN_ROOT=%FBT_ROOT%\toolchain\i686-windows"
+
+
+if not exist "%FBT_TOOLCHAIN_ROOT%" (
+    powershell -ExecutionPolicy Bypass -File %FBT_ROOT%\scripts\toolchain\windows-toolchain-download.ps1 "%flipper_toolchain_version%"
+)
+if not exist "%FBT_TOOLCHAIN_ROOT%\VERSION" (
+    powershell -ExecutionPolicy Bypass -File %FBT_ROOT%\scripts\toolchain\windows-toolchain-download.ps1 "%flipper_toolchain_version%"
+)
+set /p REAL_TOOLCHAIN_VERSION=<%FBT_TOOLCHAIN_ROOT%\VERSION
+if not "%REAL_TOOLCHAIN_VERSION%" == "%FLIPPER_TOOLCHAIN_VERSION%" (
+    powershell -ExecutionPolicy Bypass -File %FBT_ROOT%\scripts\toolchain\windows-toolchain-download.ps1 "%flipper_toolchain_version%"
+)
+
+
+set "HOME=%USERPROFILE%"
+set "PYTHONHOME=%FBT_TOOLCHAIN_ROOT%\python"
+set "PATH=%FBT_TOOLCHAIN_ROOT%\python;%FBT_TOOLCHAIN_ROOT%\bin;%FBT_TOOLCHAIN_ROOT%\protoc\bin;%FBT_TOOLCHAIN_ROOT%\openocd\bin;%PATH%"
+set "PROMPT=(fbt) %PROMPT%"
+
+:already_set
+
+if not "%1" == "env" (
+    echo *********************************
+    echo *     fbt build environment     *
+    echo *********************************
+    cd %FBT_ROOT%
+    cmd /k
+)

scripts/toolchain/fbtenv.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+# unofficial strict mode
+set -eu;
+
+FLIPPER_TOOLCHAIN_VERSION="3";
+
+get_kernel_type()
+{
+    SYS_TYPE="$(uname -s)"
+    if [ "$SYS_TYPE" = "Darwin" ]; then
+        TOOLCHAIN_PATH="toolchain/x86_64-darwin";
+    elif [ "$SYS_TYPE" = "Linux" ]; then
+        TOOLCHAIN_PATH="toolchain/x86_64-linux";
+    elif echo "$SYS_TYPE" | grep -q "MINGW"; then
+        echo "In MinGW shell use \"fbt.cmd\" instead of \"fbt\"";
+        exit 1;
+    else
+        echo "Your system is not supported. Sorry. Please report us your configuration.";
+        exit 1;
+    fi
+}
+
+check_download_toolchain()
+{
+    if [ ! -d "$SCRIPT_PATH/$TOOLCHAIN_PATH" ]; then
+        download_toolchain;
+    elif [ ! -f "$SCRIPT_PATH/$TOOLCHAIN_PATH/VERSION" ]; then
+        download_toolchain;
+    elif [ "$(cat "$SCRIPT_PATH/$TOOLCHAIN_PATH/VERSION")" -ne "$FLIPPER_TOOLCHAIN_VERSION" ]; then
+        download_toolchain;
+    fi
+}
+
+download_toolchain()
+{
+    chmod 755 "$SCRIPT_PATH/scripts/toolchain/unix-toolchain-download.sh";
+    "$SCRIPT_PATH/scripts/toolchain/unix-toolchain-download.sh" "$FLIPPER_TOOLCHAIN_VERSION" || exit 1;
+}
+
+main()
+{
+    if [ -z "${SCRIPT_PATH:-}" ]; then
+        echo "Mannual running this script is now allowed.";
+        exit 1;
+    fi
+    get_kernel_type;  # sets TOOLCHAIN_PATH
+    check_download_toolchain;
+    PATH="$SCRIPT_PATH/$TOOLCHAIN_PATH/python/bin:$PATH";
+    PATH="$SCRIPT_PATH/$TOOLCHAIN_PATH/bin:$PATH";
+    PATH="$SCRIPT_PATH/$TOOLCHAIN_PATH/protobuf/bin:$PATH";
+    PATH="$SCRIPT_PATH/$TOOLCHAIN_PATH/openocd/bin:$PATH";
+}
+main;

scripts/toolchain/unix-toolchain-download.sh

@@ -0,0 +1,135 @@
+#!/bin/sh
+# shellcheck disable=SC2086,SC2034
+
+# unofficial strict mode
+set -eu;
+
+check_system()
+{
+    VER="$1";  # toolchain version
+    printf "Checking kernel type..";
+    SYS_TYPE="$(uname -s)"
+    if [ "$SYS_TYPE" = "Darwin" ]; then
+        echo "darwin";
+        TOOLCHAIN_URL="https://update.flipperzero.one/builds/toolchain/gcc-arm-none-eabi-10.3-x86_64-darwin-flipper-$VER.tar.gz";
+        TOOLCHAIN_PATH="toolchain/x86_64-darwin";
+    elif [ "$SYS_TYPE" = "Linux" ]; then
+        echo "linux";
+        TOOLCHAIN_URL="https://update.flipperzero.one/builds/toolchain/gcc-arm-none-eabi-10.3-x86_64-linux-flipper-$VER.tar.gz";
+        TOOLCHAIN_PATH="toolchain/x86_64-linux";
+    else
+        echo "unsupported.";
+        echo "Your system is unsupported.. sorry..";
+        exit 1;
+    fi
+}
+
+check_tar()
+{
+    printf "Checking tar..";
+    if ! tar --version > /dev/null 2>&1; then
+        echo "no";
+        exit 1;
+    fi
+    echo "yes";
+}
+
+
+curl_wget_check()
+{
+    printf "Checking curl..";
+    if ! curl --version > /dev/null 2>&1; then
+        echo "no";
+        printf "Checking wget..";
+        if ! wget --version > /dev/null 2>&1; then
+            echo "no";
+            echo "No curl or wget found in your PATH.";
+            echo "Please provide it or download this file:";
+            echo;
+            echo "$TOOLCHAIN_URL";
+            echo;
+            echo "And place in repo root dir mannualy.";
+            exit 1;
+        fi
+        echo "yes"
+        DOWNLOADER="wget";
+        DOWNLOADER_ARGS="--show-progress --progress=bar:force -qO";
+        return;
+    fi
+    echo "yes"
+    DOWNLOADER="curl";
+    DOWNLOADER_ARGS="--progress-bar -SLo";
+}
+
+check_downloaded_toolchain()
+{
+    printf "Checking downloaded toolchain tgz..";
+    if [ -f "$REPO_ROOT/$TOOLCHAIN_TAR" ]; then
+        echo "yes";
+        return 0;
+    fi
+    echo "no";
+    return 1;
+}
+
+download_toolchain()
+{
+    echo "Downloading toolchain:";
+    "$DOWNLOADER" $DOWNLOADER_ARGS "$REPO_ROOT/$TOOLCHAIN_TAR" "$TOOLCHAIN_URL";
+    echo "done";
+}
+
+remove_old_tooclhain()
+{
+    printf "Removing old toolchain (if exist)..";
+    rm -rf "${REPO_ROOT:?}/$TOOLCHAIN_PATH";
+    echo "done";
+}
+
+show_unpack_percentage()
+{
+    LINE=0;
+    while read -r line; do
+        LINE=$(( LINE + 1 ));
+        if [ $(( LINE % 300 )) -eq 0 ]; then
+            printf "#";
+        fi
+    done
+    echo " 100.0%";
+}
+
+unpack_toolchain()
+{
+    echo "Unpacking toolchain:";
+    tar -xvf "$REPO_ROOT/$TOOLCHAIN_TAR" -C "$REPO_ROOT/" 2>&1 | show_unpack_percentage;
+    mkdir -p "$REPO_ROOT/toolchain";
+    mv "$REPO_ROOT/$TOOLCHAIN_DIR" "$REPO_ROOT/$TOOLCHAIN_PATH/";
+    echo "done";
+}
+
+clearing()
+{
+    printf "Clearing..";
+    rm -rf "${REPO_ROOT:?}/$TOOLCHAIN_TAR";
+    echo "done";
+}
+
+main()
+{
+    SCRIPT_PATH="$(cd "$(dirname "$0")" && pwd -P)"
+    REPO_ROOT="$(cd "$SCRIPT_PATH/../../" && pwd)";
+    check_system "$1";  # recives TOOLCHAIN_VERSION, defines TOOLCHAIN_URL and TOOLCHAIN_PATH
+    check_tar;
+    TOOLCHAIN_TAR="$(basename "$TOOLCHAIN_URL")";
+    TOOLCHAIN_DIR="$(echo "$TOOLCHAIN_TAR" | sed "s/-$VER.tar.gz//g")";
+    if ! check_downloaded_toolchain; then
+        curl_wget_check;
+        download_toolchain;
+    fi
+    remove_old_tooclhain;
+    unpack_toolchain;
+}
+
+trap clearing EXIT;
+trap clearing 2;  # SIGINT not coverable by EXIT
+main "$1";  # toochain version

scripts/toolchain/windows-toolchain-download.ps1

@@ -0,0 +1,34 @@
+Set-StrictMode -Version 2.0
+$ErrorActionPreference = "Stop"
+[Net.ServicePointManager]::SecurityProtocol = "tls12, tls11, tls"
+$repo_root = (Get-Item "$PSScriptRoot\..\..").FullName
+$toolchain_version = $args[0]
+$toolchain_url = "https://update.flipperzero.one/builds/toolchain/gcc-arm-none-eabi-10.3-i686-windows-flipper-$toolchain_version.zip"
+$toolchain_zip = "gcc-arm-none-eabi-10.3-i686-windows-flipper-$toolchain_version.zip"
+$toolchain_dir = "gcc-arm-none-eabi-10.3-i686-windows-flipper"
+
+if (Test-Path -LiteralPath "$repo_root\toolchain\i686-windows") {
+	Write-Host -NoNewline "Removing old Windows toolchain.."
+	Remove-Item -LiteralPath "$repo_root\toolchain\i686-windows" -Force -Recurse
+	Write-Host "done!"
+}
+if (!(Test-Path -Path "$repo_root\$toolchain_zip" -PathType Leaf)) {
+    Write-Host -NoNewline "Downloading Windows toolchain.."
+    $wc = New-Object net.webclient
+    $wc.Downloadfile("$toolchain_url", "$repo_root\$toolchain_zip")
+    Write-Host "done!"
+}
+
+if (!(Test-Path -LiteralPath "$repo_root\toolchain")) {
+    New-Item "$repo_root\toolchain" -ItemType Directory
+}
+
+Write-Host -NoNewline "Unziping Windows toolchain.."
+Add-Type -Assembly "System.IO.Compression.Filesystem"
+[System.IO.Compression.ZipFile]::ExtractToDirectory("$toolchain_zip", "$repo_root\")
+Move-Item -Path "$repo_root\$toolchain_dir" -Destination "$repo_root\toolchain\i686-windows"
+Write-Host "done!"
+
+Write-Host -NoNewline "Clearing temporary files.."
+Remove-Item -LiteralPath "$repo_root\$toolchain_zip" -Force
+Write-Host "done!"

site_scons/fbt/appmanifest.py

@@ -63,8 +63,13 @@ class AppManager:
             nonlocal app_manifests
             app_manifests.append(FlipperApplication(*args, **kw, _appdir=app_dir_name))
 
-        with open(app_manifest_path, "rt") as manifest_file:
-            exec(manifest_file.read())
+        try:
+            with open(app_manifest_path, "rt") as manifest_file:
+                exec(manifest_file.read())
+        except Exception as e:
+            raise FlipperManifestException(
+                f"Failed parsing manifest '{app_manifest_path}' : {e}"
+            )
 
         if len(app_manifests) == 0:
             raise FlipperManifestException(

site_scons/fbt/util.py

@@ -1,5 +1,6 @@
 import SCons
 from SCons.Subst import quote_spaces
+from SCons.Errors import StopError
 
 import re
 import os
@@ -30,7 +31,7 @@ def link_dir(target_path, source_path, is_windows):
         import _winapi
 
         if not os.path.isdir(source_path):
-            raise Exception(f"Source directory {source_path} is not a directory")
+            raise StopError(f"Source directory {source_path} is not a directory")
 
         if not os.path.exists(target_path):
             _winapi.CreateJunction(source_path, target_path)

site_scons/site_tools/crosscc.py

@@ -1,3 +1,4 @@
+from SCons.Errors import StopError
 from SCons.Tool import asm
 from SCons.Tool import gcc
 from SCons.Tool import gxx
@@ -65,7 +66,7 @@ def generate(env, **kw):
         # print("CC version =", cc_version)
         # print(list(filter(lambda v: v in cc_version, whitelisted_versions)))
         if not any(filter(lambda v: v in cc_version, whitelisted_versions)):
-            raise Exception(
+            raise StopError(
                 f"Toolchain version is not supported. Allowed: {whitelisted_versions}, toolchain: {cc_version} "
             )
 

site_scons/site_tools/fbt_apps.py

@@ -1,8 +1,14 @@
 from SCons.Builder import Builder
 from SCons.Action import Action
+from SCons.Errors import UserError
 
 import SCons
-from fbt.appmanifest import FlipperAppType, AppManager, ApplicationsCGenerator
+from fbt.appmanifest import (
+    FlipperAppType,
+    AppManager,
+    ApplicationsCGenerator,
+    FlipperManifestException,
+)
 
 # Adding objects for application management to env
 #  AppManager env["APPMGR"] - loads all manifests; manages list of known apps
@@ -13,7 +19,10 @@ def LoadApplicationManifests(env):
     appmgr = env["APPMGR"] = AppManager()
     for entry in env.Glob("#/applications/*"):
         if isinstance(entry, SCons.Node.FS.Dir) and not str(entry).startswith("."):
-            appmgr.load_manifest(entry.File("application.fam").abspath, entry.name)
+            try:
+                appmgr.load_manifest(entry.File("application.fam").abspath, entry.name)
+            except FlipperManifestException as e:
+                raise UserError(e)
 
 
 def PrepareApplicationsBuild(env):