WiFi board: fixed update script on Windows (unfortunately also Mac and Linux) (#3485)

* WiFi board update: windows fix

* Scripts: add python\Scripts to the path

* Windows: another way to call esptool

* Aaaaaand once more!

* logz

* wifi board scripts: udev rule, dev channel

* fbt: added ARGS variable for passing extra options to certain scripts; removed `devboard_flash_dev` in favor of `devboard_flash ARGS="-c dev"`

* fbt: fully removed `devboard_flash_dev

* scripts: wifi_board.py: cleanup

* ufbt: ported ARGS for supported targets to ufbt

* docs: updated for ARGS=...

---------

Co-authored-by: hedger <hedger@nanode.su>
Co-authored-by: hedger <hedger@users.noreply.github.com>

SConstruct

@@ -66,6 +66,7 @@ if GetOption("fullenv") or any(
 
     # Target for self-update package
     dist_basic_arguments = [
+        "${ARGS}",
         "--bundlever",
         "${UPDATE_VERSION_STRING}",
     ]
@@ -182,6 +183,7 @@ fap_deploy = distenv.PhonyTarget(
                 "send",
                 "${SOURCE}",
                 "/ext/apps",
+                "${ARGS}",
             ]
         ]
     ),
@@ -208,7 +210,7 @@ distenv.Alias("jflash", firmware_jflash)
 
 distenv.PhonyTarget(
     "gdb_trace_all",
-    "$GDB $GDBOPTS $SOURCES $GDBFLASH",
+    [["${GDB}", "${GDBOPTS}", "${SOURCES}", "${GDBFLASH}"]],
     source=firmware_env["FW_ELF"],
     GDBOPTS="${GDBOPTS_BASE}",
     GDBREMOTE="${OPENOCD_GDB_PIPE}",
@@ -272,19 +274,35 @@ distenv.PhonyTarget(
 # Just start OpenOCD
 distenv.PhonyTarget(
     "openocd",
-    "${OPENOCDCOM}",
+    [["${OPENOCDCOM}", "${ARGS}"]],
 )
 
 # Linter
 distenv.PhonyTarget(
     "lint",
-    [["${PYTHON3}", "${FBT_SCRIPT_DIR}/lint.py", "check", "${LINT_SOURCES}"]],
+    [
+        [
+            "${PYTHON3}",
+            "${FBT_SCRIPT_DIR}/lint.py",
+            "check",
+            "${LINT_SOURCES}",
+            "${ARGS}",
+        ]
+    ],
     LINT_SOURCES=[n.srcnode() for n in firmware_env["LINT_SOURCES"]],
 )
 
 distenv.PhonyTarget(
     "format",
-    [["${PYTHON3}", "${FBT_SCRIPT_DIR}/lint.py", "format", "${LINT_SOURCES}"]],
+    [
+        [
+            "${PYTHON3}",
+            "${FBT_SCRIPT_DIR}/lint.py",
+            "format",
+            "${LINT_SOURCES}",
+            "${ARGS}",
+        ]
+    ],
     LINT_SOURCES=[n.srcnode() for n in firmware_env["LINT_SOURCES"]],
 )
 
@@ -307,7 +325,16 @@ firmware_env.Append(
 )
 
 
-black_commandline = "@${PYTHON3} -m black ${PY_BLACK_ARGS} ${PY_LINT_SOURCES}"
+black_commandline = [
+    [
+        "@${PYTHON3}",
+        "-m",
+        "black",
+        "${PY_BLACK_ARGS}",
+        "${PY_LINT_SOURCES}",
+        "${ARGS}",
+    ]
+]
 black_base_args = [
     "--include",
     '"(\\.scons|\\.py|SConscript|SConstruct|\\.fam)$"',
@@ -333,12 +360,28 @@ distenv.PhonyTarget(
 
 # Start Flipper CLI via PySerial's miniterm
 distenv.PhonyTarget(
-    "cli", [["${PYTHON3}", "${FBT_SCRIPT_DIR}/serial_cli.py", "-p", "${FLIP_PORT}"]]
+    "cli",
+    [
+        [
+            "${PYTHON3}",
+            "${FBT_SCRIPT_DIR}/serial_cli.py",
+            "-p",
+            "${FLIP_PORT}",
+            "${ARGS}",
+        ]
+    ],
 )
 
-# Update WiFi devboard firmware
+# Update WiFi devboard firmware with release channel
 distenv.PhonyTarget(
-    "devboard_flash", [["${PYTHON3}", "${FBT_SCRIPT_DIR}/wifi_board.py"]]
+    "devboard_flash",
+    [
+        [
+            "${PYTHON3}",
+            "${FBT_SCRIPT_DIR}/wifi_board.py",
+            "${ARGS}",
+        ]
+    ],
 )
 
 
@@ -353,7 +396,7 @@ distenv.PhonyTarget(
 distenv.PhonyTarget(
     "get_stlink",
     distenv.Action(
-        lambda **kw: distenv.GetDevices(),
+        lambda **_: distenv.GetDevices(),
         None,
     ),
 )

documentation/fbt.md

@@ -71,13 +71,13 @@ To use language servers other than the default VS Code C/C++ language server, us
 - `debug` - build and flash firmware, then attach with gdb with firmware's .elf loaded.
 - `debug_other`, `debug_other_blackmagic` - attach GDB without loading any `.elf`. It will allow you to manually add external `.elf` files with `add-symbol-file` in GDB.
 - `updater_debug` - attach GDB with the updater's `.elf` loaded.
-- `devboard_flash` - update WiFi dev board with the latest firmware.
+- `devboard_flash` - Update WiFi dev board. Supports `ARGS="..."` to pass extra arguments to the update script, e.g. `ARGS="-c dev"`.
 - `blackmagic` - debug firmware with Blackmagic probe (WiFi dev board).
-- `openocd` - just start OpenOCD.
+- `openocd` - just start OpenOCD. You can pass extra arguments with `ARGS="..."`.
 - `get_blackmagic` - output the blackmagic address in the GDB remote format. Useful for IDE integration.
 - `get_stlink` - output serial numbers for attached STLink probes. Used for specifying an adapter with `SWD_TRANSPORT_SERIAL=...`.
-- `lint`, `format` - run clang-format on the C source code to check and reformat it according to the `.clang-format` specs.
-- `lint_py`, `format_py` - run [black](https://black.readthedocs.io/en/stable/index.html) on the Python source code, build system files & application manifests.
+- `lint`, `format` - run clang-format on the C source code to check and reformat it according to the `.clang-format` specs. Supports `ARGS="..."` to pass extra arguments to clang-format.
+- `lint_py`, `format_py` - run [black](https://black.readthedocs.io/en/stable/index.html) on the Python source code, build system files & application manifests. Supports `ARGS="..."` to pass extra arguments to black.
 - `firmware_pvs` - generate a PVS Studio report for the firmware. Requires PVS Studio to be available on your system's `PATH`.
 - `cli` - start a Flipper CLI session over USB.
 

scripts/fbt_tools/fbt_dist.py

@@ -150,6 +150,7 @@ def generate(env):
                         "--interface=${SWD_TRANSPORT}",
                         "--serial=${SWD_TRANSPORT_SERIAL}",
                         "${SOURCE}",
+                        "${ARGS}",
                     ],
                     Touch("${TARGET}"),
                 ]
@@ -162,6 +163,7 @@ def generate(env):
                         "-p",
                         "${FLIP_PORT}",
                         "${UPDATE_BUNDLE_DIR}/update.fuf",
+                        "${ARGS}",
                     ],
                     Touch("${TARGET}"),
                 ]
@@ -180,6 +182,7 @@ def generate(env):
                             "--stack_type=${COPRO_STACK_TYPE}",
                             "--stack_file=${COPRO_STACK_BIN}",
                             "--stack_addr=${COPRO_STACK_ADDR}",
+                            "${ARGS}",
                         ]
                     ],
                     "${COPROCOMSTR}",

scripts/ufbt/SConstruct

@@ -315,26 +315,56 @@ else:
 
 appenv.PhonyTarget(
     "cli",
-    [["${PYTHON3}", "${FBT_SCRIPT_DIR}/serial_cli.py", "-p", "${FLIP_PORT}"]],
+    [
+        [
+            "${PYTHON3}",
+            "${FBT_SCRIPT_DIR}/serial_cli.py",
+            "-p",
+            "${FLIP_PORT}",
+            "${ARGS}",
+        ]
+    ],
 )
 
 # Update WiFi devboard firmware
 dist_env.PhonyTarget(
-    "devboard_flash", [["${PYTHON3}", "${FBT_SCRIPT_DIR}/wifi_board.py"]]
+    "devboard_flash",
+    [
+        [
+            "${PYTHON3}",
+            "${FBT_SCRIPT_DIR}/wifi_board.py",
+            "${ARGS}",
+        ]
+    ],
 )
 
 # Linter
-
 dist_env.PhonyTarget(
     "lint",
-    [["${PYTHON3}", "${FBT_SCRIPT_DIR}/lint.py", "check", "${LINT_SOURCES}"]],
+    [
+        [
+            "${PYTHON3}",
+            "${FBT_SCRIPT_DIR}/lint.py",
+            "check",
+            "${LINT_SOURCES}",
+            "${ARGS}",
+        ]
+    ],
     source=original_app_dir.File(".clang-format"),
     LINT_SOURCES=[original_app_dir],
 )
 
 dist_env.PhonyTarget(
     "format",
-    [["${PYTHON3}", "${FBT_SCRIPT_DIR}/lint.py", "format", "${LINT_SOURCES}"]],
+    [
+        [
+            "${PYTHON3}",
+            "${FBT_SCRIPT_DIR}/lint.py",
+            "format",
+            "${LINT_SOURCES}",
+            "${ARGS}",
+        ]
+    ],
     source=original_app_dir.File(".clang-format"),
     LINT_SOURCES=[original_app_dir],
 )
@@ -456,6 +486,7 @@ if dolphin_src_dir.exists():
                 "send",
                 "${SOURCE}",
                 "/ext/dolphin",
+                "${ARGS}",
             ]
         ],
         source=ufbt_build_dir.Dir("dolphin"),

scripts/ufbt/site_tools/ufbt_help.py

@@ -29,7 +29,8 @@ Flashing & debugging:
     debug, debug_other, blackmagic:
         Start GDB
     devboard_flash:
-        Update WiFi dev board with the latest firmware
+        Update WiFi dev board. 
+        Supports ARGS="..." to pass extra arguments to the update script, e.g. ARGS="-c dev"
 
 Other:
     cli:

scripts/wifi_board.py

@@ -1,16 +1,16 @@
 #!/usr/bin/env python3
 
-from flipper.app import App
-from serial.tools.list_ports_common import ListPortInfo
-
+import json
 import logging
 import os
-import tempfile
 import subprocess
-import serial.tools.list_ports as list_ports
-import json
-import requests
 import tarfile
+import tempfile
+
+import requests
+import serial.tools.list_ports as list_ports
+from flipper.app import App
+from serial.tools.list_ports_common import ListPortInfo
 
 
 class UpdateDownloader:
@@ -29,15 +29,15 @@ class UpdateDownloader:
     def __init__(self):
         self.logger = logging.getLogger()
 
-    def download(self, channel_id: str, dir: str) -> bool:
+    def download(self, channel_id: str, target_dir: str) -> bool:
         # Aliases
         if channel_id in self.CHANNEL_ID_ALIAS:
             channel_id = self.CHANNEL_ID_ALIAS[channel_id]
 
         # Make directory
-        if not os.path.exists(dir):
-            self.logger.info(f"Creating directory {dir}")
-            os.makedirs(dir)
+        if not os.path.exists(target_dir):
+            self.logger.info(f"Creating directory {target_dir}")
+            os.makedirs(target_dir)
 
         # Download json index
         self.logger.info(f"Downloading {self.UPDATE_INDEX}")
@@ -79,19 +79,14 @@ class UpdateDownloader:
         self.logger.info(f"Using version '{version['version']}'")
 
         # Get changelog
-        changelog = None
-        try:
-            changelog = version["changelog"]
-        except Exception as e:
-            self.logger.error(f"Failed to get changelog: {e}")
-
-        # print changelog
-        if changelog is not None:
+        if changelog := version.get("changelog"):
             self.logger.info(f"Changelog:")
             for line in changelog.split("\n"):
                 if line.strip() == "":
                     continue
                 self.logger.info(f"  {line}")
+        else:
+            self.logger.warning(f"Changelog not found")
 
         # Find file
         file_url = None
@@ -106,7 +101,7 @@ class UpdateDownloader:
 
         # Make file path
         file_name = file_url.split("/")[-1]
-        file_path = os.path.join(dir, file_name)
+        file_path = os.path.join(target_dir, file_name)
 
         # Download file
         self.logger.info(f"Downloading {file_url} to {file_path}")
@@ -117,7 +112,7 @@ class UpdateDownloader:
         # Unzip tgz
         self.logger.info(f"Unzipping {file_path}")
         with tarfile.open(file_path, "r") as tar:
-            tar.extractall(dir)
+            tar.extractall(target_dir)
 
         return True
 
@@ -133,16 +128,24 @@ class Main(App):
         # logging
         self.logger = logging.getLogger()
 
-    def find_wifi_board(self) -> bool:
+    @staticmethod
+    def _grep_ports(regexp: str) -> list[ListPortInfo]:
         # idk why, but python thinks that list_ports.grep returns tuple[str, str, str]
-        blackmagics: list[ListPortInfo] = list(list_ports.grep("blackmagic"))  # type: ignore
-        daps: list[ListPortInfo] = list(list_ports.grep("CMSIS-DAP"))  # type: ignore
+        return list(list_ports.grep(regexp))  # type: ignore
 
-        return len(blackmagics) > 0 or len(daps) > 0
+    def is_wifi_board_connected(self) -> bool:
+        return (
+            len(self._grep_ports("ESP32-S2")) > 0
+            or len(self._grep_ports("CMSIS-DAP")) > 0
+        )
 
-    def find_wifi_board_bootloader(self):
-        # idk why, but python thinks that list_ports.grep returns tuple[str, str, str]
-        ports: list[ListPortInfo] = list(list_ports.grep("ESP32-S2"))  # type: ignore
+    @staticmethod
+    def is_windows() -> bool:
+        return os.name == "nt"
+
+    @classmethod
+    def find_port(cls, regexp: str) -> str:
+        ports: list[ListPortInfo] = cls._grep_ports(regexp)
 
         if len(ports) == 0:
             # Blackmagic probe serial port not found, will be handled later
@@ -151,27 +154,28 @@ class Main(App):
             raise Exception("More than one WiFi board found")
         else:
             port = ports[0]
-            if os.name == "nt":
-                port.device = f"\\\\.\\{port.device}"
-            return port.device
+            return f"\\\\.\\{port.device}" if cls.is_windows() else port.device
+
+    def find_wifi_board_bootloader_port(self):
+        return self.find_port("ESP32-S2")
+
+    def find_wifi_board_bootloader_port_damn_windows(self):
+        self.logger.info("Trying to find WiFi board using VID:PID")
+        return self.find_port("VID:PID=303A:0002")
 
     def update(self):
         try:
-            port = self.find_wifi_board_bootloader()
+            port = self.find_wifi_board_bootloader_port()
+
+            # Damn windows fix
+            if port is None and self.is_windows():
+                port = self.find_wifi_board_bootloader_port_damn_windows()
         except Exception as e:
             self.logger.error(f"{e}")
             return 1
 
-        if self.args.port != "auto":
-            port = self.args.port
-
-            available_ports = [p[0] for p in list(list_ports.comports())]
-            if port not in available_ports:
-                self.logger.error(f"Port {port} not found")
-                return 1
-
         if port is None:
-            if self.find_wifi_board():
+            if self.is_wifi_board_connected():
                 self.logger.error("WiFi board found, but not in bootloader mode.")
                 self.logger.info("Please hold down BOOT button and press RESET button")
             else:
@@ -179,6 +183,13 @@ class Main(App):
                 self.logger.info(
                     "Please connect WiFi board to your computer, hold down BOOT button and press RESET button"
                 )
+                if not self.is_windows():
+                    self.logger.info(
+                        "If you are using Linux, you may need to add udev rules to access the device"
+                    )
+                    self.logger.info(
+                        "Check out 41-flipper.rules & README in scripts/debug folder"
+                    )
             return 1
 
         # get temporary dir
@@ -197,24 +208,29 @@ class Main(App):
             with open(os.path.join(temp_dir, "flash.command"), "r") as f:
                 flash_command = f.read()
 
-            flash_command = flash_command.replace("\n", "").replace("\r", "")
-            flash_command = flash_command.replace("(PORT)", port)
-
-            # We can't reset the board after flashing via usb
-            flash_command = flash_command.replace(
-                "--after hard_reset", "--after no_reset_stub"
+            replacements = (
+                ("\n", ""),
+                ("\r", ""),
+                ("(PORT)", port),
+                # We can't reset the board after flashing via usb
+                ("--after hard_reset", "--after no_reset_stub"),
             )
 
-            args = flash_command.split(" ")[0:]
-            args = list(filter(None, args))
+            # hellish toolchain fix
+            if self.is_windows():
+                replacements += (("esptool.py", "python -m esptool"),)
+            else:
+                replacements += (("esptool.py", "python3 -m esptool"),)
 
-            esptool_params = []
-            esptool_params.extend(args)
+            for old, new in replacements:
+                flash_command = flash_command.replace(old, new)
+
+            args = list(filter(None, flash_command.split()))
 
             self.logger.info(f'Running command: "{" ".join(args)}" in "{temp_dir}"')
 
             process = subprocess.Popen(
-                esptool_params,
+                args,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 cwd=temp_dir,

site_scons/commandline.scons

@@ -274,6 +274,11 @@ vars.AddVariables(
         help="Enable strict import check for .faps",
         default=True,
     ),
+    (
+        "ARGS",
+        "Extra arguments to pass to certain scripts supporting it",
+        "",
+    ),
 )
 
 Return("vars")