From 0c7a8edf518c77788ada147134c64513fbcde873 Mon Sep 17 00:00:00 2001
From: rusdacent <57439765+rusdacent@users.noreply.github.com>
Date: Sat, 24 Jul 2021 16:53:02 +0300
Subject: [PATCH] [FL-1533] CI refactoring (#601)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* rename artifacts
* add workflow with date
* add debug
* Add branch name to artifact name and rm genpic job
* Add branch name to artifact name for upload job
* Add branch restrictions for dev and user branches
* Add branch restrictions for dev and user branches
* Add cleanup workspace for upload job
* Test ignore branch
* Test release CI workflow
* Rename release CI
* Add pre-release trigger
* Rm branch restriction for CI
* Add release trigger
* rm branch restriction for CI
* rm genpic job
* Github: refactor actions, move linting to parallel actions, separate forlder for artifacts, build cleanup
* Github: build date and commit hash on branches and tag on tags
* Github: minor artifacts name cleanup
* Github: decontaminate previous build leftovers
* Github: better decontamination and release build SUFFIX
* Github: fetch before decontamitation
* Github: change decontamination logic, jump to first commit if submodules are broken
* ReadMe: update links to latest firmware and images

Co-authored-by: あく <alleteam@gmail.com>
---
 .github/workflows/build.yml                   | 180 ++++++++++++++++++
 .github/workflows/ci.yml                      | 177 -----------------
 .github/workflows/lint_c.yml                  |  40 ++++
 .../workflows/{black.yml => lint_python.yml}  |   6 +-
 .github/workflows/reindex.yml                 |  15 ++
 ReadMe.md                                     |   6 +-
 6 files changed, 241 insertions(+), 183 deletions(-)
 create mode 100644 .github/workflows/build.yml
 delete mode 100644 .github/workflows/ci.yml
 create mode 100644 .github/workflows/lint_c.yml
 rename .github/workflows/{black.yml => lint_python.yml} (85%)
 create mode 100644 .github/workflows/reindex.yml

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 000000000..7f8653b06
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,180 @@
+name: 'Build'
+
+on: push
+
+env:
+  TARGETS: f6
+
+jobs:
+  build:
+    runs-on: [self-hosted]
+    steps:
+      - name: 'Cleanup workspace'
+        uses: AutoModality/action-clean@v1
+
+      - name: 'Decontaminate previous build leftovers'
+        run: |
+          git submodule status \
+            || git checkout `git rev-list --max-parents=0 HEAD | tail -n 1`
+
+      - name: 'Checkout code'
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: 'Docker cache'
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+        with:
+          key: docker-cache-${{ hashFiles('docker/**') }}-{hash}
+          restore-keys: docker-cache-${{ hashFiles('docker/**') }}-
+
+      - name: 'Build docker image'
+        uses: ./.github/actions/docker
+
+      - name: 'Make artifacts directory'
+        run: |
+          test -d artifacts && rm -rf artifacts || true
+          mkdir artifacts
+
+      - name: 'Generate tag suffix'
+        if: startsWith(github.ref, 'refs/tags/') == true
+        run: echo "SUFFIX=$(git describe --tags --abbrev=0)"  >> $GITHUB_ENV
+
+      - name: 'Generate branch suffix'
+        if: startsWith(github.ref, 'refs/tags/') != true
+        run: echo "SUFFIX=$(date +'%Y-%m-%d')-$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: 'Build bootloader in docker'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              make -j$(nproc) -C bootloader TARGET=${TARGET}
+            done
+
+      - name: 'Generate dfu file for bootloader'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              hex2dfu \
+                -i bootloader/.obj/${TARGET}/bootloader.hex \
+                -o bootloader/.obj/${TARGET}/bootloader.dfu
+            done
+
+      - name: 'Build firmware in docker'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              make -j$(nproc) -C firmware TARGET=${TARGET}
+            done
+
+      - name: 'Generate dfu file for firmware'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              hex2dfu \
+                -i firmware/.obj/${TARGET}/firmware.hex \
+                -o firmware/.obj/${TARGET}/firmware.dfu
+            done
+
+      - name: 'Generate full hex file'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              srec_cat \
+                bootloader/.obj/${TARGET}/bootloader.hex -Intel \
+                firmware/.obj/${TARGET}/firmware.hex -Intel \
+                -o firmware/.obj/${TARGET}/full.hex -Intel
+            done
+
+      - name: 'Move upload files'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              mv bootloader/.obj/${TARGET}/bootloader.dfu \
+                artifacts/flipper-z-${TARGET}-bootloader-${SUFFIX}.dfu
+              mv bootloader/.obj/${TARGET}/bootloader.bin \
+                artifacts/flipper-z-${TARGET}-bootloader-${SUFFIX}.bin
+              mv bootloader/.obj/${TARGET}/bootloader.elf \
+                artifacts/flipper-z-${TARGET}-bootloader-${SUFFIX}.elf
+              mv firmware/.obj/${TARGET}/firmware.dfu \
+                artifacts/flipper-z-${TARGET}-firmware-${SUFFIX}.dfu
+              mv firmware/.obj/${TARGET}/firmware.bin \
+                artifacts/flipper-z-${TARGET}-firmware-${SUFFIX}.bin
+              mv firmware/.obj/${TARGET}/firmware.elf \
+                artifacts/flipper-z-${TARGET}-firmware-${SUFFIX}.elf
+            done
+
+      - name: 'Generate full dfu file'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              hex2dfu \
+                -i firmware/.obj/${TARGET}/full.hex \
+                -o artifacts/flipper-z-${TARGET}-full-${SUFFIX}.dfu
+            done
+
+      - name: 'Full flash asssembly: bootloader as base'
+        run: |
+          for TARGET in ${TARGETS}
+          do
+            cp \
+              artifacts/flipper-z-${TARGET}-bootloader-${SUFFIX}.bin \
+              artifacts/flipper-z-${TARGET}-full-${SUFFIX}.bin
+          done
+
+      - name: 'Full flash asssembly: bootloader padding'
+        run: |
+          for TARGET in ${TARGETS}
+          do
+            truncate -s 32768 artifacts/flipper-z-${TARGET}-full-${SUFFIX}.bin
+          done
+      
+      - name: 'Full flash asssembly: append firmware'
+        run: |
+          for TARGET in ${TARGETS}
+          do
+            cat \
+              artifacts/flipper-z-${TARGET}-firmware-${SUFFIX}.bin \
+              >> artifacts/flipper-z-${TARGET}-full-${SUFFIX}.bin
+          done
+
+      - name: 'Publish artifacts'
+        uses: actions/upload-artifact@v2
+        with:
+          name: artifacts
+          path: artifacts/*
+          if-no-files-found: error
+          retention-days: 7
+
+      - name: 'Upload artifacts to update server'
+        uses: burnett01/rsync-deployments@4.1
+        with:
+          switches: -avzP --delete
+          path: artifacts/
+          remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${GITHUB_REF##*/}/"
+          remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
+          remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
+          remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}
+          remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }}
+
+      - name: 'Trigger update server reindex'
+        uses: wei/curl@master
+        with:
+          args: -X POST -F 'key=${{ secrets.REINDEX_KEY }}' ${{ secrets.REINDEX_URL }}
+
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
deleted file mode 100644
index d5108a598..000000000
--- a/.github/workflows/ci.yml
+++ /dev/null
@@ -1,177 +0,0 @@
-name: 'CI'
-
-on: push
-
-env:
-  TARGETS: f6
-
-jobs:
-  build:
-    runs-on: [self-hosted]
-    steps:
-      - name: Cleanup workspace
-        uses: AutoModality/action-clean@v1
-
-      - name: Checkout code
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Checkout submodules
-        run: git submodule update --init --recursive
-      - name: Checkout submodules
-        run: git submodule sync
-
-      - uses: satackey/action-docker-layer-caching@v0.0.11
-        continue-on-error: true
-        with:
-          key: docker-cache-${{ hashFiles('docker/**') }}-{hash}
-          restore-keys: docker-cache-${{ hashFiles('docker/**') }}-
-      - name: Build docker image
-        uses: ./.github/actions/docker
-
-      - name: Check syntax
-        uses: ./.github/actions/docker
-        continue-on-error: false
-        with:
-          run: /syntax_check.sh
-
-      - name: Build bootloader in docker
-        uses: ./.github/actions/docker
-        with:
-          run: for TARGET in ${TARGETS}; do make -j$(nproc) -C bootloader TARGET=${TARGET}; done
-
-      - name: Generate dfu file for bootloader
-        uses: ./.github/actions/docker
-        with:
-          run: for TARGET in ${TARGETS}; do hex2dfu -i bootloader/.obj/${TARGET}/bootloader.hex -o bootloader/.obj/${TARGET}/bootloader.dfu; done
-
-      - name: Build firmware in docker
-        uses: ./.github/actions/docker
-        with:
-          run: for TARGET in ${TARGETS}; do make -j$(nproc) -C firmware TARGET=${TARGET}; done
-
-      - name: Generate dfu file for firmware
-        uses: ./.github/actions/docker
-        with:
-          run: for TARGET in ${TARGETS}; do hex2dfu -i firmware/.obj/${TARGET}/firmware.hex -o firmware/.obj/${TARGET}/firmware.dfu; done
-
-      - name: Generate full hex file
-        uses: ./.github/actions/docker
-        with:
-          run: |
-            for TARGET in ${TARGETS}; do
-            srec_cat bootloader/.obj/${TARGET}/bootloader.hex -Intel firmware/.obj/${TARGET}/firmware.hex -Intel -o ${TARGET}_full.hex -Intel
-            done
-
-      - name: Move upload files
-        uses: ./.github/actions/docker
-        with:
-          run: |
-            for TARGET in ${TARGETS}; do
-            mv bootloader/.obj/${TARGET}/bootloader.dfu ${TARGET}_bootloader.dfu
-            mv bootloader/.obj/${TARGET}/bootloader.bin ${TARGET}_bootloader.bin
-            mv bootloader/.obj/${TARGET}/bootloader.elf ${TARGET}_bootloader.elf
-            mv firmware/.obj/${TARGET}/firmware.dfu ${TARGET}_firmware.dfu
-            mv firmware/.obj/${TARGET}/firmware.bin ${TARGET}_firmware.bin
-            mv firmware/.obj/${TARGET}/firmware.elf ${TARGET}_firmware.elf
-            done
-
-      - name: Generate full dfu file
-        uses: ./.github/actions/docker
-        with:
-          run: for TARGET in ${TARGETS}; do hex2dfu -i ${TARGET}_full.hex -o ${TARGET}_full.dfu; done
-
-      - name: Copy bootloader for full.bin 
-        run: for TARGET in ${TARGETS}; do cp ${TARGET}_bootloader.bin ${TARGET}_full.bin; done
-      - name: Truncate full.bin 
-        run: for TARGET in ${TARGETS}; do truncate -s 32768 ${TARGET}_full.bin; done
-      - name: Add firmware.bin at full.bin
-        run: for TARGET in ${TARGETS}; do cat ${TARGET}_firmware.bin >> ${TARGET}_full.bin; done
-
-      - name: Copy last target to full.dfu
-        run: cp ${TARGETS##*" "}_full.dfu full.dfu
-
-      - name: Publish artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: artifacts
-          path: |
-            *_bootloader.dfu
-            *_bootloader.bin
-            *_firmware.dfu
-            *_firmware.bin
-            *_full.dfu
-            *_full.bin
-            full.dfu
-          if-no-files-found: error
-          retention-days: 7
-
-  upload:
-    name: Upload artifacts to external storage
-    needs: build
-    runs-on: [self-hosted]
-    steps:
-
-      - name: Make empty dir
-        run: mkdir -p empty
-
-      - name: Clear upload dir
-        uses: burnett01/rsync-deployments@4.1
-        with:
-          switches: -cvzr --delete
-          path: empty/
-          remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${GITHUB_REF##*/}/"
-          remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
-          remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
-          remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}
-          remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }}
-
-      - name: Generate files list
-        run: ls *_bootloader.dfu *_firmware.dfu *_full.dfu *_bootloader.bin *_firmware.bin *_full.bin *_bootloader.elf *_firmware.elf full.dfu > uploadlist.txt
-
-      - name: Upload artifacts
-        uses: burnett01/rsync-deployments@4.1
-        with:
-          switches: -cvzr --delete --files-from=uploadlist.txt
-          remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${GITHUB_REF##*/}/"
-          remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
-          remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
-          remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}
-          remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }}
-
-      - name: Trigger reindex
-        uses: wei/curl@master
-        with:
-          args: -X POST -F 'key=${{ secrets.REINDEX_KEY }}' ${{ secrets.REINDEX_URL }}
-
-  genpic:
-    name: Generate pic
-    needs: upload
-    if: github.ref == 'refs/heads/release'
-    runs-on: [self-hosted]
-    steps:
-      - name: Set test env
-        run: echo "NUMBER_OF_COMMITS=$(git rev-list --count HEAD)" >> $GITHUB_ENV
-      - name: Test output NUMBER_OF_COMMITS
-        run: echo $NUMBER_OF_COMMITS
-      - name: Test output date
-        run: |
-          GET_DATE=$(date +"%b %d")
-          PREP_DATE=$(echo "${GET_DATE^^}")
-          echo "PREP_DATE=$PREP_DATE" >> $GITHUB_ENV
-      - name: Test output PREP_DATE
-        run: echo $PREP_DATE
-      - name: Gen pic
-        uses: ./.github/actions/docker
-        with:
-          run: convert ./.github/assets/latest-firmware-template.png -font ./.github/assets/Born2bSportyV2.ttf -weight 700 -pointsize 140 -annotate +900+330 "$NUMBER_OF_COMMITS $PREP_DATE" latest-firmware-banner.png
-      - name: Upload pic
-        uses: burnett01/rsync-deployments@4.1
-        with:
-          switches: -avzp --delete
-          path: latest-firmware-banner.png
-          remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}/"
-          remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
-          remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
-          remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}
-          remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }}
diff --git a/.github/workflows/lint_c.yml b/.github/workflows/lint_c.yml
new file mode 100644
index 000000000..e1955a7e7
--- /dev/null
+++ b/.github/workflows/lint_c.yml
@@ -0,0 +1,40 @@
+name: 'Lint C/C++ with clang-format'
+
+on: push
+
+env:
+  TARGETS: f6
+
+jobs:
+  lint_c_cpp:
+    runs-on: [self-hosted]
+    steps:
+      - name: 'Cleanup workspace'
+        uses: AutoModality/action-clean@v1
+
+      - name: 'Checkout code'
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: 'Checkout submodules: update'
+        run: git submodule update --init --recursive
+
+      - name: 'Checkout submodules: sync'
+        run: git submodule sync
+
+      - name: 'Docker cache'
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+        with:
+          key: docker-cache-${{ hashFiles('docker/**') }}-{hash}
+          restore-keys: docker-cache-${{ hashFiles('docker/**') }}-
+
+      - name: 'Build docker image'
+        uses: ./.github/actions/docker
+
+      - name: 'Check syntax'
+        uses: ./.github/actions/docker
+        continue-on-error: false
+        with:
+          run: /syntax_check.sh
diff --git a/.github/workflows/black.yml b/.github/workflows/lint_python.yml
similarity index 85%
rename from .github/workflows/black.yml
rename to .github/workflows/lint_python.yml
index c16e972e0..dc0fefd1d 100644
--- a/.github/workflows/black.yml
+++ b/.github/workflows/lint_python.yml
@@ -1,13 +1,13 @@
-name: Python Lint
+name: 'Python Lint'
 
 on: 
   push:
     paths:
       - '**.py'
   pull_request:
-  
+
 jobs:
-  lint:
+  lint_python:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
diff --git a/.github/workflows/reindex.yml b/.github/workflows/reindex.yml
new file mode 100644
index 000000000..56971eec9
--- /dev/null
+++ b/.github/workflows/reindex.yml
@@ -0,0 +1,15 @@
+name: 'Reindex'
+
+on:
+  release:
+    types: [prereleased,released]
+
+jobs:
+  reindex:
+    name: 'Reindex updates'
+    runs-on: [self-hosted]
+    steps:
+      - name: Trigger reindex
+        uses: wei/curl@master
+        with:
+          args: -X POST -F 'key=${{ secrets.REINDEX_KEY }}' ${{ secrets.REINDEX_URL }}
diff --git a/ReadMe.md b/ReadMe.md
index beedd496a..5b26bd81b 100644
--- a/ReadMe.md
+++ b/ReadMe.md
@@ -10,12 +10,12 @@ This repo will become completely public closer to the device shipping date.
 
 # Update firmware
 
-<a href="https://update.flipperzero.one/release/full.dfu"><img width="300" src="https://update.flipperzero.one/latest-firmware-banner.png" /></a>
+<a href="https://update.flipperzero.one/">Get Latest Firmware from Update Server</a>
 
 
 Flipper Zero's firmware consists of three components:
 
-- Core2 firmware - proprietary componenets by ST: FUS + radio stack.
+- Core2 firmware set - proprietary componenets by ST: FUS + radio stack.
 - Core1 Bootloader - controls basic hardware initialization and loads firmware
 - Core1 Firmware - HAL + OS + Drivers + Applications
 
@@ -46,7 +46,7 @@ One liner: `./flash_core1_main.sh`
 
 ## With USB DFU 
 
-1. Download latest [Firmware](https://update.flipperzero.one/release/full.dfu)
+1. Download latest [Firmware](https://update.flipperzero.one)
 
 2. Reboot Flipper to Bootloader 
  - Press and hold `← Left` + `↩ Back` for reset