Share import mac code signing certificate steps for release (#851)

.github/scripts/apple-signing/cleanup.sh

@@ -6,5 +6,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 . "$SCRIPT_DIR"/utils.sh
 
 # cleanup any dev certs left behind
+. "$SCRIPT_DIR"/setup-import-cert.sh # defines KEYCHAIN_NAME and KEYCHAIN_PATH
 . "$SCRIPT_DIR"/setup-dev.sh
-cleanup_signing
+cleanup_dev_signing

.github/scripts/apple-signing/setup-dev.sh

@@ -20,12 +20,6 @@ P12_FILE=$FILE_PREFIX.p12
 
 EXT_SECTION=codesign_reqext
 
-## Keychain material
-
-KEYCHAIN_NAME=$NAME
-KEYCHAIN_PATH=$HOME/Library/Keychains/$KEYCHAIN_NAME-db
-KEYCHAIN_PASSWORD="topsykretts"
-
 # setup_signing
 #
 # preps the MAC_SIGNING_IDENTITY env var for use in the signing process, using ephemeral developer certificate material
@@ -46,7 +40,7 @@ function setup_signing() {
   mkdir -p "${DIR}"
 
   # configure the openssl extensions
-  cat << EOF > $EXT_FILE
+  cat << EOF > "$EXT_FILE"
   [ req ]
   default_bits          = 2048                  # RSA key size
   encrypt_key           = yes                   # Protect private key
@@ -105,7 +99,7 @@ EOF
             -extensions $EXT_SECTION
 
   commentary "verify the certificate: we should see our extensions"
-  openssl x509 -text -noout -in $CERT_FILE | grep -A1 'X509v3' || exit_with_error "could not find x509 extensions in certificate"
+  openssl x509 -text -noout -in "$CERT_FILE" | grep -A1 'X509v3' || exit_with_error "could not find x509 extensions in certificate"
 
   title "export cert and private key to .p12 file"
   # note: this step may be entirely optional, however, I found it useful to follow the prod path which goes the route of using a p12
@@ -117,55 +111,18 @@ EOF
             -passin "pass:$KEY_PASSWORD" \
             -passout "pass:$P12_PASSWORD"
 
-
-  title "create the dev keychain"
-
   # delete the keychain if it already exists
-  if [ -f "$(KEYCHAIN_PATH)" ]; then
-    security delete-keychain "$KEYCHAIN_NAME" &> /dev/null
+  if [ -f "${KEYCHAIN_PATH}" ]; then
+    cleanup_dev_signing
   fi
 
-  security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_NAME"
-
-  set +e
-  if ! security verify-cert -k "$KEYCHAIN_PATH" -c "$CERT_FILE" &> /dev/null; then
-    set -e
-    title "import the cert into the dev keychain if it is not already trusted by the system"
-
-    security import "$P12_FILE" -P $P12_PASSWORD -f pkcs12 -k "$KEYCHAIN_PATH" -T /usr/bin/codesign
-
-    # note: set the partition list for this certificate's private key to include "apple-tool:" and "apple:" allows the codesign command to access this keychain item without an interactive user prompt.
-    security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
-
-    # note: add-trusted-cert requires user interaction
-    commentary "adding the developer certificate as a trusted certificate... (requires user interaction)"
-    security add-trusted-cert -d -r trustRoot -k "$KEYCHAIN_PATH" "$CERT_FILE"
-  else
-    set -e
-    commentary "...dev cert has already been imported onto the dev keychain"
-  fi
-
-  # remove any generated cert material since the keychain now has all of this material loaded
-  rm -rf "${DIR}"
-
-  commentary "make certain there are identities that can be used for code signing"
-  security find-identity -p codesigning "$KEYCHAIN_PATH" | grep -C 30 "$IDENTITY" || exit_with_error "could not find identity that can be used with codesign"
-
-  title "add the dev keychain to the search path for codesign"
-  add_keychain $KEYCHAIN_NAME
-
-  commentary "verify the keychain actually shows up"
-  security list-keychains | grep "$KEYCHAIN_NAME" || exit_with_error "could not find new keychain"
-
-  export MAC_SIGNING_IDENTITY=$IDENTITY
-  commentary "setting MAC_SIGNING_IDENTITY=${IDENTITY}"
-
+  import_signing_certificate "$P12_FILE" "$P12_PASSWORD" "$IDENTITY"
 }
 
-function cleanup_signing() {
+function cleanup_dev_signing() {
   title "delete the dev keychain and all certificate material"
   set -xue
-  security delete-keychain "$KEYCHAIN_NAME"
-  rm -f "$KEYCHAIN_PATH"
-  rm -rf "${DIR}"
+  security delete-keychain "$KEYCHAIN_NAME" || true
+  rm -f "$KEYCHAIN_PATH" || true
+  rm -rf "${DIR}" || true
 }

.github/scripts/apple-signing/setup-import-cert.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -eu
+
+KEYCHAIN_NAME="syft-ephemeral-keychain"
+KEYCHAIN_PATH="$HOME/Library/Keychains/${KEYCHAIN_NAME}-db"
+
+# import_signing_certificate
+#
+# imports a cert from a p12 file into a keychain used for codesigning
+#
+function import_signing_certificate() {
+  p12_file=$1
+  p12_password=$2
+  identity=$3
+
+  keychain_password="$(openssl rand -base64 100)"
+
+  title "create the a new keychain"
+
+  security create-keychain -p "$keychain_password" "$KEYCHAIN_NAME"
+  security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH"
+  security unlock-keychain -p "$keychain_password" "$KEYCHAIN_PATH"
+
+  if [ ! -f "$KEYCHAIN_PATH" ]; then
+    exit_with_error "cannot find keychain '$KEYCHAIN_PATH'"
+  fi
+
+  set +e
+  if ! security verify-cert -k "$KEYCHAIN_PATH" -c "$p12_file" &> /dev/null; then
+    set -e
+    title "import the cert into the new keychain if it is not already trusted by the system"
+
+    # '-t cert' is vital since it side-steps the need for user interaction with "security add-trusted-cert" (which has wider security implications)
+    security import "$p12_file" -P "$p12_password" -t cert -f pkcs12 -k "$KEYCHAIN_PATH" -T /usr/bin/codesign
+
+    # note: set the partition list for this certificate's private key to include "apple-tool:" and "apple:" allows the codesign command to access this keychain item without an interactive user prompt.
+    security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "$keychain_password" "$KEYCHAIN_PATH"
+  else
+    set -e
+    commentary "...cert has already been imported onto the new keychain"
+  fi
+
+  commentary "make certain there are identities that can be used for code signing"
+  security find-identity -p codesigning "$KEYCHAIN_PATH" | grep -C 30 "$identity" || exit_with_error "could not find identity that can be used with codesign"
+
+  title "add the new keychain to the search path for codesign"
+  add_keychain "$KEYCHAIN_NAME"
+
+  commentary "verify the new keychain can be found by the security sub-system"
+  security list-keychains | grep "$KEYCHAIN_NAME" || exit_with_error "could not find new keychain"
+
+  export MAC_SIGNING_IDENTITY=$identity
+  commentary "setting MAC_SIGNING_IDENTITY=${identity}"
+
+}

.github/scripts/apple-signing/setup-prod.sh

@@ -3,6 +3,8 @@ set -eu
 
 assert_in_ci
 
+IDENTITY="Developer ID Application: ANCHORE, INC. (9MJHKYX5AT)"
+
 set +xu
 if [ -z "$APPLE_DEVELOPER_ID_CERT" ]; then
   exit_with_error "APPLE_DEVELOPER_ID_CERT not set"
@@ -29,27 +31,13 @@ setup_signing() {
   title "setting up production certificate material"
 
   # Write signing certificate to disk from environment variable.
-  cert_file="$HOME/developer_id_certificate.p12"
-  echo -n "$APPLE_DEVELOPER_ID_CERT" | base64 --decode > "$cert_file"
+  p12_file="$HOME/developer_id_certificate.p12"
+  echo -n "$APPLE_DEVELOPER_ID_CERT" | base64 --decode > "$p12_file"
 
-  # In order to have all keychain interactions avoid an interactive user prompt, we need to control the password for the keychain in question, which means we need to create a new keychain into which we'll import the signing certificate and from which we'll later access this certificate during code signing.
-  ephemeral_keychain="ci-ephemeral-keychain"
-  ephemeral_keychain_password="$(openssl rand -base64 100)"
-  security create-keychain -p "${ephemeral_keychain_password}" "${ephemeral_keychain}"
-
-  # Import signing certificate into the keychain. (This is a pre-requisite for gon, which is invoked via goreleaser.)
-  ephemeral_keychain_full_path="$HOME/Library/Keychains/${ephemeral_keychain}-db"
-  security import "${cert_file}" -k "${ephemeral_keychain_full_path}" -P "${APPLE_DEVELOPER_ID_CERT_PASS}" -T "$(command -v codesign)"
-
-  # Setting the partition list for this certificate's private key to include "apple-tool:" and "apple:" allows the codesign command to access this keychain item without an interactive user prompt. (codesign is invoked by gon.)
-  security set-key-partition-list -S "apple-tool:,apple:" -s -k "${ephemeral_keychain_password}" "${ephemeral_keychain_full_path}"
+  import_signing_certificate "$p12_file" "$APPLE_DEVELOPER_ID_CERT_PASS" "$IDENTITY"
 
   # Make this new keychain the user's default keychain, so that codesign will be able to find this certificate when we specify it during signing.
-  security default-keychain -d "user" -s "${ephemeral_keychain_full_path}"
-
-  # TODO: extract this from the certificate material itself
-  export MAC_SIGNING_IDENTITY="Developer ID Application: ANCHORE, INC. (9MJHKYX5AT)"
-  commentary "setting MAC_SIGNING_IDENTITY=${MAC_SIGNING_IDENTITY}"
+  security default-keychain -d "user" -s "${KEYCHAIN_PATH}"
 
   commentary "log into docker -- required for publishing (since the default keychain has now been replaced)"
   echo "${DOCKER_PASSWORD}" | docker login docker.io -u "${DOCKER_USERNAME}"  --password-stdin

.github/scripts/apple-signing/setup.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -eu
+set -eu -o pipefail
 
 IS_SNAPSHOT="$1"
 
@@ -9,6 +9,8 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 mkdir -p "$SCRIPT_DIR/log"
 
 main() {
+  # defines KEYCHAIN_NAME and KEYCHAIN_PATH
+  . "$SCRIPT_DIR"/setup-import-cert.sh
 
   case "$IS_SNAPSHOT" in
 

.github/scripts/apple-signing/sign.sh

@@ -27,19 +27,23 @@ sign_binary() {
       return 0
   fi
 
+  set -x
+
   codesign \
     -s "$identity" \
     -f \
     --verbose=4 \
     --timestamp \
     --options runtime \
-      $exe_path
+      "$exe_path"
 
   if [ $? -ne 0 ]; then
       exit_with_error "signing failed"
   fi
 
   codesign --verify "$exe_path"  --verbose=4
+
+  set +x
 }
 
 

.gitignore

@@ -19,6 +19,7 @@ CHANGELOG.md
 .tmp/
 coverage.txt
 bin/
+.goreleaser.yaml
 
 # Binaries for programs and plugins
 *.exe

.goreleaser/Makefile

@@ -0,0 +1,24 @@
+TARGET=../goreleaser.yaml
+
+.PHONY: snapshot-config
+snapshot-config:
+	cat snapshot.yaml > $(TARGET)
+	cat env-skip-signing.yaml >> $(TARGET)
+	cat main.yaml >> $(TARGET)
+	cat docker.yaml >> $(TARGET)
+	cat $(TARGET)
+
+.PHONY: snapshot-with-signing-config
+snapshot-with-signing-config:
+	# we leave out docker entirely such that this can easily be tested in CI
+	cat snapshot.yaml > $(TARGET)
+	cat env-default.yaml >> $(TARGET)
+	cat main.yaml >> $(TARGET)
+	cat $(TARGET)
+
+.PHONY: release-config
+release-config:
+	cat env-default.yaml > $(TARGET)
+	cat docker.yaml >> $(TARGET)
+	cat main.yaml >> $(TARGET)
+	cat $(TARGET)

.goreleaser/docker.yaml

@@ -0,0 +1,39 @@
+
+dockers:
+  - image_templates:
+      - "anchore/syft:latest"
+      - "anchore/syft:{{ .Tag }}-amd64"
+      - "anchore/syft:v{{ .Major }}-amd64"
+      - "anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64"
+    dockerfile: Dockerfile
+    use: buildx
+    build_flag_templates:
+      - "--platform=linux/amd64"
+      - "--build-arg=BUILD_DATE={{.Date}}"
+      - "--build-arg=BUILD_VERSION={{.Version}}"
+      - "--build-arg=VCS_REF={{.FullCommit}}"
+      - "--build-arg=VCS_URL={{.GitURL}}"
+
+  - image_templates:
+      - "anchore/syft:{{ .Tag }}-arm64v8"
+      - "anchore/syft:v{{ .Major }}-arm64v8"
+      - "anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8"
+    goarch: arm64
+    dockerfile: Dockerfile
+    use: buildx
+    build_flag_templates:
+      - "--platform=linux/arm64/v8"
+      - "--build-arg=BUILD_DATE={{.Date}}"
+      - "--build-arg=BUILD_VERSION={{.Version}}"
+      - "--build-arg=VCS_REF={{.FullCommit}}"
+      - "--build-arg=VCS_URL={{.GitURL}}"
+
+docker_manifests:
+  - name_template: anchore/syft:{{ .Tag }}
+    image_templates:
+      - anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64
+      - anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8
+  - name_template: anchore/syft:latest
+    image_templates:
+      - anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64
+      - anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8

.goreleaser/env-default.yaml

@@ -0,0 +1,4 @@
+
+env:
+  # required to support multi architecture docker builds
+  - DOCKER_CLI_EXPERIMENTAL=enabled

.goreleaser/env-skip-signing.yaml

@@ -0,0 +1,5 @@
+
+env:
+  # required to support multi architecture docker builds
+  - DOCKER_CLI_EXPERIMENTAL=enabled
+  - SKIP_SIGNING=true

.goreleaser/main.yaml

@@ -0,0 +1,91 @@
+release:
+  prerelease: auto
+  draft: true
+
+before:
+  hooks:
+    - ./.github/scripts/apple-signing/setup.sh {{ .IsSnapshot }}
+
+builds:
+  - id: linux-build
+    binary: syft
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+    # set the modified timestamp on the output binary to the git timestamp to ensure a reproducible build
+    mod_timestamp: &build-timestamp '{{ .CommitTimestamp }}'
+    env: &build-env
+      - CGO_ENABLED=0
+    ldflags: &build-ldflags |
+      -w
+      -s
+      -extldflags '-static'
+      -X github.com/anchore/syft/internal/version.version={{.Version}}
+      -X github.com/anchore/syft/internal/version.gitCommit={{.Commit}}
+      -X github.com/anchore/syft/internal/version.buildDate={{.Date}}
+      -X github.com/anchore/syft/internal/version.gitDescription={{.Summary}}
+
+  - id: darwin-build
+    binary: syft
+    goos:
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+    mod_timestamp: *build-timestamp
+    env: *build-env
+    ldflags: *build-ldflags
+    hooks:
+      post:
+        # we must have signing as a build hook instead of the signs section. The signs section must register a new
+        # asset, where we want to replace an existing asset. A post-build hook has the advantage of not needing to
+        # unpackage and repackage a tar.gz with a signed binary
+        - ./.github/scripts/apple-signing/sign.sh "{{ .Path }}" "{{ .IsSnapshot }}" "{{ .Target }}"
+
+  - id: windows-build
+    binary: syft
+    goos:
+      - windows
+    goarch:
+      - amd64
+    mod_timestamp: *build-timestamp
+    env: *build-env
+    ldflags: *build-ldflags
+
+archives:
+  - id: linux-archives
+    builds:
+      - linux-build
+
+  # note: the signing process is depending on tar.gz archives. If this format changes then .github/scripts/apple-signing/*.sh will need to be adjusted
+  - id: darwin-archives
+    builds:
+      - darwin-build
+
+  - id: windows-archives
+    format: zip
+    builds:
+      - windows-build
+
+nfpms:
+  - license: "Apache 2.0"
+    maintainer: "Anchore, Inc"
+    homepage: &website "https://github.com/anchore/syft"
+    description: &description "A tool that generates a Software Bill Of Materials (SBOM) from container images and filesystems"
+    formats:
+      - rpm
+      - deb
+
+brews:
+  - tap:
+      owner: anchore
+      name: homebrew-syft
+    ids:
+      - darwin-archives
+      - linux-archives
+    homepage: *website
+    description: *description
+    license: "Apache License 2.0"
+

.goreleaser/snapshot.yaml

@@ -0,0 +1 @@
+dist: snapshot

Makefile

@@ -4,7 +4,7 @@ RESULTSDIR = test/results
 COVER_REPORT = $(RESULTSDIR)/unit-coverage-details.txt
 COVER_TOTAL = $(RESULTSDIR)/unit-coverage-summary.txt
 LINTCMD = $(TEMPDIR)/golangci-lint run --tests=false --timeout=4m --config .golangci.yaml
-RELEASE_CMD=$(TEMPDIR)/goreleaser release --rm-dist --timeout 60m
+RELEASE_CMD=$(TEMPDIR)/goreleaser release --rm-dist
 SNAPSHOT_CMD=$(RELEASE_CMD) --skip-publish --snapshot
 VERSION=$(shell git describe --dirty --always --tags)
 COMPARE_TEST_IMAGE = centos:8.2.2004
@@ -234,25 +234,24 @@ build: $(SNAPSHOTDIR) ## Build release snapshot binaries and packages
 $(SNAPSHOTDIR): ## Build snapshot release binaries and packages
 	$(call title,Building snapshot artifacts)
 
-	# create a config with the dist dir overridden
-	echo "dist: $(SNAPSHOTDIR)" > $(TEMPDIR)/goreleaser.yaml
-	cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml
+	cd .goreleaser && make snapshot-config
 
 	# build release snapshots
-	bash -c "SKIP_SIGNING=true $(SNAPSHOT_CMD) --skip-sign --config $(TEMPDIR)/goreleaser.yaml"
+	bash -c "$(SNAPSHOT_CMD) --skip-sign"
 
 .PHONY: snapshot-with-signing
 snapshot-with-signing: ## Build snapshot release binaries and packages (with dummy signing)
 	$(call title,Building snapshot artifacts (+ signing))
 
-	# create a config with the dist dir overridden
-	echo "dist: $(SNAPSHOTDIR)" > $(TEMPDIR)/goreleaser.yaml
-	cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml
+	cd .goreleaser && make snapshot-with-signing-config
 
 	rm -f .github/scripts/apple-signing/log/*.txt
 
+	# remove the keychain with the trusted self-signed cert automatically (from failed previous runs)
+	.github/scripts/apple-signing/cleanup.sh
+
 	# build release snapshots
-	bash -c "$(SNAPSHOT_CMD) --config $(TEMPDIR)/goreleaser.yaml || (cat .github/scripts/apple-signing/log/*.txt && false)"
+	bash -c "$(SNAPSHOT_CMD) || (cat .github/scripts/apple-signing/log/*.txt && false)"
 
 	# remove the keychain with the trusted self-signed cert automatically
 	.github/scripts/apple-signing/cleanup.sh
@@ -316,16 +315,13 @@ CHANGELOG.md:
 release: clean-dist CHANGELOG.md  ## Build and publish final binaries and packages. Intended to be run only on macOS.
 	$(call title,Publishing release artifacts)
 
-	# create a config with the dist dir overridden
-	echo "dist: $(DISTDIR)" > $(TEMPDIR)/goreleaser.yaml
-	cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml
+	cd .goreleaser && make release-config
 
 	rm -f .github/scripts/apple-signing/log/*.txt
 
 	# note: notarization cannot be done in parallel, thus --parallelism 1
 	bash -c "\
 		$(RELEASE_CMD) \
-			--config $(TEMPDIR)/goreleaser.yaml \
 			--parallelism 1 \
 			--release-notes <(cat CHANGELOG.md)\
 				 || (cat .github/scripts/apple-signing/log/*.txt && false)"

goreleaser.yaml

@@ -1,10 +1,11 @@
-release:
-  prerelease: auto
-  draft: true
+dist: snapshot
 
 env:
   # required to support multi architecture docker builds
   - DOCKER_CLI_EXPERIMENTAL=enabled
+release:
+  prerelease: auto
+  draft: true
 
 before:
   hooks:
@@ -93,41 +94,3 @@ brews:
     description: *description
     license: "Apache License 2.0"
 
-dockers:
-  - image_templates:
-      - "anchore/syft:latest"
-      - "anchore/syft:{{ .Tag }}-amd64"
-      - "anchore/syft:v{{ .Major }}-amd64"
-      - "anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64"
-    dockerfile: Dockerfile
-    use: buildx
-    build_flag_templates:
-      - "--platform=linux/amd64"
-      - "--build-arg=BUILD_DATE={{.Date}}"
-      - "--build-arg=BUILD_VERSION={{.Version}}"
-      - "--build-arg=VCS_REF={{.FullCommit}}"
-      - "--build-arg=VCS_URL={{.GitURL}}"
-
-  - image_templates:
-      - "anchore/syft:{{ .Tag }}-arm64v8"
-      - "anchore/syft:v{{ .Major }}-arm64v8"
-      - "anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8"
-    goarch: arm64
-    dockerfile: Dockerfile
-    use: buildx
-    build_flag_templates:
-      - "--platform=linux/arm64/v8"
-      - "--build-arg=BUILD_DATE={{.Date}}"
-      - "--build-arg=BUILD_VERSION={{.Version}}"
-      - "--build-arg=VCS_REF={{.FullCommit}}"
-      - "--build-arg=VCS_URL={{.GitURL}}"
-
-docker_manifests:
-  - name_template: anchore/syft:{{ .Tag }}
-    image_templates:
-      - anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64
-      - anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8
-  - name_template: anchore/syft:latest
-    image_templates:
-      - anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64
-      - anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8