Update Go, use go tool covdata for coverage files?

.github/workflows/schedules.yaml

@@ -12,6 +12,7 @@ concurrency:
 jobs:
   # run Sytest in different variations
   sytest:
+    if: ${{ false }}
     timeout-minutes: 60
     name: "Sytest (${{ matrix.label }})"
     runs-on: ubuntu-latest
@@ -75,7 +76,7 @@ jobs:
     name: "Sytest Coverage"
     runs-on: ubuntu-latest
     needs: sytest # only run once Sytest is done
-    if: ${{ always() }}
+    if: ${{ false }}
     steps:
       - uses: actions/checkout@v3
       - name: Install Go
@@ -167,7 +168,7 @@ jobs:
           cat <<EOF > /tmp/posttest.sh
           #!/bin/bash
           mkdir -p /tmp/Complement/logs/\$2/\$1/
-          docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/
+          docker cp \$1:/tmp/covdatafiles/. /tmp/Complement/logs/\$2/\$1/
           EOF
           
           chmod +x /tmp/posttest.sh
@@ -190,7 +191,7 @@ jobs:
         with:
           name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }})
           path: |
-            /tmp/Complement/**/complementcover.log
+            /tmp/Complement/logs/**
 
   complement-coverage:
     timeout-minutes: 5
@@ -199,7 +200,6 @@ jobs:
     needs: complement # only run once Complement is done
     if: ${{ always() }}
     steps:
-      - uses: actions/checkout@v3
       - name: Install Go
         uses: actions/setup-go@v3
         with:
@@ -207,20 +207,19 @@ jobs:
           cache: true
       - name: Download all artifacts
         uses: actions/download-artifact@v3
-      - name: Install gocovmerge
+      - name: Collect coverage
-        run: go install github.com/wadey/gocovmerge@latest
-      - name: Run gocovmerge
         run: |
-          find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov
+          go tool covdata textfmt -i=$(find . -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) -o complement.cov
-          go tool cover -func=complement.cov
+          grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          files: ./complement.cov
+          files: ./final.cov
           flags: complement
           fail_ci_if_error: true
 
   element_web:
+    if: ${{ false }}
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:

build/scripts/Complement.Dockerfile

@@ -1,6 +1,6 @@
 #syntax=docker/dockerfile:1.2
 
-FROM golang:1.18-stretch as build
+FROM golang:1.20-bullseye as build
 RUN apt-get update && apt-get install -y sqlite3
 WORKDIR /build
 
@@ -17,7 +17,7 @@ RUN --mount=target=. \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
     CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
-    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
+    CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
     cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite

build/scripts/ComplementPostgres.Dockerfile

@@ -1,19 +1,19 @@
 #syntax=docker/dockerfile:1.2
 
-FROM golang:1.18-stretch as build
+FROM golang:1.20-bullseye as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 
 # No password when connecting over localhost
-RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/9.6/main/pg_hba.conf && \
+RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/13/main/pg_hba.conf && \
     # Bump up max conns for moar concurrency
-    sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf
+    sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/13/main/postgresql.conf
 
 # This entry script starts postgres, waits for it to be up then starts dendrite
 RUN echo '\
     #!/bin/bash -eu \n\
     pg_lsclusters \n\
-    pg_ctlcluster 9.6 main start \n\
+    pg_ctlcluster 13 main start \n\
     \n\
     until pg_isready \n\
     do \n\
@@ -35,7 +35,7 @@ RUN --mount=target=. \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
     CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
-    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
+    CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
     cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite

build/scripts/complement-cmd.sh

@@ -2,14 +2,15 @@
 
 # This script is intended to be used inside a docker container for Complement
 
+export GOCOVERDIR=/tmp/covdatafiles
+mkdir -p "${GOCOVERDIR}"
 if [[ "${COVER}" -eq 1 ]]; then
   echo "Running with coverage"
   exec /dendrite/dendrite-cover \
     --really-enable-open-registration \
     --tls-cert server.crt \
     --tls-key server.key \
-    --config dendrite.yaml \
+    --config dendrite.yaml
-    --test.coverprofile=complementcover.log
 else
   echo "Not running with coverage"
   exec /dendrite/dendrite \

internal/fulltext/bleve.go

@@ -18,10 +18,11 @@
 package fulltext
 
 import (
-	"context"
 	"strings"
 
 	"github.com/blevesearch/bleve/v2"
+	"github.com/matrix-org/dendrite/setup/process"
+
 	// side effect imports to allow all possible languages
 	_ "github.com/blevesearch/bleve/v2/analysis/lang/ar"
 	_ "github.com/blevesearch/bleve/v2/analysis/lang/cjk"
@@ -84,17 +85,18 @@ func (i *IndexElement) SetContentType(v string) {
 }
 
 // New opens a new/existing fulltext index
-func New(ctx context.Context, cfg config.Fulltext) (fts *Search, err error) {
+func New(processCtx *process.ProcessContext, cfg config.Fulltext) (fts *Search, err error) {
 	fts = &Search{}
 	fts.FulltextIndex, err = openIndex(cfg)
 	if err != nil {
 		return nil, err
 	}
 	go func() {
-		// Wait for the context (should be from process.ProcessContext) to be
+		processCtx.ComponentStarted()
-		// done, indicating that Dendrite is shutting down.
+		// Wait for the processContext to be done, indicating that Dendrite is shutting down.
-		<-ctx.Done()
+		<-processCtx.WaitForShutdown()
 		_ = fts.Close()
+		processCtx.ComponentFinished()
 	}()
 	return fts, nil
 }

internal/fulltext/bleve_test.go

@@ -38,7 +38,7 @@ func mustOpenIndex(t *testing.T, tempDir string) (*fulltext.Search, *process.Pro
 		cfg.InMemory = false
 	}
 	ctx := process.NewProcessContext()
-	fts, err := fulltext.New(ctx.Context(), cfg)
+	fts, err := fulltext.New(ctx, cfg)
 	if err != nil {
 		t.Fatal("failed to open fulltext index:", err)
 	}

syncapi/syncapi.go

@@ -69,11 +69,10 @@ func AddPublicRoutes(
 
 	var fts *fulltext.Search
 	if dendriteCfg.SyncAPI.Fulltext.Enabled {
-		fts, err = fulltext.New(processContext.Context(), dendriteCfg.SyncAPI.Fulltext)
+		fts, err = fulltext.New(processContext, dendriteCfg.SyncAPI.Fulltext)
 		if err != nil {
 			logrus.WithError(err).Panicf("failed to create full text")
 		}
-		processContext.ComponentStarted()
 	}
 
 	federationPresenceProducer := &producers.FederationAPIPresenceProducer{

test/testrig/base.go

@@ -51,6 +51,7 @@ func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process.
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
 		cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)
+		cfg.SyncAPI.Fulltext.InMemory = true
 
 		connStr, closeDb := test.PrepareDBConnectionString(t, dbType)
 		cfg.Global.DatabaseOptions = config.DatabaseOptions{
@@ -70,7 +71,7 @@ func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process.
 			SingleDatabase: false,
 		})
 		cfg.Global.ServerName = "test"
-
+		cfg.SyncAPI.Fulltext.InMemory = true
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
 		cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)