2021-07-07 11:06:17 +00:00
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
2022-08-05 09:26:59 +00:00
"io"
2021-07-07 11:06:17 +00:00
"log"
"net/http"
"os"
"path"
"regexp"
"runtime"
"sort"
2021-07-07 13:20:07 +00:00
"strconv"
2021-07-07 11:06:17 +00:00
"strings"
"sync"
"time"
"github.com/Masterminds/semver/v3"
"github.com/codeclysm/extract"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
2021-07-07 13:20:07 +00:00
"github.com/docker/docker/api/types/filters"
2021-07-07 11:06:17 +00:00
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/volume"
"github.com/docker/docker/client"
"github.com/docker/go-connections/nat"
)
var (
flagTempDir = flag . String ( "tmp" , "tmp" , "Path to temporary directory to dump tarballs to" )
2021-07-07 13:20:07 +00:00
flagFrom = flag . String ( "from" , "HEAD-1" , "The version to start from e.g '0.3.1'. If 'HEAD-N' then starts N versions behind HEAD." )
2021-07-07 11:06:17 +00:00
flagTo = flag . String ( "to" , "HEAD" , "The version to end on e.g '0.3.3'." )
flagBuildConcurrency = flag . Int ( "build-concurrency" , runtime . NumCPU ( ) , "The amount of build concurrency when building images" )
2021-07-08 11:28:04 +00:00
flagHead = flag . String ( "head" , "" , "Location to a dendrite repository to treat as HEAD instead of Github" )
flagDockerHost = flag . String ( "docker-host" , "localhost" , "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker." )
2022-07-25 09:39:22 +00:00
flagDirect = flag . Bool ( "direct" , false , "If a direct upgrade from the defined FROM version to TO should be done" )
2021-07-07 11:06:17 +00:00
alphaNumerics = regexp . MustCompile ( "[^a-zA-Z0-9]+" )
)
2021-07-08 11:28:04 +00:00
const HEAD = "HEAD"
2021-07-07 11:06:17 +00:00
// Embed the Dockerfile to use when building dendrite versions.
// We cannot use the dockerfile associated with the repo with each version sadly due to changes in
// Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
// due to the error:
2022-08-05 09:12:41 +00:00
// When using COPY with more than one source file, the destination must be a directory and end with a /
2021-07-07 11:06:17 +00:00
// We need to run a postgres anyway, so use the dockerfile associated with Complement instead.
2022-07-11 13:31:31 +00:00
const Dockerfile = ` FROM golang : 1.18 - stretch as build
2021-07-07 11:06:17 +00:00
RUN apt - get update && apt - get install - y postgresql
WORKDIR / build
# Copy the build context to the repo as this is the right dendrite code . This is different to the
# Complement Dockerfile which wgets a branch .
COPY . .
RUN go build . / cmd / dendrite - monolith - server
RUN go build . / cmd / generate - keys
RUN go build . / cmd / generate - config
RUN . / generate - config -- ci > dendrite . yaml
RUN . / generate - keys -- private - key matrix_key . pem -- tls - cert server . crt -- tls - key server . key
# Replace the connection string with a single postgres DB , using user / db = ' postgres ' and no password
RUN sed - i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite . yaml
# No password when connecting over localhost
RUN sed - i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" / etc / postgresql / 9.6 / main / pg_hba . conf
# Bump up max conns for moar concurrency
RUN sed - i ' s / max_connections = 100 / max_connections = 2000 / g ' / etc / postgresql / 9.6 / main / postgresql . conf
RUN sed - i ' s / max_open_conns : . * $ / max_open_conns : 100 / g ' dendrite . yaml
# This entry script starts postgres , waits for it to be up then starts dendrite
RUN echo ' \
# ! / bin / bash - eu \ n \
pg_lsclusters \ n \
pg_ctlcluster 9.6 main start \ n \
\ n \
until pg_isready \ n \
do \ n \
echo "Waiting for postgres" ; \ n \
sleep 1 ; \ n \
done \ n \
\ n \
sed - i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite . yaml \ n \
2022-04-29 07:31:11 +00:00
PARAMS = "--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \ n \
. / dendrite - monolith - server -- really - enable - open - registration $ { PARAMS } || . / dendrite - monolith - server $ { PARAMS } \ n \
2021-07-07 11:06:17 +00:00
' > run_dendrite . sh && chmod + x run_dendrite . sh
ENV SERVER_NAME = localhost
EXPOSE 8008 8448
CMD / build / run_dendrite . sh `
2021-07-07 13:20:07 +00:00
const dendriteUpgradeTestLabel = "dendrite_upgrade_test"
2021-07-07 11:06:17 +00:00
// downloadArchive downloads an arbitrary github archive of the form:
2022-08-05 09:12:41 +00:00
//
// https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz
//
2021-07-07 11:06:17 +00:00
// and re-tarballs it without the top-level directory which contains branch information. It inserts
// the contents of `dockerfile` as a root file `Dockerfile` in the re-tarballed directory such that
// you can directly feed the retarballed archive to `ImageBuild` to have it run said dockerfile.
// Returns the tarball buffer on success.
func downloadArchive ( cli * http . Client , tmpDir , archiveURL string , dockerfile [ ] byte ) ( * bytes . Buffer , error ) {
resp , err := cli . Get ( archiveURL )
if err != nil {
return nil , err
}
// nolint:errcheck
defer resp . Body . Close ( )
if resp . StatusCode != 200 {
return nil , fmt . Errorf ( "got HTTP %d" , resp . StatusCode )
}
2021-07-08 09:07:39 +00:00
_ = os . RemoveAll ( tmpDir )
2021-07-07 11:06:17 +00:00
if err = os . Mkdir ( tmpDir , os . ModePerm ) ; err != nil {
return nil , fmt . Errorf ( "failed to make temporary directory: %s" , err )
}
// nolint:errcheck
defer os . RemoveAll ( tmpDir )
// dump the tarball temporarily, stripping the top-level directory
err = extract . Archive ( context . Background ( ) , resp . Body , tmpDir , func ( inPath string ) string {
// remove top level
segments := strings . Split ( inPath , "/" )
return strings . Join ( segments [ 1 : ] , "/" )
} )
if err != nil {
return nil , err
}
// add top level Dockerfile
2022-08-05 09:26:59 +00:00
err = os . WriteFile ( path . Join ( tmpDir , "Dockerfile" ) , dockerfile , os . ModePerm )
2021-07-07 11:06:17 +00:00
if err != nil {
return nil , fmt . Errorf ( "failed to inject /Dockerfile: %w" , err )
}
// now re-tarball it :/
var tarball bytes . Buffer
err = compress ( tmpDir , & tarball )
if err != nil {
return nil , err
}
return & tarball , nil
}
// buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error
func buildDendrite ( httpClient * http . Client , dockerClient * client . Client , tmpDir , branchOrTagName string ) ( string , error ) {
2021-07-08 11:28:04 +00:00
var tarball * bytes . Buffer
var err error
// If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI
// where we want to use the working directory.
if branchOrTagName == HEAD && * flagHead != "" {
log . Printf ( "%s: Using %s as HEAD" , branchOrTagName , * flagHead )
// add top level Dockerfile
2022-08-05 09:26:59 +00:00
err = os . WriteFile ( path . Join ( * flagHead , "Dockerfile" ) , [ ] byte ( Dockerfile ) , os . ModePerm )
2021-07-08 11:28:04 +00:00
if err != nil {
2021-09-08 16:31:03 +00:00
return "" , fmt . Errorf ( "custom HEAD: failed to inject /Dockerfile: %w" , err )
2021-07-08 11:28:04 +00:00
}
// now tarball it
var buffer bytes . Buffer
err = compress ( * flagHead , & buffer )
if err != nil {
return "" , fmt . Errorf ( "failed to tarball custom HEAD %s : %s" , * flagHead , err )
}
tarball = & buffer
} else {
log . Printf ( "%s: Downloading version %s to %s\n" , branchOrTagName , branchOrTagName , tmpDir )
// pull an archive, this contains a top-level directory which screws with the build context
// which we need to fix up post download
u := fmt . Sprintf ( "https://github.com/matrix-org/dendrite/archive/%s.tar.gz" , branchOrTagName )
tarball , err = downloadArchive ( httpClient , tmpDir , u , [ ] byte ( Dockerfile ) )
if err != nil {
return "" , fmt . Errorf ( "failed to download archive %s: %w" , u , err )
}
log . Printf ( "%s: %s => %d bytes\n" , branchOrTagName , u , tarball . Len ( ) )
2021-07-07 11:06:17 +00:00
}
2021-07-08 11:28:04 +00:00
2021-07-07 11:06:17 +00:00
log . Printf ( "%s: Building version %s\n" , branchOrTagName , branchOrTagName )
res , err := dockerClient . ImageBuild ( context . Background ( ) , tarball , types . ImageBuildOptions {
Tags : [ ] string { "dendrite-upgrade" } ,
} )
if err != nil {
return "" , fmt . Errorf ( "failed to start building image: %s" , err )
}
// nolint:errcheck
defer res . Body . Close ( )
decoder := json . NewDecoder ( res . Body )
// {"aux":{"ID":"sha256:247082c717963bc2639fc2daed08838d67811ea12356cd4fda43e1ffef94f2eb"}}
var imageID string
for decoder . More ( ) {
var dl struct {
Stream string ` json:"stream" `
Aux map [ string ] interface { } ` json:"aux" `
}
if err := decoder . Decode ( & dl ) ; err != nil {
return "" , fmt . Errorf ( "failed to decode build image output line: %w" , err )
}
2022-01-21 09:56:06 +00:00
if len ( strings . TrimSpace ( dl . Stream ) ) > 0 {
log . Printf ( "%s: %s" , branchOrTagName , dl . Stream )
}
2021-07-07 11:06:17 +00:00
if dl . Aux != nil {
imgID , ok := dl . Aux [ "ID" ]
if ok {
imageID = imgID . ( string )
}
}
}
return imageID , nil
}
func getAndSortVersionsFromGithub ( httpClient * http . Client ) ( semVers [ ] * semver . Version , err error ) {
u := "https://api.github.com/repos/matrix-org/dendrite/tags"
res , err := httpClient . Get ( u )
if err != nil {
return nil , err
}
if res . StatusCode != 200 {
return nil , fmt . Errorf ( "%s returned HTTP %d" , u , res . StatusCode )
}
resp := [ ] struct {
Name string ` json:"name" `
} { }
if err = json . NewDecoder ( res . Body ) . Decode ( & resp ) ; err != nil {
return nil , err
}
for _ , r := range resp {
v , err := semver . NewVersion ( r . Name )
if err != nil {
continue // not a semver, that's ok and isn't an error, we allow tags that aren't semvers
}
semVers = append ( semVers , v )
}
sort . Sort ( semver . Collection ( semVers ) )
return semVers , nil
}
2022-07-25 09:39:22 +00:00
func calculateVersions ( cli * http . Client , from , to string , direct bool ) [ ] string {
2021-07-07 11:06:17 +00:00
semvers , err := getAndSortVersionsFromGithub ( cli )
if err != nil {
log . Fatalf ( "failed to collect semvers from github: %s" , err )
}
// snip the lower bound depending on --from
if from != "" {
2021-07-07 13:20:07 +00:00
if strings . HasPrefix ( from , "HEAD-" ) {
var headN int
headN , err = strconv . Atoi ( strings . TrimPrefix ( from , "HEAD-" ) )
if err != nil {
log . Fatalf ( "invalid --from, try 'HEAD-1'" )
2021-07-07 11:06:17 +00:00
}
2021-07-07 13:20:07 +00:00
if headN >= len ( semvers ) {
log . Fatalf ( "only have %d versions, but asked to go to HEAD-%d" , len ( semvers ) , headN )
}
if headN > 0 {
semvers = semvers [ len ( semvers ) - headN : ]
}
} else {
fromVer , err := semver . NewVersion ( from )
if err != nil {
log . Fatalf ( "invalid --from: %s" , err )
}
i := 0
for i = 0 ; i < len ( semvers ) ; i ++ {
if semvers [ i ] . LessThan ( fromVer ) {
continue
}
break
}
semvers = semvers [ i : ]
2021-07-07 11:06:17 +00:00
}
}
2021-07-08 11:28:04 +00:00
if to != "" && to != HEAD {
2021-07-07 11:06:17 +00:00
toVer , err := semver . NewVersion ( to )
if err != nil {
log . Fatalf ( "invalid --to: %s" , err )
}
var i int
for i = len ( semvers ) - 1 ; i >= 0 ; i -- {
if semvers [ i ] . GreaterThan ( toVer ) {
continue
}
break
}
semvers = semvers [ : i + 1 ]
}
var versions [ ] string
for _ , sv := range semvers {
versions = append ( versions , sv . Original ( ) )
}
2021-07-08 11:28:04 +00:00
if to == HEAD {
versions = append ( versions , HEAD )
2021-07-07 11:06:17 +00:00
}
2022-07-25 09:39:22 +00:00
if direct {
versions = [ ] string { versions [ 0 ] , versions [ len ( versions ) - 1 ] }
}
2021-07-07 11:06:17 +00:00
return versions
}
func buildDendriteImages ( httpClient * http . Client , dockerClient * client . Client , baseTempDir string , concurrency int , branchOrTagNames [ ] string ) map [ string ] string {
// concurrently build all versions, this can be done in any order. The mutex protects the map
branchToImageID := make ( map [ string ] string )
var mu sync . Mutex
var wg sync . WaitGroup
wg . Add ( concurrency )
ch := make ( chan string , len ( branchOrTagNames ) )
for _ , branchName := range branchOrTagNames {
ch <- branchName
}
close ( ch )
for i := 0 ; i < concurrency ; i ++ {
go func ( ) {
defer wg . Done ( )
for branchName := range ch {
tmpDir := baseTempDir + alphaNumerics . ReplaceAllString ( branchName , "" )
imgID , err := buildDendrite ( httpClient , dockerClient , tmpDir , branchName )
if err != nil {
log . Fatalf ( "%s: failed to build dendrite image: %s" , branchName , err )
}
mu . Lock ( )
branchToImageID [ branchName ] = imgID
mu . Unlock ( )
}
} ( )
}
wg . Wait ( )
return branchToImageID
}
func runImage ( dockerClient * client . Client , volumeName , version , imageID string ) ( csAPIURL , containerID string , err error ) {
log . Printf ( "%s: running image %s\n" , version , imageID )
ctx , cancel := context . WithTimeout ( context . Background ( ) , 3 * time . Minute )
defer cancel ( )
body , err := dockerClient . ContainerCreate ( ctx , & container . Config {
Image : imageID ,
Env : [ ] string { "SERVER_NAME=hs1" } ,
2021-07-07 13:20:07 +00:00
Labels : map [ string ] string {
dendriteUpgradeTestLabel : "yes" ,
} ,
2021-07-07 11:06:17 +00:00
} , & container . HostConfig {
PublishAllPorts : true ,
Mounts : [ ] mount . Mount {
{
Type : mount . TypeVolume ,
Source : volumeName ,
Target : "/var/lib/postgresql/9.6/main" ,
} ,
} ,
} , nil , nil , "dendrite_upgrade_test_" + version )
if err != nil {
return "" , "" , fmt . Errorf ( "failed to ContainerCreate: %s" , err )
}
containerID = body . ID
err = dockerClient . ContainerStart ( ctx , containerID , types . ContainerStartOptions { } )
if err != nil {
return "" , "" , fmt . Errorf ( "failed to ContainerStart: %s" , err )
}
inspect , err := dockerClient . ContainerInspect ( ctx , containerID )
if err != nil {
return "" , "" , err
}
csapiPortInfo , ok := inspect . NetworkSettings . Ports [ nat . Port ( "8008/tcp" ) ]
if ! ok {
return "" , "" , fmt . Errorf ( "port 8008 not exposed - exposed ports: %v" , inspect . NetworkSettings . Ports )
}
2021-07-08 11:28:04 +00:00
baseURL := fmt . Sprintf ( "http://%s:%s" , * flagDockerHost , csapiPortInfo [ 0 ] . HostPort )
2021-07-07 11:06:17 +00:00
versionsURL := fmt . Sprintf ( "%s/_matrix/client/versions" , baseURL )
// hit /versions to check it is up
var lastErr error
for i := 0 ; i < 500 ; i ++ {
res , err := http . Get ( versionsURL )
if err != nil {
lastErr = fmt . Errorf ( "GET %s => error: %s" , versionsURL , err )
time . Sleep ( 50 * time . Millisecond )
continue
}
if res . StatusCode != 200 {
lastErr = fmt . Errorf ( "GET %s => HTTP %s" , versionsURL , res . Status )
time . Sleep ( 50 * time . Millisecond )
continue
}
lastErr = nil
break
}
if lastErr != nil {
logs , err := dockerClient . ContainerLogs ( context . Background ( ) , containerID , types . ContainerLogsOptions {
ShowStdout : true ,
ShowStderr : true ,
} )
// ignore errors when cannot get logs, it's just for debugging anyways
if err == nil {
2022-08-05 09:26:59 +00:00
logbody , err := io . ReadAll ( logs )
2021-07-07 11:06:17 +00:00
if err == nil {
log . Printf ( "Container logs:\n\n%s\n\n" , string ( logbody ) )
}
}
}
return baseURL , containerID , lastErr
}
func destroyContainer ( dockerClient * client . Client , containerID string ) {
err := dockerClient . ContainerRemove ( context . TODO ( ) , containerID , types . ContainerRemoveOptions {
Force : true ,
} )
if err != nil {
log . Printf ( "failed to remove container %s : %s" , containerID , err )
}
}
func loadAndRunTests ( dockerClient * client . Client , volumeName , v string , branchToImageID map [ string ] string ) error {
csAPIURL , containerID , err := runImage ( dockerClient , volumeName , v , branchToImageID [ v ] )
if err != nil {
return fmt . Errorf ( "failed to run container for branch %v: %v" , v , err )
}
defer destroyContainer ( dockerClient , containerID )
log . Printf ( "URL %s -> %s \n" , csAPIURL , containerID )
if err = runTests ( csAPIURL , v ) ; err != nil {
return fmt . Errorf ( "failed to run tests on version %s: %s" , v , err )
}
return nil
}
func verifyTests ( dockerClient * client . Client , volumeName string , versions [ ] string , branchToImageID map [ string ] string ) error {
lastVer := versions [ len ( versions ) - 1 ]
csAPIURL , containerID , err := runImage ( dockerClient , volumeName , lastVer , branchToImageID [ lastVer ] )
if err != nil {
return fmt . Errorf ( "failed to run container for branch %v: %v" , lastVer , err )
}
defer destroyContainer ( dockerClient , containerID )
return verifyTestsRan ( csAPIURL , versions )
}
2021-07-07 13:20:07 +00:00
// cleanup old containers/volumes from a previous run
func cleanup ( dockerClient * client . Client ) {
// ignore all errors, we are just cleaning up and don't want to fail just because we fail to cleanup
containers , _ := dockerClient . ContainerList ( context . Background ( ) , types . ContainerListOptions {
Filters : label ( dendriteUpgradeTestLabel ) ,
2022-01-21 09:56:06 +00:00
All : true ,
2021-07-07 13:20:07 +00:00
} )
for _ , c := range containers {
2022-01-21 09:56:06 +00:00
log . Printf ( "Removing container: %v %v\n" , c . ID , c . Names )
2021-07-07 13:20:07 +00:00
s := time . Second
_ = dockerClient . ContainerStop ( context . Background ( ) , c . ID , & s )
_ = dockerClient . ContainerRemove ( context . Background ( ) , c . ID , types . ContainerRemoveOptions {
Force : true ,
} )
}
_ = dockerClient . VolumeRemove ( context . Background ( ) , "dendrite_upgrade_test" , true )
}
func label ( in string ) filters . Args {
f := filters . NewArgs ( )
f . Add ( "label" , in )
return f
}
2021-07-07 11:06:17 +00:00
func main ( ) {
flag . Parse ( )
httpClient := & http . Client {
Timeout : 60 * time . Second ,
}
dockerClient , err := client . NewClientWithOpts ( client . FromEnv )
if err != nil {
log . Fatalf ( "failed to make docker client: %s" , err )
}
if * flagFrom == "" {
flag . Usage ( )
os . Exit ( 1 )
}
2021-07-07 13:20:07 +00:00
cleanup ( dockerClient )
2022-07-25 09:39:22 +00:00
versions := calculateVersions ( httpClient , * flagFrom , * flagTo , * flagDirect )
2021-07-07 11:06:17 +00:00
log . Printf ( "Testing dendrite versions: %v\n" , versions )
branchToImageID := buildDendriteImages ( httpClient , dockerClient , * flagTempDir , * flagBuildConcurrency , versions )
// make a shared postgres volume
volume , err := dockerClient . VolumeCreate ( context . Background ( ) , volume . VolumeCreateBody {
Name : "dendrite_upgrade_test" ,
2021-07-07 13:20:07 +00:00
Labels : map [ string ] string {
dendriteUpgradeTestLabel : "yes" ,
} ,
2021-07-07 11:06:17 +00:00
} )
if err != nil {
log . Fatalf ( "failed to make docker volume: %s" , err )
}
failed := false
defer func ( ) {
perr := recover ( )
log . Println ( "removing postgres volume" )
verr := dockerClient . VolumeRemove ( context . Background ( ) , volume . Name , true )
if perr == nil {
perr = verr
}
if perr != nil {
panic ( perr )
}
if failed {
os . Exit ( 1 )
}
} ( )
// run through images sequentially
for _ , v := range versions {
if err = loadAndRunTests ( dockerClient , volume . Name , v , branchToImageID ) ; err != nil {
log . Printf ( "failed to run tests for %v: %s\n" , v , err )
failed = true
break
}
}
if err := verifyTests ( dockerClient , volume . Name , versions , branchToImageID ) ; err != nil {
log . Printf ( "failed to verify test results: %s" , err )
failed = true
}
}