status deletes, profile updates (#30)

1. Proper DELETE of federated statuses (not yet deleting all the media and stuff -- i still have to implement this -- but the actual status is toast).
2. Proper UPDATE of profiles. When you change your profile picture on your remote instance, that will now register properly in GoToSocial.
3. Scrolling down the home timeline - it no longer just sort of ends, and will keep loading older statuses as you scroll.
4. Little bugfixes -- still had some nil pointer errors when dereferencing remote accounts.

internal/api/s2s/user/following.go

@@ -0,0 +1,58 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package user
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+)
+
+func (m *Module) FollowingGETHandler(c *gin.Context) {
+	l := m.log.WithFields(logrus.Fields{
+		"func": "FollowingGETHandler",
+		"url":  c.Request.RequestURI,
+	})
+
+	requestedUsername := c.Param(UsernameKey)
+	if requestedUsername == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "no username specified in request"})
+		return
+	}
+
+	// make sure this actually an AP request
+	format := c.NegotiateFormat(ActivityPubAcceptHeaders...)
+	if format == "" {
+		c.JSON(http.StatusNotAcceptable, gin.H{"error": "could not negotiate format with given Accept header(s)"})
+		return
+	}
+	l.Tracef("negotiated format: %s", format)
+
+	// make a copy of the context to pass along so we don't break anything
+	cp := c.Copy()
+	user, err := m.processor.GetFediFollowing(requestedUsername, cp.Request) // handles auth as well
+	if err != nil {
+		l.Info(err.Error())
+		c.JSON(err.Code(), gin.H{"error": err.Safe()})
+		return
+	}
+
+	c.JSON(http.StatusOK, user)
+}

internal/api/s2s/user/user.go

@@ -44,6 +44,8 @@ const (
 	UsersInboxPath = UsersBasePathWithUsername + "/" + util.InboxPath
 	// UsersFollowersPath is for serving GET request's to a user's followers list, with the given username key.
 	UsersFollowersPath = UsersBasePathWithUsername + "/" + util.FollowersPath
+	// UsersFollowingPath is for serving GET request's to a user's following list, with the given username key.
+	UsersFollowingPath = UsersBasePathWithUsername + "/" + util.FollowingPath
 	// UsersStatusPath is for serving GET requests to a particular status by a user, with the given username key and status ID
 	UsersStatusPath = UsersBasePathWithUsername + "/" + util.StatusesPath + "/:" + StatusIDKey
 )
@@ -76,6 +78,7 @@ func (m *Module) Route(s router.Router) error {
 	s.AttachHandler(http.MethodGet, UsersBasePathWithUsername, m.UsersGETHandler)
 	s.AttachHandler(http.MethodPost, UsersInboxPath, m.InboxPOSTHandler)
 	s.AttachHandler(http.MethodGet, UsersFollowersPath, m.FollowersGETHandler)
+	s.AttachHandler(http.MethodGet, UsersFollowingPath, m.FollowingGETHandler)
 	s.AttachHandler(http.MethodGet, UsersStatusPath, m.StatusGETHandler)
 	return nil
 }

internal/api/s2s/user/userget_test.go

@@ -145,7 +145,7 @@ func (suite *UserGetTestSuite) TestGetUser() {
 
 	// convert person to account
 	// since this account is already known, we should get a pretty full model of it from the conversion
-	a, err := suite.tc.ASRepresentationToAccount(person)
+	a, err := suite.tc.ASRepresentationToAccount(person, false)
 	assert.NoError(suite.T(), err)
 	assert.EqualValues(suite.T(), targetAccount.Username, a.Username)
 }

internal/db/pg/pg.go

@@ -500,6 +500,13 @@ func (ps *postgresService) GetStatusesByTimeDescending(accountID string, statuse
 			return q.Where("? IS NOT NULL", pg.Ident("attachments")).Where("attachments != '{}'"), nil
 		})
 	}
+	if maxID != "" {
+		s := &gtsmodel.Status{}
+		if err := ps.conn.Model(s).Where("id = ?", maxID).Select(); err != nil {
+			return err
+		}
+		q = q.Where("status.created_at < ?", s.CreatedAt)
+	}
 	if err := q.Select(); err != nil {
 		if err == pg.ErrNoRows {
 			return db.ErrNoEntries{}
@@ -1113,6 +1120,14 @@ func (ps *postgresService) GetHomeTimelineForAccount(accountID string, maxID str
 		Limit(limit).
 		Order("status.created_at DESC")
 
+	if maxID != "" {
+		s := &gtsmodel.Status{}
+		if err := ps.conn.Model(s).Where("id = ?", maxID).Select(); err != nil {
+			return nil, err
+		}
+		q = q.Where("status.created_at < ?", s.CreatedAt)
+	}
+
 	err := q.Select()
 	if err != nil {
 		if err != pg.ErrNoRows {

internal/federation/federating_db.go

@@ -241,6 +241,40 @@ func (f *federatingDB) Owns(c context.Context, id *url.URL) (bool, error) {
 		return true, nil
 	}
 
+	if util.IsFollowersPath(id) {
+		username, err := util.ParseFollowersPath(id)
+		if err != nil {
+			return false, fmt.Errorf("error parsing statuses path for url %s: %s", id.String(), err)
+		}
+		if err := f.db.GetLocalAccountByUsername(username, &gtsmodel.Account{}); err != nil {
+			if _, ok := err.(db.ErrNoEntries); ok {
+				// there are no entries for this username
+				return false, nil
+			}
+			// an actual error happened
+			return false, fmt.Errorf("database error fetching account with username %s: %s", username, err)
+		}
+		l.Debug("we DO own this")
+		return true, nil
+	}
+
+	if util.IsFollowingPath(id) {
+		username, err := util.ParseFollowingPath(id)
+		if err != nil {
+			return false, fmt.Errorf("error parsing statuses path for url %s: %s", id.String(), err)
+		}
+		if err := f.db.GetLocalAccountByUsername(username, &gtsmodel.Account{}); err != nil {
+			if _, ok := err.(db.ErrNoEntries); ok {
+				// there are no entries for this username
+				return false, nil
+			}
+			// an actual error happened
+			return false, fmt.Errorf("database error fetching account with username %s: %s", username, err)
+		}
+		l.Debug("we DO own this")
+		return true, nil
+	}
+
 	return false, fmt.Errorf("could not match activityID: %s", id.String())
 }
 
@@ -502,6 +536,15 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
 		l.Error("receiving account was set on context but couldn't be parsed")
 	}
 
+	requestingAcctI := ctx.Value(util.APRequestingAccount)
+	if receivingAcctI == nil {
+		l.Error("requesting account wasn't set on context")
+	}
+	requestingAcct, ok := requestingAcctI.(*gtsmodel.Account)
+	if !ok {
+		l.Error("requesting account was set on context but couldn't be parsed")
+	}
+
 	fromFederatorChanI := ctx.Value(util.APFromFederatorChanKey)
 	if fromFederatorChanI == nil {
 		l.Error("from federator channel wasn't set on context")
@@ -511,22 +554,64 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
 		l.Error("from federator channel was set on context but couldn't be parsed")
 	}
 
+	typeName := asType.GetTypeName()
+	if typeName == gtsmodel.ActivityStreamsApplication ||
+		typeName == gtsmodel.ActivityStreamsGroup ||
+		typeName == gtsmodel.ActivityStreamsOrganization ||
+		typeName == gtsmodel.ActivityStreamsPerson ||
+		typeName == gtsmodel.ActivityStreamsService {
+		// it's an UPDATE to some kind of account
+		var accountable typeutils.Accountable
+
 		switch asType.GetTypeName() {
-	case gtsmodel.ActivityStreamsUpdate:
+		case gtsmodel.ActivityStreamsApplication:
-		update, ok := asType.(vocab.ActivityStreamsCreate)
+			l.Debug("got update for APPLICATION")
+			i, ok := asType.(vocab.ActivityStreamsApplication)
 			if !ok {
-			return errors.New("could not convert type to create")
+				return errors.New("could not convert type to application")
 			}
-		object := update.GetActivityStreamsObject()
+			accountable = i
-		for objectIter := object.Begin(); objectIter != object.End(); objectIter = objectIter.Next() {
+		case gtsmodel.ActivityStreamsGroup:
-			switch objectIter.GetType().GetTypeName() {
+			l.Debug("got update for GROUP")
-			case string(gtsmodel.ActivityStreamsPerson):
+			i, ok := asType.(vocab.ActivityStreamsGroup)
-				person := objectIter.GetActivityStreamsPerson()
+			if !ok {
-				updatedAcct, err := f.typeConverter.ASRepresentationToAccount(person)
+				return errors.New("could not convert type to group")
+			}
+			accountable = i
+		case gtsmodel.ActivityStreamsOrganization:
+			l.Debug("got update for ORGANIZATION")
+			i, ok := asType.(vocab.ActivityStreamsOrganization)
+			if !ok {
+				return errors.New("could not convert type to organization")
+			}
+			accountable = i
+		case gtsmodel.ActivityStreamsPerson:
+			l.Debug("got update for PERSON")
+			i, ok := asType.(vocab.ActivityStreamsPerson)
+			if !ok {
+				return errors.New("could not convert type to person")
+			}
+			accountable = i
+		case gtsmodel.ActivityStreamsService:
+			l.Debug("got update for SERVICE")
+			i, ok := asType.(vocab.ActivityStreamsService)
+			if !ok {
+				return errors.New("could not convert type to service")
+			}
+			accountable = i
+		}
+
+		updatedAcct, err := f.typeConverter.ASRepresentationToAccount(accountable, true)
 		if err != nil {
-					return fmt.Errorf("error converting person to account: %s", err)
+			return fmt.Errorf("error converting to account: %s", err)
 		}
-				if err := f.db.Put(updatedAcct); err != nil {
+
+		if requestingAcct.URI != updatedAcct.URI {
+			return fmt.Errorf("update for account %s was requested by account %s, this is not valid", updatedAcct.URI, requestingAcct.URI)
+		}
+
+		updatedAcct.ID = requestingAcct.ID // set this here so the db will update properly instead of trying to PUT this and getting constraint issues
+		if err := f.db.UpdateByID(requestingAcct.ID, updatedAcct); err != nil {
 			return fmt.Errorf("database error inserting updated account: %s", err)
 		}
 
@@ -537,25 +622,8 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
 			ReceivingAccount: receivingAcct,
 		}
 
-			case string(gtsmodel.ActivityStreamsApplication):
-				application := objectIter.GetActivityStreamsApplication()
-				updatedAcct, err := f.typeConverter.ASRepresentationToAccount(application)
-				if err != nil {
-					return fmt.Errorf("error converting person to account: %s", err)
-				}
-				if err := f.db.Put(updatedAcct); err != nil {
-					return fmt.Errorf("database error inserting updated account: %s", err)
 	}
 
-				fromFederatorChan <- gtsmodel.FromFederator{
-					APObjectType:     gtsmodel.ActivityStreamsProfile,
-					APActivityType:   gtsmodel.ActivityStreamsUpdate,
-					GTSModel:         updatedAcct,
-					ReceivingAccount: receivingAcct,
-				}
-			}
-		}
-	}
 	return nil
 }
 
@@ -565,7 +633,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
 // Protocol instead call Update to create a Tombstone.
 //
 // The library makes this call only after acquiring a lock first.
-func (f *federatingDB) Delete(c context.Context, id *url.URL) error {
+func (f *federatingDB) Delete(ctx context.Context, id *url.URL) error {
 	l := f.log.WithFields(
 		logrus.Fields{
 			"func": "Delete",
@@ -573,6 +641,63 @@ func (f *federatingDB) Delete(c context.Context, id *url.URL) error {
 		},
 	)
 	l.Debugf("received DELETE id %s", id.String())
+
+	inboxAcctI := ctx.Value(util.APAccount)
+	if inboxAcctI == nil {
+		l.Error("inbox account wasn't set on context")
+		return nil
+	}
+	inboxAcct, ok := inboxAcctI.(*gtsmodel.Account)
+	if !ok {
+		l.Error("inbox account was set on context but couldn't be parsed")
+		return nil
+	}
+
+	fromFederatorChanI := ctx.Value(util.APFromFederatorChanKey)
+	if fromFederatorChanI == nil {
+		l.Error("from federator channel wasn't set on context")
+		return nil
+	}
+	fromFederatorChan, ok := fromFederatorChanI.(chan gtsmodel.FromFederator)
+	if !ok {
+		l.Error("from federator channel was set on context but couldn't be parsed")
+		return nil
+	}
+
+	// in a delete we only get the URI, we can't know if we have a status or a profile or something else,
+	// so we have to try a few different things...
+	where := []db.Where{{Key: "uri", Value: id.String()}}
+
+	s := &gtsmodel.Status{}
+	if err := f.db.GetWhere(where, s); err == nil {
+		// it's a status
+		l.Debugf("uri is for status with id: %s", s.ID)
+		if err := f.db.DeleteByID(s.ID, &gtsmodel.Status{}); err != nil {
+			return fmt.Errorf("Delete: err deleting status: %s", err)
+		}
+		fromFederatorChan <- gtsmodel.FromFederator{
+			APObjectType:     gtsmodel.ActivityStreamsNote,
+			APActivityType:   gtsmodel.ActivityStreamsDelete,
+			GTSModel:         s,
+			ReceivingAccount: inboxAcct,
+		}
+	}
+
+	a := &gtsmodel.Account{}
+	if err := f.db.GetWhere(where, a); err == nil {
+		// it's an account
+		l.Debugf("uri is for an account with id: %s", s.ID)
+		if err := f.db.DeleteByID(a.ID, &gtsmodel.Account{}); err != nil {
+			return fmt.Errorf("Delete: err deleting account: %s", err)
+		}
+		fromFederatorChan <- gtsmodel.FromFederator{
+			APObjectType:     gtsmodel.ActivityStreamsProfile,
+			APActivityType:   gtsmodel.ActivityStreamsDelete,
+			GTSModel:         a,
+			ReceivingAccount: inboxAcct,
+		}
+	}
+
 	return nil
 }
 

internal/federation/federatingprotocol.go

@@ -136,7 +136,7 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
 			return ctx, false, fmt.Errorf("error dereferencing account with public key id %s: %s", publicKeyOwnerURI.String(), err)
 		}
 
-		a, err := f.typeConverter.ASRepresentationToAccount(person)
+		a, err := f.typeConverter.ASRepresentationToAccount(person, false)
 		if err != nil {
 			return ctx, false, fmt.Errorf("error converting person with public key id %s to account: %s", publicKeyOwnerURI.String(), err)
 		}

internal/federation/util.go

@@ -132,9 +132,11 @@ func (f *federator) AuthenticateFederatedRequest(username string, r *http.Reques
 
 	var publicKey interface{}
 	var pkOwnerURI *url.URL
-	if strings.EqualFold(requestingPublicKeyID.Host, f.config.Host) {
+	requestingRemoteAccount := &gtsmodel.Account{}
-		// the request is coming from INSIDE THE HOUSE so skip the remote dereferencing
 	requestingLocalAccount := &gtsmodel.Account{}
+	if strings.EqualFold(requestingPublicKeyID.Host, f.config.Host) {
+		// LOCAL ACCOUNT REQUEST
+		// the request is coming from INSIDE THE HOUSE so skip the remote dereferencing
 		if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingLocalAccount); err != nil {
 			return nil, fmt.Errorf("couldn't get local account with public key uri %s from the database: %s", requestingPublicKeyID.String(), err)
 		}
@@ -143,8 +145,18 @@ func (f *federator) AuthenticateFederatedRequest(username string, r *http.Reques
 		if err != nil {
 			return nil, fmt.Errorf("error parsing url %s: %s", requestingLocalAccount.URI, err)
 		}
+	} else if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingRemoteAccount); err == nil {
+		// REMOTE ACCOUNT REQUEST WITH KEY CACHED LOCALLY
+		// this is a remote account and we already have the public key for it so use that
+		publicKey = requestingRemoteAccount.PublicKey
+		pkOwnerURI, err = url.Parse(requestingRemoteAccount.URI)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing url %s: %s", requestingRemoteAccount.URI, err)
+		}
 	} else {
-		// the request is remote, so we need to authenticate the request properly by dereferencing the remote key
+		// REMOTE ACCOUNT REQUEST WITHOUT KEY CACHED LOCALLY
+		// the request is remote and we don't have the public key yet,
+		// so we need to authenticate the request properly by dereferencing the remote key
 		transport, err := f.GetTransportForUser(username)
 		if err != nil {
 			return nil, fmt.Errorf("transport err: %s", err)

internal/message/accountprocess.go

@@ -83,7 +83,7 @@ func (p *processor) AccountGet(authed *oauth.Auth, targetAccountID string) (*api
 	if authed.Account != nil {
 		requestingUsername = authed.Account.Username
 	}
-	if err := p.dereferenceAccountFields(targetAccount, requestingUsername); err != nil {
+	if err := p.dereferenceAccountFields(targetAccount, requestingUsername, false); err != nil {
 		p.log.WithField("func", "AccountGet").Debugf("dereferencing account: %s", err)
 	}
 
@@ -295,7 +295,7 @@ func (p *processor) AccountFollowersGet(authed *oauth.Auth, targetAccountID stri
 		}
 
 		// derefence account fields in case we haven't done it already
-		if err := p.dereferenceAccountFields(a, authed.Account.Username); err != nil {
+		if err := p.dereferenceAccountFields(a, authed.Account.Username, false); err != nil {
 			// don't bail if we can't fetch them, we'll try another time
 			p.log.WithField("func", "AccountFollowersGet").Debugf("error dereferencing account fields: %s", err)
 		}
@@ -346,7 +346,7 @@ func (p *processor) AccountFollowingGet(authed *oauth.Auth, targetAccountID stri
 		}
 
 		// derefence account fields in case we haven't done it already
-		if err := p.dereferenceAccountFields(a, authed.Account.Username); err != nil {
+		if err := p.dereferenceAccountFields(a, authed.Account.Username, false); err != nil {
 			// don't bail if we can't fetch them, we'll try another time
 			p.log.WithField("func", "AccountFollowingGet").Debugf("error dereferencing account fields: %s", err)
 		}

internal/message/fediprocess.go

@@ -68,7 +68,7 @@ func (p *processor) authenticateAndDereferenceFediRequest(username string, r *ht
 	}
 
 	// convert it to our internal account representation
-	requestingAccount, err = p.tc.ASRepresentationToAccount(requestingPerson)
+	requestingAccount, err = p.tc.ASRepresentationToAccount(requestingPerson, false)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't convert dereferenced uri %s to gtsmodel account: %s", requestingAccountURI.String(), err)
 	}
@@ -163,6 +163,46 @@ func (p *processor) GetFediFollowers(requestedUsername string, request *http.Req
 	return data, nil
 }
 
+func (p *processor) GetFediFollowing(requestedUsername string, request *http.Request) (interface{}, ErrorWithCode) {
+	// get the account the request is referring to
+	requestedAccount := &gtsmodel.Account{}
+	if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil {
+		return nil, NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err))
+	}
+
+	// authenticate the request
+	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	if err != nil {
+		return nil, NewErrorNotAuthorized(err)
+	}
+
+	blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID))
+	}
+
+	requestedAccountURI, err := url.Parse(requestedAccount.URI)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("error parsing url %s: %s", requestedAccount.URI, err))
+	}
+
+	requestedFollowing, err := p.federator.FederatingDB().Following(context.Background(), requestedAccountURI)
+	if err != nil {
+		return nil, NewErrorInternalError(fmt.Errorf("error fetching following for uri %s: %s", requestedAccountURI.String(), err))
+	}
+
+	data, err := streams.Serialize(requestedFollowing)
+	if err != nil {
+		return nil, NewErrorInternalError(err)
+	}
+
+	return data, nil
+}
+
 func (p *processor) GetFediStatus(requestedUsername string, requestedStatusID string, request *http.Request) (interface{}, ErrorWithCode) {
 	// get the account the request is referring to
 	requestedAccount := &gtsmodel.Account{}

internal/message/fromfederatorprocess.go

@@ -68,7 +68,7 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			}
 
 			l.Debug("will now derefence incoming account")
-			if err := p.dereferenceAccountFields(incomingAccount, ""); err != nil {
+			if err := p.dereferenceAccountFields(incomingAccount, "", false); err != nil {
 				return fmt.Errorf("error dereferencing account from federator: %s", err)
 			}
 			if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
@@ -86,13 +86,26 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			}
 
 			l.Debug("will now derefence incoming account")
-			if err := p.dereferenceAccountFields(incomingAccount, ""); err != nil {
+			if err := p.dereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil {
 				return fmt.Errorf("error dereferencing account from federator: %s", err)
 			}
 			if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
 				return fmt.Errorf("error updating dereferenced account in the db: %s", err)
 			}
 		}
+	case gtsmodel.ActivityStreamsDelete:
+		// DELETE
+		switch federatorMsg.APObjectType {
+		case gtsmodel.ActivityStreamsNote:
+			// DELETE A STATUS
+			// TODO: handle side effects of status deletion here:
+			// 1. delete all media associated with status
+			// 2. delete boosts of status
+			// 3. etc etc etc
+		case gtsmodel.ActivityStreamsProfile:
+			// DELETE A PROFILE/ACCOUNT
+			// TODO: handle side effects of account deletion here: delete all objects, statuses, media etc associated with account
+		}
 	}
 
 	return nil
@@ -220,7 +233,7 @@ func (p *processor) dereferenceStatusFields(status *gtsmodel.Status) error {
 				continue
 			}
 
-			targetAccount, err = p.tc.ASRepresentationToAccount(accountable)
+			targetAccount, err = p.tc.ASRepresentationToAccount(accountable, false)
 			if err != nil {
 				l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err)
 				continue
@@ -243,7 +256,7 @@ func (p *processor) dereferenceStatusFields(status *gtsmodel.Status) error {
 	return nil
 }
 
-func (p *processor) dereferenceAccountFields(account *gtsmodel.Account, requestingUsername string) error {
+func (p *processor) dereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error {
 	l := p.log.WithFields(logrus.Fields{
 		"func":               "dereferenceAccountFields",
 		"requestingUsername": requestingUsername,
@@ -255,7 +268,7 @@ func (p *processor) dereferenceAccountFields(account *gtsmodel.Account, requesti
 	}
 
 	// fetch the header and avatar
-	if err := p.fetchHeaderAndAviForAccount(account, t); err != nil {
+	if err := p.fetchHeaderAndAviForAccount(account, t, refresh); err != nil {
 		// if this doesn't work, just skip it -- we can do it later
 		l.Debugf("error fetching header/avi for account: %s", err)
 	}

internal/message/processor.go

@@ -140,6 +140,10 @@ type Processor interface {
 	// authentication before returning a JSON serializable interface to the caller.
 	GetFediFollowers(requestedUsername string, request *http.Request) (interface{}, ErrorWithCode)
 
+	// GetFediFollowing handles the getting of a fedi/activitypub representation of a user/account's following, performing appropriate
+	// authentication before returning a JSON serializable interface to the caller.
+	GetFediFollowing(requestedUsername string, request *http.Request) (interface{}, ErrorWithCode)
+
 	// GetFediStatus handles the getting of a fedi/activitypub representation of a particular status, performing appropriate
 	// authentication before returning a JSON serializable interface to the caller.
 	GetFediStatus(requestedUsername string, requestedStatusID string, request *http.Request) (interface{}, ErrorWithCode)

internal/message/processorutil.go

@@ -130,9 +130,11 @@ func (p *processor) processReplyToID(form *apimodel.AdvancedStatusCreateForm, th
 		return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err)
 	}
 
+	if repliedStatus.VisibilityAdvanced != nil {
 		if !repliedStatus.VisibilityAdvanced.Replyable {
 			return fmt.Errorf("status with id %s is marked as not replyable", form.InReplyToID)
 		}
+	}
 
 	// check replied account is known to us
 	if err := p.db.GetByID(repliedStatus.AccountID, repliedAccount); err != nil {
@@ -329,8 +331,8 @@ func (p *processor) updateAccountHeader(header *multipart.FileHeader, accountID
 //
 // SIDE EFFECTS: remote header and avatar will be stored in local storage, and the database will be updated
 // to reflect the creation of these new attachments.
-func (p *processor) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport) error {
+func (p *processor) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error {
-	if targetAccount.AvatarRemoteURL != "" && targetAccount.AvatarMediaAttachmentID == "" {
+	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
 		a, err := p.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
 			RemoteURL: targetAccount.AvatarRemoteURL,
 			Avatar:    true,
@@ -341,7 +343,7 @@ func (p *processor) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account,
 		targetAccount.AvatarMediaAttachmentID = a.ID
 	}
 
-	if targetAccount.HeaderRemoteURL != "" && targetAccount.HeaderMediaAttachmentID == "" {
+	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
 		a, err := p.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
 			RemoteURL: targetAccount.HeaderRemoteURL,
 			Header:    true,

internal/typeutils/asextractionutil.go

@@ -29,7 +29,6 @@ import (
 	"time"
 
 	"github.com/go-fed/activity/pub"
-	"github.com/go-fed/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
@@ -63,6 +62,9 @@ func extractName(i withName) (string, error) {
 
 func extractInReplyToURI(i withInReplyTo) (*url.URL, error) {
 	inReplyToProp := i.GetActivityStreamsInReplyTo()
+	if inReplyToProp == nil {
+		return nil, errors.New("in reply to prop was nil")
+	}
 	for iter := inReplyToProp.Begin(); iter != inReplyToProp.End(); iter = iter.Next() {
 		if iter.IsIRI() {
 			if iter.GetIRI() != nil {
@@ -76,6 +78,9 @@ func extractInReplyToURI(i withInReplyTo) (*url.URL, error) {
 func extractTos(i withTo) ([]*url.URL, error) {
 	to := []*url.URL{}
 	toProp := i.GetActivityStreamsTo()
+	if toProp == nil {
+		return nil, errors.New("toProp was nil")
+	}
 	for iter := toProp.Begin(); iter != toProp.End(); iter = iter.Next() {
 		if iter.IsIRI() {
 			if iter.GetIRI() != nil {
@@ -89,6 +94,9 @@ func extractTos(i withTo) ([]*url.URL, error) {
 func extractCCs(i withCC) ([]*url.URL, error) {
 	cc := []*url.URL{}
 	ccProp := i.GetActivityStreamsCc()
+	if ccProp == nil {
+		return cc, nil
+	}
 	for iter := ccProp.Begin(); iter != ccProp.End(); iter = iter.Next() {
 		if iter.IsIRI() {
 			if iter.GetIRI() != nil {
@@ -101,6 +109,9 @@ func extractCCs(i withCC) ([]*url.URL, error) {
 
 func extractAttributedTo(i withAttributedTo) (*url.URL, error) {
 	attributedToProp := i.GetActivityStreamsAttributedTo()
+	if attributedToProp == nil {
+		return nil, errors.New("attributedToProp was nil")
+	}
 	for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
 		if iter.IsIRI() {
 			if iter.GetIRI() != nil {
@@ -302,27 +313,21 @@ func extractContent(i withContent) (string, error) {
 
 func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) {
 	attachments := []*gtsmodel.MediaAttachment{}
-
 	attachmentProp := i.GetActivityStreamsAttachment()
+	if attachmentProp == nil {
+		return attachments, nil
+	}
 	for iter := attachmentProp.Begin(); iter != attachmentProp.End(); iter = iter.Next() {
-
 		t := iter.GetType()
 		if t == nil {
-			fmt.Printf("\n\n\nGetType() nil\n\n\n")
 			continue
 		}
-
-		m, _ := streams.Serialize(t)
-		fmt.Printf("\n\n\n%s\n\n\n", m)
-
 		attachmentable, ok := t.(Attachmentable)
 		if !ok {
-			fmt.Printf("\n\n\nnot attachmentable\n\n\n")
 			continue
 		}
 		attachment, err := extractAttachment(attachmentable)
 		if err != nil {
-			fmt.Printf("\n\n\n%s\n\n\n", err)
 			continue
 		}
 		attachments = append(attachments, attachment)
@@ -373,8 +378,10 @@ func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
 
 func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) {
 	tags := []*gtsmodel.Tag{}
-
 	tagsProp := i.GetActivityStreamsTag()
+	if tagsProp == nil {
+		return tags, nil
+	}
 	for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() {
 		t := iter.GetType()
 		if t == nil {
@@ -421,6 +428,9 @@ func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
 func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) {
 	emojis := []*gtsmodel.Emoji{}
 	tagsProp := i.GetActivityStreamsTag()
+	if tagsProp == nil {
+		return emojis, nil
+	}
 	for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() {
 		t := iter.GetType()
 		if t == nil {
@@ -478,6 +488,9 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) {
 func extractMentions(i withTag) ([]*gtsmodel.Mention, error) {
 	mentions := []*gtsmodel.Mention{}
 	tagsProp := i.GetActivityStreamsTag()
+	if tagsProp == nil {
+		return mentions, nil
+	}
 	for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() {
 		t := iter.GetType()
 		if t == nil {

internal/typeutils/astointernal.go

@@ -28,7 +28,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-func (c *converter) ASRepresentationToAccount(accountable Accountable) (*gtsmodel.Account, error) {
+func (c *converter) ASRepresentationToAccount(accountable Accountable, update bool) (*gtsmodel.Account, error) {
 	// first check if we actually already know this account
 	uriProp := accountable.GetJSONLDId()
 	if uriProp == nil || !uriProp.IsIRI() {
@@ -37,6 +37,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable) (*gtsmode
 	uri := uriProp.GetIRI()
 
 	acct := &gtsmodel.Account{}
+	if !update {
 		err := c.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, acct)
 		if err == nil {
 			// we already know this account so we can skip generating it
@@ -46,8 +47,9 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable) (*gtsmode
 			// we don't know the account and there's been a real error
 			return nil, fmt.Errorf("error getting account with uri %s from the database: %s", uri.String(), err)
 		}
+	}
 
-	// we don't know the account so we need to generate it from the person -- at least we already have the URI!
+	// we don't know the account, or we're being told to update it, so we need to generate it from the person -- at least we already have the URI!
 	acct = &gtsmodel.Account{}
 	acct.URI = uri.String()
 

internal/typeutils/astointernal_test.go

@@ -349,7 +349,7 @@ func (suite *ASToInternalTestSuite) TestParsePerson() {
 
 	testPerson := suite.people["new_person_1"]
 
-	acct, err := suite.typeconverter.ASRepresentationToAccount(testPerson)
+	acct, err := suite.typeconverter.ASRepresentationToAccount(testPerson, false)
 	assert.NoError(suite.T(), err)
 
 	fmt.Printf("%+v", acct)
@@ -367,7 +367,7 @@ func (suite *ASToInternalTestSuite) TestParseGargron() {
 	rep, ok := t.(typeutils.Accountable)
 	assert.True(suite.T(), ok)
 
-	acct, err := suite.typeconverter.ASRepresentationToAccount(rep)
+	acct, err := suite.typeconverter.ASRepresentationToAccount(rep, false)
 	assert.NoError(suite.T(), err)
 
 	fmt.Printf("%+v", acct)

internal/typeutils/converter.go

@@ -95,8 +95,12 @@ type TypeConverter interface {
 		ACTIVITYSTREAMS MODEL TO INTERNAL (gts) MODEL
 	*/
 
-	// ASPersonToAccount converts a remote account/person/application representation into a gts model account
+	// ASPersonToAccount converts a remote account/person/application representation into a gts model account.
-	ASRepresentationToAccount(accountable Accountable) (*gtsmodel.Account, error)
+	//
+	// If update is false, and the account is already known in the database, then the existing account entry will be returned.
+	// If update is true, then even if the account is already known, all fields in the accountable will be parsed and a new *gtsmodel.Account
+	// will be generated. This is useful when one needs to force refresh of an account, eg., during an Update of a Profile.
+	ASRepresentationToAccount(accountable Accountable, update bool) (*gtsmodel.Account, error)
 	// ASStatus converts a remote activitystreams 'status' representation into a gts model status.
 	ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error)
 	// ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request.

internal/typeutils/internaltofrontend.go

@@ -233,9 +233,9 @@ func (c *converter) MentionToMasto(m *gtsmodel.Mention) (model.Mention, error) {
 
 	var acct string
 	if local {
-		acct = fmt.Sprintf("@%s", target.Username)
+		acct = target.Username
 	} else {
-		acct = fmt.Sprintf("@%s@%s", target.Username, target.Domain)
+		acct = fmt.Sprintf("%s@%s", target.Username, target.Domain)
 	}
 
 	return model.Mention{

internal/util/uri.go

@@ -232,3 +232,25 @@ func ParseOutboxPath(id *url.URL) (username string, err error) {
 	username = matches[1]
 	return
 }
+
+// ParseFollowersPath returns the username from a path such as /users/example_username/followers
+func ParseFollowersPath(id *url.URL) (username string, err error) {
+	matches := followersPathRegex.FindStringSubmatch(id.Path)
+	if len(matches) != 2 {
+		err = fmt.Errorf("expected 2 matches but matches length was %d", len(matches))
+		return
+	}
+	username = matches[1]
+	return
+}
+
+// ParseFollowingPath returns the username from a path such as /users/example_username/following
+func ParseFollowingPath(id *url.URL) (username string, err error) {
+	matches := followingPathRegex.FindStringSubmatch(id.Path)
+	if len(matches) != 2 {
+		err = fmt.Errorf("expected 2 matches but matches length was %d", len(matches))
+		return
+	}
+	username = matches[1]
+	return
+}