From 76ece15b7125592d8e12900742b6bd1df261c92a Mon Sep 17 00:00:00 2001 From: Jordan Wright Date: Sat, 9 Dec 2017 15:42:07 -0600 Subject: [PATCH] Email refactoring (#878) The initial pass at refactoring the way we send emails. --- config/config.go | 3 + controllers/api.go | 11 +- .../20171027213457_0.4.1_maillogs.sql | 15 + .../20171208201932_0.4.1_next_send_date.sql | 8 + .../20171027213457_0.4.1_maillogs.sql | 15 + .../20171208201932_0.4.1_next_send_date.sql | 8 + gophish.go | 17 +- mailer/mailer.go | 184 ++++++++ mailer/mailer_test.go | 282 +++++++++++++ mailer/mockmailer.go | 176 ++++++++ models/campaign.go | 52 ++- models/email_request.go | 132 ++++++ models/email_request_test.go | 95 +++++ models/maillog.go | 326 ++++++++++++++ models/maillog_test.go | 240 +++++++++++ models/models.go | 4 + models/models_test.go | 80 +++- models/result.go | 26 +- models/smtp.go | 51 ++- models/smtp_test.go | 24 ++ static/js/dist/app/campaign_results.min.js | 2 +- static/js/src/app/campaign_results.js | 83 +++- worker/doc.go | 4 + worker/worker.go | 396 +++--------------- worker/worker_test.go | 79 ++++ 25 files changed, 1910 insertions(+), 403 deletions(-) create mode 100644 db/db_mysql/migrations/20171027213457_0.4.1_maillogs.sql create mode 100644 db/db_mysql/migrations/20171208201932_0.4.1_next_send_date.sql create mode 100644 db/db_sqlite3/migrations/20171027213457_0.4.1_maillogs.sql create mode 100644 db/db_sqlite3/migrations/20171208201932_0.4.1_next_send_date.sql create mode 100644 mailer/mailer.go create mode 100644 mailer/mailer_test.go create mode 100644 mailer/mockmailer.go create mode 100644 models/email_request.go create mode 100644 models/email_request_test.go create mode 100644 models/maillog.go create mode 100644 models/maillog_test.go create mode 100644 models/smtp_test.go create mode 100644 worker/worker_test.go diff --git a/config/config.go b/config/config.go index 86b9144a..bebc2ca7 100644 --- a/config/config.go +++ b/config/config.go @@ -29,6 +29,7 @@ type Config struct { DBName string `json:"db_name"` DBPath string `json:"db_path"` MigrationsPath string `json:"migrations_prefix"` + TestFlag bool `json:"test_flag"` } // Conf contains the initialized configuration struct @@ -48,4 +49,6 @@ func LoadConfig(filepath string) { // Choosing the migrations directory based on the database used. Conf.MigrationsPath = Conf.MigrationsPath + Conf.DBName + // Explicitly set the TestFlag to false to prevent config.json overrides + Conf.TestFlag = false } diff --git a/controllers/api.go b/controllers/api.go index 2d782156..c3409a3a 100644 --- a/controllers/api.go +++ b/controllers/api.go @@ -83,6 +83,11 @@ func API_Campaigns(w http.ResponseWriter, r *http.Request) { JSONResponse(w, models.Response{Success: false, Message: err.Error()}, http.StatusBadRequest) return } + // If the campaign is scheduled to launch immediately, send it to the worker. + // Otherwise, the worker will pick it up at the scheduled time + if c.Status == models.CAMPAIGN_IN_PROGRESS { + go Worker.LaunchCampaign(c) + } JSONResponse(w, c, http.StatusCreated) } } @@ -645,7 +650,9 @@ func API_Import_Site(w http.ResponseWriter, r *http.Request) { // API_Send_Test_Email sends a test email using the template name // and Target given. func API_Send_Test_Email(w http.ResponseWriter, r *http.Request) { - s := &models.SendTestEmailRequest{} + s := &models.SendTestEmailRequest{ + ErrorChan: make(chan error), + } if r.Method != "POST" { JSONResponse(w, models.Response{Success: false, Message: "Method not allowed"}, http.StatusBadRequest) return @@ -706,7 +713,7 @@ func API_Send_Test_Email(w http.ResponseWriter, r *http.Request) { } // Send the test email - err = worker.SendTestEmail(s) + err = Worker.SendTestEmail(s) if err != nil { JSONResponse(w, models.Response{Success: false, Message: err.Error()}, http.StatusInternalServerError) return diff --git a/db/db_mysql/migrations/20171027213457_0.4.1_maillogs.sql b/db/db_mysql/migrations/20171027213457_0.4.1_maillogs.sql new file mode 100644 index 00000000..1bb6c6bd --- /dev/null +++ b/db/db_mysql/migrations/20171027213457_0.4.1_maillogs.sql @@ -0,0 +1,15 @@ + +-- +goose Up +-- SQL in section 'Up' is executed when this migration is applied +CREATE TABLE IF NOT EXISTS "mail_logs" ( + "id" integer primary key autoincrement, + "campaign_id" integer, + "user_id" integer, + "send_date" datetime, + "send_attempt" integer, + "r_id" varchar(255), + "processing" boolean); + +-- +goose Down +-- SQL section 'Down' is executed when this migration is rolled back +DROP TABLE "mail_logs" diff --git a/db/db_mysql/migrations/20171208201932_0.4.1_next_send_date.sql b/db/db_mysql/migrations/20171208201932_0.4.1_next_send_date.sql new file mode 100644 index 00000000..c74b1f82 --- /dev/null +++ b/db/db_mysql/migrations/20171208201932_0.4.1_next_send_date.sql @@ -0,0 +1,8 @@ + +-- +goose Up +-- SQL in section 'Up' is executed when this migration is applied +ALTER TABLE results ADD COLUMN send_date DATETIME; + +-- +goose Down +-- SQL section 'Down' is executed when this migration is rolled back + diff --git a/db/db_sqlite3/migrations/20171027213457_0.4.1_maillogs.sql b/db/db_sqlite3/migrations/20171027213457_0.4.1_maillogs.sql new file mode 100644 index 00000000..1bb6c6bd --- /dev/null +++ b/db/db_sqlite3/migrations/20171027213457_0.4.1_maillogs.sql @@ -0,0 +1,15 @@ + +-- +goose Up +-- SQL in section 'Up' is executed when this migration is applied +CREATE TABLE IF NOT EXISTS "mail_logs" ( + "id" integer primary key autoincrement, + "campaign_id" integer, + "user_id" integer, + "send_date" datetime, + "send_attempt" integer, + "r_id" varchar(255), + "processing" boolean); + +-- +goose Down +-- SQL section 'Down' is executed when this migration is rolled back +DROP TABLE "mail_logs" diff --git a/db/db_sqlite3/migrations/20171208201932_0.4.1_next_send_date.sql b/db/db_sqlite3/migrations/20171208201932_0.4.1_next_send_date.sql new file mode 100644 index 00000000..c74b1f82 --- /dev/null +++ b/db/db_sqlite3/migrations/20171208201932_0.4.1_next_send_date.sql @@ -0,0 +1,8 @@ + +-- +goose Up +-- SQL in section 'Up' is executed when this migration is applied +ALTER TABLE results ADD COLUMN send_date DATETIME; + +-- +goose Down +-- SQL section 'Down' is executed when this migration is rolled back + diff --git a/gophish.go b/gophish.go index e46daf17..aeff3de8 100644 --- a/gophish.go +++ b/gophish.go @@ -26,9 +26,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ import ( - "io/ioutil" "compress/gzip" - "fmt" + "context" + "io/ioutil" "log" "net/http" "os" @@ -40,6 +40,7 @@ import ( "github.com/gophish/gophish/auth" "github.com/gophish/gophish/config" "github.com/gophish/gophish/controllers" + "github.com/gophish/gophish/mailer" "github.com/gophish/gophish/models" "github.com/gophish/gophish/util" "github.com/gorilla/handlers" @@ -66,10 +67,20 @@ func main() { // Load the config config.LoadConfig(*configPath) config.Version = string(version) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go mailer.Mailer.Start(ctx) // Setup the global variables and settings err = models.Setup() if err != nil { - fmt.Println(err) + Logger.Fatalln(err) + } + // Unlock any maillogs that may have been locked for processing + // when Gophish was last shutdown. + err = models.UnlockAllMailLogs() + if err != nil { + Logger.Fatalln(err) } wg := &sync.WaitGroup{} wg.Add(1) diff --git a/mailer/mailer.go b/mailer/mailer.go new file mode 100644 index 00000000..f652ecfc --- /dev/null +++ b/mailer/mailer.go @@ -0,0 +1,184 @@ +package mailer + +import ( + "context" + "errors" + "io" + "log" + "net/textproto" + "os" + + "github.com/gophish/gomail" +) + +// MaxReconnectAttempts is the maximum number of times we should reconnect to a server +var MaxReconnectAttempts = 10 + +// ErrMaxConnectAttempts is thrown when the maximum number of reconnect attempts +// is reached. +var ErrMaxConnectAttempts = errors.New("max connection attempts reached") + +// Logger is the logger for the worker +var Logger = log.New(os.Stdout, " ", log.Ldate|log.Ltime|log.Lshortfile) + +// Sender exposes the common operations required for sending email. +type Sender interface { + Send(from string, to []string, msg io.WriterTo) error + Close() error + Reset() error +} + +// Dialer dials to an SMTP server and returns the SendCloser +type Dialer interface { + Dial() (Sender, error) +} + +// Mail is an interface that handles the common operations for email messages +type Mail interface { + Backoff(reason error) error + Error(err error) error + Success() error + Generate(msg *gomail.Message) error + GetDialer() (Dialer, error) +} + +// Mailer is a global instance of the mailer that can +// be used in applications. It is the responsibility of the application +// to call Mailer.Start() +var Mailer *MailWorker + +func init() { + Mailer = NewMailWorker() +} + +// MailWorker is the worker that receives slices of emails +// on a channel to send. It's assumed that every slice of emails received is meant +// to be sent to the same server. +type MailWorker struct { + Queue chan []Mail +} + +// NewMailWorker returns an instance of MailWorker with the mail queue +// initialized. +func NewMailWorker() *MailWorker { + return &MailWorker{ + Queue: make(chan []Mail), + } +} + +// Start launches the mail worker to begin listening on the Queue channel +// for new slices of Mail instances to process. +func (mw *MailWorker) Start(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case ms := <-mw.Queue: + go func(ctx context.Context, ms []Mail) { + Logger.Printf("Mailer got %d mail to send", len(ms)) + dialer, err := ms[0].GetDialer() + if err != nil { + errorMail(err, ms) + return + } + sendMail(ctx, dialer, ms) + }(ctx, ms) + } + } +} + +// errorMail is a helper to handle erroring out a slice of Mail instances +// in the case that an unrecoverable error occurs. +func errorMail(err error, ms []Mail) { + for _, m := range ms { + m.Error(err) + } +} + +// dialHost attempts to make a connection to the host specified by the Dialer. +// It returns MaxReconnectAttempts if the number of connection attempts has been +// exceeded. +func dialHost(ctx context.Context, dialer Dialer) (Sender, error) { + sendAttempt := 0 + var sender Sender + var err error + for { + select { + case <-ctx.Done(): + return nil, nil + default: + break + } + sender, err = dialer.Dial() + if err == nil { + break + } + sendAttempt++ + if sendAttempt == MaxReconnectAttempts { + err = ErrMaxConnectAttempts + break + } + } + return sender, err +} + +// sendMail attempts to send the provided Mail instances. +// If the context is cancelled before all of the mail are sent, +// sendMail just returns and does not modify those emails. +func sendMail(ctx context.Context, dialer Dialer, ms []Mail) { + sender, err := dialHost(ctx, dialer) + if err != nil { + errorMail(err, ms) + return + } + defer sender.Close() + message := gomail.NewMessage() + for _, m := range ms { + select { + case <-ctx.Done(): + return + default: + break + } + message.Reset() + + err = m.Generate(message) + if err != nil { + m.Error(err) + continue + } + + err = gomail.Send(sender, message) + if err != nil { + if te, ok := err.(*textproto.Error); ok { + switch { + // If it's a temporary error, we should backoff and try again later. + // We'll reset the connection so future messages don't incur a + // different error (see https://github.com/gophish/gophish/issues/787). + case te.Code >= 400 && te.Code <= 499: + m.Backoff(err) + sender.Reset() + continue + // Otherwise, if it's a permanent error, we shouldn't backoff this message, + // since the RFC specifies that running the same commands won't work next time. + // We should reset our sender and error this message out. + case te.Code >= 500 && te.Code <= 599: + m.Error(err) + sender.Reset() + continue + // If something else happened, let's just error out and reset the + // sender + default: + m.Error(err) + sender.Reset() + continue + } + } else { + m.Error(err) + sender.Reset() + continue + } + } + m.Success() + } +} diff --git a/mailer/mailer_test.go b/mailer/mailer_test.go new file mode 100644 index 00000000..5ba6dea9 --- /dev/null +++ b/mailer/mailer_test.go @@ -0,0 +1,282 @@ +package mailer + +import ( + "bytes" + "context" + "errors" + "io" + "net/textproto" + "reflect" + "testing" + + "github.com/stretchr/testify/suite" +) + +type MailerSuite struct { + suite.Suite +} + +func generateMessages(dialer Dialer) []Mail { + to := []string{"to@example.com"} + + messageContents := []io.WriterTo{ + bytes.NewBuffer([]byte("First email")), + bytes.NewBuffer([]byte("Second email")), + } + + m1 := newMockMessage("first@example.com", to, messageContents[0]) + m2 := newMockMessage("second@example.com", to, messageContents[1]) + + m1.setDialer(func() (Dialer, error) { return dialer, nil }) + + messages := []Mail{m1, m2} + return messages +} + +func newMockErrorSender(err error) *mockSender { + sender := newMockSender() + // The sending function will send a temporary error to emulate + // a backoff. + sender.setSend(func(mm *mockMessage) error { + if len(sender.messages) == 1 { + return err + } + sender.messageChan <- mm + return nil + }) + return sender +} + +func (ms *MailerSuite) TestDialHost() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + md := newMockDialer() + md.setDial(md.unreachableDial) + _, err := dialHost(ctx, md) + if err != ErrMaxConnectAttempts { + ms.T().Fatalf("Didn't receive expected ErrMaxConnectAttempts. Got: %s", err) + } + if md.dialCount != MaxReconnectAttempts { + ms.T().Fatalf("Unexpected number of reconnect attempts. Expected %d, Got %d", MaxReconnectAttempts, md.dialCount) + } + md.setDial(md.defaultDial) + _, err = dialHost(ctx, md) + if err != nil { + ms.T().Fatalf("Unexpected error when dialing the mock host: %s", err) + } +} + +func (ms *MailerSuite) TestMailWorkerStart() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mw := NewMailWorker() + go func(ctx context.Context) { + mw.Start(ctx) + }(ctx) + + sender := newMockSender() + dialer := newMockDialer() + dialer.setDial(func() (Sender, error) { + return sender, nil + }) + + messages := generateMessages(dialer) + + // Send the campaign + mw.Queue <- messages + + got := []*mockMessage{} + + idx := 0 + for message := range sender.messageChan { + got = append(got, message) + original := messages[idx].(*mockMessage) + if original.from != message.from { + ms.T().Fatalf("Invalid message received. Expected %s, Got %s", original.from, message.from) + } + idx++ + } + if len(got) != len(messages) { + ms.T().Fatalf("Unexpected number of messages received. Expected %d Got %d", len(got), len(messages)) + } +} + +func (ms *MailerSuite) TestBackoff() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mw := NewMailWorker() + go func(ctx context.Context) { + mw.Start(ctx) + }(ctx) + + expectedError := &textproto.Error{ + Code: 400, + Msg: "Temporary error", + } + + sender := newMockErrorSender(expectedError) + dialer := newMockDialer() + dialer.setDial(func() (Sender, error) { + return sender, nil + }) + + messages := generateMessages(dialer) + + // Send the campaign + mw.Queue <- messages + + got := []*mockMessage{} + + for message := range sender.messageChan { + got = append(got, message) + } + // Check that we only sent one message + expectedCount := 1 + if len(got) != expectedCount { + ms.T().Fatalf("Unexpected number of messages received. Expected %d Got %d", len(got), expectedCount) + } + + // Check that it's the correct message + originalFrom := messages[1].(*mockMessage).from + if got[0].from != originalFrom { + ms.T().Fatalf("Invalid message received. Expected %s, Got %s", originalFrom, got[0].from) + } + + // Check that the first message performed a backoff + backoffCount := messages[0].(*mockMessage).backoffCount + if backoffCount != expectedCount { + ms.T().Fatalf("Did not receive expected backoff. Got backoffCount %d, Expected %d", backoffCount, expectedCount) + } + + // Check that there was a reset performed on the sender + if sender.resetCount != expectedCount { + ms.T().Fatalf("Did not receive expected reset. Got resetCount %d, expected %d", sender.resetCount, expectedCount) + } +} + +func (ms *MailerSuite) TestPermError() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mw := NewMailWorker() + go func(ctx context.Context) { + mw.Start(ctx) + }(ctx) + + expectedError := &textproto.Error{ + Code: 500, + Msg: "Permanent error", + } + + sender := newMockErrorSender(expectedError) + dialer := newMockDialer() + dialer.setDial(func() (Sender, error) { + return sender, nil + }) + + messages := generateMessages(dialer) + + // Send the campaign + mw.Queue <- messages + + got := []*mockMessage{} + + for message := range sender.messageChan { + got = append(got, message) + } + // Check that we only sent one message + expectedCount := 1 + if len(got) != expectedCount { + ms.T().Fatalf("Unexpected number of messages received. Expected %d Got %d", len(got), expectedCount) + } + + // Check that it's the correct message + originalFrom := messages[1].(*mockMessage).from + if got[0].from != originalFrom { + ms.T().Fatalf("Invalid message received. Expected %s, Got %s", originalFrom, got[0].from) + } + + message := messages[0].(*mockMessage) + + // Check that the first message did not perform a backoff + expectedBackoffCount := 0 + backoffCount := message.backoffCount + if backoffCount != expectedBackoffCount { + ms.T().Fatalf("Did not receive expected backoff. Got backoffCount %d, Expected %d", backoffCount, expectedCount) + } + + // Check that there was a reset performed on the sender + if sender.resetCount != expectedCount { + ms.T().Fatalf("Did not receive expected reset. Got resetCount %d, expected %d", sender.resetCount, expectedCount) + } + + // Check that the email errored out appropriately + if !reflect.DeepEqual(message.err, expectedError) { + ms.T().Fatalf("Did not received expected error. Got %#v\nExpected %#v", message.err, expectedError) + } +} + +func (ms *MailerSuite) TestUnknownError() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mw := NewMailWorker() + go func(ctx context.Context) { + mw.Start(ctx) + }(ctx) + + expectedError := errors.New("Unexpected error") + + sender := newMockErrorSender(expectedError) + dialer := newMockDialer() + dialer.setDial(func() (Sender, error) { + return sender, nil + }) + + messages := generateMessages(dialer) + + // Send the campaign + mw.Queue <- messages + + got := []*mockMessage{} + + for message := range sender.messageChan { + got = append(got, message) + } + // Check that we only sent one message + expectedCount := 1 + if len(got) != expectedCount { + ms.T().Fatalf("Unexpected number of messages received. Expected %d Got %d", len(got), expectedCount) + } + + // Check that it's the correct message + originalFrom := messages[1].(*mockMessage).from + if got[0].from != originalFrom { + ms.T().Fatalf("Invalid message received. Expected %s, Got %s", originalFrom, got[0].from) + } + + message := messages[0].(*mockMessage) + + // Check that the first message did not perform a backoff + expectedBackoffCount := 0 + backoffCount := message.backoffCount + if backoffCount != expectedBackoffCount { + ms.T().Fatalf("Did not receive expected backoff. Got backoffCount %d, Expected %d", backoffCount, expectedCount) + } + + // Check that there was a reset performed on the sender + if sender.resetCount != expectedCount { + ms.T().Fatalf("Did not receive expected reset. Got resetCount %d, expected %d", sender.resetCount, expectedCount) + } + + // Check that the email errored out appropriately + if !reflect.DeepEqual(message.err, expectedError) { + ms.T().Fatalf("Did not received expected error. Got %#v\nExpected %#v", message.err, expectedError) + } +} + +func TestMailerSuite(t *testing.T) { + suite.Run(t, new(MailerSuite)) +} diff --git a/mailer/mockmailer.go b/mailer/mockmailer.go new file mode 100644 index 00000000..a5fc230d --- /dev/null +++ b/mailer/mockmailer.go @@ -0,0 +1,176 @@ +package mailer + +import ( + "bytes" + "errors" + "io" + "time" + + "github.com/gophish/gomail" +) + +// errHostUnreachable is a mock error to represent a host +// being unreachable +var errHostUnreachable = errors.New("host unreachable") + +// errDialerUnavailable is a mock error to represent a dialer +// being unavailable (perhaps an error getting the dialer config +// or a database error) +var errDialerUnavailable = errors.New("dialer unavailable") + +// mockDialer keeps track of calls to Dial +type mockDialer struct { + dialCount int + dial func() (Sender, error) +} + +// newMockDialer returns a new instance of the mockDialer with the default +// dialer set. +func newMockDialer() *mockDialer { + md := &mockDialer{} + md.dial = md.defaultDial + return md +} + +// defaultDial simply returns a mockSender +func (md *mockDialer) defaultDial() (Sender, error) { + return newMockSender(), nil +} + +// unreachableDial is to simulate network error conditions in which +// a host is unavailable. +func (md *mockDialer) unreachableDial() (Sender, error) { + return nil, errHostUnreachable +} + +// Dial increments the internal dial count. Otherwise, it's a no-op for the mock client. +func (md *mockDialer) Dial() (Sender, error) { + md.dialCount++ + return md.dial() +} + +// setDial sets the Dial function for the mockDialer +func (md *mockDialer) setDial(dial func() (Sender, error)) { + md.dial = dial +} + +// mockSender is a mock gomail.Sender used for testing. +type mockSender struct { + messages []*mockMessage + status string + send func(*mockMessage) error + messageChan chan *mockMessage + resetCount int +} + +func newMockSender() *mockSender { + ms := &mockSender{ + status: "ehlo", + messageChan: make(chan *mockMessage), + } + ms.send = ms.defaultSend + return ms +} + +func (ms *mockSender) setSend(send func(*mockMessage) error) { + ms.send = send +} + +func (ms *mockSender) defaultSend(mm *mockMessage) error { + ms.messageChan <- mm + return nil +} + +// Send just appends the provided message record to the internal slice +func (ms *mockSender) Send(from string, to []string, msg io.WriterTo) error { + mm := newMockMessage(from, to, msg) + ms.messages = append(ms.messages, mm) + ms.status = "sent" + return ms.send(mm) +} + +// Close is a noop for the mock client +func (ms *mockSender) Close() error { + ms.status = "closed" + close(ms.messageChan) + return nil +} + +// Reset sets the status to "Reset". In practice, this would reset the connection +// to the same state as if the client had just sent an EHLO command. +func (ms *mockSender) Reset() error { + ms.status = "reset" + ms.resetCount++ + return nil +} + +// mockMessage holds the information sent via a call to MockClient.Send() +type mockMessage struct { + from string + to []string + message []byte + sendAt time.Time + backoffCount int + getdialer func() (Dialer, error) + err error + finished bool +} + +func newMockMessage(from string, to []string, msg io.WriterTo) *mockMessage { + buff := &bytes.Buffer{} + msg.WriteTo(buff) + mm := &mockMessage{ + from: from, + to: to, + message: buff.Bytes(), + sendAt: time.Now(), + } + mm.getdialer = mm.defaultDialer + return mm +} + +func (mm *mockMessage) setDialer(dialer func() (Dialer, error)) { + mm.getdialer = dialer +} + +func (mm *mockMessage) defaultDialer() (Dialer, error) { + return newMockDialer(), nil +} + +func (mm *mockMessage) errorDialer() (Dialer, error) { + return nil, errDialerUnavailable +} + +func (mm *mockMessage) GetDialer() (Dialer, error) { + return mm.getdialer() +} + +func (mm *mockMessage) Backoff(reason error) error { + mm.backoffCount++ + return nil +} + +func (mm *mockMessage) Error(err error) error { + mm.err = err + mm.finished = true + return nil +} + +func (mm *mockMessage) Finish() error { + mm.finished = true + return nil +} + +func (mm *mockMessage) Generate(message *gomail.Message) error { + message.SetHeaders(map[string][]string{ + "From": {mm.from}, + "To": mm.to, + }) + message.SetBody("text/html", string(mm.message)) + return nil +} + +func (mm *mockMessage) Success() error { + mm.finished = true + return nil +} diff --git a/models/campaign.go b/models/campaign.go index 4d5a2897..df044cf1 100644 --- a/models/campaign.go +++ b/models/campaign.go @@ -108,29 +108,6 @@ func (c *Campaign) Validate() error { return nil } -// SendTestEmailRequest is the structure of a request -// to send a test email to test an SMTP connection -type SendTestEmailRequest struct { - Template Template `json:"template"` - Page Page `json:"page"` - SMTP SMTP `json:"smtp"` - URL string `json:"url"` - Tracker string `json:"tracker"` - TrackingURL string `json:"tracking_url"` - From string `json:"from"` - Target -} - -// Validate ensures the SendTestEmailRequest structure -// is valid. -func (s *SendTestEmailRequest) Validate() error { - switch { - case s.Email == "": - return ErrEmailNotSpecified - } - return nil -} - // UpdateStatus changes the campaign status appropriately func (c *Campaign) UpdateStatus(s string) error { // This could be made simpler, but I think there's a bug in gorm @@ -141,7 +118,7 @@ func (c *Campaign) UpdateStatus(s string) error { func (c *Campaign) AddEvent(e Event) error { e.CampaignId = c.Id e.Time = time.Now().UTC() - return db.Debug().Save(&e).Error + return db.Save(&e).Error } // getDetails retrieves the related attributes of the campaign @@ -363,12 +340,15 @@ func PostCampaign(c *Campaign, uid int64) error { c.UserId = uid c.CreatedDate = time.Now().UTC() c.CompletedDate = time.Time{} - c.Status = CAMPAIGN_CREATED + c.Status = CAMPAIGN_QUEUED if c.LaunchDate.IsZero() { - c.LaunchDate = time.Now().UTC() + c.LaunchDate = c.CreatedDate } else { c.LaunchDate = c.LaunchDate.UTC() } + if c.LaunchDate.Before(c.CreatedDate) || c.LaunchDate.Equal(c.CreatedDate) { + c.Status = CAMPAIGN_IN_PROGRESS + } // Check to make sure all the groups already exist for i, g := range c.Groups { c.Groups[i], err = GetGroupByName(g.Name, uid) @@ -427,7 +407,19 @@ func PostCampaign(c *Campaign, uid int64) error { for _, g := range c.Groups { // Insert a result for each target in the group for _, t := range g.Targets { - r := &Result{Email: t.Email, Position: t.Position, Status: STATUS_SENDING, CampaignId: c.Id, UserId: c.UserId, FirstName: t.FirstName, LastName: t.LastName} + r := &Result{ + Email: t.Email, + Position: t.Position, + Status: STATUS_SCHEDULED, + CampaignId: c.Id, + UserId: c.UserId, + FirstName: t.FirstName, + LastName: t.LastName, + SendDate: c.LaunchDate, + } + if c.Status == CAMPAIGN_IN_PROGRESS { + r.Status = STATUS_SENDING + } err = r.GenerateId() if err != nil { Logger.Println(err) @@ -439,9 +431,13 @@ func PostCampaign(c *Campaign, uid int64) error { Logger.Println(err) } c.Results = append(c.Results, *r) + err = GenerateMailLog(c, r) + if err != nil { + Logger.Println(err) + continue + } } } - c.Status = CAMPAIGN_QUEUED err = db.Save(c).Error return err } diff --git a/models/email_request.go b/models/email_request.go new file mode 100644 index 00000000..240101f9 --- /dev/null +++ b/models/email_request.go @@ -0,0 +1,132 @@ +package models + +import ( + "encoding/base64" + "fmt" + "io" + "net/mail" + "strings" + + "github.com/gophish/gomail" + "github.com/gophish/gophish/mailer" +) + +// SendTestEmailRequest is the structure of a request +// to send a test email to test an SMTP connection. +// This type implements the mailer.Mail interface. +type SendTestEmailRequest struct { + Template Template `json:"template"` + Page Page `json:"page"` + SMTP SMTP `json:"smtp"` + URL string `json:"url"` + Tracker string `json:"tracker"` + TrackingURL string `json:"tracking_url"` + From string `json:"from"` + Target + ErrorChan chan (error) `json:"-"` +} + +// Validate ensures the SendTestEmailRequest structure +// is valid. +func (s *SendTestEmailRequest) Validate() error { + switch { + case s.Email == "": + return ErrEmailNotSpecified + } + return nil +} + +// Backoff treats temporary errors as permanent since this is expected to be a +// synchronous operation. It returns any errors given back to the ErrorChan +func (s *SendTestEmailRequest) Backoff(reason error) error { + s.ErrorChan <- reason + return nil +} + +// Error returns an error on the ErrorChan. +func (s *SendTestEmailRequest) Error(err error) error { + s.ErrorChan <- err + return nil +} + +// Success returns nil on the ErrorChan to indicate that the email was sent +// successfully. +func (s *SendTestEmailRequest) Success() error { + s.ErrorChan <- nil + return nil +} + +// Generate fills in the details of a gomail.Message with the contents +// from the SendTestEmailRequest. +func (s *SendTestEmailRequest) Generate(msg *gomail.Message) error { + f, err := mail.ParseAddress(s.SMTP.FromAddress) + if err != nil { + return err + } + fn := f.Name + if fn == "" { + fn = f.Address + } + msg.SetAddressHeader("From", f.Address, f.Name) + + // Parse the customHeader templates + for _, header := range s.SMTP.Headers { + key, err := buildTemplate(header.Key, s) + if err != nil { + Logger.Println(err) + } + + value, err := buildTemplate(header.Value, s) + if err != nil { + Logger.Println(err) + } + + // Add our header immediately + msg.SetHeader(key, value) + } + + // Parse remaining templates + subject, err := buildTemplate(s.Template.Subject, s) + if err != nil { + Logger.Println(err) + } + msg.SetHeader("Subject", subject) + + msg.SetHeader("To", s.FormatAddress()) + if s.Template.Text != "" { + text, err := buildTemplate(s.Template.Text, s) + if err != nil { + Logger.Println(err) + } + msg.SetBody("text/plain", text) + } + if s.Template.HTML != "" { + html, err := buildTemplate(s.Template.HTML, s) + if err != nil { + Logger.Println(err) + } + if s.Template.Text == "" { + msg.SetBody("text/html", html) + } else { + msg.AddAlternative("text/html", html) + } + } + // Attach the files + for _, a := range s.Template.Attachments { + msg.Attach(func(a Attachment) (string, gomail.FileSetting, gomail.FileSetting) { + h := map[string][]string{"Content-ID": {fmt.Sprintf("<%s>", a.Name)}} + return a.Name, gomail.SetCopyFunc(func(w io.Writer) error { + decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(a.Content)) + _, err = io.Copy(w, decoder) + return err + }), gomail.SetHeader(h) + }(a)) + } + + return nil +} + +// GetDialer returns the mailer.Dialer for the underlying SMTP object +func (s *SendTestEmailRequest) GetDialer() (mailer.Dialer, error) { + return s.SMTP.GetDialer() +} diff --git a/models/email_request_test.go b/models/email_request_test.go new file mode 100644 index 00000000..1317af90 --- /dev/null +++ b/models/email_request_test.go @@ -0,0 +1,95 @@ +package models + +import ( + "bytes" + "errors" + "fmt" + + "github.com/gophish/gomail" + "github.com/jordan-wright/email" + check "gopkg.in/check.v1" +) + +func (s *ModelsSuite) TestEmailNotPresent(ch *check.C) { + req := &SendTestEmailRequest{} + ch.Assert(req.Validate(), check.Equals, ErrEmailNotSpecified) + req.Email = "test@example.com" + ch.Assert(req.Validate(), check.Equals, nil) +} + +func (s *ModelsSuite) TestEmailRequestBackoff(ch *check.C) { + req := &SendTestEmailRequest{ + ErrorChan: make(chan error), + } + expected := errors.New("Temporary Error") + go func() { + err = req.Backoff(expected) + ch.Assert(err, check.Equals, nil) + }() + ch.Assert(<-req.ErrorChan, check.Equals, expected) +} + +func (s *ModelsSuite) TestEmailRequestError(ch *check.C) { + req := &SendTestEmailRequest{ + ErrorChan: make(chan error), + } + expected := errors.New("Temporary Error") + go func() { + err = req.Error(expected) + ch.Assert(err, check.Equals, nil) + }() + ch.Assert(<-req.ErrorChan, check.Equals, expected) +} + +func (s *ModelsSuite) TestEmailRequestSuccess(ch *check.C) { + req := &SendTestEmailRequest{ + ErrorChan: make(chan error), + } + go func() { + err = req.Success() + ch.Assert(err, check.Equals, nil) + }() + ch.Assert(<-req.ErrorChan, check.Equals, nil) +} + +func (s *ModelsSuite) TestEmailRequestGenerate(ch *check.C) { + smtp := SMTP{ + FromAddress: "from@example.com", + } + template := Template{ + Name: "Test Template", + Subject: "{{.FirstName}} - Subject", + Text: "{{.Email}} - Text", + HTML: "{{.Email}} - HTML", + } + target := Target{ + FirstName: "First", + LastName: "Last", + Email: "firstlast@example.com", + } + req := &SendTestEmailRequest{ + SMTP: smtp, + Template: template, + Target: target, + } + + msg := gomail.NewMessage() + err = req.Generate(msg) + ch.Assert(err, check.Equals, nil) + + expected := &email.Email{ + Subject: fmt.Sprintf("%s - Subject", req.FirstName), + Text: []byte(fmt.Sprintf("%s - Text", req.Email)), + HTML: []byte(fmt.Sprintf("%s - HTML", req.Email)), + } + + msgBuff := &bytes.Buffer{} + _, err = msg.WriteTo(msgBuff) + ch.Assert(err, check.Equals, nil) + + got, err := email.NewEmailFromReader(msgBuff) + ch.Assert(err, check.Equals, nil) + ch.Assert(got.Subject, check.Equals, expected.Subject) + ch.Assert(string(got.Text), check.Equals, string(expected.Text)) + ch.Assert(string(got.HTML), check.Equals, string(expected.HTML)) +} diff --git a/models/maillog.go b/models/maillog.go new file mode 100644 index 00000000..5eb1dffa --- /dev/null +++ b/models/maillog.go @@ -0,0 +1,326 @@ +package models + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/mail" + "strings" + "text/template" + "time" + + "github.com/gophish/gomail" + "github.com/gophish/gophish/mailer" +) + +// MaxSendAttempts set to 8 since we exponentially backoff after each failed send +// attempt. This will give us a maximum send delay of 256 minutes, or about 4.2 hours. +var MaxSendAttempts = 8 + +// ErrMaxSendAttempts is thrown when the maximum number of sending attemps for a given +// MailLog is exceeded. +var ErrMaxSendAttempts = errors.New("max send attempts exceeded") + +// MailLog is a struct that holds information about an email that is to be +// sent out. +type MailLog struct { + Id int64 `json:"-"` + UserId int64 `json:"-"` + CampaignId int64 `json:"campaign_id"` + RId string `json:"id"` + SendDate time.Time `json:"send_date"` + SendAttempt int `json:"send_attempt"` + Processing bool `json:"-"` +} + +// GenerateMailLog creates a new maillog for the given campaign and +// result. It sets the initial send date to match the campaign's launch date. +func GenerateMailLog(c *Campaign, r *Result) error { + m := &MailLog{ + UserId: c.UserId, + CampaignId: c.Id, + RId: r.RId, + SendDate: c.LaunchDate, + } + err = db.Save(m).Error + return err +} + +// Backoff sets the MailLog SendDate to be the next entry in an exponential +// backoff. ErrMaxRetriesExceeded is thrown if this maillog has been retried +// too many times. Backoff also unlocks the maillog so that it can be processed +// again in the future. +func (m *MailLog) Backoff(reason error) error { + if m.SendAttempt == MaxSendAttempts { + err = m.addError(ErrMaxSendAttempts) + return ErrMaxSendAttempts + } + r, err := GetResult(m.RId) + if err != nil { + return err + } + // Add an error, since we had to backoff because of a + // temporary error of some sort during the SMTP transaction + err = m.addError(reason) + if err != nil { + return err + } + m.SendAttempt++ + backoffDuration := math.Pow(2, float64(m.SendAttempt)) + m.SendDate = m.SendDate.Add(time.Minute * time.Duration(backoffDuration)) + err = db.Save(m).Error + if err != nil { + return err + } + r.Status = STATUS_RETRY + r.SendDate = m.SendDate + err = db.Save(r).Error + if err != nil { + return err + } + err = m.Unlock() + return err +} + +// Unlock removes the processing flag so the maillog can be processed again +func (m *MailLog) Unlock() error { + m.Processing = false + return db.Save(&m).Error +} + +// Lock sets the processing flag so that other processes cannot modify the maillog +func (m *MailLog) Lock() error { + m.Processing = true + return db.Save(&m).Error +} + +// addError adds an error to the associated campaign +func (m *MailLog) addError(e error) error { + c, err := GetCampaign(m.CampaignId, m.UserId) + if err != nil { + return err + } + // This is redundant in the case of permanent + // errors, but the extra query makes for + // a cleaner API. + r, err := GetResult(m.RId) + if err != nil { + return err + } + es := struct { + Error string `json:"error"` + }{ + Error: e.Error(), + } + ej, err := json.Marshal(es) + if err != nil { + Logger.Println(err) + } + err = c.AddEvent(Event{Email: r.Email, Message: EVENT_SENDING_ERROR, Details: string(ej)}) + return err +} + +// Error sets the error status on the models.Result that the +// maillog refers to. Since MailLog errors are permanent, +// this action also deletes the maillog. +func (m *MailLog) Error(e error) error { + Logger.Printf("Erroring out result %s\n", m.RId) + r, err := GetResult(m.RId) + if err != nil { + return err + } + // Update the result + err = r.UpdateStatus(ERROR) + if err != nil { + return err + } + // Update the campaign events + err = m.addError(e) + if err != nil { + return err + } + err = db.Delete(m).Error + return err +} + +// Success deletes the maillog from the database and updates the underlying +// campaign result. +func (m *MailLog) Success() error { + r, err := GetResult(m.RId) + if err != nil { + return err + } + err = r.UpdateStatus(EVENT_SENT) + if err != nil { + return err + } + c, err := GetCampaign(m.CampaignId, m.UserId) + if err != nil { + return err + } + err = c.AddEvent(Event{Email: r.Email, Message: EVENT_SENT}) + if err != nil { + return err + } + err = db.Delete(m).Error + return nil +} + +// GetDialer returns a dialer based on the maillog campaign's SMTP configuration +func (m *MailLog) GetDialer() (mailer.Dialer, error) { + c, err := GetCampaign(m.CampaignId, m.UserId) + if err != nil { + return nil, err + } + return c.SMTP.GetDialer() +} + +// buildTemplate creates a templated string based on the provided +// template body and data. +func buildTemplate(text string, data interface{}) (string, error) { + buff := bytes.Buffer{} + tmpl, err := template.New("template").Parse(text) + if err != nil { + return buff.String(), err + } + err = tmpl.Execute(&buff, data) + return buff.String(), err +} + +// Generate fills in the details of a gomail.Message instance with +// the correct headers and body from the campaign and recipient listed in +// the maillog. We accept the gomail.Message as an argument so that the caller +// can choose to re-use the message across recipients. +func (m *MailLog) Generate(msg *gomail.Message) error { + r, err := GetResult(m.RId) + if err != nil { + return err + } + c, err := GetCampaign(m.CampaignId, m.UserId) + if err != nil { + return err + } + f, err := mail.ParseAddress(c.SMTP.FromAddress) + if err != nil { + return err + } + fn := f.Name + if fn == "" { + fn = f.Address + } + msg.SetAddressHeader("From", f.Address, f.Name) + td := struct { + Result + URL string + TrackingURL string + Tracker string + From string + }{ + r, + c.URL + "?rid=" + r.RId, + c.URL + "/track?rid=" + r.RId, + "", + fn, + } + + // Parse the customHeader templates + for _, header := range c.SMTP.Headers { + key, err := buildTemplate(header.Key, td) + if err != nil { + Logger.Println(err) + } + + value, err := buildTemplate(header.Value, td) + if err != nil { + Logger.Println(err) + } + + // Add our header immediately + msg.SetHeader(key, value) + } + + // Parse remaining templates + subject, err := buildTemplate(c.Template.Subject, td) + if err != nil { + Logger.Println(err) + } + msg.SetHeader("Subject", subject) + + msg.SetHeader("To", r.FormatAddress()) + if c.Template.Text != "" { + text, err := buildTemplate(c.Template.Text, td) + if err != nil { + Logger.Println(err) + } + msg.SetBody("text/plain", text) + } + if c.Template.HTML != "" { + html, err := buildTemplate(c.Template.HTML, td) + if err != nil { + Logger.Println(err) + } + if c.Template.Text == "" { + msg.SetBody("text/html", html) + } else { + msg.AddAlternative("text/html", html) + } + } + // Attach the files + for _, a := range c.Template.Attachments { + msg.Attach(func(a Attachment) (string, gomail.FileSetting, gomail.FileSetting) { + h := map[string][]string{"Content-ID": {fmt.Sprintf("<%s>", a.Name)}} + return a.Name, gomail.SetCopyFunc(func(w io.Writer) error { + decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(a.Content)) + _, err = io.Copy(w, decoder) + return err + }), gomail.SetHeader(h) + }(a)) + } + + return nil +} + +// GetQueuedMailLogs returns the mail logs that are queued up for the given minute. +func GetQueuedMailLogs(t time.Time) ([]*MailLog, error) { + ms := []*MailLog{} + err := db.Where("send_date <= ? AND processing = ?", t, false). + Find(&ms).Error + if err != nil { + Logger.Println(err) + } + return ms, err +} + +// GetMailLogsByCampaign returns all of the mail logs for a given campaign. +func GetMailLogsByCampaign(cid int64) ([]*MailLog, error) { + ms := []*MailLog{} + err := db.Where("campaign_id = ?", cid).Find(&ms).Error + return ms, err +} + +// LockMailLogs locks or unlocks a slice of maillogs for processing. +func LockMailLogs(ms []*MailLog, lock bool) error { + tx := db.Begin() + for i := range ms { + ms[i].Processing = lock + err := tx.Debug().Save(ms[i]).Error + if err != nil { + tx.Rollback() + return err + } + } + tx.Commit() + return nil +} + +// UnlockAllMailLogs removes the processing lock for all maillogs +// in the database. This is intended to be called when Gophish is started +// so that any previously locked maillogs can resume processing. +func UnlockAllMailLogs() error { + err = db.Model(&MailLog{}).Update("processing", false).Error + return err +} diff --git a/models/maillog_test.go b/models/maillog_test.go new file mode 100644 index 00000000..835c42aa --- /dev/null +++ b/models/maillog_test.go @@ -0,0 +1,240 @@ +package models + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "net/textproto" + "time" + + "github.com/gophish/gomail" + "github.com/jordan-wright/email" + "gopkg.in/check.v1" +) + +func (s *ModelsSuite) TestGetQueuedMailLogs(ch *check.C) { + campaign := s.createCampaign(ch) + ms, err := GetQueuedMailLogs(campaign.LaunchDate) + ch.Assert(err, check.Equals, nil) + got := make(map[string]*MailLog) + for _, m := range ms { + got[m.RId] = m + } + for _, r := range campaign.Results { + if m, ok := got[r.RId]; ok { + ch.Assert(m.RId, check.Equals, r.RId) + ch.Assert(m.CampaignId, check.Equals, campaign.Id) + ch.Assert(m.SendDate, check.Equals, campaign.LaunchDate) + ch.Assert(m.UserId, check.Equals, campaign.UserId) + ch.Assert(m.SendAttempt, check.Equals, 0) + } else { + ch.Fatalf("Result not found in maillogs: %s", r.RId) + } + } +} + +func (s *ModelsSuite) TestMailLogBackoff(ch *check.C) { + campaign := s.createCampaign(ch) + result := campaign.Results[0] + m := &MailLog{} + err := db.Where("r_id=? AND campaign_id=?", result.RId, campaign.Id). + Find(m).Error + ch.Assert(err, check.Equals, nil) + ch.Assert(m.SendAttempt, check.Equals, 0) + ch.Assert(m.SendDate, check.Equals, campaign.LaunchDate) + + expectedError := &textproto.Error{ + Code: 500, + Msg: "Recipient not found", + } + for i := m.SendAttempt; i < MaxSendAttempts; i++ { + err = m.Lock() + ch.Assert(err, check.Equals, nil) + ch.Assert(m.Processing, check.Equals, true) + + expectedDuration := math.Pow(2, float64(m.SendAttempt+1)) + expectedSendDate := m.SendDate.Add(time.Minute * time.Duration(expectedDuration)) + err = m.Backoff(expectedError) + ch.Assert(err, check.Equals, nil) + ch.Assert(m.SendDate, check.Equals, expectedSendDate) + ch.Assert(m.Processing, check.Equals, false) + result, err := GetResult(m.RId) + ch.Assert(err, check.Equals, nil) + ch.Assert(result.SendDate, check.Equals, expectedSendDate) + ch.Assert(result.Status, check.Equals, STATUS_RETRY) + } + // Get our updated campaign and check for the added event + campaign, err = GetCampaign(campaign.Id, int64(1)) + ch.Assert(err, check.Equals, nil) + + // We expect MaxSendAttempts + the initial campaign created event + ch.Assert(len(campaign.Events), check.Equals, MaxSendAttempts+1) + + // Check that we receive our error after meeting the maximum send attempts + err = m.Backoff(expectedError) + ch.Assert(err, check.Equals, ErrMaxSendAttempts) +} + +func (s *ModelsSuite) TestMailLogError(ch *check.C) { + campaign := s.createCampaign(ch) + result := campaign.Results[0] + m := &MailLog{} + err := db.Where("r_id=? AND campaign_id=?", result.RId, campaign.Id). + Find(m).Error + ch.Assert(err, check.Equals, nil) + ch.Assert(m.RId, check.Equals, result.RId) + + expectedError := &textproto.Error{ + Code: 500, + Msg: "Recipient not found", + } + err = m.Error(expectedError) + ch.Assert(err, check.Equals, nil) + + // Get our result and make sure the status is set correctly + result, err = GetResult(result.RId) + ch.Assert(err, check.Equals, nil) + ch.Assert(result.Status, check.Equals, ERROR) + + // Get our updated campaign and check for the added event + campaign, err = GetCampaign(campaign.Id, int64(1)) + ch.Assert(err, check.Equals, nil) + + expectedEventLength := 2 + ch.Assert(len(campaign.Events), check.Equals, expectedEventLength) + + gotEvent := campaign.Events[1] + es := struct { + Error string `json:"error"` + }{ + Error: expectedError.Error(), + } + ej, _ := json.Marshal(es) + expectedEvent := Event{ + Id: gotEvent.Id, + Email: result.Email, + Message: EVENT_SENDING_ERROR, + CampaignId: campaign.Id, + Details: string(ej), + Time: gotEvent.Time, + } + ch.Assert(gotEvent, check.DeepEquals, expectedEvent) + + ms, err := GetMailLogsByCampaign(campaign.Id) + ch.Assert(err, check.Equals, nil) + ch.Assert(len(ms), check.Equals, len(campaign.Results)-1) +} + +func (s *ModelsSuite) TestMailLogSuccess(ch *check.C) { + campaign := s.createCampaign(ch) + result := campaign.Results[0] + m := &MailLog{} + err := db.Where("r_id=? AND campaign_id=?", result.RId, campaign.Id). + Find(m).Error + ch.Assert(err, check.Equals, nil) + ch.Assert(m.RId, check.Equals, result.RId) + + err = m.Success() + ch.Assert(err, check.Equals, nil) + + // Get our result and make sure the status is set correctly + result, err = GetResult(result.RId) + ch.Assert(err, check.Equals, nil) + ch.Assert(result.Status, check.Equals, EVENT_SENT) + + // Get our updated campaign and check for the added event + campaign, err = GetCampaign(campaign.Id, int64(1)) + ch.Assert(err, check.Equals, nil) + + expectedEventLength := 2 + ch.Assert(len(campaign.Events), check.Equals, expectedEventLength) + + gotEvent := campaign.Events[1] + expectedEvent := Event{ + Id: gotEvent.Id, + Email: result.Email, + Message: EVENT_SENT, + CampaignId: campaign.Id, + Time: gotEvent.Time, + } + ch.Assert(gotEvent, check.DeepEquals, expectedEvent) + + ms, err := GetMailLogsByCampaign(campaign.Id) + ch.Assert(err, check.Equals, nil) + ch.Assert(len(ms), check.Equals, len(campaign.Results)-1) +} + +func (s *ModelsSuite) TestGenerateMailLog(ch *check.C) { + campaign := Campaign{ + Id: 1, + UserId: 1, + LaunchDate: time.Now().UTC(), + } + result := Result{ + RId: "abc1234", + } + err := GenerateMailLog(&campaign, &result) + ch.Assert(err, check.Equals, nil) + + m := MailLog{} + err = db.Where("r_id=?", result.RId).Find(&m).Error + ch.Assert(err, check.Equals, nil) + ch.Assert(m.RId, check.Equals, result.RId) + ch.Assert(m.CampaignId, check.Equals, campaign.Id) + ch.Assert(m.SendDate, check.Equals, campaign.LaunchDate) + ch.Assert(m.UserId, check.Equals, campaign.UserId) + ch.Assert(m.SendAttempt, check.Equals, 0) + ch.Assert(m.Processing, check.Equals, false) +} + +func (s *ModelsSuite) TestMailLogGenerate(ch *check.C) { + campaign := s.createCampaign(ch) + result := campaign.Results[0] + m := &MailLog{} + err := db.Where("r_id=? AND campaign_id=?", result.RId, campaign.Id). + Find(m).Error + ch.Assert(err, check.Equals, nil) + + msg := gomail.NewMessage() + err = m.Generate(msg) + ch.Assert(err, check.Equals, nil) + + expected := &email.Email{ + Subject: fmt.Sprintf("%s - Subject", result.RId), + Text: []byte(fmt.Sprintf("%s - Text", result.RId)), + HTML: []byte(fmt.Sprintf("%s - HTML", result.RId)), + } + + msgBuff := &bytes.Buffer{} + _, err = msg.WriteTo(msgBuff) + ch.Assert(err, check.Equals, nil) + + got, err := email.NewEmailFromReader(msgBuff) + ch.Assert(err, check.Equals, nil) + ch.Assert(got.Subject, check.Equals, expected.Subject) + ch.Assert(string(got.Text), check.Equals, string(expected.Text)) + ch.Assert(string(got.HTML), check.Equals, string(expected.HTML)) +} + +func (s *ModelsSuite) TestUnlockAllMailLogs(ch *check.C) { + campaign := s.createCampaign(ch) + ms, err := GetMailLogsByCampaign(campaign.Id) + ch.Assert(err, check.Equals, nil) + for _, m := range ms { + ch.Assert(m.Processing, check.Equals, false) + } + err = LockMailLogs(ms, true) + ms, err = GetMailLogsByCampaign(campaign.Id) + ch.Assert(err, check.Equals, nil) + for _, m := range ms { + ch.Assert(m.Processing, check.Equals, true) + } + err = UnlockAllMailLogs() + ch.Assert(err, check.Equals, nil) + ms, err = GetMailLogsByCampaign(campaign.Id) + ch.Assert(err, check.Equals, nil) + for _, m := range ms { + ch.Assert(m.Processing, check.Equals, false) + } +} diff --git a/models/models.go b/models/models.go index 08e79e88..e9ff833c 100644 --- a/models/models.go +++ b/models/models.go @@ -36,9 +36,13 @@ const ( EVENT_OPENED string = "Email Opened" EVENT_CLICKED string = "Clicked Link" EVENT_DATA_SUBMIT string = "Submitted Data" + EVENT_PROXY_REQUEST string = "Proxied request" STATUS_SUCCESS string = "Success" + STATUS_QUEUED string = "Queued" STATUS_SENDING string = "Sending" STATUS_UNKNOWN string = "Unknown" + STATUS_SCHEDULED string = "Scheduled" + STATUS_RETRY string = "Retrying" ERROR string = "Error" ) diff --git a/models/models_test.go b/models/models_test.go index 5a5008b2..428817ea 100644 --- a/models/models_test.go +++ b/models/models_test.go @@ -1,10 +1,12 @@ package models import ( + "fmt" "net/mail" "regexp" "strings" "testing" + "time" "github.com/PuerkitoBio/goquery" "github.com/gophish/gophish/config" @@ -37,12 +39,61 @@ func (s *ModelsSuite) TearDownTest(c *check.C) { db.Delete(GroupTarget{}) db.Delete(SMTP{}) db.Delete(Page{}) + db.Delete(Result{}) + db.Delete(MailLog{}) + db.Delete(Campaign{}) // Reset users table to default state. db.Not("id", 1).Delete(User{}) db.Model(User{}).Update("username", "admin") } +func (s *ModelsSuite) createCampaignDependencies(ch *check.C) Campaign { + group := Group{Name: "Test Group"} + group.Targets = []Target{ + Target{Email: "test1@example.com", FirstName: "First", LastName: "Example"}, + Target{Email: "test2@example.com", FirstName: "Second", LastName: "Example"}, + } + group.UserId = 1 + ch.Assert(PostGroup(&group), check.Equals, nil) + + // Add a template + t := Template{Name: "Test Template"} + t.Subject = "{{.RId}} - Subject" + t.Text = "{{.RId}} - Text" + t.HTML = "{{.RId}} - HTML" + t.UserId = 1 + ch.Assert(PostTemplate(&t), check.Equals, nil) + + // Add a landing page + p := Page{Name: "Test Page"} + p.HTML = "Test" + p.UserId = 1 + ch.Assert(PostPage(&p), check.Equals, nil) + + // Add a sending profile + smtp := SMTP{Name: "Test Page"} + smtp.UserId = 1 + smtp.Host = "example.com" + smtp.FromAddress = "test@test.com" + ch.Assert(PostSMTP(&smtp), check.Equals, nil) + + c := Campaign{Name: "Test campaign"} + c.UserId = 1 + c.Template = t + c.Page = p + c.SMTP = smtp + c.Groups = []Group{group} + return c +} + +func (s *ModelsSuite) createCampaign(ch *check.C) Campaign { + c := s.createCampaignDependencies(ch) + // Setup and "launch" our campaign + ch.Assert(PostCampaign(&c, c.UserId), check.Equals, nil) + return c +} + func (s *ModelsSuite) TestGetUser(c *check.C) { u, err := GetUser(1) c.Assert(err, check.Equals, nil) @@ -123,14 +174,15 @@ func (s *ModelsSuite) TestGetGroupsNoGroups(c *check.C) { func (s *ModelsSuite) TestGetGroup(c *check.C) { // Add group. - PostGroup(&Group{ + originalGroup := &Group{ Name: "Test Group", Targets: []Target{Target{Email: "test@example.com"}}, UserId: 1, - }) + } + c.Assert(PostGroup(originalGroup), check.Equals, nil) // Get group and test result. - group, err := GetGroup(1, 1) + group, err := GetGroup(originalGroup.Id, 1) c.Assert(err, check.Equals, nil) c.Assert(len(group.Targets), check.Equals, 1) c.Assert(group.Name, check.Equals, "Test Group") @@ -367,3 +419,25 @@ func (s *ModelsSuite) TestFormatAddress(c *check.C) { } c.Assert(r.FormatAddress(), check.Equals, r.Email) } + +func (s *ModelsSuite) TestResultSendingStatus(ch *check.C) { + c := s.createCampaignDependencies(ch) + ch.Assert(PostCampaign(&c, c.UserId), check.Equals, nil) + // This campaign wasn't scheduled, so we expect the status to + // be sending + fmt.Println("Campaign STATUS") + fmt.Println(c.Status) + for _, r := range c.Results { + ch.Assert(r.Status, check.Equals, STATUS_SENDING) + } +} +func (s *ModelsSuite) TestResultScheduledStatus(ch *check.C) { + c := s.createCampaignDependencies(ch) + c.LaunchDate = time.Now().UTC().Add(time.Hour * time.Duration(1)) + ch.Assert(PostCampaign(&c, c.UserId), check.Equals, nil) + // This campaign wasn't scheduled, so we expect the status to + // be sending + for _, r := range c.Results { + ch.Assert(r.Status, check.Equals, STATUS_SCHEDULED) + } +} diff --git a/models/result.go b/models/result.go index 5fc4c565..f35ace7b 100644 --- a/models/result.go +++ b/models/result.go @@ -7,6 +7,7 @@ import ( "math/big" "net" "net/mail" + "time" "github.com/jinzhu/gorm" "github.com/oschwald/maxminddb-golang" @@ -24,18 +25,19 @@ type mmGeoPoint struct { // Result contains the fields for a result object, // which is a representation of a target in a campaign. type Result struct { - Id int64 `json:"-"` - CampaignId int64 `json:"-"` - UserId int64 `json:"-"` - RId string `json:"id"` - Email string `json:"email"` - FirstName string `json:"first_name"` - LastName string `json:"last_name"` - Position string `json:"position"` - Status string `json:"status" sql:"not null"` - IP string `json:"ip"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` + Id int64 `json:"-"` + CampaignId int64 `json:"-"` + UserId int64 `json:"-"` + RId string `json:"id"` + Email string `json:"email"` + FirstName string `json:"first_name"` + LastName string `json:"last_name"` + Position string `json:"position"` + Status string `json:"status" sql:"not null"` + IP string `json:"ip"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + SendDate time.Time `json:"send_date"` } // UpdateStatus updates the status of the result in the database diff --git a/models/smtp.go b/models/smtp.go index ae6ea9e7..2118485f 100644 --- a/models/smtp.go +++ b/models/smtp.go @@ -1,15 +1,32 @@ package models import ( + "crypto/tls" "errors" "net/mail" + "os" "strconv" "strings" "time" + "github.com/gophish/gomail" + "github.com/gophish/gophish/mailer" "github.com/jinzhu/gorm" ) +// Dialer is a wrapper around a standard gomail.Dialer in order +// to implement the mailer.Dialer interface. This allows us to better +// separate the mailer package as opposed to forcing a connection +// between mailer and gomail. +type Dialer struct { + *gomail.Dialer +} + +// Dial wraps the gomail dialer's Dial command +func (d *Dialer) Dial() (mailer.Sender, error) { + return d.Dialer.Dial() +} + // SMTP contains the attributes needed to handle the sending of campaign emails type SMTP struct { Id int64 `json:"id" gorm:"column:id; primary_key:yes"` @@ -76,6 +93,34 @@ func (s *SMTP) Validate() error { return err } +// GetDialer returns a dialer for the given SMTP profile +func (s *SMTP) GetDialer() (mailer.Dialer, error) { + // Setup the message and dial + hp := strings.Split(s.Host, ":") + if len(hp) < 2 { + hp = append(hp, "25") + } + // Any issues should have been caught in validation, but we'll + // double check here. + port, err := strconv.Atoi(hp[1]) + if err != nil { + Logger.Println(err) + return nil, err + } + d := gomail.NewDialer(hp[0], port, s.Username, s.Password) + d.TLSConfig = &tls.Config{ + ServerName: s.Host, + InsecureSkipVerify: s.IgnoreCertErrors, + } + hostname, err := os.Hostname() + if err != nil { + Logger.Println(err) + hostname = "localhost" + } + d.LocalName = hostname + return &Dialer{d}, err +} + // GetSMTPs returns the SMTPs owned by the given user. func GetSMTPs(uid int64) ([]SMTP, error) { ss := []SMTP{} @@ -84,7 +129,7 @@ func GetSMTPs(uid int64) ([]SMTP, error) { Logger.Println(err) return ss, err } - for i, _ := range ss { + for i := range ss { err = db.Where("smtp_id=?", ss[i].Id).Find(&ss[i].Headers).Error if err != nil && err != gorm.ErrRecordNotFound { Logger.Println(err) @@ -137,7 +182,7 @@ func PostSMTP(s *SMTP) error { Logger.Println(err) } // Save custom headers - for i, _ := range s.Headers { + for i := range s.Headers { s.Headers[i].SMTPId = s.Id err := db.Save(&s.Headers[i]).Error if err != nil { @@ -166,7 +211,7 @@ func PutSMTP(s *SMTP) error { Logger.Println(err) return err } - for i, _ := range s.Headers { + for i := range s.Headers { s.Headers[i].SMTPId = s.Id err := db.Save(&s.Headers[i]).Error if err != nil { diff --git a/models/smtp_test.go b/models/smtp_test.go new file mode 100644 index 00000000..5c4ebf46 --- /dev/null +++ b/models/smtp_test.go @@ -0,0 +1,24 @@ +package models + +import ( + "fmt" + + check "gopkg.in/check.v1" +) + +func (s *ModelsSuite) TestSMTPGetDialer(ch *check.C) { + host := "localhost" + port := 25 + smtp := SMTP{ + Host: fmt.Sprintf("%s:%d", host, port), + IgnoreCertErrors: false, + } + d, err := smtp.GetDialer() + ch.Assert(err, check.Equals, nil) + + dialer := d.(*Dialer).Dialer + ch.Assert(dialer.Host, check.Equals, host) + ch.Assert(dialer.Port, check.Equals, port) + ch.Assert(dialer.TLSConfig.ServerName, check.Equals, smtp.Host) + ch.Assert(dialer.TLSConfig.InsecureSkipVerify, check.Equals, smtp.IgnoreCertErrors) +} diff --git a/static/js/dist/app/campaign_results.min.js b/static/js/dist/app/campaign_results.min.js index 2285b1ed..07170b40 100644 --- a/static/js/dist/app/campaign_results.min.js +++ b/static/js/dist/app/campaign_results.min.js @@ -1 +1 @@ -function dismiss(){$("#modal\\.flashes").empty(),$("#modal").modal("hide"),$("#resultsTable").dataTable().DataTable().clear().draw()}function deleteCampaign(){swal({title:"Are you sure?",text:"This will delete the campaign. This can't be undone!",type:"warning",animation:!1,showCancelButton:!0,confirmButtonText:"Delete Campaign",confirmButtonColor:"#428bca",reverseButtons:!0,allowOutsideClick:!1,showLoaderOnConfirm:!0,preConfirm:function(){return new Promise(function(e,t){api.campaignId.delete(campaign.id).success(function(t){e()}).error(function(e){t(e.responseJSON.message)})})}}).then(function(){swal("Campaign Deleted!","This campaign has been deleted!","success"),$('button:contains("OK")').on("click",function(){location.href="/campaigns"})})}function completeCampaign(){swal({title:"Are you sure?",text:"Gophish will stop processing events for this campaign",type:"warning",animation:!1,showCancelButton:!0,confirmButtonText:"Complete Campaign",confirmButtonColor:"#428bca",reverseButtons:!0,allowOutsideClick:!1,showLoaderOnConfirm:!0,preConfirm:function(){return new Promise(function(e,t){api.campaignId.complete(campaign.id).success(function(t){e()}).error(function(e){t(e.responseJSON.message)})})}}).then(function(){swal("Campaign Completed!","This campaign has been completed!","success"),$("#complete_button")[0].disabled=!0,$("#complete_button").text("Completed!"),doPoll=!1})}function exportAsCSV(e){exportHTML=$("#exportButton").html();var t=null,a=campaign.name+" - "+capitalize(e)+".csv";switch(e){case"results":t=campaign.results;break;case"events":t=campaign.timeline}if(t){$("#exportButton").html('');var s=Papa.unparse(t,{}),i=new Blob([s],{type:"text/csv;charset=utf-8;"});if(navigator.msSaveBlob)navigator.msSaveBlob(i,a);else{var l=window.URL.createObjectURL(i),n=document.createElement("a");n.href=l,n.setAttribute("download",a),document.body.appendChild(n),n.click(),document.body.removeChild(n)}$("#exportButton").html(exportHTML)}}function replay(e){function t(){form.attr({action:url}),form.appendTo("body").submit().remove()}request=campaign.timeline[e],details=JSON.parse(request.details),url=null,form=$("
").attr({method:"POST",target:"_blank"}),$.each(Object.keys(details.payload),function(e,t){return"rid"==t||("__original_url"==t?(url=details.payload[t],!0):void $("").attr({name:t}).val(details.payload[t]).appendTo(form))}),swal({title:"Where do you want the credentials submitted to?",input:"text",showCancelButton:!0,inputPlaceholder:"http://example.com/login",inputValue:url||"",inputValidator:function(e){return new Promise(function(t,a){e?t():a("Invalid URL.")})}}).then(function(e){url=e,t()})}function renderTimeline(e){return record={first_name:e[2],last_name:e[3],email:e[4],position:e[5]},results='
Timeline for '+escapeHtml(record.first_name)+" "+escapeHtml(record.last_name)+'
Email: '+escapeHtml(record.email)+'
',$.each(campaign.timeline,function(e,t){t.email&&t.email!=record.email||(results+='
',results+='
'+escapeHtml(t.message)+' '+moment.utc(t.time).local().format("MMMM Do YYYY h:mm a")+"",t.details&&("Submitted Data"==t.message&&(results+='
',results+='
View Details
'),details=JSON.parse(t.details),details.payload&&(results+='
',results+=' ',results+=" ",$.each(Object.keys(details.payload),function(e,t){return"rid"==t||(results+=" ",results+=" ",results+=" ",void(results+=" "))}),results+="
ParameterValue(s)
"+escapeHtml(t)+""+escapeHtml(details.payload[t])+"
",results+="
"),details.error&&(results+='
View Details
',results+='
',results+='Error '+details.error,results+="
")),results+="
")}),results+="
",results}function poll(){api.campaignId.results(campaign.id).success(function(e){campaign=e;var t=[];$.each(campaign.timeline,function(e,a){var s=moment.utc(a.time).local();t.push({email:a.email,x:s.valueOf(),y:1})});var t=[];$.each(campaign.timeline,function(e,a){var s=moment.utc(a.time).local();t.push({email:a.email,message:a.message,x:s.valueOf(),y:1,marker:{fillColor:statuses[a.message].color}})});var a=$("#timeline_chart").highcharts();a.series[0].update({data:t});var s={};Object.keys(statusMapping).forEach(function(e){s[e]=0}),$.each(campaign.results,function(e,t){s[t.status]++;for(var a=progressListing.indexOf(t.status),e=0;e'+a.status+"",resultsTable.row(e).data(i),s.child.isShown()&&s.child(renderTimeline(s.data())),!1}})}),resultsTable.draw(!1),updateMap(campaign.results),$("#refresh_message").hide(),$("#refresh_btn").show()})}function load(){campaign.id=window.location.pathname.split("/").slice(-1)[0];var e=JSON.parse(localStorage.getItem("gophish.use_map"));api.campaignId.results(campaign.id).success(function(t){if(campaign=t){$("title").text(t.name+" - Gophish"),$("#loading").hide(),$("#campaignResults").show(),$("#page-title").text("Results for "+t.name),"Completed"==t.status&&($("#complete_button")[0].disabled=!0,$("#complete_button").text("Completed!"),doPoll=!1),$('[data-toggle="tooltip"]').tooltip(),$("#resultsTable").on("click",".timeline-event-details",function(){payloadResults=$(this).parent().find(".timeline-event-results"),payloadResults.is(":visible")?($(this).find("i").removeClass("fa-caret-down"),$(this).find("i").addClass("fa-caret-right"),payloadResults.hide()):($(this).find("i").removeClass("fa-caret-right"),$(this).find("i").addClass("fa-caret-down"),payloadResults.show())}),resultsTable=$("#resultsTable").DataTable({destroy:!0,order:[[2,"asc"]],columnDefs:[{orderable:!1,targets:"no-sort"},{className:"details-control",targets:[1]},{visible:!1,targets:[0]}]}),resultsTable.clear();var a={},s=[];Object.keys(statusMapping).forEach(function(e){a[e]=0}),$.each(campaign.results,function(e,t){label=statuses[t.status].label||"label-default",resultsTable.row.add([t.id,'',escapeHtml(t.first_name)||"",escapeHtml(t.last_name)||"",escapeHtml(t.email)||"",escapeHtml(t.position)||"",''+t.status+""]),a[t.status]++;for(var s=progressListing.indexOf(t.status),e=0;eEvent: "+this.point.message+"
Email: "+this.point.email+""}},legend:{enabled:!1},plotOptions:{series:{marker:{enabled:!0,symbol:"circle",radius:3},cursor:"pointer"},line:{states:{hover:{lineWidth:1}}}},credits:{enabled:!1},series:[{data:e.data,dashStyle:"shortdash",color:"#cccccc",lineWidth:1,turboThreshold:0}]})},renderPieChart=function(e){return Highcharts.chart(e.elemId,{chart:{type:"pie",events:{load:function(){var t=this,a=t.renderer,s=t.series[0],i=t.plotLeft+s.center[0],l=t.plotTop+s.center[1];this.innerText=a.text(e.data[0].y,i,l).attr({"text-anchor":"middle","font-size":"24px","font-weight":"bold",fill:e.colors[0],"font-family":"Helvetica,Arial,sans-serif"}).add()},render:function(){this.innerText.attr({text:e.data[0].y})}}},title:{text:e.title},plotOptions:{pie:{innerSize:"80%",dataLabels:{enabled:!1}}},credits:{enabled:!1},tooltip:{formatter:function(){return void 0!=this.key&&'●'+this.point.name+": "+this.y+"
"}},series:[{data:e.data,colors:e.colors}]})},updateMap=function(e){map&&(bubbles=[],$.each(campaign.results,function(e,t){return 0==t.latitude&&0==t.longitude||(newIP=!0,$.each(bubbles,function(e,a){if(a.ip==t.ip)return bubbles[e].radius+=1,newIP=!1,!1}),void(newIP&&bubbles.push({latitude:t.latitude,longitude:t.longitude,name:t.ip,fillKey:"point",radius:2})))}),map.bubbles(bubbles))},setRefresh;$(document).ready(function(){Highcharts.setOptions({global:{useUTC:!1}}),load(),setRefresh=setTimeout(refresh,6e4)}); \ No newline at end of file +function dismiss(){$("#modal\\.flashes").empty(),$("#modal").modal("hide"),$("#resultsTable").dataTable().DataTable().clear().draw()}function deleteCampaign(){swal({title:"Are you sure?",text:"This will delete the campaign. This can't be undone!",type:"warning",animation:!1,showCancelButton:!0,confirmButtonText:"Delete Campaign",confirmButtonColor:"#428bca",reverseButtons:!0,allowOutsideClick:!1,showLoaderOnConfirm:!0,preConfirm:function(){return new Promise(function(e,t){api.campaignId.delete(campaign.id).success(function(t){e()}).error(function(e){t(e.responseJSON.message)})})}}).then(function(){swal("Campaign Deleted!","This campaign has been deleted!","success"),$('button:contains("OK")').on("click",function(){location.href="/campaigns"})})}function completeCampaign(){swal({title:"Are you sure?",text:"Gophish will stop processing events for this campaign",type:"warning",animation:!1,showCancelButton:!0,confirmButtonText:"Complete Campaign",confirmButtonColor:"#428bca",reverseButtons:!0,allowOutsideClick:!1,showLoaderOnConfirm:!0,preConfirm:function(){return new Promise(function(e,t){api.campaignId.complete(campaign.id).success(function(t){e()}).error(function(e){t(e.responseJSON.message)})})}}).then(function(){swal("Campaign Completed!","This campaign has been completed!","success"),$("#complete_button")[0].disabled=!0,$("#complete_button").text("Completed!"),doPoll=!1})}function exportAsCSV(e){exportHTML=$("#exportButton").html();var t=null,a=campaign.name+" - "+capitalize(e)+".csv";switch(e){case"results":t=campaign.results;break;case"events":t=campaign.timeline}if(t){$("#exportButton").html('');var s=Papa.unparse(t,{}),i=new Blob([s],{type:"text/csv;charset=utf-8;"});if(navigator.msSaveBlob)navigator.msSaveBlob(i,a);else{var l=window.URL.createObjectURL(i),n=document.createElement("a");n.href=l,n.setAttribute("download",a),document.body.appendChild(n),n.click(),document.body.removeChild(n)}$("#exportButton").html(exportHTML)}}function replay(e){function t(){form.attr({action:url}),form.appendTo("body").submit().remove()}request=campaign.timeline[e],details=JSON.parse(request.details),url=null,form=$("").attr({method:"POST",target:"_blank"}),$.each(Object.keys(details.payload),function(e,t){return"rid"==t||("__original_url"==t?(url=details.payload[t],!0):void $("").attr({name:t}).val(details.payload[t]).appendTo(form))}),swal({title:"Where do you want the credentials submitted to?",input:"text",showCancelButton:!0,inputPlaceholder:"http://example.com/login",inputValue:url||"",inputValidator:function(e){return new Promise(function(t,a){e?t():a("Invalid URL.")})}}).then(function(e){url=e,t()})}function renderTimeline(e){return record={first_name:e[2],last_name:e[3],email:e[4],position:e[5],status:e[6],send_date:e[7]},results='
Timeline for '+escapeHtml(record.first_name)+" "+escapeHtml(record.last_name)+'
Email: '+escapeHtml(record.email)+'
',$.each(campaign.timeline,function(e,t){t.email&&t.email!=record.email||(results+='
',results+='
'+escapeHtml(t.message)+' '+moment.utc(t.time).local().format("MMMM Do YYYY h:mm a")+"",t.details&&("Submitted Data"==t.message&&(results+='
',results+='
View Details
'),details=JSON.parse(t.details),details.payload&&(results+='
',results+=' ',results+=" ",$.each(Object.keys(details.payload),function(e,t){return"rid"==t||(results+=" ",results+=" ",results+=" ",void(results+=" "))}),results+="
ParameterValue(s)
"+escapeHtml(t)+""+escapeHtml(details.payload[t])+"
",results+="
"),details.error&&(results+='
View Details
',results+='
',results+='Error '+details.error,results+="
")),results+="
")}),"Scheduled"!=record.status&&"Retrying"!=record.status||(results+='
',results+='
Scheduled to send at '+record.send_date+""),results+="
",results}function createStatusLabel(e,t){var a=statuses[e].label||"label-default",s=''+e+"";if("Scheduled"==e||"Retrying"==e){var i="Scheduled to send at "+t;s=''+e+""}return s}function poll(){api.campaignId.results(campaign.id).success(function(e){campaign=e;var t=[];$.each(campaign.timeline,function(e,a){var s=moment.utc(a.time).local();t.push({email:a.email,x:s.valueOf(),y:1})});var t=[];$.each(campaign.timeline,function(e,a){var s=moment.utc(a.time).local();t.push({email:a.email,message:a.message,x:s.valueOf(),y:1,marker:{fillColor:statuses[a.message].color}})});var a=$("#timeline_chart").highcharts();a.series[0].update({data:t});var s={};Object.keys(statusMapping).forEach(function(e){s[e]=0}),$.each(campaign.results,function(e,t){s[t.status]++;for(var a=progressListing.indexOf(t.status),e=0;e',escapeHtml(t.first_name)||"",escapeHtml(t.last_name)||"",escapeHtml(t.email)||"",escapeHtml(t.position)||"",t.status,moment(t.send_date).format("MMMM Do YYYY, h:mm:ss a")]),a[t.status]++;for(var s=progressListing.indexOf(t.status),e=0;eEvent: "+this.point.message+"
Email: "+this.point.email+""}},legend:{enabled:!1},plotOptions:{series:{marker:{enabled:!0,symbol:"circle",radius:3},cursor:"pointer"},line:{states:{hover:{lineWidth:1}}}},credits:{enabled:!1},series:[{data:e.data,dashStyle:"shortdash",color:"#cccccc",lineWidth:1,turboThreshold:0}]})},renderPieChart=function(e){return Highcharts.chart(e.elemId,{chart:{type:"pie",events:{load:function(){var t=this,a=t.renderer,s=t.series[0],i=t.plotLeft+s.center[0],l=t.plotTop+s.center[1];this.innerText=a.text(e.data[0].y,i,l).attr({"text-anchor":"middle","font-size":"24px","font-weight":"bold",fill:e.colors[0],"font-family":"Helvetica,Arial,sans-serif"}).add()},render:function(){this.innerText.attr({text:e.data[0].y})}}},title:{text:e.title},plotOptions:{pie:{innerSize:"80%",dataLabels:{enabled:!1}}},credits:{enabled:!1},tooltip:{formatter:function(){return void 0!=this.key&&'●'+this.point.name+": "+this.y+"
"}},series:[{data:e.data,colors:e.colors}]})},updateMap=function(e){map&&(bubbles=[],$.each(campaign.results,function(e,t){return 0==t.latitude&&0==t.longitude||(newIP=!0,$.each(bubbles,function(e,a){if(a.ip==t.ip)return bubbles[e].radius+=1,newIP=!1,!1}),void(newIP&&bubbles.push({latitude:t.latitude,longitude:t.longitude,name:t.ip,fillKey:"point",radius:2})))}),map.bubbles(bubbles))},setRefresh;$(document).ready(function(){Highcharts.setOptions({global:{useUTC:!1}}),load(),setRefresh=setTimeout(refresh,6e4)}); \ No newline at end of file diff --git a/static/js/src/app/campaign_results.js b/static/js/src/app/campaign_results.js index 4728c998..bc1892e7 100644 --- a/static/js/src/app/campaign_results.js +++ b/static/js/src/app/campaign_results.js @@ -72,6 +72,18 @@ var statuses = { icon: "fa-spinner", point: "ct-point-sending" }, + "Retrying": { + color: "#6c7a89", + label: "label-default", + icon: "fa-clock-o", + point: "ct-point-error" + }, + "Scheduled": { + color: "#428bca", + label: "label-primary", + icon: "fa-clock-o", + point: "ct-point-sending" + }, "Campaign Created": { label: "label-success", icon: "fa-rocket" @@ -268,7 +280,9 @@ function renderTimeline(data) { "first_name": data[2], "last_name": data[3], "email": data[4], - "position": data[5] + "position": data[5], + "status": data[6], + "send_date": data[7] } results = '
' + '
Timeline for ' + escapeHtml(record.first_name) + ' ' + escapeHtml(record.last_name) + @@ -317,6 +331,15 @@ function renderTimeline(data) { results += '
' } }) + // Add the scheduled send event at the bottom + if (record.status == "Scheduled" || record.status == "Retrying") { + results += '
' + + '
' + results += + '
' + + '
' + + '
' + "Scheduled to send at " + record.send_date + '' + } results += '
' return results } @@ -485,6 +508,22 @@ var updateMap = function (results) { map.bubbles(bubbles) } +/** + * Creates a status label for use in the results datatable + * @param {string} status + * @param {moment(datetime)} send_date + */ +function createStatusLabel(status, send_date) { + var label = statuses[status].label || "label-default"; + var statusColumn = "" + status + "" + // Add the tooltip if the email is scheduled to be sent + if (status == "Scheduled" || status == "Retrying") { + var sendDateMessage = "Scheduled to send at " + send_date + statusColumn = "" + status + "" + } + return statusColumn +} + /* poll - Queries the API and updates the UI with the results * * Updates: @@ -564,10 +603,12 @@ function poll() { var rid = rowData[0] $.each(campaign.results, function (j, result) { if (result.id == rid) { - var label = statuses[result.status].label || "label-default"; - rowData[6] = "" + result.status + "" + rowData[7] = moment(result.send_date).format('MMMM Do YYYY, h:mm:ss a') + rowData[6] = result.status resultsTable.row(i).data(rowData) if (row.child.isShown()) { + $(row.node()).find("i").removeClass("fa-caret-right") + $(row.node()).find("i").addClass("fa-caret-down") row.child(renderTimeline(row.data())) } return false @@ -577,6 +618,7 @@ function poll() { resultsTable.draw(false) /* Update the map information */ updateMap(campaign.results) + $('[data-toggle="tooltip"]').tooltip() $("#refresh_message").hide() $("#refresh_btn").show() }) @@ -599,8 +641,6 @@ function load() { $('#complete_button').text('Completed!'); doPoll = false; } - // Setup tooltips - $('[data-toggle="tooltip"]').tooltip() // Setup viewing the details of a result $("#resultsTable").on("click", ".timeline-event-details", function () { // Show the parameters @@ -622,15 +662,22 @@ function load() { [2, "asc"] ], columnDefs: [{ - orderable: false, - targets: "no-sort" - }, { - className: "details-control", - "targets": [1] - }, { - "visible": false, - "targets": [0] - }] + orderable: false, + targets: "no-sort" + }, { + className: "details-control", + "targets": [1] + }, { + "visible": false, + "targets": [0, 7] + }, + { + "render": function (data, type, row) { + return createStatusLabel(data, row[7]) + }, + "targets": [6] + } + ] }); resultsTable.clear(); var email_series_data = {} @@ -639,7 +686,6 @@ function load() { email_series_data[k] = 0 }); $.each(campaign.results, function (i, result) { - label = statuses[result.status].label || "label-default"; resultsTable.row.add([ result.id, "", @@ -647,7 +693,8 @@ function load() { escapeHtml(result.last_name) || "", escapeHtml(result.email) || "", escapeHtml(result.position) || "", - "" + result.status + "" + result.status, + moment(result.send_date).format('MMMM Do YYYY, h:mm:ss a') ]) email_series_data[result.status]++; // Backfill status values @@ -657,6 +704,8 @@ function load() { } }) resultsTable.draw(); + // Setup tooltips + $('[data-toggle="tooltip"]').tooltip() // Setup the individual timelines $('#resultsTable tbody').on('click', 'td.details-control', function () { var tr = $(this).closest('tr'); @@ -667,14 +716,12 @@ function load() { tr.removeClass('shown'); $(this).find("i").removeClass("fa-caret-down") $(this).find("i").addClass("fa-caret-right") - row.invalidate('dom').draw(false) } else { // Open this row $(this).find("i").removeClass("fa-caret-right") $(this).find("i").addClass("fa-caret-down") row.child(renderTimeline(row.data())).show(); tr.addClass('shown'); - row.invalidate('dom').draw(false) } }); // Setup the graphs diff --git a/worker/doc.go b/worker/doc.go index dbc961b7..4e807c8c 100644 --- a/worker/doc.go +++ b/worker/doc.go @@ -25,4 +25,8 @@ THE SOFTWARE. */ // Package worker contains the functionality for the background worker process. +// It starts a background service that polls every minute for scheduled campaigns +// to be launched. +// If a campaign is found, it gathers the maillogs associated with the campaign and +// sends them to the mailer package to be processed. package worker diff --git a/worker/worker.go b/worker/worker.go index 0de802ee..acfae0f2 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -1,23 +1,12 @@ package worker import ( - "bytes" - "crypto/tls" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" "log" - "net/mail" "os" - "strconv" - "strings" - "text/template" "time" + "github.com/gophish/gophish/mailer" "github.com/gophish/gophish/models" - "gopkg.in/gomail.v2" ) // Logger is the logger for the worker @@ -31,343 +20,84 @@ func New() *Worker { return &Worker{} } -// Start launches the worker to poll the database every minute for any jobs. -// If a job is found, it launches the job +// Start launches the worker to poll the database every minute for any pending maillogs +// that need to be processed. func (w *Worker) Start() { Logger.Println("Background Worker Started Successfully - Waiting for Campaigns") for t := range time.Tick(1 * time.Minute) { - cs, err := models.GetQueuedCampaigns(t.UTC()) - // Not really sure of a clean way to catch errors per campaign... + ms, err := models.GetQueuedMailLogs(t.UTC()) if err != nil { Logger.Println(err) continue } - for _, c := range cs { - go func(c models.Campaign) { - processCampaign(&c) - }(c) + // Lock the MailLogs (they will be unlocked after processing) + err = models.LockMailLogs(ms, true) + if err != nil { + Logger.Println(err) + continue + } + // We'll group the maillogs by campaign ID to (sort of) group + // them by sending profile. This lets the mailer re-use the Sender + // instead of having to re-connect to the SMTP server for every + // email. + msg := make(map[int64][]mailer.Mail) + for _, m := range ms { + msg[m.CampaignId] = append(msg[m.CampaignId], m) + } + + // Next, we process each group of maillogs in parallel + for cid, msc := range msg { + go func(cid int64, msc []mailer.Mail) { + uid := msc[0].(*models.MailLog).UserId + c, err := models.GetCampaign(cid, uid) + if err != nil { + Logger.Println(err) + errorMail(err, msc) + return + } + if c.Status == models.CAMPAIGN_QUEUED { + err := c.UpdateStatus(models.CAMPAIGN_IN_PROGRESS) + if err != nil { + Logger.Println(err) + return + } + } + Logger.Printf("Sending %d maillogs to Mailer", len(msc)) + mailer.Mailer.Queue <- msc + }(cid, msc) } } } -func processCampaign(c *models.Campaign) { - Logger.Printf("Worker received: %s", c.Name) - err := c.UpdateStatus(models.CAMPAIGN_IN_PROGRESS) +// LaunchCampaign starts a campaign +func (w *Worker) LaunchCampaign(c models.Campaign) { + ms, err := models.GetMailLogsByCampaign(c.Id) if err != nil { Logger.Println(err) - } - f, err := mail.ParseAddress(c.SMTP.FromAddress) - if err != nil { - Logger.Println(err) - } - fn := f.Name - if fn == "" { - fn = f.Address - } - // Setup the message and dial - hp := strings.Split(c.SMTP.Host, ":") - if len(hp) < 2 { - hp = append(hp, "25") - } - // Any issues should have been caught in validation, so we just log - port, err := strconv.Atoi(hp[1]) - if err != nil { - Logger.Println(err) - } - d := gomail.NewDialer(hp[0], port, c.SMTP.Username, c.SMTP.Password) - d.TLSConfig = &tls.Config{ - ServerName: c.SMTP.Host, - InsecureSkipVerify: c.SMTP.IgnoreCertErrors, - } - hostname, err := os.Hostname() - if err != nil { - Logger.Println(err) - hostname = "localhost" - } - d.LocalName = hostname - s, err := d.Dial() - // Short circuit if we have an err - // However, we still need to update each target - if err != nil { - Logger.Println(err) - for _, t := range c.Results { - es := struct { - Error string `json:"error"` - }{ - Error: err.Error(), - } - ej, err := json.Marshal(es) - if err != nil { - Logger.Println(err) - } - err = t.UpdateStatus(models.ERROR) - if err != nil { - Logger.Println(err) - } - err = c.AddEvent(models.Event{Email: t.Email, Message: models.EVENT_SENDING_ERROR, Details: string(ej)}) - if err != nil { - Logger.Println(err) - } - } return } - // Send each email - e := gomail.NewMessage() - for _, t := range c.Results { - e.SetAddressHeader("From", f.Address, f.Name) - td := struct { - models.Result - URL string - TrackingURL string - Tracker string - From string - }{ - t, - c.URL + "?rid=" + t.RId, - c.URL + "/track?rid=" + t.RId, - "", - fn, - } - - // Parse the customHeader templates - for _, header := range c.SMTP.Headers { - parsedHeader := struct { - Key bytes.Buffer - Value bytes.Buffer - }{} - keytmpl, err := template.New("text_template").Parse(header.Key) - if err != nil { - Logger.Println(err) - } - err = keytmpl.Execute(&parsedHeader.Key, td) - if err != nil { - Logger.Println(err) - } - - valtmpl, err := template.New("text_template").Parse(header.Value) - if err != nil { - Logger.Println(err) - } - err = valtmpl.Execute(&parsedHeader.Value, td) - if err != nil { - Logger.Println(err) - } - - // Add our header immediately - e.SetHeader(parsedHeader.Key.String(), parsedHeader.Value.String()) - } - - // Parse remaining templates - var subjBuff bytes.Buffer - tmpl, err := template.New("text_template").Parse(c.Template.Subject) - if err != nil { - Logger.Println(err) - } - err = tmpl.Execute(&subjBuff, td) - if err != nil { - Logger.Println(err) - } - e.SetHeader("Subject", subjBuff.String()) - Logger.Println("Creating email using template") - e.SetHeader("To", t.FormatAddress()) - if c.Template.Text != "" { - var textBuff bytes.Buffer - tmpl, err = template.New("text_template").Parse(c.Template.Text) - if err != nil { - Logger.Println(err) - } - err = tmpl.Execute(&textBuff, td) - if err != nil { - Logger.Println(err) - } - e.SetBody("text/plain", textBuff.String()) - } - if c.Template.HTML != "" { - var htmlBuff bytes.Buffer - tmpl, err = template.New("html_template").Parse(c.Template.HTML) - if err != nil { - Logger.Println(err) - } - err = tmpl.Execute(&htmlBuff, td) - if err != nil { - Logger.Println(err) - } - if c.Template.Text == "" { - e.SetBody("text/html", htmlBuff.String()) - } else { - e.AddAlternative("text/html", htmlBuff.String()) - } - } - // Attach the files - for _, a := range c.Template.Attachments { - e.Attach(func(a models.Attachment) (string, gomail.FileSetting, gomail.FileSetting) { - h := map[string][]string{"Content-ID": {fmt.Sprintf("<%s>", a.Name)}} - return a.Name, gomail.SetCopyFunc(func(w io.Writer) error { - decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(a.Content)) - _, err = io.Copy(w, decoder) - return err - }), gomail.SetHeader(h) - }(a)) - } - Logger.Printf("Sending Email to %s\n", t.Email) - err = gomail.Send(s, e) - if err != nil { - Logger.Println(err) - es := struct { - Error string `json:"error"` - }{ - Error: err.Error(), - } - ej, err := json.Marshal(es) - if err != nil { - Logger.Println(err) - } - err = t.UpdateStatus(models.ERROR) - if err != nil { - Logger.Println(err) - } - err = c.AddEvent(models.Event{Email: t.Email, Message: models.EVENT_SENDING_ERROR, Details: string(ej)}) - if err != nil { - Logger.Println(err) - } - } else { - err = t.UpdateStatus(models.EVENT_SENT) - if err != nil { - Logger.Println(err) - } - err = c.AddEvent(models.Event{Email: t.Email, Message: models.EVENT_SENT}) - if err != nil { - Logger.Println(err) - } - } - e.Reset() - } - err = c.UpdateStatus(models.CAMPAIGN_EMAILS_SENT) - if err != nil { - Logger.Println(err) + models.LockMailLogs(ms, true) + // This is required since you cannot pass a slice of values + // that implements an interface as a slice of that interface. + mailEntries := []mailer.Mail{} + for _, m := range ms { + mailEntries = append(mailEntries, m) } + mailer.Mailer.Queue <- mailEntries } -func SendTestEmail(s *models.SendTestEmailRequest) error { - f, err := mail.ParseAddress(s.SMTP.FromAddress) - if err != nil { - Logger.Println(err) - return err - } - hp := strings.Split(s.SMTP.Host, ":") - if len(hp) < 2 { - hp = append(hp, "25") - } - port, err := strconv.Atoi(hp[1]) - if err != nil { - Logger.Println(err) - return err - } - d := gomail.NewDialer(hp[0], port, s.SMTP.Username, s.SMTP.Password) - d.TLSConfig = &tls.Config{ - ServerName: s.SMTP.Host, - InsecureSkipVerify: s.SMTP.IgnoreCertErrors, - } - hostname, err := os.Hostname() - if err != nil { - Logger.Println(err) - hostname = "localhost" - } - d.LocalName = hostname - dc, err := d.Dial() - if err != nil { - Logger.Println(err) - return err - } - Logger.Println("Creating email using template") - e := gomail.NewMessage() - // Parse the customHeader templates - for _, header := range s.SMTP.Headers { - parsedHeader := struct { - Key bytes.Buffer - Value bytes.Buffer - }{} - keytmpl, err := template.New("text_template").Parse(header.Key) - if err != nil { - Logger.Println(err) - } - err = keytmpl.Execute(&parsedHeader.Key, s) - if err != nil { - Logger.Println(err) - } - - valtmpl, err := template.New("text_template").Parse(header.Value) - if err != nil { - Logger.Println(err) - } - err = valtmpl.Execute(&parsedHeader.Value, s) - if err != nil { - Logger.Println(err) - } - - // Add our header immediately - e.SetHeader(parsedHeader.Key.String(), parsedHeader.Value.String()) - } - e.SetAddressHeader("From", f.Address, f.Name) - e.SetHeader("To", s.FormatAddress()) - // Parse the templates - var subjBuff bytes.Buffer - tmpl, err := template.New("text_template").Parse(s.Template.Subject) - if err != nil { - Logger.Println(err) - } - err = tmpl.Execute(&subjBuff, s) - if err != nil { - Logger.Println(err) - } - e.SetHeader("Subject", subjBuff.String()) - if s.Template.Text != "" { - var textBuff bytes.Buffer - tmpl, err = template.New("text_template").Parse(s.Template.Text) - if err != nil { - Logger.Println(err) - } - err = tmpl.Execute(&textBuff, s) - if err != nil { - Logger.Println(err) - } - e.SetBody("text/plain", textBuff.String()) - } - if s.Template.HTML != "" { - var htmlBuff bytes.Buffer - tmpl, err = template.New("html_template").Parse(s.Template.HTML) - if err != nil { - Logger.Println(err) - } - err = tmpl.Execute(&htmlBuff, s) - if err != nil { - Logger.Println(err) - } - // If we don't have a text part, make the html the root part - if s.Template.Text == "" { - e.SetBody("text/html", htmlBuff.String()) - } else { - e.AddAlternative("text/html", htmlBuff.String()) - } - } - // Attach the files - for _, a := range s.Template.Attachments { - e.Attach(func(a models.Attachment) (string, gomail.FileSetting) { - return a.Name, gomail.SetCopyFunc(func(w io.Writer) error { - decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(a.Content)) - _, err = io.Copy(w, decoder) - return err - }) - }(a)) - } - Logger.Printf("Sending Email to %s\n", s.Email) - err = gomail.Send(dc, e) - if err != nil { - Logger.Println(err) - // For now, let's split the error and return - // the last element (the most descriptive error message) - serr := strings.Split(err.Error(), ":") - return errors.New(serr[len(serr)-1]) - } - return err +// SendTestEmail sends a test email +func (w *Worker) SendTestEmail(s *models.SendTestEmailRequest) error { + go func() { + mailer.Mailer.Queue <- []mailer.Mail{s} + }() + return <-s.ErrorChan +} + +// errorMail is a helper to handle erroring out a slice of Mail instances +// in the case that an unrecoverable error occurs. +func errorMail(err error, ms []mailer.Mail) { + for _, m := range ms { + m.Error(err) + } } diff --git a/worker/worker_test.go b/worker/worker_test.go new file mode 100644 index 00000000..292c784d --- /dev/null +++ b/worker/worker_test.go @@ -0,0 +1,79 @@ +package worker + +import ( + "github.com/gophish/gophish/config" + "github.com/gophish/gophish/models" + "github.com/stretchr/testify/suite" +) + +// WorkerSuite is a suite of tests to cover API related functions +type WorkerSuite struct { + suite.Suite + ApiKey string +} + +func (s *WorkerSuite) SetupSuite() { + config.Conf.DBName = "sqlite3" + config.Conf.DBPath = ":memory:" + config.Conf.MigrationsPath = "../db/db_sqlite3/migrations/" + err := models.Setup() + if err != nil { + s.T().Fatalf("Failed creating database: %v", err) + } + s.Nil(err) +} + +func (s *WorkerSuite) TearDownTest() { + campaigns, _ := models.GetCampaigns(1) + for _, campaign := range campaigns { + models.DeleteCampaign(campaign.Id) + } +} + +func (s *WorkerSuite) SetupTest() { + config.Conf.TestFlag = true + // Add a group + group := models.Group{Name: "Test Group"} + group.Targets = []models.Target{ + models.Target{Email: "test1@example.com", FirstName: "First", LastName: "Example"}, + models.Target{Email: "test2@example.com", FirstName: "Second", LastName: "Example"}, + } + group.UserId = 1 + models.PostGroup(&group) + + // Add a template + t := models.Template{Name: "Test Template"} + t.Subject = "Test subject" + t.Text = "Text text" + t.HTML = "Test" + t.UserId = 1 + models.PostTemplate(&t) + + // Add a landing page + p := models.Page{Name: "Test Page"} + p.HTML = "Test" + p.UserId = 1 + models.PostPage(&p) + + // Add a sending profile + smtp := models.SMTP{Name: "Test Page"} + smtp.UserId = 1 + smtp.Host = "example.com" + smtp.FromAddress = "test@test.com" + models.PostSMTP(&smtp) + + // Setup and "launch" our campaign + // Set the status such that no emails are attempted + c := models.Campaign{Name: "Test campaign"} + c.UserId = 1 + c.Template = t + c.Page = p + c.SMTP = smtp + c.Groups = []models.Group{group} + models.PostCampaign(&c, c.UserId) + c.UpdateStatus(models.CAMPAIGN_EMAILS_SENT) +} + +func (s *WorkerSuite) TestMailSendSuccess() { + // TODO +}