Generalize UI events for cataloging tasks (#2369)

* generalize ui events for cataloging tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* moderate review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* incorporate review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename cataloger task progress object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* migrate cataloger task fn to bus helper

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman 2023-11-30 11:25:50 -05:00 committed by GitHub
parent b943da6433
commit 4adfbeb5f0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 427 additions and 807 deletions

View file

@ -1,16 +1,24 @@
[TestHandler_handleCatalogerTaskStarted/cataloging_task_in_progress - 1] [TestHandler_handleCatalogerTaskStarted/cataloging_task_in_progress - 1]
some task title [some value] ⠙ Cataloging contents
⠙ some task title ━━━━━━━━━━━━━━━━━━━━ [some stage]
--- ---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_in_progress - 1] [TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_in_progress - 1]
└── some task title [some value] ⠙ Cataloging contents
└── ⠙ some task title ━━━━━━━━━━━━━━━━━━━━ [some stage]
--- ---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete - 1] [TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete - 1]
✔ └── some task done [some value] ⠙ Cataloging contents
└── ✔ some task done [some stage]
---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete_--_hide_stage - 1]
⠙ Cataloging contents
└── ✔ some task done
--- ---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete_with_removal - 1] [TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete_with_removal - 1]
⠙ Cataloging contents
--- ---

View file

@ -100,18 +100,21 @@ func TestHandler_handleAttestationStarted(t *testing.T) {
Height: 80, Height: 80,
} }
models := handler.Handle(event) models, _ := handler.Handle(event)
require.Len(t, models, 2) require.Len(t, models, 2)
t.Run("task line", func(t *testing.T) { t.Run("task line", func(t *testing.T) {
tsk, ok := models[0].(taskprogress.Model) tsk, ok := models[0].(taskprogress.Model)
require.True(t, ok) require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(), Time: time.Now(),
Sequence: tsk.Sequence(), Sequence: tsk.Sequence(),
ID: tsk.ID(), ID: tsk.ID(),
}) })
got := gotModel.View()
t.Log(got) t.Log(got)
snaps.MatchSnapshot(t, got) snaps.MatchSnapshot(t, got)
}) })
@ -119,11 +122,15 @@ func TestHandler_handleAttestationStarted(t *testing.T) {
t.Run("log", func(t *testing.T) { t.Run("log", func(t *testing.T) {
log, ok := models[1].(attestLogFrame) log, ok := models[1].(attestLogFrame)
require.True(t, ok) require.True(t, ok)
got := runModel(t, log, tt.iterations, attestLogFrameTickMsg{
gotModel := runModel(t, log, tt.iterations, attestLogFrameTickMsg{
Time: time.Now(), Time: time.Now(),
Sequence: log.sequence, Sequence: log.sequence,
ID: log.id, ID: log.id,
}, log.reader.running) }, log.reader.running)
got := gotModel.View()
t.Log(got) t.Log(got)
snaps.MatchSnapshot(t, got) snaps.MatchSnapshot(t, got)
}) })

View file

@ -3,70 +3,121 @@ package ui
import ( import (
tea "github.com/charmbracelet/bubbletea" tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss" "github.com/charmbracelet/lipgloss"
"github.com/google/uuid"
"github.com/wagoodman/go-partybus" "github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress" "github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress" "github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/bubbly/bubbles/tree"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/event/monitor"
syftEventParsers "github.com/anchore/syft/syft/event/parsers" syftEventParsers "github.com/anchore/syft/syft/event/parsers"
) )
var _ progress.Stager = (*catalogerTaskStageAdapter)(nil) // we standardize how rows are instantiated to ensure consistency in the appearance across the UI
type taskModelFactory func(title taskprogress.Title, opts ...taskprogress.Option) taskprogress.Model
type catalogerTaskStageAdapter struct { var _ tea.Model = (*catalogerTaskModel)(nil)
mon *monitor.CatalogerTask
type catalogerTaskModel struct {
model tree.Model
modelFactory taskModelFactory
} }
func newCatalogerTaskStageAdapter(mon *monitor.CatalogerTask) *catalogerTaskStageAdapter { func newCatalogerTaskTreeModel(f taskModelFactory) *catalogerTaskModel {
return &catalogerTaskStageAdapter{ t := tree.NewModel()
mon: mon, t.Padding = " "
t.RootsWithoutPrefix = true
return &catalogerTaskModel{
modelFactory: f,
model: t,
} }
} }
func (c catalogerTaskStageAdapter) Stage() string { type newCatalogerTaskRowEvent struct {
return c.mon.GetValue() info monitor.GenericTask
prog progress.StagedProgressable
} }
func (m *Handler) handleCatalogerTaskStarted(e partybus.Event) []tea.Model { func (cts catalogerTaskModel) Init() tea.Cmd {
mon, err := syftEventParsers.ParseCatalogerTaskStarted(e) return cts.model.Init()
if err != nil { }
log.WithFields("error", err).Warn("unable to parse event")
return nil func (cts catalogerTaskModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
event, ok := msg.(newCatalogerTaskRowEvent)
if !ok {
model, cmd := cts.model.Update(msg)
cts.model = model.(tree.Model)
return cts, cmd
} }
var prefix string info, prog := event.info, event.prog
if mon.SubStatus {
// TODO: support list of sub-statuses, not just a single leaf
prefix = "└── "
}
tsk := m.newTaskProgress( tsk := cts.modelFactory(
taskprogress.Title{ taskprogress.Title{
// TODO: prefix should not be part of the title, but instead a separate field that is aware of the tree structure Default: info.Title.Default,
Default: prefix + mon.Title, Running: info.Title.WhileRunning,
Running: prefix + mon.Title, Success: info.Title.OnSuccess,
Success: prefix + mon.TitleOnCompletion,
}, },
taskprogress.WithStagedProgressable( taskprogress.WithStagedProgressable(prog),
struct {
progress.Stager
progress.Progressable
}{
Progressable: mon.GetMonitor(),
Stager: newCatalogerTaskStageAdapter(mon),
},
),
) )
// TODO: this isn't ideal since the model stays around after it is no longer needed, but it works for now if info.Context != "" {
tsk.HideOnSuccess = mon.RemoveOnCompletion tsk.Context = []string{info.Context}
tsk.HideStageOnSuccess = false }
tsk.HideProgressOnSuccess = false
tsk.TitleStyle = lipgloss.NewStyle() tsk.HideOnSuccess = info.HideOnSuccess
// TODO: this is a hack to get the spinner to not show up, but ideally the component would support making the spinner optional tsk.HideStageOnSuccess = info.HideStageOnSuccess
tsk.Spinner.Spinner.Frames = []string{" "} tsk.HideProgressOnSuccess = true
return []tea.Model{tsk} if info.ParentID != "" {
tsk.TitleStyle = lipgloss.NewStyle()
}
if err := cts.model.Add(info.ParentID, info.ID, tsk); err != nil {
log.WithFields("error", err).Error("unable to add cataloger task to tree model")
}
return cts, tsk.Init()
}
func (cts catalogerTaskModel) View() string {
return cts.model.View()
}
func (m *Handler) handleCatalogerTaskStarted(e partybus.Event) ([]tea.Model, tea.Cmd) {
mon, info, err := syftEventParsers.ParseCatalogerTaskStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil, nil
}
var models []tea.Model
// only create the new cataloger task tree once to manage all cataloger task events
m.onNewCatalogerTask.Do(func() {
models = append(models, newCatalogerTaskTreeModel(m.newTaskProgress))
})
// we need to update the cataloger task model with a new row. We should never update the model outside of the
// bubbletea update-render event loop. Instead, we return a command that will be executed by the bubbletea runtime,
// producing a message that is passed to the cataloger task model. This is the prescribed way to update models
// in bubbletea.
if info.ID == "" {
// ID is optional from the consumer perspective, but required internally
info.ID = uuid.Must(uuid.NewRandom()).String()
}
cmd := func() tea.Msg {
// this message will cause the cataloger task model to add a new row to the output based on the given task
// information and progress data.
return newCatalogerTaskRowEvent{
info: *info,
prog: mon,
}
}
return models, cmd
} }

View file

@ -2,19 +2,22 @@ package ui
import ( import (
"testing" "testing"
"time"
tea "github.com/charmbracelet/bubbletea" tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps" "github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus" "github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event" syftEvent "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/event/monitor"
) )
func TestHandler_handleCatalogerTaskStarted(t *testing.T) { func TestHandler_handleCatalogerTaskStarted(t *testing.T) {
title := monitor.Title{
Default: "some task title",
OnSuccess: "some task done",
}
tests := []struct { tests := []struct {
name string name string
eventFn func(*testing.T) partybus.Event eventFn func(*testing.T) partybus.Event
@ -23,99 +26,171 @@ func TestHandler_handleCatalogerTaskStarted(t *testing.T) {
{ {
name: "cataloging task in progress", name: "cataloging task in progress",
eventFn: func(t *testing.T) partybus.Event { eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{ value := &monitor.CatalogerTaskProgress{
SubStatus: false, AtomicStage: progress.NewAtomicStage("some stage"),
RemoveOnCompletion: false, Manual: progress.NewManual(100),
Title: "some task title",
TitleOnCompletion: "some task done",
} }
src.SetValue("some value") value.Manual.Add(50)
return partybus.Event{ return partybus.Event{
Type: syftEvent.CatalogerTaskStarted, Type: syftEvent.CatalogerTaskStarted,
Source: src, Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: false,
ID: "my-id",
},
Value: value,
} }
}, },
}, },
{ {
name: "cataloging sub task in progress", name: "cataloging sub task in progress",
eventFn: func(t *testing.T) partybus.Event { eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{ value := &monitor.CatalogerTaskProgress{
SubStatus: true, AtomicStage: progress.NewAtomicStage("some stage"),
RemoveOnCompletion: false, Manual: progress.NewManual(100),
Title: "some task title",
TitleOnCompletion: "some task done",
} }
src.SetValue("some value") value.Manual.Add(50)
return partybus.Event{ return partybus.Event{
Type: syftEvent.CatalogerTaskStarted, Type: syftEvent.CatalogerTaskStarted,
Source: src, Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: false,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
} }
}, },
}, },
{ {
name: "cataloging sub task complete", name: "cataloging sub task complete",
eventFn: func(t *testing.T) partybus.Event { eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{ value := &monitor.CatalogerTaskProgress{
SubStatus: true, AtomicStage: progress.NewAtomicStage("some stage"),
RemoveOnCompletion: false, Manual: progress.NewManual(100),
Title: "some task title",
TitleOnCompletion: "some task done",
} }
src.SetValue("some value") value.SetCompleted()
src.SetCompleted()
return partybus.Event{ return partybus.Event{
Type: syftEvent.CatalogerTaskStarted, Type: syftEvent.CatalogerTaskStarted,
Source: src, Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: false,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
}
},
},
{
name: "cataloging sub task complete -- hide stage",
eventFn: func(t *testing.T) partybus.Event {
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
value.SetCompleted()
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: true,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
} }
}, },
}, },
{ {
name: "cataloging sub task complete with removal", name: "cataloging sub task complete with removal",
eventFn: func(t *testing.T) partybus.Event { eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{ value := &monitor.CatalogerTaskProgress{
SubStatus: true, AtomicStage: progress.NewAtomicStage("some stage"),
RemoveOnCompletion: true, Manual: progress.NewManual(100),
Title: "some task title",
TitleOnCompletion: "some task done",
} }
src.SetValue("some value") value.SetCompleted()
src.SetCompleted()
return partybus.Event{ return partybus.Event{
Type: syftEvent.CatalogerTaskStarted, Type: syftEvent.CatalogerTaskStarted,
Source: src, Source: monitor.GenericTask{
Title: title,
HideOnSuccess: true,
HideStageOnSuccess: false,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
} }
}, },
}, },
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t) // need to be able to get the initial newCatalogerTaskRowEvent + initialize the nested taskprogress model
if tt.iterations == 0 {
tt.iterations = 2
}
e := tt.eventFn(t)
handler := New(DefaultHandlerConfig()) handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{ handler.WindowSize = tea.WindowSizeMsg{
Width: 100, Width: 100,
Height: 80, Height: 80,
} }
models := handler.Handle(event) info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog contents",
WhileRunning: "Cataloging contents",
OnSuccess: "Cataloged contents",
},
ID: "top-level-task",
}
// note: this line / event is not under test, only needed to show a sub status
kickoffEvent := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage(""),
Manual: progress.NewManual(-1),
}
models, cmd := handler.Handle(
partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: info,
Value: progress.StagedProgressable(kickoffEvent),
},
)
require.Len(t, models, 1) require.Len(t, models, 1)
require.NotNil(t, cmd)
model := models[0] model := models[0]
tsk, ok := model.(taskprogress.Model) tr, ok := model.(*catalogerTaskModel)
require.True(t, ok) require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ gotModel := runModel(t, tr, tt.iterations, cmd())
Time: time.Now(),
Sequence: tsk.Sequence(), models, cmd = handler.Handle(e)
ID: tsk.ID(), require.Len(t, models, 0)
}) require.NotNil(t, cmd)
gotModel = runModel(t, gotModel, tt.iterations, cmd())
got := gotModel.View()
t.Log(got) t.Log(got)
snaps.MatchSnapshot(t, got) snaps.MatchSnapshot(t, got)
}) })

View file

@ -80,18 +80,21 @@ func TestHandler_handleFetchImage(t *testing.T) {
Height: 80, Height: 80,
} }
models := handler.Handle(event) models, _ := handler.Handle(event)
require.Len(t, models, 1) require.Len(t, models, 1)
model := models[0] model := models[0]
tsk, ok := model.(taskprogress.Model) tsk, ok := model.(taskprogress.Model)
require.True(t, ok) require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(), Time: time.Now(),
Sequence: tsk.Sequence(), Sequence: tsk.Sequence(),
ID: tsk.ID(), ID: tsk.ID(),
}) })
got := gotModel.View()
t.Log(got) t.Log(got)
snaps.MatchSnapshot(t, got) snaps.MatchSnapshot(t, got)
}) })

View file

@ -1,28 +0,0 @@
package ui
import (
tea "github.com/charmbracelet/bubbletea"
"github.com/wagoodman/go-partybus"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/syft/internal/log"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
)
func (m *Handler) handleFileDigestsCatalogerStarted(e partybus.Event) []tea.Model {
prog, err := syftEventParsers.ParseFileDigestsCatalogingStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
}
tsk := m.newTaskProgress(
taskprogress.Title{
Default: "Catalog file digests",
Running: "Cataloging file digests",
Success: "Cataloged file digests",
}, taskprogress.WithStagedProgressable(prog),
)
return []tea.Model{tsk}
}

View file

@ -1,97 +0,0 @@
package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
)
func TestHandler_handleFileDigestsCatalogerStarted(t *testing.T) {
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
iterations int
}{
{
name: "cataloging in progress",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileDigestsCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileDigestsCatalogerStarted,
Value: mon,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
t.Log(got)
snaps.MatchSnapshot(t, got)
})
}
}

View file

@ -80,18 +80,21 @@ func TestHandler_handleFileIndexingStarted(t *testing.T) {
Height: 80, Height: 80,
} }
models := handler.Handle(event) models, _ := handler.Handle(event)
require.Len(t, models, 1) require.Len(t, models, 1)
model := models[0] model := models[0]
tsk, ok := model.(taskprogress.Model) tsk, ok := model.(taskprogress.Model)
require.True(t, ok) require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(), Time: time.Now(),
Sequence: tsk.Sequence(), Sequence: tsk.Sequence(),
ID: tsk.ID(), ID: tsk.ID(),
}) })
got := gotModel.View()
t.Log(got) t.Log(got)
snaps.MatchSnapshot(t, got) snaps.MatchSnapshot(t, got)
}) })

View file

@ -1,29 +0,0 @@
package ui
import (
tea "github.com/charmbracelet/bubbletea"
"github.com/wagoodman/go-partybus"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/syft/internal/log"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
)
func (m *Handler) handleFileMetadataCatalogerStarted(e partybus.Event) []tea.Model {
prog, err := syftEventParsers.ParseFileMetadataCatalogingStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
}
tsk := m.newTaskProgress(
taskprogress.Title{
Default: "Catalog file metadata",
Running: "Cataloging file metadata",
Success: "Cataloged file metadata",
},
taskprogress.WithStagedProgressable(prog),
)
return []tea.Model{tsk}
}

View file

@ -1,97 +0,0 @@
package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
)
func TestHandler_handleFileMetadataCatalogerStarted(t *testing.T) {
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
iterations int
}{
{
name: "cataloging in progress",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileMetadataCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileMetadataCatalogerStarted,
Value: mon,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
t.Log(got)
snaps.MatchSnapshot(t, got)
})
}
}

View file

@ -1,87 +0,0 @@
package ui
import (
"fmt"
tea "github.com/charmbracelet/bubbletea"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/syft/internal/log"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
"github.com/anchore/syft/syft/pkg/cataloger"
)
var _ progress.StagedProgressable = (*packageCatalogerProgressAdapter)(nil)
type packageCatalogerProgressAdapter struct {
monitor *cataloger.Monitor
monitors []progress.Monitorable
}
func newPackageCatalogerProgressAdapter(monitor *cataloger.Monitor) packageCatalogerProgressAdapter {
return packageCatalogerProgressAdapter{
monitor: monitor,
monitors: []progress.Monitorable{
monitor.FilesProcessed,
monitor.PackagesDiscovered,
},
}
}
func (p packageCatalogerProgressAdapter) Stage() string {
return fmt.Sprintf("%d packages", p.monitor.PackagesDiscovered.Current())
}
func (p packageCatalogerProgressAdapter) Current() int64 {
return p.monitor.PackagesDiscovered.Current()
}
func (p packageCatalogerProgressAdapter) Error() error {
completedMonitors := 0
for _, monitor := range p.monitors {
err := monitor.Error()
if err == nil {
continue
}
if progress.IsErrCompleted(err) {
completedMonitors++
continue
}
// something went wrong
return err
}
if completedMonitors == len(p.monitors) && len(p.monitors) > 0 {
return p.monitors[0].Error()
}
return nil
}
func (p packageCatalogerProgressAdapter) Size() int64 {
// this is an inherently unknown value (indeterminate total number of packages to discover)
return -1
}
func (m *Handler) handlePackageCatalogerStarted(e partybus.Event) []tea.Model {
monitor, err := syftEventParsers.ParsePackageCatalogerStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
}
tsk := m.newTaskProgress(
taskprogress.Title{
Default: "Catalog packages",
Running: "Cataloging packages",
Success: "Cataloged packages",
},
taskprogress.WithStagedProgressable(
newPackageCatalogerProgressAdapter(monitor),
),
)
tsk.HideStageOnSuccess = false
return []tea.Model{tsk}
}

View file

@ -1,133 +0,0 @@
package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/pkg/cataloger"
)
func TestHandler_handlePackageCatalogerStarted(t *testing.T) {
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
iterations int
}{
{
name: "cataloging in progress",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
mon := cataloger.Monitor{
FilesProcessed: progress.NewManual(-1),
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging only files complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
files := progress.NewManual(-1)
files.SetCompleted()
mon := cataloger.Monitor{
FilesProcessed: files,
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging only packages complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
files := progress.NewManual(-1)
mon := cataloger.Monitor{
FilesProcessed: files,
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
files := progress.NewManual(-1)
files.SetCompleted()
mon := cataloger.Monitor{
FilesProcessed: files,
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
t.Log(got)
snaps.MatchSnapshot(t, got)
})
}
}

View file

@ -98,18 +98,21 @@ func TestHandler_handleReadImage(t *testing.T) {
Height: 80, Height: 80,
} }
models := handler.Handle(event) models, _ := handler.Handle(event)
require.Len(t, models, 1) require.Len(t, models, 1)
model := models[0] model := models[0]
tsk, ok := model.(taskprogress.Model) tsk, ok := model.(taskprogress.Model)
require.True(t, ok) require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(), Time: time.Now(),
Sequence: tsk.Sequence(), Sequence: tsk.Sequence(),
ID: tsk.ID(), ID: tsk.ID(),
}) })
got := gotModel.View()
t.Log(got) t.Log(got)
snaps.MatchSnapshot(t, got) snaps.MatchSnapshot(t, got)
}) })

View file

@ -29,6 +29,8 @@ type Handler struct {
Config HandlerConfig Config HandlerConfig
bubbly.EventHandler bubbly.EventHandler
onNewCatalogerTask *sync.Once
} }
func DefaultHandlerConfig() HandlerConfig { func DefaultHandlerConfig() HandlerConfig {
@ -41,28 +43,32 @@ func New(cfg HandlerConfig) *Handler {
d := bubbly.NewEventDispatcher() d := bubbly.NewEventDispatcher()
h := &Handler{ h := &Handler{
EventHandler: d, EventHandler: d,
Running: &sync.WaitGroup{}, Running: &sync.WaitGroup{},
Config: cfg, Config: cfg,
onNewCatalogerTask: &sync.Once{},
} }
// register all supported event types with the respective handler functions // register all supported event types with the respective handler functions
d.AddHandlers(map[partybus.EventType]bubbly.EventHandlerFn{ d.AddHandlers(map[partybus.EventType]bubbly.EventHandlerFn{
stereoscopeEvent.PullDockerImage: h.handlePullDockerImage, stereoscopeEvent.PullDockerImage: simpleHandler(h.handlePullDockerImage),
stereoscopeEvent.PullContainerdImage: h.handlePullContainerdImage, stereoscopeEvent.PullContainerdImage: simpleHandler(h.handlePullContainerdImage),
stereoscopeEvent.ReadImage: h.handleReadImage, stereoscopeEvent.ReadImage: simpleHandler(h.handleReadImage),
stereoscopeEvent.FetchImage: h.handleFetchImage, stereoscopeEvent.FetchImage: simpleHandler(h.handleFetchImage),
syftEvent.PackageCatalogerStarted: h.handlePackageCatalogerStarted, syftEvent.FileIndexingStarted: simpleHandler(h.handleFileIndexingStarted),
syftEvent.FileDigestsCatalogerStarted: h.handleFileDigestsCatalogerStarted, syftEvent.AttestationStarted: simpleHandler(h.handleAttestationStarted),
syftEvent.FileMetadataCatalogerStarted: h.handleFileMetadataCatalogerStarted, syftEvent.CatalogerTaskStarted: h.handleCatalogerTaskStarted,
syftEvent.FileIndexingStarted: h.handleFileIndexingStarted,
syftEvent.AttestationStarted: h.handleAttestationStarted,
syftEvent.CatalogerTaskStarted: h.handleCatalogerTaskStarted,
}) })
return h return h
} }
func simpleHandler(fn func(partybus.Event) []tea.Model) bubbly.EventHandlerFn {
return func(e partybus.Event) ([]tea.Model, tea.Cmd) {
return fn(e), nil
}
}
func (m *Handler) OnMessage(msg tea.Msg) { func (m *Handler) OnMessage(msg tea.Msg) {
if msg, ok := msg.(tea.WindowSizeMsg); ok { if msg, ok := msg.(tea.WindowSizeMsg); ok {
m.WindowSize = msg m.WindowSize = msg

View file

@ -4,12 +4,11 @@ import (
"reflect" "reflect"
"sync" "sync"
"testing" "testing"
"unsafe"
tea "github.com/charmbracelet/bubbletea" tea "github.com/charmbracelet/bubbletea"
) )
func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*sync.WaitGroup) string { func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*sync.WaitGroup) tea.Model {
t.Helper() t.Helper()
if iterations == 0 { if iterations == 0 {
iterations = 1 iterations = 1
@ -37,34 +36,21 @@ func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*
cmd = tea.Batch(nextCmds...) cmd = tea.Batch(nextCmds...)
} }
return m.View() return m
} }
func flatten(p tea.Msg) (msgs []tea.Msg) { func flatten(ps ...tea.Msg) (msgs []tea.Msg) {
if reflect.TypeOf(p).Name() == "batchMsg" { for _, p := range ps {
partials := extractBatchMessages(p) if bm, ok := p.(tea.BatchMsg); ok {
for _, m := range partials { for _, m := range bm {
msgs = append(msgs, flatten(m)...) if m == nil {
continue
}
msgs = append(msgs, flatten(m())...)
}
} else {
msgs = []tea.Msg{p}
} }
} else {
msgs = []tea.Msg{p}
} }
return msgs return msgs
} }
func extractBatchMessages(m tea.Msg) (ret []tea.Msg) {
sliceMsgType := reflect.SliceOf(reflect.TypeOf(tea.Cmd(nil)))
value := reflect.ValueOf(m) // note: this is technically unaddressable
// make our own instance that is addressable
valueCopy := reflect.New(value.Type()).Elem()
valueCopy.Set(value)
cmds := reflect.NewAt(sliceMsgType, unsafe.Pointer(valueCopy.UnsafeAddr())).Elem()
for i := 0; i < cmds.Len(); i++ {
item := cmds.Index(i)
r := item.Call(nil)
ret = append(ret, r[0].Interface().(tea.Msg))
}
return ret
}

View file

@ -155,7 +155,11 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, nil return m, nil
} }
for _, newModel := range m.handler.Handle(msg) { models, cmd := m.handler.Handle(msg)
if cmd != nil {
cmds = append(cmds, cmd)
}
for _, newModel := range models {
if newModel == nil { if newModel == nil {
continue continue
} }

2
go.mod
View file

@ -8,7 +8,7 @@ require (
github.com/Masterminds/sprig/v3 v3.2.3 github.com/Masterminds/sprig/v3 v3.2.3
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/acobaugh/osrelease v0.1.0 github.com/acobaugh/osrelease v0.1.0
github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461 github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9
github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc
github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e
github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a

4
go.sum
View file

@ -89,8 +89,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461 h1:xGu4/uMWucwWV0YV3fpFIQZ6KVfS/Wfhmma8t0s0vRo= github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 h1:p0ZIe0htYOX284Y4axJaGBvXHU0VCCzLN5Wf5XbKStU=
github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461/go.mod h1:Ger02eh5NpPm2IqkPAy396HU1KlK3BhOeCljDYXySSk= github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9/go.mod h1:3ZsFB9tzW3vl4gEiUeuSOMDnwroWxIxJelOOHUp8dSw=
github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc h1:A1KFO+zZZmbNlz1+WKsCF0RKVx6XRoxsAG3lrqH9hUQ= github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc h1:A1KFO+zZZmbNlz1+WKsCF0RKVx6XRoxsAG3lrqH9hUQ=
github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc/go.mod h1:QeWvNzxsrUNxcs6haQo3OtISfXUXW0qAuiG4EQiz0GU= github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc/go.mod h1:QeWvNzxsrUNxcs6haQo3OtISfXUXW0qAuiG4EQiz0GU=
github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e h1:O8ZubApaSl7dRzKNvyfGq9cLIPLQ5v3Iz0Y3huHKCgg= github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e h1:O8ZubApaSl7dRzKNvyfGq9cLIPLQ5v3Iz0Y3huHKCgg=

View file

@ -2,10 +2,12 @@ package bus
import ( import (
"github.com/wagoodman/go-partybus" "github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/clio" "github.com/anchore/clio"
"github.com/anchore/syft/internal/redact" "github.com/anchore/syft/internal/redact"
"github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
) )
func Exit() { func Exit() {
@ -33,3 +35,18 @@ func Notify(message string) {
Value: message, Value: message,
}) })
} }
func StartCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.CatalogerTaskProgress {
t := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage(initialStage),
Manual: progress.NewManual(size),
}
Publish(partybus.Event{
Type: event.CatalogerTaskStarted,
Source: info,
Value: progress.StagedProgressable(t),
})
return t
}

View file

@ -14,15 +14,6 @@ const (
// Events from the syft library // Events from the syft library
// PackageCatalogerStarted is a partybus event that occurs when the package cataloging has begun
PackageCatalogerStarted partybus.EventType = typePrefix + "-package-cataloger-started-event"
// FileMetadataCatalogerStarted is a partybus event that occurs when the file metadata cataloging has begun
FileMetadataCatalogerStarted partybus.EventType = typePrefix + "-file-metadata-cataloger-started-event"
// FileDigestsCatalogerStarted is a partybus event that occurs when the file digests cataloging has begun
FileDigestsCatalogerStarted partybus.EventType = typePrefix + "-file-digests-cataloger-started-event"
// FileIndexingStarted is a partybus event that occurs when the directory resolver begins indexing a filesystem // FileIndexingStarted is a partybus event that occurs when the directory resolver begins indexing a filesystem
FileIndexingStarted partybus.EventType = typePrefix + "-file-indexing-started-event" FileIndexingStarted partybus.EventType = typePrefix + "-file-indexing-started-event"

View file

@ -1,55 +0,0 @@
package monitor
import (
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/syft/event"
)
// TODO: this should be refactored to support read-only/write-only access using idioms of the progress lib
type CatalogerTask struct {
prog *progress.Manual
// Title
Title string
// TitleOnCompletion a string to use as title when completed
TitleOnCompletion string
// SubStatus indicates this progress should be rendered as a sub-item
SubStatus bool
// RemoveOnCompletion indicates this progress line will be removed when completed
RemoveOnCompletion bool
// value is the value to display -- not public as SetValue needs to be called to initialize this progress
value string
}
func (e *CatalogerTask) init() {
e.prog = progress.NewManual(-1)
bus.Publish(partybus.Event{
Type: event.CatalogerTaskStarted,
Source: e,
})
}
func (e *CatalogerTask) SetCompleted() {
if e.prog != nil {
e.prog.SetCompleted()
}
}
func (e *CatalogerTask) SetValue(value string) {
if e.prog == nil {
e.init()
}
e.value = value
}
func (e *CatalogerTask) GetValue() string {
return e.value
}
func (e *CatalogerTask) GetMonitor() *progress.Manual {
return e.prog
}

View file

@ -0,0 +1,10 @@
package monitor
import (
"github.com/wagoodman/go-progress"
)
type CatalogerTaskProgress struct {
*progress.AtomicStage
*progress.Manual
}

View file

@ -18,6 +18,19 @@ type Title struct {
} }
type GenericTask struct { type GenericTask struct {
Title Title
Context string // required fields
Title Title
// optional format fields
HideOnSuccess bool
HideStageOnSuccess bool
// optional fields
ID string
ParentID string
Context string
} }

View file

@ -12,7 +12,6 @@ import (
"github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/pkg/cataloger"
) )
type ErrBadPayload struct { type ErrBadPayload struct {
@ -40,45 +39,6 @@ func checkEventType(actual, expected partybus.EventType) error {
return nil return nil
} }
func ParsePackageCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) {
if err := checkEventType(e.Type, event.PackageCatalogerStarted); err != nil {
return nil, err
}
monitor, ok := e.Value.(cataloger.Monitor)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return &monitor, nil
}
func ParseFileMetadataCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileMetadataCatalogerStarted); err != nil {
return nil, err
}
prog, ok := e.Value.(progress.StagedProgressable)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return prog, nil
}
func ParseFileDigestsCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileDigestsCatalogerStarted); err != nil {
return nil, err
}
prog, ok := e.Value.(progress.StagedProgressable)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return prog, nil
}
func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgressable, error) { func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileIndexingStarted); err != nil { if err := checkEventType(e.Type, event.FileIndexingStarted); err != nil {
return "", nil, err return "", nil, err
@ -97,17 +57,24 @@ func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgress
return path, prog, nil return path, prog, nil
} }
func ParseCatalogerTaskStarted(e partybus.Event) (*monitor.CatalogerTask, error) { func ParseCatalogerTaskStarted(e partybus.Event) (progress.StagedProgressable, *monitor.GenericTask, error) {
if err := checkEventType(e.Type, event.CatalogerTaskStarted); err != nil { if err := checkEventType(e.Type, event.CatalogerTaskStarted); err != nil {
return nil, err return nil, nil, err
} }
source, ok := e.Source.(*monitor.CatalogerTask) var mon progress.StagedProgressable
source, ok := e.Source.(monitor.GenericTask)
if !ok { if !ok {
return nil, newPayloadErr(e.Type, "Source", e.Source) return nil, nil, newPayloadErr(e.Type, "Source", e.Source)
} }
return source, nil mon, ok = e.Value.(progress.StagedProgressable)
if !ok {
mon = nil
}
return mon, &source, nil
} }
func ParseAttestationStartedEvent(e partybus.Event) (io.Reader, progress.Progressable, *monitor.GenericTask, error) { func ParseAttestationStartedEvent(e partybus.Event) (io.Reader, progress.Progressable, *monitor.GenericTask, error) {

View file

@ -3,16 +3,16 @@ package filedigest
import ( import (
"crypto" "crypto"
"errors" "errors"
"fmt"
"github.com/wagoodman/go-partybus" "github.com/dustin/go-humanize"
"github.com/wagoodman/go-progress"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
intFile "github.com/anchore/syft/internal/file" intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
intCataloger "github.com/anchore/syft/syft/file/cataloger/internal" intCataloger "github.com/anchore/syft/syft/file/cataloger/internal"
) )
@ -41,9 +41,11 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
} }
} }
stage, prog := digestsCatalogingProgress(int64(len(locations))) prog := digestsCatalogingProgress(int64(len(locations)))
for _, location := range locations { for _, location := range locations {
stage.Current = location.RealPath prog.Increment()
prog.AtomicStage.Set(location.Path())
result, err := i.catalogLocation(resolver, location) result, err := i.catalogLocation(resolver, location)
if errors.Is(err, ErrUndigestableFile) { if errors.Is(err, ErrUndigestableFile) {
@ -61,8 +63,12 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
prog.Increment() prog.Increment()
results[location.Coordinates] = result results[location.Coordinates] = result
} }
log.Debugf("file digests cataloger processed %d files", prog.Current()) log.Debugf("file digests cataloger processed %d files", prog.Current())
prog.AtomicStage.Set(fmt.Sprintf("%s digests", humanize.Comma(prog.Current())))
prog.SetCompleted() prog.SetCompleted()
return results, nil return results, nil
} }
@ -91,20 +97,14 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
return digests, nil return digests, nil
} }
func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) { func digestsCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
stage := &progress.Stage{} info := monitor.GenericTask{
prog := progress.NewManual(locations) Title: monitor.Title{
Default: "Catalog file digests",
bus.Publish(partybus.Event{ WhileRunning: "Cataloging file digests",
Type: event.FileDigestsCatalogerStarted, OnSuccess: "Cataloged file digests",
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
}, },
}) }
return stage, prog return bus.StartCatalogerTask(info, locations, "")
} }

View file

@ -1,12 +1,13 @@
package filemetadata package filemetadata
import ( import (
"github.com/wagoodman/go-partybus" "fmt"
"github.com/wagoodman/go-progress"
"github.com/dustin/go-humanize"
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
) )
@ -20,7 +21,6 @@ func NewCataloger() *Cataloger {
func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates]file.Metadata, error) { func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates]file.Metadata, error) {
results := make(map[file.Coordinates]file.Metadata) results := make(map[file.Coordinates]file.Metadata)
var locations <-chan file.Location var locations <-chan file.Location
if len(coordinates) == 0 { if len(coordinates) == 0 {
locations = resolver.AllLocations() locations = resolver.AllLocations()
} else { } else {
@ -43,36 +43,35 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
}() }()
} }
stage, prog := metadataCatalogingProgress(int64(len(locations))) prog := metadataCatalogingProgress(int64(len(locations)))
for location := range locations { for location := range locations {
stage.Current = location.RealPath prog.Increment()
prog.AtomicStage.Set(location.Path())
metadata, err := resolver.FileMetadataByLocation(location) metadata, err := resolver.FileMetadataByLocation(location)
if err != nil { if err != nil {
return nil, err return nil, err
} }
results[location.Coordinates] = metadata results[location.Coordinates] = metadata
prog.Increment()
} }
log.Debugf("file metadata cataloger processed %d files", prog.Current()) log.Debugf("file metadata cataloger processed %d files", prog.Current())
prog.AtomicStage.Set(fmt.Sprintf("%s locations", humanize.Comma(prog.Current())))
prog.SetCompleted() prog.SetCompleted()
return results, nil return results, nil
} }
func metadataCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) { func metadataCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
stage := &progress.Stage{} info := monitor.GenericTask{
prog := progress.NewManual(locations) Title: monitor.Title{
Default: "Catalog file metadata",
bus.Publish(partybus.Event{ WhileRunning: "Cataloging file metadata",
Type: event.FileMetadataCatalogerStarted, OnSuccess: "Cataloged file metadata",
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
}, },
}) }
return stage, prog return bus.StartCatalogerTask(info, locations, "")
} }

View file

@ -6,14 +6,14 @@ import (
"runtime/debug" "runtime/debug"
"sync" "sync"
"github.com/dustin/go-humanize"
"github.com/hashicorp/go-multierror" "github.com/hashicorp/go-multierror"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress" "github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
@ -35,21 +35,6 @@ type catalogResult struct {
Error error Error error
} }
// newMonitor creates a new Monitor object and publishes the object on the bus as a PackageCatalogerStarted event.
func newMonitor() (*progress.Manual, *progress.Manual) {
filesProcessed := progress.Manual{}
packagesDiscovered := progress.Manual{}
bus.Publish(partybus.Event{
Type: event.PackageCatalogerStarted,
Value: Monitor{
FilesProcessed: progress.Monitorable(&filesProcessed),
PackagesDiscovered: progress.Monitorable(&packagesDiscovered),
},
})
return &filesProcessed, &packagesDiscovered
}
func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerResult *catalogResult, err error) { func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerResult *catalogResult, err error) {
// handle individual cataloger panics // handle individual cataloger panics
defer func() { defer func() {
@ -101,6 +86,7 @@ func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerRes
} }
catalogerResult.Packages = append(catalogerResult.Packages, p) catalogerResult.Packages = append(catalogerResult.Packages, p)
} }
catalogerResult.Relationships = append(catalogerResult.Relationships, relationships...) catalogerResult.Relationships = append(catalogerResult.Relationships, relationships...)
log.WithFields("cataloger", cataloger.Name()).Trace("cataloging complete") log.WithFields("cataloger", cataloger.Name()).Trace("cataloging complete")
return catalogerResult, err return catalogerResult, err
@ -116,9 +102,7 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
catalog := pkg.NewCollection() catalog := pkg.NewCollection()
var allRelationships []artifact.Relationship var allRelationships []artifact.Relationship
filesProcessed, packagesDiscovered := newMonitor() prog := monitorPackageCatalogingTask()
defer filesProcessed.SetCompleted()
defer packagesDiscovered.SetCompleted()
// perform analysis, accumulating errors for each failed analysis // perform analysis, accumulating errors for each failed analysis
var errs error var errs error
@ -131,10 +115,11 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
jobs := make(chan pkg.Cataloger, nCatalogers) jobs := make(chan pkg.Cataloger, nCatalogers)
results := make(chan *catalogResult, nCatalogers) results := make(chan *catalogResult, nCatalogers)
discoveredPackages := make(chan int64, nCatalogers)
waitGroup := sync.WaitGroup{} waitGroup := sync.WaitGroup{}
var totalPackagesDiscovered int64
for i := 0; i < parallelism; i++ { for i := 0; i < parallelism; i++ {
waitGroup.Add(1) waitGroup.Add(1)
@ -148,20 +133,16 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
// ensure we set the error to be aggregated // ensure we set the error to be aggregated
result.Error = err result.Error = err
discoveredPackages <- result.Discovered prog.Add(result.Discovered)
totalPackagesDiscovered += result.Discovered
count := humanize.Comma(totalPackagesDiscovered)
prog.AtomicStage.Set(fmt.Sprintf("%s packages", count))
results <- result results <- result
} }
}() }()
} }
// dynamically show updated discovered package status
go func() {
for discovered := range discoveredPackages {
packagesDiscovered.Add(discovered)
}
}()
// Enqueue the jobs // Enqueue the jobs
for _, cataloger := range catalogers { for _, cataloger := range catalogers {
jobs <- cataloger jobs <- cataloger
@ -171,7 +152,6 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
// Wait for the jobs to finish // Wait for the jobs to finish
waitGroup.Wait() waitGroup.Wait()
close(results) close(results)
close(discoveredPackages)
// collect the results // collect the results
for result := range results { for result := range results {
@ -186,6 +166,12 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...) allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...)
if errs != nil {
prog.SetError(errs)
} else {
prog.SetCompleted()
}
return catalog, allRelationships, errs return catalog, allRelationships, errs
} }
@ -228,3 +214,16 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
} }
return relationships, nil return relationships, nil
} }
func monitorPackageCatalogingTask() *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog packages",
WhileRunning: "Cataloging packages",
OnSuccess: "Cataloged packages",
},
HideOnSuccess: false,
}
return bus.StartCatalogerTask(info, -1, "")
}

View file

@ -20,6 +20,7 @@ import (
"github.com/go-git/go-git/v5/storage/memory" "github.com/go-git/go-git/v5/storage/memory"
"github.com/scylladb/go-set/strset" "github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/licenses"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/event/monitor"
@ -31,7 +32,6 @@ import (
type goLicenses struct { type goLicenses struct {
opts CatalogerConfig opts CatalogerConfig
localModCacheResolver file.WritableResolver localModCacheResolver file.WritableResolver
progress *monitor.CatalogerTask
lowerLicenseFileNames *strset.Set lowerLicenseFileNames *strset.Set
} }
@ -39,11 +39,6 @@ func newGoLicenses(opts CatalogerConfig) goLicenses {
return goLicenses{ return goLicenses{
opts: opts, opts: opts,
localModCacheResolver: modCacheResolver(opts.LocalModCacheDir), localModCacheResolver: modCacheResolver(opts.LocalModCacheDir),
progress: &monitor.CatalogerTask{
SubStatus: true,
RemoveOnCompletion: true,
Title: "Downloading go mod",
},
lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...), lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...),
} }
} }
@ -123,7 +118,16 @@ func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]
proxies := remotesForModule(c.opts.Proxies, c.opts.NoProxy, moduleName) proxies := remotesForModule(c.opts.Proxies, c.opts.NoProxy, moduleName)
fsys, err := getModule(c.progress, proxies, moduleName, moduleVersion) prog := bus.StartCatalogerTask(monitor.GenericTask{
Title: monitor.Title{
Default: "Download go mod",
WhileRunning: "Downloading go mod",
OnSuccess: "Downloaded go mod",
},
HideOnSuccess: true,
}, -1, "")
fsys, err := getModule(prog, proxies, moduleName, moduleVersion)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -206,7 +210,7 @@ func processCaps(s string) string {
}) })
} }
func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) { func getModule(progress *monitor.CatalogerTaskProgress, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) {
for _, proxy := range proxies { for _, proxy := range proxies {
u, _ := url.Parse(proxy) u, _ := url.Parse(proxy)
if proxy == "direct" { if proxy == "direct" {
@ -218,7 +222,7 @@ func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, mo
fsys, err = getModuleProxy(progress, proxy, moduleName, moduleVersion) fsys, err = getModuleProxy(progress, proxy, moduleName, moduleVersion)
case "file": case "file":
p := filepath.Join(u.Path, moduleName, "@v", moduleVersion) p := filepath.Join(u.Path, moduleName, "@v", moduleVersion)
progress.SetValue(fmt.Sprintf("file: %s", p)) progress.AtomicStage.Set(fmt.Sprintf("file: %s", p))
fsys = os.DirFS(p) fsys = os.DirFS(p)
} }
if fsys != nil { if fsys != nil {
@ -228,9 +232,9 @@ func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, mo
return return
} }
func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) { func getModuleProxy(progress *monitor.CatalogerTaskProgress, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) {
u := fmt.Sprintf("%s/%s/@v/%s.zip", proxy, moduleName, moduleVersion) u := fmt.Sprintf("%s/%s/@v/%s.zip", proxy, moduleName, moduleVersion)
progress.SetValue(u) progress.AtomicStage.Set(u)
// get the module zip // get the module zip
resp, err := http.Get(u) //nolint:gosec resp, err := http.Get(u) //nolint:gosec
@ -241,7 +245,7 @@ func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName st
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
u = fmt.Sprintf("%s/%s/@v/%s.zip", proxy, strings.ToLower(moduleName), moduleVersion) u = fmt.Sprintf("%s/%s/@v/%s.zip", proxy, strings.ToLower(moduleName), moduleVersion)
progress.SetValue(u) progress.AtomicStage.Set(u)
// try lowercasing it; some packages have mixed casing that really messes up the proxy // try lowercasing it; some packages have mixed casing that really messes up the proxy
resp, err = http.Get(u) //nolint:gosec resp, err = http.Get(u) //nolint:gosec
@ -284,14 +288,14 @@ func findVersionPath(f fs.FS, dir string) string {
return "" return ""
} }
func getModuleRepository(progress *monitor.CatalogerTask, moduleName string, moduleVersion string) (fs.FS, error) { func getModuleRepository(progress *monitor.CatalogerTaskProgress, moduleName string, moduleVersion string) (fs.FS, error) {
repoName := moduleName repoName := moduleName
parts := strings.Split(moduleName, "/") parts := strings.Split(moduleName, "/")
if len(parts) > 2 { if len(parts) > 2 {
repoName = fmt.Sprintf("%s/%s/%s", parts[0], parts[1], parts[2]) repoName = fmt.Sprintf("%s/%s/%s", parts[0], parts[1], parts[2])
} }
progress.SetValue(fmt.Sprintf("git: %s", repoName)) progress.AtomicStage.Set(fmt.Sprintf("git: %s", repoName))
f := memfs.New() f := memfs.New()
buf := &bytes.Buffer{} buf := &bytes.Buffer{}