Mirrors/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft synced 2024-11-10 06:14:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

Generalize UI events for cataloging tasks (#2369)

Browse source 
* generalize ui events for cataloging tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* moderate review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* incorporate review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename cataloger task progress object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* migrate cataloger task fn to bus helper

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman 2023-11-30 11:25:50 -05:00 committed by GitHub
parent b943da6433
commit 4adfbeb5f0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 427 additions and 807 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
cmd/syft/cli/ui/__snapshots__/handle_cataloger_task_test.snap
View file

@ -1,16 +1,24 @@
[TestHandler_handleCatalogerTaskStarted/cataloging_task_in_progress - 1]
some task title [some value]
⠙ Cataloging contents
⠙ some task title ━━━━━━━━━━━━━━━━━━━━ [some stage]
---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_in_progress - 1]
└── some task title [some value]
⠙ Cataloging contents
└── ⠙ some task title ━━━━━━━━━━━━━━━━━━━━ [some stage]
---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete - 1]
✔ └── some task done [some value]
⠙ Cataloging contents
└── ✔ some task done [some stage]
---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete_--_hide_stage - 1]
⠙ Cataloging contents
└── ✔ some task done
---
[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete_with_removal - 1]
⠙ Cataloging contents
---

13
cmd/syft/cli/ui/handle_attestation_test.go
View file

@ -100,18 +100,21 @@ func TestHandler_handleAttestationStarted(t *testing.T) {
Height: 80,
}
models := handler.Handle(event)
models, _ := handler.Handle(event)
require.Len(t, models, 2)
t.Run("task line", func(t *testing.T) {
tsk, ok := models[0].(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})
@ -119,11 +122,15 @@ func TestHandler_handleAttestationStarted(t *testing.T) {
t.Run("log", func(t *testing.T) {
log, ok := models[1].(attestLogFrame)
require.True(t, ok)
got := runModel(t, log, tt.iterations, attestLogFrameTickMsg{
gotModel := runModel(t, log, tt.iterations, attestLogFrameTickMsg{
Time: time.Now(),
Sequence: log.sequence,
ID: log.id,
}, log.reader.running)
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})

131
cmd/syft/cli/ui/handle_cataloger_task.go
View file

@ -3,70 +3,121 @@ package ui
import (
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"github.com/google/uuid"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/bubbly/bubbles/tree"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event/monitor"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
)
var _ progress.Stager = (*catalogerTaskStageAdapter)(nil)
// we standardize how rows are instantiated to ensure consistency in the appearance across the UI
type taskModelFactory func(title taskprogress.Title, opts ...taskprogress.Option) taskprogress.Model
type catalogerTaskStageAdapter struct {
mon *monitor.CatalogerTask
var _ tea.Model = (*catalogerTaskModel)(nil)
type catalogerTaskModel struct {
model tree.Model
modelFactory taskModelFactory
}
func newCatalogerTaskStageAdapter(mon *monitor.CatalogerTask) *catalogerTaskStageAdapter {
return &catalogerTaskStageAdapter{
mon: mon,
func newCatalogerTaskTreeModel(f taskModelFactory) *catalogerTaskModel {
t := tree.NewModel()
t.Padding = " "
t.RootsWithoutPrefix = true
return &catalogerTaskModel{
modelFactory: f,
model: t,
}
}
func (c catalogerTaskStageAdapter) Stage() string {
return c.mon.GetValue()
type newCatalogerTaskRowEvent struct {
info monitor.GenericTask
prog progress.StagedProgressable
}
func (m *Handler) handleCatalogerTaskStarted(e partybus.Event) []tea.Model {
mon, err := syftEventParsers.ParseCatalogerTaskStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
func (cts catalogerTaskModel) Init() tea.Cmd {
return cts.model.Init()
}
func (cts catalogerTaskModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
event, ok := msg.(newCatalogerTaskRowEvent)
if !ok {
model, cmd := cts.model.Update(msg)
cts.model = model.(tree.Model)
return cts, cmd
}
var prefix string
if mon.SubStatus {
// TODO: support list of sub-statuses, not just a single leaf
prefix = "└── "
}
info, prog := event.info, event.prog
tsk := m.newTaskProgress(
tsk := cts.modelFactory(
taskprogress.Title{
// TODO: prefix should not be part of the title, but instead a separate field that is aware of the tree structure
Default: prefix + mon.Title,
Running: prefix + mon.Title,
Success: prefix + mon.TitleOnCompletion,
Default: info.Title.Default,
Running: info.Title.WhileRunning,
Success: info.Title.OnSuccess,
},
taskprogress.WithStagedProgressable(
struct {
progress.Stager
progress.Progressable
}{
Progressable: mon.GetMonitor(),
Stager: newCatalogerTaskStageAdapter(mon),
},
),
taskprogress.WithStagedProgressable(prog),
)
// TODO: this isn't ideal since the model stays around after it is no longer needed, but it works for now
tsk.HideOnSuccess = mon.RemoveOnCompletion
tsk.HideStageOnSuccess = false
tsk.HideProgressOnSuccess = false
if info.Context != "" {
tsk.Context = []string{info.Context}
}
tsk.TitleStyle = lipgloss.NewStyle()
// TODO: this is a hack to get the spinner to not show up, but ideally the component would support making the spinner optional
tsk.Spinner.Spinner.Frames = []string{" "}
tsk.HideOnSuccess = info.HideOnSuccess
tsk.HideStageOnSuccess = info.HideStageOnSuccess
tsk.HideProgressOnSuccess = true
return []tea.Model{tsk}
if info.ParentID != "" {
tsk.TitleStyle = lipgloss.NewStyle()
}
if err := cts.model.Add(info.ParentID, info.ID, tsk); err != nil {
log.WithFields("error", err).Error("unable to add cataloger task to tree model")
}
return cts, tsk.Init()
}
func (cts catalogerTaskModel) View() string {
return cts.model.View()
}
func (m *Handler) handleCatalogerTaskStarted(e partybus.Event) ([]tea.Model, tea.Cmd) {
mon, info, err := syftEventParsers.ParseCatalogerTaskStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil, nil
}
var models []tea.Model
// only create the new cataloger task tree once to manage all cataloger task events
m.onNewCatalogerTask.Do(func() {
models = append(models, newCatalogerTaskTreeModel(m.newTaskProgress))
})
// we need to update the cataloger task model with a new row. We should never update the model outside of the
// bubbletea update-render event loop. Instead, we return a command that will be executed by the bubbletea runtime,
// producing a message that is passed to the cataloger task model. This is the prescribed way to update models
// in bubbletea.
if info.ID == "" {
// ID is optional from the consumer perspective, but required internally
info.ID = uuid.Must(uuid.NewRandom()).String()
}
cmd := func() tea.Msg {
// this message will cause the cataloger task model to add a new row to the output based on the given task
// information and progress data.
return newCatalogerTaskRowEvent{
info: *info,
prog: mon,
}
}
return models, cmd
}

163
cmd/syft/cli/ui/handle_cataloger_task_test.go
View file

@ -2,19 +2,22 @@ package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
)
func TestHandler_handleCatalogerTaskStarted(t *testing.T) {
title := monitor.Title{
Default: "some task title",
OnSuccess: "some task done",
}
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
@ -23,99 +26,171 @@ func TestHandler_handleCatalogerTaskStarted(t *testing.T) {
{
name: "cataloging task in progress",
eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{
SubStatus: false,
RemoveOnCompletion: false,
Title: "some task title",
TitleOnCompletion: "some task done",
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
src.SetValue("some value")
value.Manual.Add(50)
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: src,
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: false,
ID: "my-id",
},
Value: value,
}
},
},
{
name: "cataloging sub task in progress",
eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{
SubStatus: true,
RemoveOnCompletion: false,
Title: "some task title",
TitleOnCompletion: "some task done",
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
src.SetValue("some value")
value.Manual.Add(50)
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: src,
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: false,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
}
},
},
{
name: "cataloging sub task complete",
eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{
SubStatus: true,
RemoveOnCompletion: false,
Title: "some task title",
TitleOnCompletion: "some task done",
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
src.SetValue("some value")
src.SetCompleted()
value.SetCompleted()
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: src,
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: false,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
}
},
},
{
name: "cataloging sub task complete -- hide stage",
eventFn: func(t *testing.T) partybus.Event {
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
value.SetCompleted()
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: false,
HideStageOnSuccess: true,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
}
},
},
{
name: "cataloging sub task complete with removal",
eventFn: func(t *testing.T) partybus.Event {
src := &monitor.CatalogerTask{
SubStatus: true,
RemoveOnCompletion: true,
Title: "some task title",
TitleOnCompletion: "some task done",
value := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage("some stage"),
Manual: progress.NewManual(100),
}
src.SetValue("some value")
src.SetCompleted()
value.SetCompleted()
return partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: src,
Type: syftEvent.CatalogerTaskStarted,
Source: monitor.GenericTask{
Title: title,
HideOnSuccess: true,
HideStageOnSuccess: false,
ID: "my-id",
ParentID: "top-level-task",
},
Value: value,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
// need to be able to get the initial newCatalogerTaskRowEvent + initialize the nested taskprogress model
if tt.iterations == 0 {
tt.iterations = 2
}
e := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog contents",
WhileRunning: "Cataloging contents",
OnSuccess: "Cataloged contents",
},
ID: "top-level-task",
}
// note: this line / event is not under test, only needed to show a sub status
kickoffEvent := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage(""),
Manual: progress.NewManual(-1),
}
models, cmd := handler.Handle(
partybus.Event{
Type: syftEvent.CatalogerTaskStarted,
Source: info,
Value: progress.StagedProgressable(kickoffEvent),
},
)
require.Len(t, models, 1)
require.NotNil(t, cmd)
model := models[0]
tsk, ok := model.(taskprogress.Model)
tr, ok := model.(*catalogerTaskModel)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
gotModel := runModel(t, tr, tt.iterations, cmd())
models, cmd = handler.Handle(e)
require.Len(t, models, 0)
require.NotNil(t, cmd)
gotModel = runModel(t, gotModel, tt.iterations, cmd())
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})

7
cmd/syft/cli/ui/handle_fetch_image_test.go
View file

@ -80,18 +80,21 @@ func TestHandler_handleFetchImage(t *testing.T) {
Height: 80,
}
models := handler.Handle(event)
models, _ := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})

28
cmd/syft/cli/ui/handle_file_digests_cataloger.go
View file

@ -1,28 +0,0 @@
package ui
import (
tea "github.com/charmbracelet/bubbletea"
"github.com/wagoodman/go-partybus"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/syft/internal/log"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
)
func (m *Handler) handleFileDigestsCatalogerStarted(e partybus.Event) []tea.Model {
prog, err := syftEventParsers.ParseFileDigestsCatalogingStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
}
tsk := m.newTaskProgress(
taskprogress.Title{
Default: "Catalog file digests",
Running: "Cataloging file digests",
Success: "Cataloged file digests",
}, taskprogress.WithStagedProgressable(prog),
)
return []tea.Model{tsk}
}

97
cmd/syft/cli/ui/handle_file_digests_cataloger_test.go
View file

@ -1,97 +0,0 @@
package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
)
func TestHandler_handleFileDigestsCatalogerStarted(t *testing.T) {
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
iterations int
}{
{
name: "cataloging in progress",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileDigestsCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileDigestsCatalogerStarted,
Value: mon,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
t.Log(got)
snaps.MatchSnapshot(t, got)
})
}
}

7
cmd/syft/cli/ui/handle_file_indexing_test.go
View file

@ -80,18 +80,21 @@ func TestHandler_handleFileIndexingStarted(t *testing.T) {
Height: 80,
}
models := handler.Handle(event)
models, _ := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})

29
cmd/syft/cli/ui/handle_file_metadata_cataloger.go
View file

@ -1,29 +0,0 @@
package ui
import (
tea "github.com/charmbracelet/bubbletea"
"github.com/wagoodman/go-partybus"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/syft/internal/log"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
)
func (m *Handler) handleFileMetadataCatalogerStarted(e partybus.Event) []tea.Model {
prog, err := syftEventParsers.ParseFileMetadataCatalogingStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
}
tsk := m.newTaskProgress(
taskprogress.Title{
Default: "Catalog file metadata",
Running: "Cataloging file metadata",
Success: "Cataloged file metadata",
},
taskprogress.WithStagedProgressable(prog),
)
return []tea.Model{tsk}
}

97
cmd/syft/cli/ui/handle_file_metadata_cataloger_test.go
View file

@ -1,97 +0,0 @@
package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
)
func TestHandler_handleFileMetadataCatalogerStarted(t *testing.T) {
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
iterations int
}{
{
name: "cataloging in progress",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileMetadataCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
mon := struct {
progress.Progressable
progress.Stager
}{
Progressable: prog,
Stager: &progress.Stage{
Current: "current",
},
}
return partybus.Event{
Type: syftEvent.FileMetadataCatalogerStarted,
Value: mon,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
t.Log(got)
snaps.MatchSnapshot(t, got)
})
}
}

87
cmd/syft/cli/ui/handle_package_cataloger.go
View file

@ -1,87 +0,0 @@
package ui
import (
"fmt"
tea "github.com/charmbracelet/bubbletea"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
"github.com/anchore/syft/internal/log"
syftEventParsers "github.com/anchore/syft/syft/event/parsers"
"github.com/anchore/syft/syft/pkg/cataloger"
)
var _ progress.StagedProgressable = (*packageCatalogerProgressAdapter)(nil)
type packageCatalogerProgressAdapter struct {
monitor *cataloger.Monitor
monitors []progress.Monitorable
}
func newPackageCatalogerProgressAdapter(monitor *cataloger.Monitor) packageCatalogerProgressAdapter {
return packageCatalogerProgressAdapter{
monitor: monitor,
monitors: []progress.Monitorable{
monitor.FilesProcessed,
monitor.PackagesDiscovered,
},
}
}
func (p packageCatalogerProgressAdapter) Stage() string {
return fmt.Sprintf("%d packages", p.monitor.PackagesDiscovered.Current())
}
func (p packageCatalogerProgressAdapter) Current() int64 {
return p.monitor.PackagesDiscovered.Current()
}
func (p packageCatalogerProgressAdapter) Error() error {
completedMonitors := 0
for _, monitor := range p.monitors {
err := monitor.Error()
if err == nil {
continue
}
if progress.IsErrCompleted(err) {
completedMonitors++
continue
}
// something went wrong
return err
}
if completedMonitors == len(p.monitors) && len(p.monitors) > 0 {
return p.monitors[0].Error()
}
return nil
}
func (p packageCatalogerProgressAdapter) Size() int64 {
// this is an inherently unknown value (indeterminate total number of packages to discover)
return -1
}
func (m *Handler) handlePackageCatalogerStarted(e partybus.Event) []tea.Model {
monitor, err := syftEventParsers.ParsePackageCatalogerStarted(e)
if err != nil {
log.WithFields("error", err).Warn("unable to parse event")
return nil
}
tsk := m.newTaskProgress(
taskprogress.Title{
Default: "Catalog packages",
Running: "Cataloging packages",
Success: "Cataloged packages",
},
taskprogress.WithStagedProgressable(
newPackageCatalogerProgressAdapter(monitor),
),
)
tsk.HideStageOnSuccess = false
return []tea.Model{tsk}
}

133
cmd/syft/cli/ui/handle_package_cataloger_test.go
View file

@ -1,133 +0,0 @@
package ui
import (
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/gkampitakis/go-snaps/snaps"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/bubbly/bubbles/taskprogress"
syftEvent "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/pkg/cataloger"
)
func TestHandler_handlePackageCatalogerStarted(t *testing.T) {
tests := []struct {
name string
eventFn func(*testing.T) partybus.Event
iterations int
}{
{
name: "cataloging in progress",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
mon := cataloger.Monitor{
FilesProcessed: progress.NewManual(-1),
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging only files complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(50)
files := progress.NewManual(-1)
files.SetCompleted()
mon := cataloger.Monitor{
FilesProcessed: files,
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging only packages complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
files := progress.NewManual(-1)
mon := cataloger.Monitor{
FilesProcessed: files,
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
{
name: "cataloging complete",
eventFn: func(t *testing.T) partybus.Event {
prog := &progress.Manual{}
prog.SetTotal(100)
prog.Set(100)
prog.SetCompleted()
files := progress.NewManual(-1)
files.SetCompleted()
mon := cataloger.Monitor{
FilesProcessed: files,
PackagesDiscovered: prog,
}
return partybus.Event{
Type: syftEvent.PackageCatalogerStarted,
Value: mon,
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event := tt.eventFn(t)
handler := New(DefaultHandlerConfig())
handler.WindowSize = tea.WindowSizeMsg{
Width: 100,
Height: 80,
}
models := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
t.Log(got)
snaps.MatchSnapshot(t, got)
})
}
}

7
cmd/syft/cli/ui/handle_read_image_test.go
View file

@ -98,18 +98,21 @@ func TestHandler_handleReadImage(t *testing.T) {
Height: 80,
}
models := handler.Handle(event)
models, _ := handler.Handle(event)
require.Len(t, models, 1)
model := models[0]
tsk, ok := model.(taskprogress.Model)
require.True(t, ok)
got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{
Time: time.Now(),
Sequence: tsk.Sequence(),
ID: tsk.ID(),
})
got := gotModel.View()
t.Log(got)
snaps.MatchSnapshot(t, got)
})

32
cmd/syft/cli/ui/handler.go
View file

@ -29,6 +29,8 @@ type Handler struct {
Config HandlerConfig
bubbly.EventHandler
onNewCatalogerTask *sync.Once
}
func DefaultHandlerConfig() HandlerConfig {
@ -41,28 +43,32 @@ func New(cfg HandlerConfig) *Handler {
d := bubbly.NewEventDispatcher()
h := &Handler{
EventHandler: d,
Running: &sync.WaitGroup{},
Config: cfg,
EventHandler: d,
Running: &sync.WaitGroup{},
Config: cfg,
onNewCatalogerTask: &sync.Once{},
}
// register all supported event types with the respective handler functions
d.AddHandlers(map[partybus.EventType]bubbly.EventHandlerFn{
stereoscopeEvent.PullDockerImage: h.handlePullDockerImage,
stereoscopeEvent.PullContainerdImage: h.handlePullContainerdImage,
stereoscopeEvent.ReadImage: h.handleReadImage,
stereoscopeEvent.FetchImage: h.handleFetchImage,
syftEvent.PackageCatalogerStarted: h.handlePackageCatalogerStarted,
syftEvent.FileDigestsCatalogerStarted: h.handleFileDigestsCatalogerStarted,
syftEvent.FileMetadataCatalogerStarted: h.handleFileMetadataCatalogerStarted,
syftEvent.FileIndexingStarted: h.handleFileIndexingStarted,
syftEvent.AttestationStarted: h.handleAttestationStarted,
syftEvent.CatalogerTaskStarted: h.handleCatalogerTaskStarted,
stereoscopeEvent.PullDockerImage: simpleHandler(h.handlePullDockerImage),
stereoscopeEvent.PullContainerdImage: simpleHandler(h.handlePullContainerdImage),
stereoscopeEvent.ReadImage: simpleHandler(h.handleReadImage),
stereoscopeEvent.FetchImage: simpleHandler(h.handleFetchImage),
syftEvent.FileIndexingStarted: simpleHandler(h.handleFileIndexingStarted),
syftEvent.AttestationStarted: simpleHandler(h.handleAttestationStarted),
syftEvent.CatalogerTaskStarted: h.handleCatalogerTaskStarted,
})
return h
}
func simpleHandler(fn func(partybus.Event) []tea.Model) bubbly.EventHandlerFn {
return func(e partybus.Event) ([]tea.Model, tea.Cmd) {
return fn(e), nil
}
}
func (m *Handler) OnMessage(msg tea.Msg) {
if msg, ok := msg.(tea.WindowSizeMsg); ok {
m.WindowSize = msg

40
cmd/syft/cli/ui/util_test.go
View file

@ -4,12 +4,11 @@ import (
"reflect"
"sync"
"testing"
"unsafe"
tea "github.com/charmbracelet/bubbletea"
)
func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*sync.WaitGroup) string {
func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*sync.WaitGroup) tea.Model {
t.Helper()
if iterations == 0 {
iterations = 1
@ -37,34 +36,21 @@ func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*
cmd = tea.Batch(nextCmds...)
}
return m.View()
return m
}
func flatten(p tea.Msg) (msgs []tea.Msg) {
if reflect.TypeOf(p).Name() == "batchMsg" {
partials := extractBatchMessages(p)
for _, m := range partials {
msgs = append(msgs, flatten(m)...)
func flatten(ps ...tea.Msg) (msgs []tea.Msg) {
for _, p := range ps {
if bm, ok := p.(tea.BatchMsg); ok {
for _, m := range bm {
if m == nil {
continue
}
msgs = append(msgs, flatten(m())...)
}
} else {
msgs = []tea.Msg{p}
}
} else {
msgs = []tea.Msg{p}
}
return msgs
}
func extractBatchMessages(m tea.Msg) (ret []tea.Msg) {
sliceMsgType := reflect.SliceOf(reflect.TypeOf(tea.Cmd(nil)))
value := reflect.ValueOf(m) // note: this is technically unaddressable
// make our own instance that is addressable
valueCopy := reflect.New(value.Type()).Elem()
valueCopy.Set(value)
cmds := reflect.NewAt(sliceMsgType, unsafe.Pointer(valueCopy.UnsafeAddr())).Elem()
for i := 0; i < cmds.Len(); i++ {
item := cmds.Index(i)
r := item.Call(nil)
ret = append(ret, r[0].Interface().(tea.Msg))
}
return ret
}

6
cmd/syft/internal/ui/ui.go
View file

@ -155,7 +155,11 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, nil
}
for _, newModel := range m.handler.Handle(msg) {
models, cmd := m.handler.Handle(msg)
if cmd != nil {
cmds = append(cmds, cmd)
}
for _, newModel := range models {
if newModel == nil {
continue
}

2
go.mod
View file

@ -8,7 +8,7 @@ require (
github.com/Masterminds/sprig/v3 v3.2.3
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/acobaugh/osrelease v0.1.0
github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461
github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9
github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc
github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e
github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a

4
go.sum
View file

@ -89,8 +89,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461 h1:xGu4/uMWucwWV0YV3fpFIQZ6KVfS/Wfhmma8t0s0vRo=
github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461/go.mod h1:Ger02eh5NpPm2IqkPAy396HU1KlK3BhOeCljDYXySSk=
github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 h1:p0ZIe0htYOX284Y4axJaGBvXHU0VCCzLN5Wf5XbKStU=
github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9/go.mod h1:3ZsFB9tzW3vl4gEiUeuSOMDnwroWxIxJelOOHUp8dSw=
github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc h1:A1KFO+zZZmbNlz1+WKsCF0RKVx6XRoxsAG3lrqH9hUQ=
github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc/go.mod h1:QeWvNzxsrUNxcs6haQo3OtISfXUXW0qAuiG4EQiz0GU=
github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e h1:O8ZubApaSl7dRzKNvyfGq9cLIPLQ5v3Iz0Y3huHKCgg=

17
internal/bus/helpers.go
View file

@ -2,10 +2,12 @@ package bus
import (
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/clio"
"github.com/anchore/syft/internal/redact"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
)
func Exit() {
@ -33,3 +35,18 @@ func Notify(message string) {
Value: message,
})
}
func StartCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.CatalogerTaskProgress {
t := &monitor.CatalogerTaskProgress{
AtomicStage: progress.NewAtomicStage(initialStage),
Manual: progress.NewManual(size),
}
Publish(partybus.Event{
Type: event.CatalogerTaskStarted,
Source: info,
Value: progress.StagedProgressable(t),
})
return t
}

9
syft/event/event.go
View file

@ -14,15 +14,6 @@ const (
// Events from the syft library
// PackageCatalogerStarted is a partybus event that occurs when the package cataloging has begun
PackageCatalogerStarted partybus.EventType = typePrefix + "-package-cataloger-started-event"
// FileMetadataCatalogerStarted is a partybus event that occurs when the file metadata cataloging has begun
FileMetadataCatalogerStarted partybus.EventType = typePrefix + "-file-metadata-cataloger-started-event"
// FileDigestsCatalogerStarted is a partybus event that occurs when the file digests cataloging has begun
FileDigestsCatalogerStarted partybus.EventType = typePrefix + "-file-digests-cataloger-started-event"
// FileIndexingStarted is a partybus event that occurs when the directory resolver begins indexing a filesystem
FileIndexingStarted partybus.EventType = typePrefix + "-file-indexing-started-event"

55
syft/event/monitor/cataloger_task.go
View file

@ -1,55 +0,0 @@
package monitor
import (
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/syft/event"
)
// TODO: this should be refactored to support read-only/write-only access using idioms of the progress lib
type CatalogerTask struct {
prog *progress.Manual
// Title
Title string
// TitleOnCompletion a string to use as title when completed
TitleOnCompletion string
// SubStatus indicates this progress should be rendered as a sub-item
SubStatus bool
// RemoveOnCompletion indicates this progress line will be removed when completed
RemoveOnCompletion bool
// value is the value to display -- not public as SetValue needs to be called to initialize this progress
value string
}
func (e *CatalogerTask) init() {
e.prog = progress.NewManual(-1)
bus.Publish(partybus.Event{
Type: event.CatalogerTaskStarted,
Source: e,
})
}
func (e *CatalogerTask) SetCompleted() {
if e.prog != nil {
e.prog.SetCompleted()
}
}
func (e *CatalogerTask) SetValue(value string) {
if e.prog == nil {
e.init()
}
e.value = value
}
func (e *CatalogerTask) GetValue() string {
return e.value
}
func (e *CatalogerTask) GetMonitor() *progress.Manual {
return e.prog
}

10
syft/event/monitor/cataloger_task_progress.go Normal file
View file

@ -0,0 +1,10 @@
package monitor
import (
"github.com/wagoodman/go-progress"
)
type CatalogerTaskProgress struct {
*progress.AtomicStage
*progress.Manual
}

17
syft/event/monitor/generic_task.go
View file

@ -18,6 +18,19 @@ type Title struct {
}
type GenericTask struct {
Title Title
Context string
// required fields
Title Title
// optional format fields
HideOnSuccess bool
HideStageOnSuccess bool
// optional fields
ID string
ParentID string
Context string
}

57
syft/event/parsers/parsers.go
View file

@ -12,7 +12,6 @@ import (
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/pkg/cataloger"
)
type ErrBadPayload struct {
@ -40,45 +39,6 @@ func checkEventType(actual, expected partybus.EventType) error {
return nil
}
func ParsePackageCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) {
if err := checkEventType(e.Type, event.PackageCatalogerStarted); err != nil {
return nil, err
}
monitor, ok := e.Value.(cataloger.Monitor)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return &monitor, nil
}
func ParseFileMetadataCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileMetadataCatalogerStarted); err != nil {
return nil, err
}
prog, ok := e.Value.(progress.StagedProgressable)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return prog, nil
}
func ParseFileDigestsCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileDigestsCatalogerStarted); err != nil {
return nil, err
}
prog, ok := e.Value.(progress.StagedProgressable)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return prog, nil
}
func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileIndexingStarted); err != nil {
return "", nil, err
@ -97,17 +57,24 @@ func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgress
return path, prog, nil
}
func ParseCatalogerTaskStarted(e partybus.Event) (*monitor.CatalogerTask, error) {
func ParseCatalogerTaskStarted(e partybus.Event) (progress.StagedProgressable, *monitor.GenericTask, error) {
if err := checkEventType(e.Type, event.CatalogerTaskStarted); err != nil {
return nil, err
return nil, nil, err
}
source, ok := e.Source.(*monitor.CatalogerTask)
var mon progress.StagedProgressable
source, ok := e.Source.(monitor.GenericTask)
if !ok {
return nil, newPayloadErr(e.Type, "Source", e.Source)
return nil, nil, newPayloadErr(e.Type, "Source", e.Source)
}
return source, nil
mon, ok = e.Value.(progress.StagedProgressable)
if !ok {
mon = nil
}
return mon, &source, nil
}
func ParseAttestationStartedEvent(e partybus.Event) (io.Reader, progress.Progressable, *monitor.GenericTask, error) {

38
syft/file/cataloger/filedigest/cataloger.go
View file

@ -3,16 +3,16 @@ package filedigest
import (
"crypto"
"errors"
"fmt"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/dustin/go-humanize"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
intCataloger "github.com/anchore/syft/syft/file/cataloger/internal"
)
@ -41,9 +41,11 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
}
}
stage, prog := digestsCatalogingProgress(int64(len(locations)))
prog := digestsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
prog.Increment()
prog.AtomicStage.Set(location.Path())
result, err := i.catalogLocation(resolver, location)
if errors.Is(err, ErrUndigestableFile) {
@ -61,8 +63,12 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
prog.Increment()
results[location.Coordinates] = result
}
log.Debugf("file digests cataloger processed %d files", prog.Current())
prog.AtomicStage.Set(fmt.Sprintf("%s digests", humanize.Comma(prog.Current())))
prog.SetCompleted()
return results, nil
}
@ -91,20 +97,14 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
return digests, nil
}
func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
Type: event.FileDigestsCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
func digestsCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog file digests",
WhileRunning: "Cataloging file digests",
OnSuccess: "Cataloged file digests",
},
})
}
return stage, prog
return bus.StartCatalogerTask(info, locations, "")
}

41
syft/file/cataloger/filemetadata/cataloger.go
View file

@ -1,12 +1,13 @@
package filemetadata
import (
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"fmt"
"github.com/dustin/go-humanize"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
)
@ -20,7 +21,6 @@ func NewCataloger() *Cataloger {
func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates]file.Metadata, error) {
results := make(map[file.Coordinates]file.Metadata)
var locations <-chan file.Location
if len(coordinates) == 0 {
locations = resolver.AllLocations()
} else {
@ -43,36 +43,35 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
}()
}
stage, prog := metadataCatalogingProgress(int64(len(locations)))
prog := metadataCatalogingProgress(int64(len(locations)))
for location := range locations {
stage.Current = location.RealPath
prog.Increment()
prog.AtomicStage.Set(location.Path())
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
}
results[location.Coordinates] = metadata
prog.Increment()
}
log.Debugf("file metadata cataloger processed %d files", prog.Current())
prog.AtomicStage.Set(fmt.Sprintf("%s locations", humanize.Comma(prog.Current())))
prog.SetCompleted()
return results, nil
}
func metadataCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
Type: event.FileMetadataCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
func metadataCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog file metadata",
WhileRunning: "Cataloging file metadata",
OnSuccess: "Cataloged file metadata",
},
})
}
return stage, prog
return bus.StartCatalogerTask(info, locations, "")
}

59
syft/pkg/cataloger/catalog.go
View file

@ -6,14 +6,14 @@ import (
"runtime/debug"
"sync"
"github.com/dustin/go-humanize"
"github.com/hashicorp/go-multierror"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
@ -35,21 +35,6 @@ type catalogResult struct {
Error error
}
// newMonitor creates a new Monitor object and publishes the object on the bus as a PackageCatalogerStarted event.
func newMonitor() (*progress.Manual, *progress.Manual) {
filesProcessed := progress.Manual{}
packagesDiscovered := progress.Manual{}
bus.Publish(partybus.Event{
Type: event.PackageCatalogerStarted,
Value: Monitor{
FilesProcessed: progress.Monitorable(&filesProcessed),
PackagesDiscovered: progress.Monitorable(&packagesDiscovered),
},
})
return &filesProcessed, &packagesDiscovered
}
func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerResult *catalogResult, err error) {
// handle individual cataloger panics
defer func() {
@ -101,6 +86,7 @@ func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerRes
}
catalogerResult.Packages = append(catalogerResult.Packages, p)
}
catalogerResult.Relationships = append(catalogerResult.Relationships, relationships...)
log.WithFields("cataloger", cataloger.Name()).Trace("cataloging complete")
return catalogerResult, err
@ -116,9 +102,7 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
catalog := pkg.NewCollection()
var allRelationships []artifact.Relationship
filesProcessed, packagesDiscovered := newMonitor()
defer filesProcessed.SetCompleted()
defer packagesDiscovered.SetCompleted()
prog := monitorPackageCatalogingTask()
// perform analysis, accumulating errors for each failed analysis
var errs error
@ -131,10 +115,11 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
jobs := make(chan pkg.Cataloger, nCatalogers)
results := make(chan *catalogResult, nCatalogers)
discoveredPackages := make(chan int64, nCatalogers)
waitGroup := sync.WaitGroup{}
var totalPackagesDiscovered int64
for i := 0; i < parallelism; i++ {
waitGroup.Add(1)
@ -148,20 +133,16 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
// ensure we set the error to be aggregated
result.Error = err
discoveredPackages <- result.Discovered
prog.Add(result.Discovered)
totalPackagesDiscovered += result.Discovered
count := humanize.Comma(totalPackagesDiscovered)
prog.AtomicStage.Set(fmt.Sprintf("%s packages", count))
results <- result
}
}()
}
// dynamically show updated discovered package status
go func() {
for discovered := range discoveredPackages {
packagesDiscovered.Add(discovered)
}
}()
// Enqueue the jobs
for _, cataloger := range catalogers {
jobs <- cataloger
@ -171,7 +152,6 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
// Wait for the jobs to finish
waitGroup.Wait()
close(results)
close(discoveredPackages)
// collect the results
for result := range results {
@ -186,6 +166,12 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...)
if errs != nil {
prog.SetError(errs)
} else {
prog.SetCompleted()
}
return catalog, allRelationships, errs
}
@ -228,3 +214,16 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
}
return relationships, nil
}
func monitorPackageCatalogingTask() *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog packages",
WhileRunning: "Cataloging packages",
OnSuccess: "Cataloged packages",
},
HideOnSuccess: false,
}
return bus.StartCatalogerTask(info, -1, "")
}

32
syft/pkg/cataloger/golang/licenses.go
View file

@ -20,6 +20,7 @@ import (
"github.com/go-git/go-git/v5/storage/memory"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/licenses"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event/monitor"
@ -31,7 +32,6 @@ import (
type goLicenses struct {
opts CatalogerConfig
localModCacheResolver file.WritableResolver
progress *monitor.CatalogerTask
lowerLicenseFileNames *strset.Set
}
@ -39,11 +39,6 @@ func newGoLicenses(opts CatalogerConfig) goLicenses {
return goLicenses{
opts: opts,
localModCacheResolver: modCacheResolver(opts.LocalModCacheDir),
progress: &monitor.CatalogerTask{
SubStatus: true,
RemoveOnCompletion: true,
Title: "Downloading go mod",
},
lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...),
}
}
@ -123,7 +118,16 @@ func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]
proxies := remotesForModule(c.opts.Proxies, c.opts.NoProxy, moduleName)
fsys, err := getModule(c.progress, proxies, moduleName, moduleVersion)
prog := bus.StartCatalogerTask(monitor.GenericTask{
Title: monitor.Title{
Default: "Download go mod",
WhileRunning: "Downloading go mod",
OnSuccess: "Downloaded go mod",
},
HideOnSuccess: true,
}, -1, "")
fsys, err := getModule(prog, proxies, moduleName, moduleVersion)
if err != nil {
return nil, err
}
@ -206,7 +210,7 @@ func processCaps(s string) string {
})
}
func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) {
func getModule(progress *monitor.CatalogerTaskProgress, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) {
for _, proxy := range proxies {
u, _ := url.Parse(proxy)
if proxy == "direct" {
@ -218,7 +222,7 @@ func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, mo
fsys, err = getModuleProxy(progress, proxy, moduleName, moduleVersion)
case "file":
p := filepath.Join(u.Path, moduleName, "@v", moduleVersion)
progress.SetValue(fmt.Sprintf("file: %s", p))
progress.AtomicStage.Set(fmt.Sprintf("file: %s", p))
fsys = os.DirFS(p)
}
if fsys != nil {
@ -228,9 +232,9 @@ func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, mo
return
}
func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) {
func getModuleProxy(progress *monitor.CatalogerTaskProgress, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) {
u := fmt.Sprintf("%s/%s/@v/%s.zip", proxy, moduleName, moduleVersion)
progress.SetValue(u)
progress.AtomicStage.Set(u)
// get the module zip
resp, err := http.Get(u) //nolint:gosec
@ -241,7 +245,7 @@ func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName st
if resp.StatusCode != http.StatusOK {
u = fmt.Sprintf("%s/%s/@v/%s.zip", proxy, strings.ToLower(moduleName), moduleVersion)
progress.SetValue(u)
progress.AtomicStage.Set(u)
// try lowercasing it; some packages have mixed casing that really messes up the proxy
resp, err = http.Get(u) //nolint:gosec
@ -284,14 +288,14 @@ func findVersionPath(f fs.FS, dir string) string {
return ""
}
func getModuleRepository(progress *monitor.CatalogerTask, moduleName string, moduleVersion string) (fs.FS, error) {
func getModuleRepository(progress *monitor.CatalogerTaskProgress, moduleName string, moduleVersion string) (fs.FS, error) {
repoName := moduleName
parts := strings.Split(moduleName, "/")
if len(parts) > 2 {
repoName = fmt.Sprintf("%s/%s/%s", parts[0], parts[1], parts[2])
}
progress.SetValue(fmt.Sprintf("git: %s", repoName))
progress.AtomicStage.Set(fmt.Sprintf("git: %s", repoName))
f := memfs.New()
buf := &bytes.Buffer{}