From 6ed6824d5dc4d08b3425fff4741c96c9cd565f73 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:03:35 +0000 Subject: [PATCH] bump go-structr to v0.8.5 to improve memory usage (#2955) --- go.mod | 2 +- go.sum | 4 +- vendor/codeberg.org/gruf/go-structr/cache.go | 14 ++-- vendor/codeberg.org/gruf/go-structr/index.go | 74 ++++++++------------ vendor/codeberg.org/gruf/go-structr/key.go | 4 +- vendor/codeberg.org/gruf/go-structr/queue.go | 8 +-- vendor/modules.txt | 2 +- 7 files changed, 45 insertions(+), 63 deletions(-) diff --git a/go.mod b/go.mod index d2a506f39..5b268b31a 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( codeberg.org/gruf/go-runners v1.6.2 codeberg.org/gruf/go-sched v1.2.3 codeberg.org/gruf/go-storage v0.1.1 - codeberg.org/gruf/go-structr v0.8.4 + codeberg.org/gruf/go-structr v0.8.5 codeberg.org/superseriousbusiness/exif-terminator v0.7.0 github.com/DmitriyVTitov/size v1.5.0 github.com/KimMachineGun/automemlimit v0.6.1 diff --git a/go.sum b/go.sum index 08fdd8744..010111f24 100644 --- a/go.sum +++ b/go.sum @@ -76,8 +76,8 @@ codeberg.org/gruf/go-sched v1.2.3 h1:H5ViDxxzOBR3uIyGBCf0eH8b1L8wMybOXcdtUUTXZHk codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A= codeberg.org/gruf/go-storage v0.1.1 h1:CSX1PMMg/7vqqK8aCFtq94xCrOB3xhj7eWIvzILdLpY= codeberg.org/gruf/go-storage v0.1.1/go.mod h1:145IWMUOc6YpIiZIiCIEwkkNZZPiSbwMnZxRjSc5q6c= -codeberg.org/gruf/go-structr v0.8.4 h1:2eT1VOTWG6T9gIGZwF/1Jop6k6plvfdUY5yBcvbizVg= -codeberg.org/gruf/go-structr v0.8.4/go.mod h1:c5UvVDSA3lZ1kv05V+7pXkO8u8Jea+VRWFDRFBCOxSA= +codeberg.org/gruf/go-structr v0.8.5 h1:WQuvLSQFyFwMjdU7dCWvgcjuhk07oWdSl9guShekzGQ= +codeberg.org/gruf/go-structr v0.8.5/go.mod h1:c5UvVDSA3lZ1kv05V+7pXkO8u8Jea+VRWFDRFBCOxSA= codeberg.org/superseriousbusiness/exif-terminator v0.7.0 h1:Y6VApSXhKqExG0H2hZ2JelRK4xmWdjDQjn13CpEfzko= codeberg.org/superseriousbusiness/exif-terminator v0.7.0/go.mod h1:gCWKduudUWFzsnixoMzu0FYVdxHWG+AbXnZ50DqxsUE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= diff --git a/vendor/codeberg.org/gruf/go-structr/cache.go b/vendor/codeberg.org/gruf/go-structr/cache.go index 429c6ab67..0b8a8b2c7 100644 --- a/vendor/codeberg.org/gruf/go-structr/cache.go +++ b/vendor/codeberg.org/gruf/go-structr/cache.go @@ -159,7 +159,7 @@ func (c *Cache[T]) Get(index *Index, keys ...Key) []T { for i := range keys { // Concatenate all *values* from cached items. - index.get(keys[i], func(item *indexed_item) { + index.get(keys[i].key, func(item *indexed_item) { if value, ok := item.data.(T); ok { // Append value COPY. value = c.copy(value) @@ -344,7 +344,7 @@ func (c *Cache[T]) Load(index *Index, keys []Key, load func([]Key) ([]T, error)) before := len(values) // Concatenate all *values* from cached items. - index.get(keys[i], func(item *indexed_item) { + index.get(keys[i].key, func(item *indexed_item) { if value, ok := item.data.(T); ok { // Append value COPY. value = c.copy(value) @@ -446,10 +446,10 @@ func (c *Cache[T]) Invalidate(index *Index, keys ...Key) { // Preallocate expected ret slice. values := make([]T, 0, len(keys)) - for _, key := range keys { + for i := range keys { // Delete all items under key from index, collecting // value items and dropping them from all their indices. - index.delete(key, func(item *indexed_item) { + index.delete(keys[i].key, func(item *indexed_item) { if value, ok := item.data.(T); ok { // No need to copy, as item @@ -569,7 +569,7 @@ func (c *Cache[T]) store_value(index *Index, key Key, value T) { if index != nil { // Append item to index. - index.append(key, item) + index.append(key.key, item) } // Get ptr to value data. @@ -596,7 +596,7 @@ func (c *Cache[T]) store_value(index *Index, key Key, value T) { // Calculate index key. key := idx.key(buf, parts) - if key.Zero() { + if key == "" { continue } @@ -639,7 +639,7 @@ func (c *Cache[T]) store_error(index *Index, key Key, err error) { item.data = err // Append item to index. - index.append(key, item) + index.append(key.key, item) // Add item to main lru list. c.lru.push_front(&item.elem) diff --git a/vendor/codeberg.org/gruf/go-structr/index.go b/vendor/codeberg.org/gruf/go-structr/index.go index 16f099ec6..522dca836 100644 --- a/vendor/codeberg.org/gruf/go-structr/index.go +++ b/vendor/codeberg.org/gruf/go-structr/index.go @@ -96,7 +96,10 @@ func (i *Index) Key(parts ...any) Key { buf := new_buffer() key := i.key(buf, parts) free_buffer(buf) - return key + return Key{ + raw: parts, + key: key, + } } // Keys generates []Key{} from given (multiple) parts @@ -107,10 +110,13 @@ func (i *Index) Keys(parts ...[]any) []Key { buf := new_buffer() for _, parts := range parts { key := i.key(buf, parts) - if key.Zero() { + if key == "" { continue } - keys = append(keys, key) + keys = append(keys, Key{ + raw: parts, + key: key, + }) } free_buffer(buf) return keys @@ -169,34 +175,21 @@ func (i *Index) get_one(key Key) *indexed_item { // Extract entry from first list elem. entry := (*index_entry)(l.head.data) - // Check contains expected key. - if !entry.key.Equal(key) { - return nil - } - return entry.item } // get will fetch all indexed items under key, passing each to hook. -func (i *Index) get(key Key, hook func(*indexed_item)) { +func (i *Index) get(key string, hook func(*indexed_item)) { if hook == nil { panic("nil hook") } // Get list at hash. - l, _ := i.data.Get(key.key) + l, _ := i.data.Get(key) if l == nil { return } - // Extract entry from first list elem. - entry := (*index_entry)(l.head.data) - - // Check contains expected key. - if !entry.key.Equal(key) { - return - } - // Iterate all entries in list. l.rangefn(func(elem *list_elem) { @@ -210,7 +203,7 @@ func (i *Index) get(key Key, hook func(*indexed_item)) { } // key uses hasher to generate Key{} from given raw parts. -func (i *Index) key(buf *byteutil.Buffer, parts []any) Key { +func (i *Index) key(buf *byteutil.Buffer, parts []any) string { if len(parts) != len(i.fields) { panicf("incorrect number key parts: want=%d received=%d", len(i.fields), @@ -223,7 +216,7 @@ func (i *Index) key(buf *byteutil.Buffer, parts []any) Key { before := len(buf.B) buf.B = field.mangle(buf.B, parts[x]) if string(buf.B[before:]) == field.zerostr { - return Key{} + return "" } buf.B = append(buf.B, '.') } @@ -233,24 +226,21 @@ func (i *Index) key(buf *byteutil.Buffer, parts []any) Key { buf.B = append(buf.B, '.') } } - return Key{ - raw: parts, - key: string(buf.B), - } + return string(buf.B) } // append will append the given index entry to appropriate // doubly-linked-list in index hashmap. this handles case // of key collisions and overwriting 'unique' entries. -func (i *Index) append(key Key, item *indexed_item) { +func (i *Index) append(key string, item *indexed_item) { // Look for existing. - l, _ := i.data.Get(key.key) + l, _ := i.data.Get(key) if l == nil { // Allocate new. l = new_list() - i.data.Put(key.key, l) + i.data.Put(key, l) } else if is_unique(i.flags) { @@ -280,27 +270,19 @@ func (i *Index) append(key Key, item *indexed_item) { } // delete will remove all indexed items under key, passing each to hook. -func (i *Index) delete(key Key, hook func(*indexed_item)) { +func (i *Index) delete(key string, hook func(*indexed_item)) { if hook == nil { panic("nil hook") } // Get list at hash. - l, _ := i.data.Get(key.key) + l, _ := i.data.Get(key) if l == nil { return } - // Extract entry from first list elem. - entry := (*index_entry)(l.head.data) - - // Check contains expected key. - if !entry.key.Equal(key) { - return - } - - // Delete data at hash. - i.data.Delete(key.key) + // Delete at hash. + i.data.Delete(key) // Iterate entries in list. for x := 0; x < l.len; x++ { @@ -330,7 +312,7 @@ func (i *Index) delete(key Key, hook func(*indexed_item)) { // delete_entry deletes the given index entry. func (i *Index) delete_entry(entry *index_entry) { // Get list at hash sum. - l, _ := i.data.Get(entry.key.key) + l, _ := i.data.Get(entry.key) if l == nil { return } @@ -339,8 +321,8 @@ func (i *Index) delete_entry(entry *index_entry) { l.remove(&entry.elem) if l.len == 0 { - // Remove entry list from map. - i.data.Delete(entry.key.key) + // Remove entry from map. + i.data.Delete(entry.key) // Release list. free_list(l) @@ -387,9 +369,9 @@ type index_entry struct { // elem.data is ptr to index_entry. elem list_elem - // hash checksum - // + raw key data - key Key + // raw cache key + // for this entry. + key string // index this is stored in. index *Index @@ -415,7 +397,7 @@ func new_index_entry() *index_entry { // free_index_entry releases the index_entry. func free_index_entry(entry *index_entry) { entry.elem.data = nil - entry.key = Key{} + entry.key = "" entry.index = nil entry.item = nil index_entry_pool.Put(entry) diff --git a/vendor/codeberg.org/gruf/go-structr/key.go b/vendor/codeberg.org/gruf/go-structr/key.go index d68e3fe19..099f70f99 100644 --- a/vendor/codeberg.org/gruf/go-structr/key.go +++ b/vendor/codeberg.org/gruf/go-structr/key.go @@ -22,7 +22,7 @@ func (k Key) Key() string { // Equal returns whether keys are equal. func (k Key) Equal(o Key) bool { - return k.key == o.key + return (k.key == o.key) } // Value returns the raw slice of @@ -33,7 +33,7 @@ func (k Key) Values() []any { // Zero indicates a zero value key. func (k Key) Zero() bool { - return k.raw == nil + return (k.raw == nil) } var buf_pool sync.Pool diff --git a/vendor/codeberg.org/gruf/go-structr/queue.go b/vendor/codeberg.org/gruf/go-structr/queue.go index ed5d78b5a..bdead6ea8 100644 --- a/vendor/codeberg.org/gruf/go-structr/queue.go +++ b/vendor/codeberg.org/gruf/go-structr/queue.go @@ -127,7 +127,7 @@ func (q *Queue[T]) Pop(index *Index, keys ...Key) []T { for i := range keys { // Delete all items under key from index, collecting // value items and dropping them from all their indices. - index.delete(keys[i], func(item *indexed_item) { + index.delete(keys[i].key, func(item *indexed_item) { // Append deleted to values. value := item.data.(T) @@ -179,7 +179,7 @@ func (q *Queue[T]) PushBack(values ...T) { func (q *Queue[T]) MoveFront(index *Index, keys ...Key) { q.mutex.Lock() for i := range keys { - index.get(keys[i], func(item *indexed_item) { + index.get(keys[i].key, func(item *indexed_item) { q.queue.move_front(&item.elem) }) } @@ -190,7 +190,7 @@ func (q *Queue[T]) MoveFront(index *Index, keys ...Key) { func (q *Queue[T]) MoveBack(index *Index, keys ...Key) { q.mutex.Lock() for i := range keys { - index.get(keys[i], func(item *indexed_item) { + index.get(keys[i].key, func(item *indexed_item) { q.queue.move_back(&item.elem) }) } @@ -305,7 +305,7 @@ func (q *Queue[T]) index(value T) *indexed_item { // Calculate index key. key := idx.key(buf, parts) - if key.Zero() { + if key == "" { continue } diff --git a/vendor/modules.txt b/vendor/modules.txt index f2aeb67fe..8c3eaf1d5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -68,7 +68,7 @@ codeberg.org/gruf/go-storage/disk codeberg.org/gruf/go-storage/internal codeberg.org/gruf/go-storage/memory codeberg.org/gruf/go-storage/s3 -# codeberg.org/gruf/go-structr v0.8.4 +# codeberg.org/gruf/go-structr v0.8.5 ## explicit; go 1.21 codeberg.org/gruf/go-structr # codeberg.org/superseriousbusiness/exif-terminator v0.7.0