Browse Source

fix skipbloom/skipfeed and merge initial item pop into loop logic

master
Fusl 8 months ago
parent
commit
3b37b4fe5e
1 changed files with 11 additions and 18 deletions
  1. +11
    -18
      projectbackfeedmanager.go

+ 11
- 18
projectbackfeedmanager.go View File

@@ -88,38 +88,29 @@ func (that *ProjectBackfeedManager) Do() {
break
default:
}
item, ok := that.PopItem(true)
if !ok {
break
}
keyMap := map[string][][]byte{}
var sAddItems []any
if item.SkipBloom {
sAddItems = append(sAddItems, item.Item)
continue
} else {
key := fmt.Sprintf("%s:%02x:%s", that.Name, item.PrimaryShard, item.SecondaryShard)
keyMap[key] = append(keyMap[key], item.Item)
}
wrapped := 1
skipFeedItems := map[string]struct{}{}
wrapped := 0
for wrapped < ItemWrapSize {
item, ok := that.PopItem(false)
item, ok := that.PopItem(wrapped == 0)
if !ok {
break
}
if item.SkipBloom {
sAddItems = append(sAddItems, item.Item)
continue
} else {
key := fmt.Sprintf("%s:%02x:%s", that.Name, item.PrimaryShard, item.SecondaryShard)
keyMap[key] = append(keyMap[key], item.Item)
}
if item.SkipFeed {
skipFeedItems[string(item.Item)] = Tag
if item.SkipFeed {
skipFeedItems[string(item.Item)] = Tag
}
}
wrapped++
}
if wrapped == 0 {
break
}
if len(keyMap) > 0 {
try := 0
lastTS := make([]any, 0, len(keyMap)*2)
@@ -174,7 +165,9 @@ func (that *ProjectBackfeedManager) Do() {
if len(sAddItems) > 0 && len(skipFeedItems) > 0 {
sAddItemsFiltered := make([]any, 0, len(sAddItems))
for _, item := range sAddItems {
if _, exists := skipFeedItems[item.(string)]; !exists {
itemBytes := item.([]byte)
itemString := string(itemBytes)
if _, exists := skipFeedItems[itemString]; !exists {
sAddItemsFiltered = append(sAddItemsFiltered, item)
}
}


Loading…
Cancel
Save