|
|
@@ -88,38 +88,29 @@ func (that *ProjectBackfeedManager) Do() { |
|
|
|
break |
|
|
|
default: |
|
|
|
} |
|
|
|
item, ok := that.PopItem(true) |
|
|
|
if !ok { |
|
|
|
break |
|
|
|
} |
|
|
|
keyMap := map[string][][]byte{} |
|
|
|
var sAddItems []any |
|
|
|
if item.SkipBloom { |
|
|
|
sAddItems = append(sAddItems, item.Item) |
|
|
|
continue |
|
|
|
} else { |
|
|
|
key := fmt.Sprintf("%s:%02x:%s", that.Name, item.PrimaryShard, item.SecondaryShard) |
|
|
|
keyMap[key] = append(keyMap[key], item.Item) |
|
|
|
} |
|
|
|
wrapped := 1 |
|
|
|
skipFeedItems := map[string]struct{}{} |
|
|
|
wrapped := 0 |
|
|
|
for wrapped < ItemWrapSize { |
|
|
|
item, ok := that.PopItem(false) |
|
|
|
item, ok := that.PopItem(wrapped == 0) |
|
|
|
if !ok { |
|
|
|
break |
|
|
|
} |
|
|
|
if item.SkipBloom { |
|
|
|
sAddItems = append(sAddItems, item.Item) |
|
|
|
continue |
|
|
|
} else { |
|
|
|
key := fmt.Sprintf("%s:%02x:%s", that.Name, item.PrimaryShard, item.SecondaryShard) |
|
|
|
keyMap[key] = append(keyMap[key], item.Item) |
|
|
|
} |
|
|
|
if item.SkipFeed { |
|
|
|
skipFeedItems[string(item.Item)] = Tag |
|
|
|
if item.SkipFeed { |
|
|
|
skipFeedItems[string(item.Item)] = Tag |
|
|
|
} |
|
|
|
} |
|
|
|
wrapped++ |
|
|
|
} |
|
|
|
if wrapped == 0 { |
|
|
|
break |
|
|
|
} |
|
|
|
if len(keyMap) > 0 { |
|
|
|
try := 0 |
|
|
|
lastTS := make([]any, 0, len(keyMap)*2) |
|
|
@@ -174,7 +165,9 @@ func (that *ProjectBackfeedManager) Do() { |
|
|
|
if len(sAddItems) > 0 && len(skipFeedItems) > 0 { |
|
|
|
sAddItemsFiltered := make([]any, 0, len(sAddItems)) |
|
|
|
for _, item := range sAddItems { |
|
|
|
if _, exists := skipFeedItems[item.(string)]; !exists { |
|
|
|
itemBytes := item.([]byte) |
|
|
|
itemString := string(itemBytes) |
|
|
|
if _, exists := skipFeedItems[itemString]; !exists { |
|
|
|
sAddItemsFiltered = append(sAddItemsFiltered, item) |
|
|
|
} |
|
|
|
} |
|
|
|