Skip to content

Commit

Permalink
Using filer.remote.sync concurrency in filer.remote.gateway (seaweedf…
Browse files Browse the repository at this point in the history
…s#5123)

- Chnaged ProcessEvenFn to be concurrent just like filer.remote.sync

Co-authored-by: itsfarbod <itsfarbod@hamravesh.com>
  • Loading branch information
itsfarbod and itsfarbod authored Dec 21, 2023
1 parent 06343f8 commit c278f49
Showing 1 changed file with 14 additions and 4 deletions.
18 changes: 14 additions & 4 deletions weed/command/filer_remote_gateway_buckets.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,20 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
return err
}

processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
lastTime := time.Unix(0, lastTsNs)
glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, lastTsNs)
processor := NewMetadataProcessor(eachEntryFunc, 128)

var lastLogTsNs = time.Now().UnixNano()
processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
processor.AddSyncJob(resp)
return nil
}, 3*time.Second, func(counter int64, lastTsNs int64) error {
if processor.processedTsWatermark == 0 {
return nil
}
now := time.Now().UnixNano()
glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, processor.processedTsWatermark), float64(counter)/(float64(now-lastLogTsNs)/1e9))
lastLogTsNs = now
return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, processor.processedTsWatermark)
})

lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo)
Expand Down

0 comments on commit c278f49

Please sign in to comment.