Only flush files when they were written to since the last flush

irclog.py

@@ -715,7 +715,7 @@ class Storage:
 		self.messageQueue = messageQueue
 		self.config = config
 		self.paths = {} # channel -> path from channels config
-		self.files = {} # channel|None -> (filename, fileobj); None = general raw log
+		self.files = {} # channel|None -> [filename, fileobj, lastWriteTime]; None = general raw log
 
 	def update_config(self, config):
 		channelsOld = {channel['ircchannel'] for channel in self.config['channels'].values()}
@@ -747,7 +747,7 @@ class Storage:
 		fpath = os.path.join(self.config['storage']['path'], dn, fn)
 		self.logger.debug(f'Opening file {fpath!r} for {channel!r} with mode {mode!r}')
 		os.makedirs(os.path.join(self.config['storage']['path'], dn), exist_ok = True)
-		self.files[channel] = (fn, open(fpath, mode))
+		self.files[channel] = [fn, open(fpath, mode), 0]
 
 	async def run(self, loop, sigintEvent):
 		self.update_config(self.config) # Ensure that files are open etc.
@@ -787,20 +787,27 @@ class Storage:
 			self.files[None][1].write(str(time_).encode('ascii') + b' ' + message + b'\n')
 		else:
 			self.files[channel][1].write(f'{time_} {command} {message}\n')
+		self.files[channel][2] = time.time()
 
 	async def flush_files(self, flushExitEvent):
+		lastFlushTime = 0
 		while True:
 			await asyncio.wait({asyncio.create_task(flushExitEvent.wait())}, timeout = self.config['storage']['flushTime'])
 			self.logger.debug('Flushing files')
-			for _, f in self.files.values():
-				f.flush()
+			for channel, (fn, f, fLastWriteTime) in self.files.items():
+				if fLastWriteTime > lastFlushTime:
+					self.logger.debug(f'Flushing {channel} {fn}')
+					f.flush()
+				else:
+					self.logger.debug(f'Flushing {channel} {fn} unnecessary')
 			self.logger.debug('Flushing done')
 			if flushExitEvent.is_set():
 				break
+			lastFlushTime = time.time()
 		self.logger.debug('Exiting flush_files')
 
 	def close(self):
-		for _, f in self.files.values():
+		for _, f, _ in self.files.values():
 			f.close()
 		self.files = {}