From 256a94443e487bcc07ed9502b373f4a401ea8e79 Mon Sep 17 00:00:00 2001
From: JustAnotherArchivist <JustAnotherArchivist@users.noreply.github.com>
Date: Tue, 22 Oct 2019 14:53:20 +0000
Subject: [PATCH] Fix deduplication within each section processing

---
 wiki-recursive-extract-normalise | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/wiki-recursive-extract-normalise b/wiki-recursive-extract-normalise
index ca357c2..bd9e1fa 100755
--- a/wiki-recursive-extract-normalise
+++ b/wiki-recursive-extract-normalise
@@ -51,6 +51,11 @@ do
 			url="${url%% | *}"
 		fi
 
+		if [[ "${sectionUrls[${url}]}" ]]
+		then
+			# Processed already, skip
+			continue
+		fi
 		sectionUrls["${url}"]=1
 		toProcess=("${url}")
 		while [[ ${#toProcess[@]} -gt 0 ]]