diff --git a/wiki-recursive-extract-normalise b/wiki-recursive-extract-normalise index ca357c2..bd9e1fa 100755 --- a/wiki-recursive-extract-normalise +++ b/wiki-recursive-extract-normalise @@ -51,6 +51,11 @@ do url="${url%% | *}" fi + if [[ "${sectionUrls[${url}]}" ]] + then + # Processed already, skip + continue + fi sectionUrls["${url}"]=1 toProcess=("${url}") while [[ ${#toProcess[@]} -gt 0 ]]