checked external links

code/backlog.txt

@@ -137,6 +137,7 @@ https://app.assembla.com/spaces/openrpg/wiki
 https://app.assembla.com/spaces/Project_Valkyrie2/wiki
 https://appimage.github.io/categories/Game
 https://appimage.org/
+https://arcade.academy/
 https://archive.codeplex.com/?p=turnota
 https://archive.codeplex.com/?p=voxeliq
 https://archive.org/details/Gna_code_hosting (all of them)
@@ -280,9 +281,11 @@ https://github.com/junkdog/artemis-odb
 https://github.com/jwvhewitt/gearhead-caramel.git
 https://github.com/KDE
 https://github.com/kduske/TrenchBroom
+https://github.com/kidscancode/circle_jump
 https://github.com/kimkulling/osre
 https://github.com/klaussilveira/ioquake3.js
 https://github.com/Kosmonaut3d/DeferredEngine
+https://github.com/kyzfrintin/Godot-Mixing-Desk
 https://github.com/LgLinus/StrategyGame
 https://github.com/libretro/libretro-chailove
 https://github.com/libretro/libretro-prboom
@@ -316,6 +319,7 @@ https://github.com/opensourcedesign
 https://github.com/opentrack/opentrack
 https://github.com/OSSGames
 https://github.com/OSSGames (all there, but we should have them already)
+https://github.com/ozkriff/zemeroth
 https://github.com/Patapom/GodComplex
 https://github.com/PavelDoGreat/WebGL-Fluid-Simulation
 https://github.com/perbone/luascript
@@ -393,6 +397,7 @@ https://gitlab.com/xonlegacy/xonlegacy
 https://gitlab.openrsc.com/open-rsc
 https://gitorious.org/index-list.html (are there other interesting games)
 https://gottcode.org/ (all)
+https://invent.kde.org/games
 https://itch.io/ (scrape, look for open source)
 https://jcrpg.blogspot.de/
 https://jotd.pagesperso-orange.fr/ (several games)

code/maintenance.py

@@ -144,7 +144,10 @@ def check_validity_external_links():
     regex = re.compile(r"[\s\n]<(http.+?)>|\]\((http.+?)\)|[\s\n](http[^\s\n,]+?)[\s\n\)]")
 
     # ignore the following patterns (they give false positives here)
-    ignored_urls = ('https://git.tukaani.org/xz.git', 'https://git.code.sf.net/p/')
+    ignored_urls = ('https://git.tukaani.org/xz.git', 'https://git.code.sf.net/', 'http://hg.hedgewars.org/hedgewars/', 'https://git.xiph.org/vorbis.git', 'http://svn.uktrainsim.com/svn/openrails')
+
+    # some do redirect, but we nedertheless want the original URL in the database
+    redirect_okay = ('https://octaforge.org/', 'https://svn.openttd.org/')
 
     # extract all links from entries
     urls = {}
@@ -158,14 +161,24 @@ def check_validity_external_links():
                     # github and gitlab git URLs are shortened to not contain .git
                     if any((url.startswith(x) for x in ('https://github.com/', 'https://gitlab.com/', 'https://salsa.debian.org/', 'https://src.fedoraproject.org/', 'https://gitlab.gnome.org/GNOME/'))) and url.endswith('.git'):
                         url = url[:-4]
-                    if url.startswith('https://svn.code.sf.net/p/') and url.endswith('code'):
+                    if (url.startswith('https://svn.code.sf.net/p/') or url.startswith('http://svn.code.sf.net/p/')) and url.endswith('code'):
                         url = url + '/'
                     if url.startswith('https://bitbucket.org/') and url.endswith('.git'):
                         url = url[:-4] + '/commits/'
-                    if url.startswith('https://svn.code.sf.net/p/'):
+                    if url.startswith('https://svn.code.sf.net/p/') or url.endswith('.cvs.sourceforge.net'):
                         url = 'http' + url[5:]
-                    if url.startswith('https://git.savannah.gnu.org/git/'):
+                    if url.startswith('https://git.savannah.gnu.org/git/') or url.startswith('https://git.savannah.nongnu.org/git/') or url.startswith('http://git.artsoft.org/'):
+                        url = url + '/'
+                    if url.startswith('https://anongit.freedesktop.org/git'):
                         url = url + '/'
+                    if url.startswith('http://cvs.savannah.nongnu.org:/sources/'):
+                        url = 'http://cvs.savannah.nongnu.org/viewvc/' + url[40:]
+                    if url.startswith('http://cvs.savannah.gnu.org:/sources/'):
+                        url = 'http://cvs.savannah.gnu.org/viewvc/' + url[37:]
+                    if 'bzr.sourceforge.net/bzrroot/' in url:
+                        continue
+                    if url.endswith('.git'):
+                        url = url[:-4]
 
                     if url in urls:
                         urls[url].add(entry)
@@ -177,12 +190,17 @@ def check_validity_external_links():
     # now iterate over all urls
     for index, url in enumerate(urls.keys()):
         try:
-            r = requests.head(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64)'}, timeout=10, allow_redirects=True)
+            verify = True
+            # some have an expired certificate but otherwise still work
+            if any((url.startswith(x) for x in ('https://perso.b2b2c.ca/~sarrazip/dev/', 'https://dreerally.com/', 'https://henlin.net/', 'https://www.megamek.org/', 'https://pixeldoctrine.com/', 'https://gitorious.org/'))):
+                verify = False
+            r = requests.head(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64)'}, timeout=10, allow_redirects=True, verify=verify)
             # check for bad status
             if r.status_code != requests.codes.ok:
                 print('{}: URL {} in entry {} has status {}'.format(index, url, urls[url], r.status_code))
             # check for redirect
-            if r.history:
+            if r.history and url not in redirect_okay:
+                # only / added or http->https
                 print('{}: URL {} in entry {} was redirected to {}'.format(index, url, urls[url], r.url))
         except Exception as e:
             print('{}: URL {} in entry {} gave error {}'.format(index, url, urls[url], e))