Skip to content

Commit

Permalink
entries cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@Trilarion
Trilarion committed Sep 7, 2020
1 parent ec233f2 commit cd63ad2
Show file tree
Hide file tree
Showing 463 changed files with 589 additions and 962 deletions.
1 change: 1 addition & 0 deletions code/backlog.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ https://github.com/SadConsole/SadConsole
https://github.com/tlgkccampbell/ultraviolet
https://github.com/amerkoleci/Vortice.Windows
https://github.com/horde3d/Horde3D
https://github.com/delaford/game
https://github.com/cxong/cdogs-sdl
https://moaiwebsite.github.io/
http://cyxdown.free.fr/bs/
Expand Down
259 changes: 35 additions & 224 deletions code/maintenance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,118 +18,47 @@
import utils.constants
from utils import constants as c, utils, osg


def update_readme_and_tocs(infos):
def extract_links():
"""
Parses all entries and extracts http(s) links from them
"""
Recounts entries in sub categories and writes them to the readme.
Also updates the _toc files in the categories directories.

Note: The Readme must have a specific structure at the beginning, starting with "# Open Source Games" and ending
on "A collection.."
# regex for finding urls (can be in <> or in ]() or after a whitespace
regex = re.compile(r"[\s\n]<(http.+?)>|]\((http.+?)\)|[\s\n](http[^\s\n,]+?)[\s\n,]")

Needs to be performed regularly.
"""
print('update readme and toc files')

# completely delete content of toc path
for file in os.listdir(c.tocs_path):
os.remove(os.path.join(c.tocs_path, file))

# read readme
readme_file = os.path.join(c.root_path, 'README.md')
readme_text = utils.read_text(readme_file)

# compile regex for identifying the building blocks in the readme
regex = re.compile(r"(.*?)(\[comment\]: # \(start.*?end of autogenerated content\))(.*)", re.DOTALL)

# apply regex
matches = regex.findall(readme_text)
if len(matches) != 1:
raise RuntimeError('readme file has invalid structure')
matches = matches[0]
start = matches[0]
end = matches[2]

tocs_text = ''

# split infos
infos_games, infos_tools, infos_frameworks, infos_libraries = osg.split_infos(infos)

# create games, tools, frameworks, libraries tocs
title = 'Games'
file = '_games.md'
tocs_text += '**[{}](entries/tocs/{}#{})** ({}) - '.format(title, file, title, len(infos_games))
create_toc(title, file, infos_games)

title = 'Tools'
file = '_tools.md'
tocs_text += '**[{}](entries/tocs/{}#{})** ({}) - '.format(title, file, title, len(infos_tools))
create_toc(title, file, infos_tools)

title = 'Frameworks'
file = '_frameworks.md'
tocs_text += '**[{}](entries/tocs/{}#{})** ({}) - '.format(title, file, title, len(infos_frameworks))
create_toc(title, file, infos_frameworks)

title = 'Libraries'
file = '_libraries.md'
tocs_text += '**[{}](entries/tocs/{}#{})** ({})\n'.format(title, file, title, len(infos_libraries))
create_toc(title, file, infos_libraries)

# create by category
categories_text = []
for keyword in utils.constants.recommended_keywords:
infos_filtered = [x for x in infos if keyword in x['keywords']]
title = keyword.capitalize()
name = keyword.replace(' ', '-')
file = '_{}.md'.format(name)
categories_text.append('**[{}](entries/tocs/{}#{})** ({})'.format(title, file, name, len(infos_filtered)))
create_toc(title, file, infos_filtered)
categories_text.sort()
tocs_text += '\nBy category: {}\n'.format(', '.join(categories_text))

# create by platform
platforms_text = []
for platform in utils.constants.valid_platforms:
infos_filtered = [x for x in infos if platform in x.get('platform', [])]
title = platform
name = platform.lower()
file = '_{}.md'.format(name)
platforms_text.append('**[{}](entries/tocs/{}#{})** ({})'.format(title, file, name, len(infos_filtered)))
create_toc(title, file, infos_filtered)
tocs_text += '\nBy platform: {}\n'.format(', '.join(platforms_text))

# insert new text in the middle (the \n before the second comment is necessary, otherwise Markdown displays it as part of the bullet list)
text = start + "[comment]: # (start of autogenerated content, do not edit)\n" + tocs_text + "\n[comment]: # (end of autogenerated content)" + end

# write to readme
utils.write_text(readme_file, text)


def create_toc(title, file, entries):
"""
# iterate over all entries
urls = set()
for _, _, content in entry_iterator():

# apply regex
matches = regex.findall(content)

# for each match
for match in matches:

# for each possible clause
for url in match:

# if there was something (and not a sourceforge git url)
if url:
urls.add(url)
urls = sorted(list(urls), key=str.casefold)
return urls

def split_infos(infos):
"""
# file path
toc_file = os.path.join(c.tocs_path, file)
Split into games, tools, frameworks, libraries
"""
games = [x for x in infos if not any([y in x['keywords'] for y in ('tool', 'framework', 'library')])]
tools = [x for x in infos if 'tool' in x['keywords']]
frameworks = [x for x in infos if 'framework' in x['keywords']]
libraries = [x for x in infos if 'library' in x['keywords']]
return games, tools, frameworks, libraries

# header line
text = '[comment]: # (autogenerated content, do not edit)\n# {}\n\n'.format(title)

# assemble rows
rows = []
for entry in entries:
rows.append('- **[{}]({})** ({})'.format(entry['Name'], '../' + entry['file'], ', '.join(
entry['code language'] + entry['code license'] + entry['state'])))

# sort rows (by title)
rows.sort(key=str.casefold)

# add to text
text += '\n'.join(rows)

# write to toc file
utils.write_text(toc_file, text)


def check_validity_external_links():
Expand Down Expand Up @@ -220,28 +149,6 @@ def check_validity_external_links():
print('{}: {} - exception {}'.format(names, url, error_name))


def check_template_leftovers():
"""
Checks for template leftovers.
Should be run only occasionally.
"""

print('check for template leftovers')

# load template and get all lines
text = utils.read_text(os.path.join(c.root_path, 'template.md'))
text = text.split('\n')
check_strings = [x for x in text if x and not x.startswith('##')]

# iterate over all entries
for _, entry_path, content in osg.entry_iterator():

for check_string in check_strings:
if content.find(check_string) >= 0:
raise RuntimeError('{}: found {}'.format(os.path.basename(entry_path), check_string))


def fix_entries():
"""
Fixes the keywords, code dependencies, build systems, .. entries, mostly by automatically sorting them.
Expand Down Expand Up @@ -879,72 +786,6 @@ def check_validity_backlog():
print('{} redirected to {}, {}'.format(url, r.url, r.history))


def update_inspirations(infos):
"""
"""

print('update inspirations')

# collect information
originals = {}
for info in infos:
name = info['Name']
keywords = info['keywords']
ins = [x[12:] for x in keywords if x.startswith('inspired by ')]
if ins:
ins = ins[0].split(' + ')
for original in ins:
if original in originals:
originals[original].append(name)
else:
originals[original] = [name]

inspirations_file = os.path.join(c.root_path, 'inspirations.md')
inspirations = '[comment]: # (partly autogenerated content, edit with care, read the manual before)\n'
inspirations += '# Inspirations ({})\n\n'.format(len(originals)) # add number of inspirations

# iterate through originals alphabetically sorted
for original, names in sorted(originals.items(), key=lambda x: str.casefold(x[0])):
inspirations += '## {} ({})\n\n'.format(original, len(names))
inspirations += '- Inspired entries: {}\n\n'.format(', '.join(sorted(names, key=str.casefold)))

# write to statistics file
utils.write_text(inspirations_file, inspirations)


def update_developer(infos):
"""
"""

print('update developer')

# collect information
developer = {}
for info in infos:
if 'developer' in info:
name = info['Name']
devs = info['developer']
for dev in devs:
if dev in developer:
developer[dev].append(name)
else:
developer[dev] = [name]

developer_file = os.path.join(c.root_path, 'developer.md')
content = '[comment]: # (partly autogenerated content, edit with care, read the manual before)\n'
content += '# Developer ({})\n\n'.format(len(developer)) # add number of developer

# iterate through developers alphabetically sorted
for dev, names in sorted(developer.items(), key=lambda x: str.casefold(x[0])):
content += '## {} ({})\n\n'.format(dev, len(names))
content += '- Games: {}\n\n'.format(', '.join(sorted(names, key=str.casefold)))

# write to statistics file
utils.write_text(developer_file, content)


def check_code_dependencies(infos):
"""
Expand Down Expand Up @@ -984,50 +825,20 @@ def check_code_dependencies(infos):

if __name__ == "__main__":

# check_validity_backlog()

# clean backlog
game_urls = osg.extract_links()
text = utils.read_text(os.path.join(c.root_path, 'code', 'rejected.txt'))
regex = re.compile(r"\((http.*?)\)", re.MULTILINE)
matches = regex.findall(text)
rejected_urls = []
for match in matches:
urls = match.split(',')
urls = [x.strip() for x in urls]
rejected_urls.extend(urls)
game_urls.extend(rejected_urls)
more_urls = []
for url in game_urls:
if url.startswith('https://web.archive.org/web'):
# print(url) # sometimes the http is missing in archive links (would need proper parsing)
url = url[url.index('http', 5):]
more_urls.append(url)
game_urls.extend(more_urls)
stripped_game_urls = [utils.strip_url(x) for x in game_urls]
clean_backlog(stripped_game_urls)

# check for unfilled template lines
check_template_leftovers()
check_validity_backlog()




# fix entries
fix_entries()

# assemble info
infos = osg.assemble_infos()

# recount and write to readme and to tocs
update_readme_and_tocs(infos)

# generate report
update_statistics(infos)

# update inspirations
# update_inspirations(infos)

# update developers
# update_developer(infos)

# update database for html table
export_json(infos)

Expand Down
8 changes: 4 additions & 4 deletions code/utils/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def get_config(key):

url_fields = ('Home', 'Media', 'Play', 'Download', 'Code repository')

valid_url_prefixes = ('http://', 'https://', 'git://', 'svn://', 'ftp://', 'bzr://', '@see-')
valid_url_prefixes = ('http://', 'https://', 'git://', 'svn://', 'ftp://', 'bzr://', '@see-', '@not-', '?')

valid_building_properties = ('Build system', 'Build instructions')
valid_building_fields = valid_building_properties + ('Note',)
Expand All @@ -63,8 +63,8 @@ def get_config(key):
'C++', 'Clojure', 'CoffeeScript', 'ColdFusion', 'D', 'DM', 'Dart', 'Dia', 'Elm', 'Emacs Lisp', 'F#', 'GDScript',
'Game Maker Script', 'Go', 'Groovy', 'Haskell', 'Haxe', 'Io', 'Java', 'JavaScript', 'Kotlin', 'Lisp', 'Lua',
'MegaGlest Script', 'MoonScript', 'None', 'OCaml', 'Objective-C', 'PHP', 'Pascal', 'Perl', 'Python', 'QuakeC', 'R',
"Ren'py", 'Ruby', 'Rust', 'Scala', 'Scheme', 'Script', 'Shell', 'Swift', 'TorqueScript', 'TypeScript', 'Vala',
'Visual Basic', 'XUL', 'ZenScript', 'ooc')
"Ren'Py", 'Ruby', 'Rust', 'Scala', 'Scheme', 'Script', 'Shell', 'Swift', 'TorqueScript', 'TypeScript', 'Vala',
'Visual Basic', 'XUL', 'ZenScript', 'ooc', '?')

# known licenses, anything outside of this will result in a warning during a maintenance operation
# only these will be used when gathering statistics
Expand All @@ -73,7 +73,7 @@ def get_config(key):
'Boost-1.0', 'CC-BY-NC-3.0', 'CC-BY-NC-SA-2.0', 'CC-BY-NC-SA-3.0', 'CC-BY-SA-3.0', 'CC-BY-NC-SA-4.0',
'CC-BY-SA-4.0', 'CC0', 'Custom', 'EPL-2.0', 'GPL-2.0', 'GPL-3.0', 'IJG', 'ISC', 'Java Research License', 'LGPL-2.0',
'LGPL-2.1', 'LGPL-3.0', 'MAME', 'MIT', 'MPL-1.1', 'MPL-2.0', 'MS-PL', 'MS-RL', 'NetHack General Public License',
'None', 'Proprietary', 'Public domain', 'SWIG license', 'Unlicense', 'WTFPL', 'wxWindows license', 'zlib')
'None', 'Proprietary', 'Public domain', 'SWIG license', 'Unlicense', 'WTFPL', 'wxWindows license', 'zlib', '?')

# valid multiplayer modes (can be combined with "+" )
valid_multiplayer_modes = (
Expand Down
Loading

0 comments on commit cd63ad2

Please sign in to comment.