generate.py

#!/usr/bin/env python
"""
xemu.app Static Site Generator
"""

import codecs
import json
import os
import re
import requests

from collections import defaultdict
from datetime import datetime, timezone
from functools import reduce
from github import Github
from jinja2 import Environment, FileSystemLoader
from tqdm import tqdm
from minify_html import minify as minify_html

output_dir = 'dist'
repo_url_base = 'https://raw.githubusercontent.com/mborgerson/xemu-website/master/'
compatibility_reports_url = 'https://reports.xemu.app/compatibility'
compatibility_reports_url_verify_certs = True
main_url_base = 'https://xemu.app'
# compatibility_reports_url = 'https://127.0.0.1/compatibility'
# compatibility_reports_url_verify_certs = False

develop_mode = False
disable_load_issues = develop_mode
disable_load_reports = develop_mode
disable_load_version = develop_mode

title_status_descriptions = {
    'Unknown'  : 'A compatibility test has not been recorded for this title.',
    'Broken'   : 'This title crashes very soon after launching, or displays nothing at all.',
    'Intro'    : 'This title displays an intro sequence, but fails to make it to gameplay.',
    'Starts'   : 'This title starts, but may crash or have significant issues.',
    'Playable' : 'This title is playable, with minor issues.',
    'Perfect'  : 'This title is playable from start to finish with no noticable issues.'
    }

def get_field(s,x):
    return s[x] if x in s else ''

class Issue:
    all_issues = []
    issues_by_title = defaultdict(list)

    def __init__(self, number, url, title, affected_titles, created_at, updated_at):
        self.number = number
        self.url = url
        self.title = title
        self.affected_titles = affected_titles
        self.created_at = created_at
        self.updated_at = updated_at

    def __repr__(self):
        return self.title

    @classmethod
    def load_issues(cls, title_alias_map):
        """
        Search through all GitHub issues for any title tags to construct a
        list of titles and their associated issues
        """
        if disable_load_issues:
            return
        titles_re = re.compile(r'Titles?[:/]\s*([a-fA-f0-9,\s]+)', re.IGNORECASE)
        title_id_re = re.compile(r'([a-fA-f0-9]{8})')
        for issue in Github().get_user('mborgerson').get_repo('xemu').get_issues():
            if issue.state != 'open': continue
            # Look for a titles sequence and pull out anything that looks like
            # an id
            references = ' '.join(titles_re.findall(issue.body))
            affected_titles = title_id_re.findall(references)
            cls.all_issues.append(cls(
                issue.number,
                issue.html_url,
                issue.title,
                affected_titles,
                issue.created_at,
                issue.updated_at))

        # Organize issues by title
        for issue in cls.all_issues:
            for title_id in issue.affected_titles:
                if title_id not in title_alias_map:
                    print('Warning: Issue %s references unknown title id "%s"' % (issue.url, title_id))
                    continue
                if issue not in cls.issues_by_title[title_alias_map[title_id]]:
                    cls.issues_by_title[title_alias_map[title_id]].append(issue)

class CompatibilityReport:
    all_reports = []
    reports_by_title = defaultdict(list)

    def __init__(self, info):
        self.info = info

    @property
    def created_at(self):
        return datetime.fromtimestamp(self.info['created_at'], timezone.utc)

    @classmethod
    def load_reports(cls, title_alias_map, url, verify):
        # FIXME: Ideally shouldn't load this all into memory. Instead, save to
        # disk and load on demand. But this works for now.
        if disable_load_reports:
            return
        cls.all_reports = [CompatibilityReport(i) for i in json.loads(requests.get(url, verify=verify).text)]
        for report in cls.all_reports:
            title_id = '%08x' % report.info['xbe_cert_title_id']
            if title_id not in title_alias_map:
                print('Warning: Compatibility report %s references unknown title "%s"' % (report.info['_id'], title_id))
                continue
            cls.reports_by_title[title_alias_map[title_id]].append(report)

class Title:
    def __init__(self, info_path):
        with open(info_path) as f:
            self.info = json.load(f)
        self.pubid = codecs.decode(self.info['title_id'][0:4], 'hex').decode('ascii')
        self.tid = '%03d' % (int(self.info['title_id'][4:], 16))
        self.title_name = self.info['name']
        anchor_text = ''.join([c if c.isalnum() else '-' for c in self.title_name])
        anchor_text = reduce(lambda s, c: s if (s.endswith('-') and c == '-') else s+c, anchor_text)
        anchor_text = anchor_text.lstrip('-').rstrip('-')
        self.title_url = f"/titles/{self.info['title_id']}#{anchor_text}"
        self.title_path = os.path.dirname(info_path)
        self.full_title_id_text = '%s-%s' % (self.pubid, self.tid)
        self.full_title_id_hex = self.info['title_id']
        self.full_title_id_num = int(self.info['title_id'], 16)

        # Determine cover paths
        self.have_cover = True
        self.cover_path = f'cover_front.jpg'
        if not os.path.exists(os.path.join(self.title_path, self.cover_path)):
            # Try .png extension
            self.cover_path = f'cover_front.png'
            if not os.path.exists(os.path.join(self.title_path, self.cover_path)):
                self.have_cover = False

        self.have_thumbnail = True
        self.cover_thumbnail_path = 'cover_front_thumbnail.jpg'
        if not os.path.exists(os.path.join(self.title_path, self.cover_thumbnail_path)):
            assert not self.have_cover, "Please create thumbnail for %s" % self.title_name
            self.have_thumbnail = False

        if self.have_cover:
            self.cover_url = repo_url_base + self.title_path + '/' + self.cover_path
        else:
            print('Note: Missing artwork for %s' % self.title_name)
            self.cover_url = repo_url_base + '/resources/cover_front_default.png'

        if self.have_thumbnail:
            self.cover_thumbnail_url = repo_url_base + self.title_path + '/' + self.cover_thumbnail_path
        else:
            if self.have_cover:
                print('Note: Missing thumbnail for %s' % self.title_name)
            self.cover_thumbnail_url = self.cover_url

    def process_compatibility(self):
        # FIXME:
        # - Only show reports for master branch
        # - Sort by the build version that was tested, showing most recent
        #   builds first
        # - Check version against repo for version numbers to filter out
        #   unofficial builds
        self.compatibility_tests = CompatibilityReport.reports_by_title[self.full_title_id_hex]
        if len(self.compatibility_tests) > 0:
            self.most_recent_test = sorted(self.compatibility_tests, key=lambda x:x.info['created_at'])[-1]
            self.status = self.most_recent_test.info['compat_rating']
            assert(self.status in title_status_descriptions)
        else:
            self.most_recent_test = None
            self.status = 'Unknown'
        assert(self.status in title_status_descriptions)

    @property
    def issues(self):
        return Issue.issues_by_title[self.info['title_id']]

def main():
    env = Environment(loader=FileSystemLoader(searchpath='templates'))
    game_status_counts = {
        'Unknown'  : 0,
        'Broken'   : 0,
        'Intro'    : 0,
        'Starts'   : 0,
        'Playable' : 0,
        'Perfect'  : 0,
    }

    print('Gathering info.json files...')
    titles = []
    title_alias_map = {}
    title_lookup = {}
    for root, dirs, files in os.walk('titles', topdown=True):
        for name in files:
            if name != 'info.json': continue
            title = Title(os.path.join(root,name))
            titles.append(title)
            assert(title.full_title_id_hex not in title_lookup), "Title %s defined in multiple places" % title.full_title_id_hex
            title_lookup[title.full_title_id_hex] = title
            for release in title.info['releases']:
                title_alias_map[release['title_id']] = title.info['title_id']
    print('  - Found %d' % (len(titles)))

    print('Getting GitHub Issues List...')
    Issue.load_issues(title_alias_map)
    print('  - Ok. %d issues total' % len(Issue.all_issues))

    print('Getting compatibility report list...')
    CompatibilityReport.load_reports(
        title_alias_map,
        compatibility_reports_url,
        compatibility_reports_url_verify_certs
        )
    print('  - Ok. %d reports total' % len(CompatibilityReport.all_reports))
    for title in titles:
        title.process_compatibility()
        game_status_counts[title.status] += 1

    print('Rebuilding pages...')
    template = env.get_template('template_title.html')
    count = 0
    for title_id in tqdm(title_lookup):
        title_dir = os.path.join(output_dir, 'titles', title_id)
        os.makedirs(title_dir, exist_ok=True)
        title = title_lookup[title_id]
        with open(os.path.join(title_dir, 'index.html'), 'w') as f:
            f.write(minify_html(template.render(
                title=title,
                title_status_descriptions=title_status_descriptions,
                main_url_base=main_url_base
                )))
        count += 1
    print('  - Created %d title pages' % count)

    print('Generating alias redirects...')
    count = 0
    for title_id in title_alias_map:
        if title_alias_map[title_id] != title_id:
            # This is an alias, create a redirect
            title_dir = os.path.join(output_dir, 'titles', title_id)
            os.makedirs(title_dir, exist_ok=True)
            with open(os.path.join(title_dir, 'index.html'), 'w') as f:
                url=f"/titles/{title_alias_map[title_id]}"
                f.write(f'<html><head><meta http-equiv="refresh" content="0; URL={url!s}" /></head></html>')
            count += 1
    print('  - Created %d redirect pages' % count)

    if disable_load_version:
        xemu_build_tag = 'build-202106041913'
        xemu_build_version = '0.5.2-26-g0a8e9b8db3'
        xemu_build_date = datetime(2021, 6, 4, 19, 13, 6)
    else:
        xemu_build_version = requests.get('https://raw.githubusercontent.com/mborgerson/xemu/ppa-snapshot/XEMU_VERSION').text
        latest_release = Github().get_user('mborgerson').get_repo('xemu').get_latest_release()
        xemu_build_date = latest_release.created_at

    print('Rebuilding index...')
    template = env.get_template('template_index.html')

    tmap = {t.full_title_id_num : t for t in titles}
    from rank import rank
    dorder = [tmap.pop(k) for k in rank]
    dorder.extend(sorted(tmap.values(),key=lambda title:title.title_name))

    with open(os.path.join(output_dir, 'index.html'), 'w') as f:
        f.write(minify_html(template.render(
            titles=dorder,
            title_status_descriptions=title_status_descriptions,
            game_status_counts=game_status_counts,
            xemu_build_version=xemu_build_version,
            xemu_build_date=xemu_build_date,
            main_url_base=main_url_base
            ), minify_js=True, minify_css=True))
    print('  - Ok')

if __name__ == '__main__':
    main()