From b60a7fa8011da0eab900f9cc18b231df06b4a97f Mon Sep 17 00:00:00 2001 From: davidacm Date: Tue, 2 May 2023 07:12:37 -0600 Subject: [PATCH] version 23.5.2 added a script to update version in the buildVars.py automatically if it as not been changed but is confirmed in a commit. this can be used manually, or automatically with a post commit hook. updated sconstruct to generate the required json file for the add-on store. Currently this add-on is not in the add-on store, but I prefer to use the same workflow for all my add-ons. updated buildVars to be compatible with the new sconstruct. now *.json files are ignored. fix the update function to check for the correct asset first, in case of more than one asset in the release. --- .github/workflows/upload-on-tag.yaml | 48 ++++-- .gitignore | 5 +- addon/globalPlugins/_ibmttsUtils.py | 13 +- buildVars.py | 19 ++- sconstruct | 228 +++++++++++++++++++++------ updateVersion.py | 13 ++ 6 files changed, 263 insertions(+), 63 deletions(-) create mode 100644 updateVersion.py diff --git a/.github/workflows/upload-on-tag.yaml b/.github/workflows/upload-on-tag.yaml index 6858c8e..c97ed8b 100644 --- a/.github/workflows/upload-on-tag.yaml +++ b/.github/workflows/upload-on-tag.yaml @@ -1,25 +1,31 @@ +permissions: + contents: write name: Upload on new tags - on: push: tags: - '*' + ['*'] + workflow_dispatch: jobs: - buildAndUpload: + build: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 + uses: actions/checkout@v3 + - run: echo -e "pre-commit\nscons\nmarkdown">requirements.txt + - name: Set up Python + uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 + cache: 'pip' + - name: Install dependencies run: | pip install scons markdown sudo apt update sudo apt install gettext + - name: Add add-on version run: | import re @@ -31,16 +37,34 @@ jobs: f.write(text) f.truncate() shell: python + - name: Build add-on run: scons - name: Calculate sha256 run: sha256sum *.nvda-addon >> changelog.md + + - uses: actions/upload-artifact@v3 + with: + name: packaged_addon + path: | + ./*.nvda-addon + ./*.json + + upload_release: + runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags/') }} + needs: ["build"] + steps: + - uses: actions/checkout@v3 + - name: download releases files + uses: actions/download-artifact@v3 + - name: Display structure of downloaded files + run: ls -R + - name: Release uses: softprops/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: files: | - *.nvda-addon - body_path: changelog.md - prerelease: ${{ endsWith(github.ref, '-dev') }} + packaged_addon/*.nvda-addon + fail_on_unmatched_files: true + prerelease: ${{ contains(github.ref, '-') }} diff --git a/.gitignore b/.gitignore index 1f461d4..172e42b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,11 @@ addon/doc/*.css addon/doc/en/ -addon/synthDrivers/ibmtts *_docHandler.py *.html *.ini *.mo *.py[co] *.nvda-addon -*.code-workspace -setup.cfg .sconsign.dblite +*.code-workspace +*.json \ No newline at end of file diff --git a/addon/globalPlugins/_ibmttsUtils.py b/addon/globalPlugins/_ibmttsUtils.py index 3c5319a..c8ffccc 100644 --- a/addon/globalPlugins/_ibmttsUtils.py +++ b/addon/globalPlugins/_ibmttsUtils.py @@ -217,11 +217,18 @@ def getUpdateInfo(self): if res.code != 200: raise RuntimeError(f"Checking for update failed with code {res.code} of url: {self.url}") data = json.loads(res.read()) + asset = None + for k in data['assets']: + if ".nvda-addon" in k['name']: + asset = k + break + if not asset: + raise RuntimeError("Unable to find the package addon file (.add-on) in the asset list") return { 'version': data['name'], - 'name': data['assets'][0]['name'], - 'downloadUrl': data['assets'][0]['browser_download_url'], - 'releaseDate': time.mktime(time.strptime(data['assets'][0]['updated_at'], "%Y-%m-%dT%H:%M:%SZ")) + 'name': asset['name'], + 'downloadUrl': asset['browser_download_url'], + 'releaseDate': time.mktime(time.strptime(asset['updated_at'], "%Y-%m-%dT%H:%M:%SZ")) } diff --git a/buildVars.py b/buildVars.py index 29fb61d..59f2501 100644 --- a/buildVars.py +++ b/buildVars.py @@ -19,7 +19,7 @@ # Translators: Long description to be shown for this add-on on add-on information from add-ons manager "addon_description" : _("""This is the IBMTTS synthesizer driver for NVDA."""), # version - "addon_version" : "v23.02.1", + "addon_version" : "23.5.2", # Author(s) "addon_author" : u"David CM , x0 and others", # URL for the add-on documentation support @@ -32,6 +32,12 @@ "addon_lastTestedNVDAVersion" : "2023.1.0", # Add-on update channel (default is stable or None) "addon_updateChannel" : None, + # Add-on license such as GPL 2 + "addon_license": "GPL 2", + # URL for the license document the ad-on is licensed under + "addon_licenseURL": "https://www.gnu.org/licenses/old-licenses/gpl-2.0.html", + # URL for the add-on repository where the source code can be found + "addon_sourceURL": "https://github.com/davidacm/NVDA-IBMTTS-Driver", } @@ -50,3 +56,14 @@ # Files that will be ignored when building the nvda-addon file # Paths are relative to the addon directory, not to the root directory of your addon sources. excludedFiles = [] + + +# Base language for the NVDA add-on +baseLanguage = "en" + +# Markdown extensions for add-on documentation +# Most add-ons do not require additional Markdown extensions. +# If you need to add support for markup such as tables, fill out the below list. +# Extensions string must be of the form "markdown.extensions.extensionName" +# e.g. "markdown.extensions.tables" to add tables. +markdownExtensions = [] diff --git a/sconstruct b/sconstruct index 5d4d38c..8aa914b 100644 --- a/sconstruct +++ b/sconstruct @@ -1,19 +1,41 @@ # NVDA add-on template SCONSTRUCT file -#Copyright (C) 2012, 2014 Rui Batista -#This file is covered by the GNU General Public License. -#See the file COPYING.txt for more details. +# Copyright (C) 2012-2023 Rui Batista, Noelia Martinez, Joseph Lee +# This file is covered by the GNU General Public License. +# See the file COPYING.txt for more details. import codecs import gettext import os import os.path import zipfile -import buildVars +import sys + +# While names imported below are available by default in every SConscript +# Linters aren't aware about them. +# To avoid Flake8 F821 warnings about them they are imported explicitly. +# When using other Scons functions please add them to the line below. +from SCons.Script import BoolVariable, Builder, Copy, Environment, Variables + +sys.dont_write_bytecode = True + +# Bytecode should not be written for build vars module to keep the repository root folder clean. +import buildVars # NOQA: E402 + def md2html(source, dest): import markdown + # Use extensions if defined. + mdExtensions = buildVars.markdownExtensions lang = os.path.basename(os.path.dirname(source)).replace('_', '-') - title="{addonSummary} {addonVersion}".format(addonSummary=buildVars.addon_info["addon_summary"], addonVersion=buildVars.addon_info["addon_version"]) + localeLang = os.path.basename(os.path.dirname(source)) + try: + _ = gettext.translation("nvda", localedir=os.path.join("addon", "locale"), languages=[localeLang]).gettext + summary = _(buildVars.addon_info["addon_summary"]) + except Exception: + summary = buildVars.addon_info["addon_summary"] + title = "{addonSummary} {addonVersion}".format( + addonSummary=summary, addonVersion=buildVars.addon_info["addon_version"] + ) headerDic = { "[[!meta title=\"": "# ", "\"]]": " #", @@ -22,35 +44,52 @@ def md2html(source, dest): mdText = f.read() for k, v in headerDic.items(): mdText = mdText.replace(k, v, 1) - htmlText = markdown.markdown(mdText) + htmlText = markdown.markdown(mdText, extensions=mdExtensions) + # Optimization: build resulting HTML text in one go instead of writing parts separately. + docText = "\n".join([ + "", + "" % lang, + "", + "" + "", + "", + "%s" % title, + "\n", + htmlText, + "\n" + ]) with codecs.open(dest, "w", "utf-8") as f: - f.write("\n" + - "\n" + - "\n" % (lang, lang) + - "\n" + - "\n" + - "\n" + - "%s\n" % title + - "\n\n" - ) - f.write(htmlText) - f.write("\n\n") + f.write(docText) + def mdTool(env): - mdAction=env.Action( - lambda target,source,env: md2html(source[0].path, target[0].path), - lambda target,source,env: 'Generating %s'%target[0], + mdAction = env.Action( + lambda target, source, env: md2html(source[0].path, target[0].path), + lambda target, source, env: 'Generating % s' % target[0], ) - mdBuilder=env.Builder( + mdBuilder = env.Builder( action=mdAction, suffix='.html', src_suffix='.md', ) - env['BUILDERS']['markdown']=mdBuilder + env['BUILDERS']['markdown'] = mdBuilder + + +def validateVersionNumber(key, val, env): + # Used to make sure version major.minor.patch are integers to comply with NV Access add-on store. + # Ignore all this if version number is not specified, in which case json generator will validate this info. + if val == "0.0.0": + return + versionNumber = val.split(".") + if len(versionNumber) < 3: + raise ValueError("versionNumber must have three parts (major.minor.patch)") + if not all([part.isnumeric() for part in versionNumber]): + raise ValueError("versionNumber (major.minor.patch) must be integers") + vars = Variables() vars.Add("version", "The version of this build", buildVars.addon_info["addon_version"]) +vars.Add("versionNumber", "Version number of the form major.minor.patch", "0.0.0", validateVersionNumber) vars.Add(BoolVariable("dev", "Whether this is a daily development version", False)) vars.Add("channel", "Update channel for this build", buildVars.addon_info["addon_updateChannel"]) @@ -61,36 +100,52 @@ if env["dev"]: import datetime buildDate = datetime.datetime.now() year, month, day = str(buildDate.year), str(buildDate.month), str(buildDate.day) - env["addon_version"] = "".join([year, month.zfill(2), day.zfill(2), "-dev"]) + versionTimestamp = "".join([year, month.zfill(2), day.zfill(2)]) + env["addon_version"] = f"{versionTimestamp}-dev" + env["versionNumber"] = f"{versionTimestamp}.0.0" env["channel"] = "dev" elif env["version"] is not None: env["addon_version"] = env["version"] if "channel" in env and env["channel"] is not None: env["addon_updateChannel"] = env["channel"] +buildVars.addon_info["addon_version"] = env["addon_version"] +buildVars.addon_info["addon_updateChannel"] = env["addon_updateChannel"] + addonFile = env.File("${addon_name}-${addon_version}.nvda-addon") + def addonGenerator(target, source, env, for_signature): - action = env.Action(lambda target, source, env : createAddonBundleFromPath(source[0].abspath, target[0].abspath) and None, - lambda target, source, env : "Generating Addon %s" % target[0]) + action = env.Action( + lambda target, source, env: createAddonBundleFromPath(source[0].abspath, target[0].abspath) and None, + lambda target, source, env: "Generating Addon %s" % target[0] + ) return action + def manifestGenerator(target, source, env, for_signature): - action = env.Action(lambda target, source, env : generateManifest(source[0].abspath, target[0].abspath) and None, - lambda target, source, env : "Generating manifest %s" % target[0]) + action = env.Action( + lambda target, source, env: generateManifest(source[0].abspath, target[0].abspath) and None, + lambda target, source, env: "Generating manifest %s" % target[0] + ) return action + def translatedManifestGenerator(target, source, env, for_signature): dir = os.path.abspath(os.path.join(os.path.dirname(str(source[0])), "..")) lang = os.path.basename(dir) - action = env.Action(lambda target, source, env : generateTranslatedManifest(source[1].abspath, lang, target[0].abspath) and None, - lambda target, source, env : "Generating translated manifest %s" % target[0]) + action = env.Action( + lambda target, source, env: generateTranslatedManifest(source[1].abspath, lang, target[0].abspath) and None, + lambda target, source, env: "Generating translated manifest %s" % target[0] + ) return action + env['BUILDERS']['NVDAAddon'] = Builder(generator=addonGenerator) env['BUILDERS']['NVDAManifest'] = Builder(generator=manifestGenerator) env['BUILDERS']['NVDATranslatedManifest'] = Builder(generator=translatedManifestGenerator) + def createAddonHelp(dir): docsDir = os.path.join(dir, "doc") if os.path.isfile("style.css"): @@ -98,10 +153,11 @@ def createAddonHelp(dir): cssTarget = env.Command(cssPath, "style.css", Copy("$TARGET", "$SOURCE")) env.Depends(addon, cssTarget) if os.path.isfile("readme.md"): - readmePath = os.path.join(docsDir, "en", "readme.md") + readmePath = os.path.join(docsDir, buildVars.baseLanguage, "readme.md") readmeTarget = env.Command(readmePath, "readme.md", Copy("$TARGET", "$SOURCE")) env.Depends(addon, readmeTarget) + def createAddonBundleFromPath(path, dest): """ Creates a bundle from a directory that contains an addon manifest file.""" basedir = os.path.abspath(path) @@ -112,19 +168,96 @@ def createAddonBundleFromPath(path, dest): for filename in filenames: pathInBundle = os.path.join(relativePath, filename) absPath = os.path.join(dir, filename) - if pathInBundle not in buildVars.excludedFiles: z.write(absPath, pathInBundle) + if pathInBundle not in buildVars.excludedFiles: + z.write(absPath, pathInBundle) + createAddonStoreJson(dest) return dest + +def createAddonStoreJson(bundle): + """Creates add-on store JSON file from an add-on package and manifest data.""" + import json + import hashlib + # Set different json file names and version number properties based on version number parsing results. + if env["versionNumber"] == "0.0.0": + env["versionNumber"] = buildVars.addon_info["addon_version"] + versionNumberParsed = env["versionNumber"].split(".") + if all([part.isnumeric() for part in versionNumberParsed]): + if len(versionNumberParsed) == 1: + versionNumberParsed += ["0", "0"] + elif len(versionNumberParsed) == 2: + versionNumberParsed.append("0") + else: + versionNumberParsed = [] + if len(versionNumberParsed): + major, minor, patch = [int(part) for part in versionNumberParsed] + jsonFilename = f'{major}.{minor}.{patch}.json' + else: + jsonFilename = f'{buildVars.addon_info["addon_version"]}.json' + major, minor, patch = 0, 0, 0 + print('Generating % s' % jsonFilename) + sha256 = hashlib.sha256() + with open(bundle, "rb") as f: + for byte_block in iter(lambda: f.read(65536), b""): + sha256.update(byte_block) + hashValue = sha256.hexdigest() + try: + minimumNVDAVersion = buildVars.addon_info["addon_minimumNVDAVersion"].split(".") + except AttributeError: + minimumNVDAVersion = [0, 0, 0] + minMajor, minMinor = minimumNVDAVersion[:2] + minPatch = minimumNVDAVersion[-1] if len(minimumNVDAVersion) == 3 else "0" + try: + lastTestedNVDAVersion = buildVars.addon_info["addon_lastTestedNVDAVersion"].split(".") + except AttributeError: + lastTestedNVDAVersion = [0, 0, 0] + lastTestedMajor, lastTestedMinor = lastTestedNVDAVersion[:2] + lastTestedPatch = lastTestedNVDAVersion[-1] if len(lastTestedNVDAVersion) == 3 else "0" + channel = buildVars.addon_info["addon_updateChannel"] + if channel is None: + channel = "stable" + addonStoreEntry = { + "addonId": buildVars.addon_info["addon_name"], + "displayName": buildVars.addon_info["addon_summary"], + "URL": "", + "description": buildVars.addon_info["addon_description"], + "sha256": hashValue, + "homepage": buildVars.addon_info["addon_url"], + "addonVersionName": buildVars.addon_info["addon_version"], + "addonVersionNumber": { + "major": major, + "minor": minor, + "patch": patch + }, + "minNVDAVersion": { + "major": int(minMajor), + "minor": int(minMinor), + "patch": int(minPatch) + }, + "lastTestedVersion": { + "major": int(lastTestedMajor), + "minor": int(lastTestedMinor), + "patch": int(lastTestedPatch) + }, + "channel": channel, + "publisher": "", + "sourceURL": buildVars.addon_info["addon_sourceURL"], + "license": buildVars.addon_info["addon_license"], + "licenseURL": buildVars.addon_info["addon_licenseURL"], + } + with open(jsonFilename, "w") as addonStoreJson: + json.dump(addonStoreEntry, addonStoreJson, indent="\t") + + def generateManifest(source, dest): addon_info = buildVars.addon_info - addon_info["addon_version"] = env["addon_version"] - addon_info["addon_updateChannel"] = env["addon_updateChannel"] with codecs.open(source, "r", "utf-8") as f: manifest_template = f.read() manifest = manifest_template.format(**addon_info) with codecs.open(dest, "w", "utf-8") as f: f.write(manifest) + def generateTranslatedManifest(source, language, out): _ = gettext.translation("nvda", localedir=os.path.join("addon", "locale"), languages=[language]).gettext vars = {} @@ -136,19 +269,24 @@ def generateTranslatedManifest(source, language, out): with codecs.open(out, "w", "utf-8") as f: f.write(result) + def expandGlobs(files): return [f for pattern in files for f in env.Glob(pattern)] + addon = env.NVDAAddon(addonFile, env.Dir('addon')) langDirs = [f for f in env.Glob(os.path.join("addon", "locale", "*"))] -#Allow all NVDA's gettext po files to be compiled in source/locale, and manifest files to be generated +# Allow all NVDA's gettext po files to be compiled in source/locale, and manifest files to be generated for dir in langDirs: poFile = dir.File(os.path.join("LC_MESSAGES", "nvda.po")) - moFile=env.gettextMoFile(poFile) + moFile = env.gettextMoFile(poFile) env.Depends(moFile, poFile) - translatedManifest = env.NVDATranslatedManifest(dir.File("manifest.ini"), [moFile, os.path.join("manifest-translated.ini.tpl")]) + translatedManifest = env.NVDATranslatedManifest( + dir.File("manifest.ini"), + [moFile, os.path.join("manifest-translated.ini.tpl")] + ) env.Depends(translatedManifest, ["buildVars.py"]) env.Depends(addon, [translatedManifest, moFile]) @@ -156,8 +294,9 @@ pythonFiles = expandGlobs(buildVars.pythonSources) for file in pythonFiles: env.Depends(addon, file) -#Convert markdown files to html -createAddonHelp("addon") # We need at least doc in English and should enable the Help button for the add-on in Add-ons Manager +# Convert markdown files to html +# We need at least doc in English and should enable the Help button for the add-on in Add-ons Manager +createAddonHelp("addon") for mdFile in env.Glob(os.path.join('addon', 'doc', '*', '*.md')): htmlFile = env.markdown(mdFile) env.Depends(htmlFile, mdFile) @@ -165,11 +304,11 @@ for mdFile in env.Glob(os.path.join('addon', 'doc', '*', '*.md')): # Pot target i18nFiles = expandGlobs(buildVars.i18nSources) -gettextvars={ - 'gettext_package_bugs_address' : 'nvda-translations@freelists.org', - 'gettext_package_name' : buildVars.addon_info['addon_name'], - 'gettext_package_version' : buildVars.addon_info['addon_version'] - } +gettextvars = { + 'gettext_package_bugs_address': 'nvda-translations@groups.io', + 'gettext_package_name': buildVars.addon_info['addon_name'], + 'gettext_package_version': buildVars.addon_info['addon_version'] +} pot = env.gettextPotFile("${addon_name}.pot", i18nFiles, **gettextvars) env.Alias('pot', pot) @@ -185,3 +324,4 @@ env.Depends(manifest, "buildVars.py") env.Depends(addon, manifest) env.Default(addon) +env.Clean(addon, ['.sconsign.dblite', 'addon/doc/' + buildVars.baseLanguage + '/']) diff --git a/updateVersion.py b/updateVersion.py new file mode 100644 index 0000000..c53e16f --- /dev/null +++ b/updateVersion.py @@ -0,0 +1,13 @@ +import re, sys + +if len(sys.argv) < 2: + print("the version was not detected") + exit(1) +version = sys.argv[1] +print(f"the version recognized is: {version}") +with open("buildVars.py", 'r+', encoding='utf-8') as f: + text = f.read() + text = re.sub('"addon_version" *:.*,', f'"addon_version" : "{version}",', text) + f.seek(0) + f.write(text) + f.truncate()