From d8593ea7f3481e35efe53b074c2111486e76184c Mon Sep 17 00:00:00 2001 From: Michael Reid Date: Sun, 25 Feb 2024 06:39:02 -0400 Subject: [PATCH] Improve admin webui (#112) * fix columns and filters - add formatters to remove milliseconds on dates - add column labels to be used with filters - add new filters for version and build views * fix screenshot management - remove edit function - fix file removal with row deletion * fix Babel initialisation * add service view and api check - add service view to admin interface - add service check on package upload * add and maintain md5 hashes - add a functions to get md5 hash in build model and spk - use function in build model when uploading build - use function in spk when signing/unsigning spk * remove builds from architecture create form * fix build environment and model - fix alembic setup environment - update model to match database - fix default sort for screenshot * Update nas test for major_version check * add major_version check - identify the major DSM version based on a closest match to the build - filter package versions based on match to major DSM version - include earlier "noarch" package version when major DSM version < 6 * fix download counter - rewrite catalog download links using md5 hashes for id - allow downloads of noarch builds to pass arch checks - rewrite nas tests for new url structure * Fix depopulate db function * Revert "fix download counter" This reverts commit bea771851e8789d152a0bbc22cdd75ab0ce902f6. * Amend major_version check - Add type column to Firmware table - Increase length of version column - Filter by type for closest firmware when getting catalog - Update populate_db with firmware type * add validators for firmware input * fix Popped wrong app context. * fix 500 page on users: 'NoneType' object has no attribute 'strftime' BaseModelView.index_view() got an unexpected keyword argument 'cls' Flask-SQLAlchemy uses a custom formatter to handle date formatting when displaying data in the templates. It dosn't support overriding the value. * fix column formatting --------- Co-authored-by: publicarray --- .flake8 | 2 +- migrations/alembic.ini | 2 +- migrations/env.py | 1 - .../dc7687894ba7_increase_field_sizes.py | 4 - ...add_firmware_type_and_increase_version_.py | 47 +++++++ spkrepo/app.py | 11 +- spkrepo/cli.py | 9 +- spkrepo/ext.py | 4 + spkrepo/models.py | 30 ++++- spkrepo/tests/common.py | 3 +- spkrepo/tests/test_nas.py | 21 +++- spkrepo/utils.py | 21 +++- spkrepo/views/__init__.py | 1 + spkrepo/views/admin.py | 116 +++++++++++++++--- spkrepo/views/api.py | 12 ++ spkrepo/views/nas.py | 31 ++++- 16 files changed, 275 insertions(+), 40 deletions(-) create mode 100644 migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py diff --git a/.flake8 b/.flake8 index 487c389..0e544b6 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] max-line-length = 88 extend-ignore = E203 -per-file-ignores = __init__.py:F401 spkrepo/app.py:F841 +per-file-ignores = __init__.py:F401 exclude = docs/* migrations/* diff --git a/migrations/alembic.ini b/migrations/alembic.ini index 7eae6d2..ea28df6 100644 --- a/migrations/alembic.ini +++ b/migrations/alembic.ini @@ -1,5 +1,5 @@ [alembic] -script_location = . +script_location = ./migrations [loggers] keys = root,sqlalchemy,alembic diff --git a/migrations/env.py b/migrations/env.py index 2127eba..3273e03 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -8,7 +8,6 @@ config = context.config fileConfig(config.config_file_name) - config.set_main_option( "sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI") ) diff --git a/migrations/versions/dc7687894ba7_increase_field_sizes.py b/migrations/versions/dc7687894ba7_increase_field_sizes.py index c4209fe..b1bb233 100644 --- a/migrations/versions/dc7687894ba7_increase_field_sizes.py +++ b/migrations/versions/dc7687894ba7_increase_field_sizes.py @@ -13,7 +13,6 @@ def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.alter_column( "version", "conf_dependencies", @@ -42,11 +41,9 @@ def upgrade(): type_=sa.UnicodeText(), existing_nullable=True, ) - # ### end Alembic commands ### def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.alter_column( "version", "conf_resource", @@ -75,4 +72,3 @@ def downgrade(): type_=sa.VARCHAR(length=255), existing_nullable=True, ) - # ### end Alembic commands ### diff --git a/migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py b/migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py new file mode 100644 index 0000000..564122c --- /dev/null +++ b/migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py @@ -0,0 +1,47 @@ +"""Add firmware type and increase version length + +Revision ID: f95855ce9471 +Revises: 76d559b4e873 +Create Date: 2024-01-15 13:58:34.160242 + +""" +revision = "f95855ce9471" +down_revision = "76d559b4e873" + +import sqlalchemy as sa +from alembic import op + + +def upgrade(): + op.add_column("firmware", sa.Column("type", sa.Unicode(length=4))) + # Set type based on version + op.execute( + """ + UPDATE firmware + SET type = CASE + WHEN version LIKE '1.%' THEN 'srm' + ELSE 'dsm' + END + """ + ) + # Modify the column to be NOT NULL after setting the values + op.alter_column("firmware", "type", nullable=False) + + op.alter_column( + "firmware", + "version", + existing_type=sa.VARCHAR(length=3), + type_=sa.Unicode(length=4), + existing_nullable=False, + ) + + +def downgrade(): + op.alter_column( + "firmware", + "version", + existing_type=sa.Unicode(length=4), + type_=sa.VARCHAR(length=3), + existing_nullable=False, + ) + op.drop_column("firmware", "type") diff --git a/spkrepo/app.py b/spkrepo/app.py index 8477edb..55083ea 100644 --- a/spkrepo/app.py +++ b/spkrepo/app.py @@ -2,12 +2,11 @@ import jinja2 from flask import Flask from flask_admin import Admin -from flask_babel import Babel from wtforms import HiddenField from . import config as default_config from .cli import spkrepo as spkrepo_cli -from .ext import cache, db, debug_toolbar, mail, migrate, security +from .ext import babel, cache, db, debug_toolbar, mail, migrate, security from .models import user_datastore from .views import ( ArchitectureView, @@ -16,6 +15,7 @@ IndexView, PackageView, ScreenshotView, + ServiceView, SpkrepoConfirmRegisterForm, UserView, VersionView, @@ -53,18 +53,19 @@ def create_app(config=None, register_blueprints=True, init_admin=True): admin.add_view(UserView()) admin.add_view(ArchitectureView()) admin.add_view(FirmwareView()) + admin.add_view(ServiceView()) admin.add_view(ScreenshotView()) admin.add_view(PackageView()) admin.add_view(VersionView()) admin.add_view(BuildView()) admin.init_app(app) - # Initialize Flask-Babel - babel = Babel(app) - # Commands app.cli.add_command(spkrepo_cli) + # Flask-Babel + babel.init_app(app) + # SQLAlchemy db.init_app(app) diff --git a/spkrepo/cli.py b/spkrepo/cli.py index 7c23e70..ac43884 100644 --- a/spkrepo/cli.py +++ b/spkrepo/cli.py @@ -180,8 +180,15 @@ def depopulate_db(): from spkrepo.models import Package for package in Package.query.all(): - shutil.rmtree(os.path.join(current_app.config["DATA_PATH"], package.name)) + # Delete the package and its associated versions and builds db.session.delete(package) + + # Remove the directory associated with the package (if it exists) + shutil.rmtree( + os.path.join(current_app.config["DATA_PATH"], package.name), + ignore_errors=True, + ) + db.session.commit() diff --git a/spkrepo/ext.py b/spkrepo/ext.py index a029bd7..06360a9 100644 --- a/spkrepo/ext.py +++ b/spkrepo/ext.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from flask_babel import Babel from flask_caching import Cache from flask_debugtoolbar import DebugToolbarExtension from flask_mail import Mail @@ -6,6 +7,9 @@ from flask_security import Security from flask_sqlalchemy import SQLAlchemy +# Flask-Babel +babel = Babel() + # Cache cache = Cache() diff --git a/spkrepo/models.py b/spkrepo/models.py index 6394480..2cc0613 100644 --- a/spkrepo/models.py +++ b/spkrepo/models.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import hashlib import io import os import shutil @@ -133,8 +134,9 @@ class Firmware(db.Model): # Columns id = db.Column(db.Integer, primary_key=True) - version = db.Column(db.Unicode(3), nullable=False) + version = db.Column(db.Unicode(4), nullable=False) build = db.Column(db.Integer, unique=True, nullable=False) + type = db.Column(db.Unicode(4), nullable=False) @classmethod def find(cls, build): @@ -328,7 +330,7 @@ class Build(db.Model): publisher_user_id = db.Column(db.Integer, db.ForeignKey("user.id")) checksum = db.Column(db.Unicode(32)) extract_size = db.Column(db.Integer) - path = db.Column(db.Unicode(100)) + path = db.Column(db.Unicode(2048)) md5 = db.Column(db.Unicode(32)) insert_date = db.Column(db.DateTime, default=db.func.now(), nullable=False) active = db.Column(db.Boolean(), default=False, nullable=False) @@ -343,7 +345,11 @@ class Build(db.Model): ) firmware = db.relationship("Firmware", lazy=False) publisher = db.relationship("User", foreign_keys=[publisher_user_id]) - downloads = db.relationship("Download", back_populates="build") + downloads = db.relationship( + "Download", + back_populates="build", + cascade="save-update, merge, delete, delete-orphan", + ) @classmethod def generate_filename(cls, package, version, firmware, architectures): @@ -360,6 +366,24 @@ def save(self, stream): ) as f: f.write(stream.read()) + def calculate_md5(self): + if not self.path: + raise ValueError("Path cannot be empty.") + + file_path = os.path.join(current_app.config["DATA_PATH"], self.path) + + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found at path: {file_path}") + + if self.md5 is None: + with io.open(file_path, "rb") as f: + md5_hash = hashlib.md5() + for chunk in iter(lambda: f.read(4096), b""): + md5_hash.update(chunk) + return md5_hash.hexdigest() + + return self.md5 + def _after_insert(self): assert os.path.exists(os.path.join(current_app.config["DATA_PATH"], self.path)) diff --git a/spkrepo/tests/common.py b/spkrepo/tests/common.py index cb43261..1682b9b 100644 --- a/spkrepo/tests/common.py +++ b/spkrepo/tests/common.py @@ -281,8 +281,7 @@ def create_spk(self, create, extracted, **kwargs): with create_spk(self) as spk_stream: self.save(spk_stream) if self.md5 is None: - spk_stream.seek(0) - self.md5 = hashlib.md5(spk_stream.read()).hexdigest() + self.md5 = self.calculate_md5() spk_stream.close() @classmethod diff --git a/spkrepo/tests/test_nas.py b/spkrepo/tests/test_nas.py index 123140b..eff207b 100644 --- a/spkrepo/tests/test_nas.py +++ b/spkrepo/tests/test_nas.py @@ -227,11 +227,11 @@ def test_stable_build_active_stable(self): catalog[0], build, data, dict(arch="88f628x", build="1594") ) - def test_stable_build_active_stable_5004(self): + def test_stable_noarch_build_active_stable_5004(self): build = BuildFactory( active=True, version__report_url=None, - architectures=[Architecture.find("88f6281", syno=True)], + architectures=[Architecture.find("noarch", syno=True)], firmware=Firmware.find(1594), ) db.session.commit() @@ -247,6 +247,23 @@ def test_stable_build_active_stable_5004(self): catalog["packages"][0], build, data, dict(arch="88f628x", build="5004") ) + def test_stable_arch_build_active_stable_5004(self): + BuildFactory( + active=True, + version__report_url=None, + architectures=[Architecture.find("88f6281", syno=True)], + firmware=Firmware.find(1594), + ) + db.session.commit() + data = dict(arch="88f6281", build="5004", language="enu") + response = self.client.post(url_for("nas.catalog"), data=data) + self.assert200(response) + self.assertHeader(response, "Content-Type", "application/json") + catalog = json.loads(response.data.decode()) + self.assertIn("packages", catalog) + self.assertIn("keyrings", catalog) + self.assertEqual(len(catalog["packages"]), 0) + def test_stable_build_active_stable_download_count(self): package = PackageFactory() build = BuildFactory( diff --git a/spkrepo/utils.py b/spkrepo/utils.py index c5d4043..b0b0656 100644 --- a/spkrepo/utils.py +++ b/spkrepo/utils.py @@ -58,6 +58,10 @@ class SPK(object): #: Regex for files in conf conf_filename_re = re.compile(r"^conf/.+$") + #: Regex for firmware input + firmware_version_re = re.compile(r"^\d+\.\d$") + firmware_type_re = re.compile(r"^([a-z]){3,}$") + def __init__(self, stream): self.info = {} self.icons = {} @@ -345,6 +349,18 @@ def unsign(self): self.stream.truncate() self.stream.seek(0) + def calculate_md5(self): + md5_hash = hashlib.md5() + + # Ensure the stream position is at the beginning + self.stream.seek(0) + + # Update MD5 hash directly from the stream + for chunk in iter(lambda: self.stream.read(4096), b""): + md5_hash.update(chunk) + + return md5_hash.hexdigest() + def _generate_signature(self, stream, timestamp_url, gnupghome): # pragma: no cover # generate the signature gpg = gnupg.GPG(gnupghome=gnupghome) @@ -386,7 +402,10 @@ def populate_db(): ) db.session.execute( Firmware.__table__.insert().values( - [{"version": "3.1", "build": 1594}, {"version": "5.0", "build": 4458}] + [ + {"version": "3.1", "build": 1594, "type": "dsm"}, + {"version": "5.0", "build": 4458, "type": "dsm"}, + ] ) ) db.session.execute( diff --git a/spkrepo/views/__init__.py b/spkrepo/views/__init__.py index 2312443..8425c5b 100644 --- a/spkrepo/views/__init__.py +++ b/spkrepo/views/__init__.py @@ -6,6 +6,7 @@ IndexView, PackageView, ScreenshotView, + ServiceView, UserView, VersionView, ) diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index 08d7533..d4af9b9 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -16,7 +16,16 @@ from wtforms.validators import Regexp from ..ext import db -from ..models import Architecture, Build, Firmware, Package, Screenshot, User, Version +from ..models import ( + Architecture, + Build, + Firmware, + Package, + Screenshot, + Service, + User, + Version, +) from ..utils import SPK @@ -35,6 +44,12 @@ def is_accessible(self): # View column_list = ("username", "email", "roles", "active", "confirmed_at") + column_formatters = { + "confirmed_at": lambda v, c, m, p: ( + m.confirmed_at.strftime("%Y-%m-%d %H:%M:%S") if m.confirmed_at else None + ) + } + # Form form_columns = ("username", "roles", "active") form_overrides = {"password": PasswordField} @@ -91,6 +106,9 @@ def is_accessible(self): can_delete = False + # Form + form_excluded_columns = "builds" + class FirmwareView(ModelView): """View for :class:`~spkrepo.models.Firmware`""" @@ -106,6 +124,28 @@ def is_accessible(self): can_delete = False + # Form + form_columns = ("version", "build", "type") + form_args = { + "version": {"validators": [Regexp(SPK.firmware_version_re)]}, + "type": {"validators": [Regexp(SPK.firmware_type_re)]}, + } + + +class ServiceView(ModelView): + """View for :class:`~spkrepo.models.Service`""" + + def __init__(self, **kwargs): + super(ServiceView, self).__init__(Service, db.session, **kwargs) + + # Permissions + def is_accessible(self): + return current_user.is_authenticated and current_user.has_role("package_admin") + + can_edit = False + + can_delete = False + def screenshot_namegen(obj, file_data): pattern = "screenshot_%0d%s" @@ -120,18 +160,6 @@ def screenshot_namegen(obj, file_data): return os.path.join(obj.package.name, pattern % (i, ext)) -# TODO: Not necessary with Flask-Admin>1.0.8 -# see https://github.com/mrjoes/flask-admin/pull/705 -class SpkrepoImageUploadField(ImageUploadField): - def _get_path(self, filename): - if not self.base_path: # pragma: no cover - raise ValueError("FileUploadField field requires base_path to be set.") - - if callable(self.base_path): - return os.path.join(self.base_path(), filename) - return os.path.join(self.base_path, filename) # pragma: no cover - - class ScreenshotView(ModelView): """View for :class:`~spkrepo.models.Screenshot`""" @@ -142,7 +170,14 @@ def __init__(self, **kwargs): def is_accessible(self): return current_user.is_authenticated and current_user.has_role("package_admin") + can_edit = False + # View + column_labels = { + "package.name": "Package Name", + "path": "Screenshot", + } + def _display(view, context, model, name): return Markup( 'screenshot' @@ -151,11 +186,17 @@ def _display(view, context, model, name): column_formatters = {"path": _display} column_sortable_list = (("package", "package.name"),) - column_default_sort = (Package.name, True) + column_default_sort = "package.name" column_filters = ("package.name",) + # Hooks + def on_model_delete(self, model): + build_path = os.path.join(current_app.config["DATA_PATH"], model.path) + if os.path.exists(build_path): + os.remove(build_path) + # Form - form_overrides = {"path": SpkrepoImageUploadField} + form_overrides = {"path": ImageUploadField} form_args = { "path": { "label": "Screenshot", @@ -221,6 +262,10 @@ def on_model_delete(self, model): ("insert_date", "insert_date"), ) + column_formatters = { + "insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S") + } + # Form form_columns = ("name", "author", "maintainers") form_args = {"name": {"validators": [Regexp(SPK.package_re)]}} @@ -276,21 +321,33 @@ def on_model_delete(self, model): "startable", ) column_labels = { + "package.name": "Package Name", "version_string": "Version", "dependencies": "Dependencies", "service_dependencies": "Services", } - column_filters = ("package.name", "version", "upstream_version") + column_filters = ( + "package.name", + "upstream_version", + "version", + "beta", + "all_builds_active", + ) column_sortable_list = ( ("package", "package.name"), ("upstream_version", "upstream_version"), ("version", "version"), + ("beta", "beta"), ("insert_date", "insert_date"), + ("all_builds_active", "all_builds_active"), ("install_wizard", "install_wizard"), ("upgrade_wizard", "upgrade_wizard"), ("startable", "startable"), ) - # TODO: Add beta and all_builds_active with Flask-Admin>1.0.8 + + column_formatters = { + "insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S") + } column_default_sort = (Version.insert_date, True) # Custom queries @@ -383,8 +440,11 @@ def action_sign(self, ids): current_app.config["GNUPG_TIMESTAMP_URL"], current_app.config["GNUPG_PATH"], ) + build.md5 = spk.calculate_md5() + self.session.commit() success.append(filename) except Exception: + self.session.rollback() failed.append(filename) if failed: if len(failed) == 1: @@ -434,8 +494,11 @@ def action_unsign(self, ids): continue try: spk.unsign() + build.md5 = spk.calculate_md5() + self.session.commit() success.append(filename) except Exception: + self.session.rollback() failed.append(filename) if failed: if len(failed) == 1: @@ -516,16 +579,24 @@ def can_unsign(self): ) column_labels = { "version.package": "Package", + "version.package.name": "Package Name", "version.upstream_version": "Upstream Version", "version.version": "Version", + "architectures.code": "Architecture", + "firmware.version": "Firmware Version", + "publisher.username": "Publisher Username", } column_filters = ( "version.package.name", "version.upstream_version", "version.version", + "architectures.code", + "firmware.version", "publisher.username", + "active", ) column_sortable_list = ( + ("version.package", "version.package.name"), ("version.upstream_version", "version.upstream_version"), ("version.version", "version.version"), ("firmware", "firmware.build"), @@ -533,7 +604,10 @@ def can_unsign(self): ("insert_date", "insert_date"), ("active", "active"), ) - # TODO: Add version.package with Flask-Admin>1.0.8 + + column_formatters = { + "insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S") + } column_default_sort = (Build.insert_date, True) # Custom queries @@ -627,8 +701,11 @@ def action_sign(self, ids): current_app.config["GNUPG_TIMESTAMP_URL"], current_app.config["GNUPG_PATH"], ) + build.md5 = spk.calculate_md5() + self.session.commit() success.append(filename) except Exception: + self.session.rollback() failed.append(filename) if failed: if len(failed) == 1: @@ -677,8 +754,11 @@ def action_unsign(self, ids): continue try: spk.unsign() + build.md5 = spk.calculate_md5() + self.session.commit() success.append(filename) except Exception: + self.session.rollback() failed.append(filename) if failed: if len(failed) == 1: diff --git a/spkrepo/views/api.py b/spkrepo/views/api.py index 5d4565d..593a73f 100644 --- a/spkrepo/views/api.py +++ b/spkrepo/views/api.py @@ -130,6 +130,16 @@ def post(self): if firmware is None: abort(422, message="Unknown firmware") + # Services + input_install_dep_services = spk.info.get("install_dep_services", None) + if input_install_dep_services: + for info_dep_service in input_install_dep_services.split(): + service_name = Service.find(info_dep_service) + if service_name is None: + abort( + 422, message="Unknown dependent service: %s" % info_dep_service + ) + # Package create_package = False package = Package.find(spk.info["package"]) @@ -268,6 +278,8 @@ def post(self): for size, icon in build.version.icons.items(): icon.save(spk.icons[size]) build.save(spk.stream) + # generate md5 hash + build.md5 = build.calculate_md5() except Exception as e: # pragma: no cover if create_package: shutil.rmtree(os.path.join(data_path, package.name), ignore_errors=True) diff --git a/spkrepo/views/nas.py b/spkrepo/views/nas.py index 3db8f56..9cbeeed 100644 --- a/spkrepo/views/nas.py +++ b/spkrepo/views/nas.py @@ -44,12 +44,28 @@ def is_valid_language(language): @cache.memoize(timeout=600) def get_catalog(arch, build, language, beta): - # latest version per package + # Find the closest matching firmware for the provided build + closest_firmware = ( + Firmware.query.filter(Firmware.build <= build, Firmware.type == "dsm") + .order_by(Firmware.build.desc()) + .first() + ) + + # Extract major version from the closest matching firmware + major_version = ( + int(closest_firmware.version.split(".")[0]) + if closest_firmware and closest_firmware.version + else None + ) + + # latest version per package and major version latest_version = db.session.query( Version.package_id, db.func.max(Version.version).label("latest_version") ).select_from(Version) + if not beta: latest_version = latest_version.filter(Version.report_url.is_(None)) + latest_version = ( latest_version.join(Build) .filter(Build.active) @@ -57,6 +73,19 @@ def get_catalog(arch, build, language, beta): .filter(Architecture.code.in_(["noarch", arch])) .join(Build.firmware) .filter(Firmware.build <= build) + .filter( + db.or_( + # Check if major_version is not None before applying the filter + (major_version is not None) + and Firmware.version.startswith(f"{major_version}."), + # Include earlier "noarch" version when major_version < 6 + db.and_( + Architecture.code == "noarch", + (major_version is not None) and (major_version < 6), + Firmware.version.startswith("3."), + ), + ) + ) .group_by(Version.package_id) ).subquery()