YouTube + Music

.gitignore

@@ -12,3 +12,4 @@ node_modules/
 survey.json
 .coverage
 export_deta.py
+.DS_Store

classquiz/db/models.py

@@ -136,6 +136,8 @@ class QuizQuestion(BaseModel):
     type: None | QuizQuestionType = QuizQuestionType.ABCD
     answers: list[ABCDQuizAnswer] | RangeQuizAnswer | list[TextQuizAnswer] | list[VotingQuizAnswer] | str
     image: str | None = None
+    youtubeUrl: str | None = None
+    music: str | None = None
 
     @validator("answers")
     def answers_not_none_if_abcd_type(cls, v, values):

classquiz/helpers/__init__.py

@@ -297,3 +297,12 @@ def extract_image_ids_from_quiz(quiz: Quiz) -> list[str | uuid.UUID]:
             continue
         quiz_images.append(question["image"])
     return quiz_images
+
+
+def extract_music_ids_from_quiz(quiz: Quiz) -> list[str | uuid.UUID]:
+    quiz_musics = []
+    for question in quiz.questions:
+        if question["music"] is None:
+            continue
+        quiz_musics.append(question["music"])
+    return quiz_musics

classquiz/routers/editor.py

@@ -19,7 +19,12 @@
 from datetime import datetime
 from uuid import UUID
 
-from classquiz.helpers import get_meili_data, check_image_string, extract_image_ids_from_quiz
+from classquiz.helpers import (
+    get_meili_data,
+    check_image_string,
+    extract_image_ids_from_quiz,
+    extract_music_ids_from_quiz,
+)
 from classquiz.storage.errors import DeletionFailedError
 
 settings = settings()
@@ -88,7 +93,6 @@ async def finish_edit(edit_id: str, quiz_input: QuizInput):
                         answer.answer, tags=ALLOWED_TAGS_FOR_QUIZ, strip=True
                     )
 
-    images_to_delete = []
     old_quiz_data: Quiz = await Quiz.objects.get_or_none(id=session_data.quiz_id, user_id=session_data.user_id)
 
     for i, question in enumerate(quiz_input.questions):
@@ -127,12 +131,6 @@ async def finish_edit(edit_id: str, quiz_input: QuizInput):
         quiz.background_color = quiz_input.background_color
         quiz.background_image = quiz_input.background_image
         quiz.mod_rating = None
-        for image in images_to_delete:
-            if image is not None:
-                try:
-                    await storage.delete([image])
-                except DeletionFailedError:
-                    pass
         await redis.srem("edit_sessions", edit_id)
         await redis.delete(f"edit_session:{edit_id}")
         await redis.delete(f"edit_session:{edit_id}:images")
@@ -158,8 +156,14 @@ async def finish_edit(edit_id: str, quiz_input: QuizInput):
         except asyncpg.exceptions.UniqueViolationError:
             raise HTTPException(status_code=400, detail="The quiz already exists")
         new_images = extract_image_ids_from_quiz(quiz)
+        new_musics = extract_music_ids_from_quiz(quiz)
         for image in new_images:
             item = await StorageItem.objects.get_or_none(id=uuid.UUID(image))
             if item is None:
                 continue
             await quiz.storageitems.add(item)
+        for music in new_musics:
+            item = await StorageItem.objects.get_or_none(id=uuid.UUID(music))
+            if item is None:
+                continue
+            await quiz.storageitems.add(item)

classquiz/routers/storage.py

@@ -151,8 +151,10 @@ async def upload_raw_file(request: Request, user: User = Depends(get_current_use
     if user.storage_used > settings.free_storage_limit:
         raise HTTPException(status_code=409, detail="Storage limit reached")
     file_id = uuid4()
+    body_len = 0
     data_file = SpooledTemporaryFile(max_size=1000)
     async for chunk in request.stream():
+        body_len += len(chunk)
         data_file.write(chunk)
     data_file.seek(0)
     file_obj = StorageItem(
@@ -161,7 +163,41 @@ async def upload_raw_file(request: Request, user: User = Depends(get_current_use
         mime_type=request.headers.get("Content-Type"),
         hash=None,
         user=user,
-        size=0,
+        size=body_len,
+        deleted_at=None,
+        alt_text=None,
+    )
+    # https://github.com/VirusTotal/vt-py/issues/119#issuecomment-1261246867
+    await storage.upload(
+        file_name=file_id.hex,
+        # skipcq: PYL-W0212
+        file_data=data_file._file,
+        mime_type=request.headers.get("Content-Type"),
+    )
+    await file_obj.save()
+    await arq.enqueue_job("calculate_hash", file_id.hex)
+    return PublicStorageItem.from_db_model(file_obj)
+
+
+@router.post("/raw/{filename}")
+async def upload_raw_file(filename: str, request: Request, user: User = Depends(get_current_user)) -> PublicStorageItem:
+    if user.storage_used > settings.free_storage_limit:
+        raise HTTPException(status_code=409, detail="Storage limit reached")
+    file_id = uuid4()
+    body_len = 0
+    data_file = SpooledTemporaryFile(max_size=1000)
+    async for chunk in request.stream():
+        body_len += len(chunk)
+        data_file.write(chunk)
+    data_file.seek(0)
+    file_obj = StorageItem(
+        id=file_id,
+        uploaded_at=datetime.now(),
+        mime_type=request.headers.get("Content-Type"),
+        hash=None,
+        user=user,
+        size=body_len,
+        filename=filename,
         deleted_at=None,
         alt_text=None,
     )
@@ -243,6 +279,7 @@ async def get_latest_images(count: int = 50, user: User = Depends(get_current_us
     count = min(count, 50)
     items = (
         await StorageItem.objects.filter(user=user)
+        .filter(StorageItem.deleted_at == None)  # noqa: E711
         .limit(count)
         .select_related([StorageItem.quizzes, StorageItem.quiztivities])
         .order_by(StorageItem.uploaded_at.desc())

classquiz/worker/storage.py

@@ -8,19 +8,20 @@
 import ormar.exceptions
 from arq.worker import Retry
 import xxhash
+import logging
 
 from classquiz.config import redis, storage
 from tempfile import SpooledTemporaryFile
 
 from classquiz.db.models import StorageItem, Quiz, User
-from classquiz.helpers import extract_image_ids_from_quiz
+from classquiz.helpers import extract_image_ids_from_quiz, extract_music_ids_from_quiz
 from classquiz.storage.errors import DeletionFailedError
 from thumbhash import image_to_thumbhash
 
 
 # skipcq: PYL-W0613
 async def clean_editor_images_up(ctx):
-    print("Cleaning images up")
+    logging.critical("Cleaning images up")
     edit_sessions = await redis.smembers("edit_sessions")
     for session_id in edit_sessions:
         session = await redis.get(f"edit_session:{session_id}")
@@ -30,43 +31,16 @@ async def clean_editor_images_up(ctx):
                 try:
                     await storage.delete(images)
                 except DeletionFailedError:
-                    print("Deletion Error", images)
+                    logging.critical("Deletion Error", images)
             await redis.srem("edit_sessions", session_id)
             await redis.delete(f"edit_session:{session_id}:images")
 
 
 async def calculate_hash(ctx, file_id_as_str: str):
     file_id = uuid.UUID(file_id_as_str)
-    file_data: StorageItem = await StorageItem.objects.select_related(StorageItem.user).get(id=file_id)
-    file_path = file_id.hex
-    if file_data.storage_path is not None:
-        file_path = file_data.storage_path
-    file = SpooledTemporaryFile()
-    file_data.size = await storage.get_file_size(file_name=file_path)
-    if file_data.size is None:
-        file_data.size = 0
-    file_bytes = storage.download(file_path)
-    if file_bytes is None:
-        print("Retry raised!")
-        raise Retry(defer=ctx["job_try"] * 10)
-    async for chunk in file_bytes:
-        file.write(chunk)
-    try:
-        if 0 < file_data.size < 20_970_000:  # greater than 0 but smaller than 20mbytes
-            file_data.thumbhash = image_to_thumbhash(file)
-    # skipcq: PYL-W0703
-    except Exception:
-        pass
-    hash_obj = xxhash.xxh3_128()
-
-    # skipcq: PY-W0069
-    # assert hash_obj.block_size == 64
-    while chunk := file.read(6400):
-        hash_obj.update(chunk)
-    file_data.hash = hash_obj.digest()
-    await file_data.update()
-    file.close()
-    user: User | None = await User.objects.get_or_none(id=file_data.user.id)
+    file_data: StorageItem = await StorageItem.objects.select_related(StorageItem).get(id=file_id)
+
+    user: User | None = await User.objects.get_or_none(id=file_data.id)
     if user is None:
         return
     user.storage_used += file_data.size
@@ -78,14 +52,20 @@ async def quiz_update(ctx, old_quiz: Quiz, quiz_id: uuid.UUID):
     new_quiz: Quiz = await Quiz.objects.get(id=quiz_id)
     old_images = extract_image_ids_from_quiz(old_quiz)
     new_images = extract_image_ids_from_quiz(new_quiz)
+    old_musics = extract_music_ids_from_quiz(old_quiz)
+    new_musics = extract_music_ids_from_quiz(new_quiz)
 
     # If images are identical, then return
-    if sorted(old_images) == sorted(new_images):
-        print("Nothing's changed")
+    if sorted(old_images) == sorted(new_images) and sorted(old_musics) == sorted(new_musics):
+        logging.info("Nothing's changed")
         return
-    print("Change detected")
+    logging.info("Change detected")
+
     removed_images = list(set(old_images) - set(new_images))
+    removed_musics = list(set(old_musics) - set(new_musics))
     added_images = list(set(new_images) - set(old_images))
+    added_musics = list(set(new_musics) - set(old_musics))
+
     change_made = False
     for image in removed_images:
         if "--" in image:
@@ -96,7 +76,21 @@ async def quiz_update(ctx, old_quiz: Quiz, quiz_id: uuid.UUID):
                 continue
             try:
                 await new_quiz.storageitems.remove(item)
-            except ormar.exceptions.NoMatch:
+            except ormar.exceptions.NoMatch as e:
+                logging.critical(e)
+                continue
+            change_made = True
+    for music in removed_musics:
+        if "--" in music:
+            await storage.delete([music])
+        else:
+            item = await StorageItem.objects.get_or_none(id=uuid.UUID(music))
+            if item is None:
+                continue
+            try:
+                await new_quiz.storageitems.remove(item)
+            except ormar.exceptions.NoMatch as e:
+                logging.critical(e)
                 continue
             change_made = True
     for image in added_images:
@@ -106,5 +100,13 @@ async def quiz_update(ctx, old_quiz: Quiz, quiz_id: uuid.UUID):
                 continue
             await new_quiz.storageitems.add(item)
             change_made = True
+    for music in added_musics:
+        if "--" not in music:
+            item = await StorageItem.objects.get_or_none(id=uuid.UUID(music))
+            if item is None:
+                continue
+            await new_quiz.storageitems.add(item)
+            change_made = True
+
     if change_made:
         await new_quiz.update()