diff --git a/.github/workflows/meet.yml b/.github/workflows/meet.yml index 26e6d3dd..54878b34 100644 --- a/.github/workflows/meet.yml +++ b/.github/workflows/meet.yml @@ -122,6 +122,9 @@ jobs: DB_PASSWORD: pass DB_PORT: 5432 STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage + AWS_S3_ENDPOINT_URL: http://localhost:9000 + AWS_S3_ACCESS_KEY_ID: meet + AWS_S3_SECRET_ACCESS_KEY: password LIVEKIT_API_SECRET: secret LIVEKIT_API_KEY: devkey diff --git a/docker-compose.yml b/docker-compose.yml index 3b997d87..41849f1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,30 @@ services: ports: - "1081:1080" + minio: + user: ${DOCKER_USER:-1000} + image: minio/minio + environment: + - MINIO_ROOT_USER=meet + - MINIO_ROOT_PASSWORD=password + ports: + - '9000:9000' + - '9001:9001' + entrypoint: "" + command: minio server --console-address :9001 /data + volumes: + - ./data/media:/data + + createbuckets: + image: minio/mc + depends_on: + - minio + entrypoint: > + sh -c " + /usr/bin/mc alias set meet http://minio:9000 meet password && \ + /usr/bin/mc mb meet/meet-media-storage && \ + exit 0;" + app-dev: build: context: . @@ -40,6 +64,7 @@ services: - mailcatcher - redis - nginx + - createbuckets - livekit celery-dev: @@ -73,6 +98,7 @@ services: depends_on: - postgresql - redis + - createbuckets - livekit celery: diff --git a/env.d/development/common.dist b/env.d/development/common.dist index 5018b1e4..275579a1 100644 --- a/env.d/development/common.dist +++ b/env.d/development/common.dist @@ -18,6 +18,9 @@ MEET_BASE_URL="http://localhost:8072" # Media STORAGES_STATICFILES_BACKEND=django.contrib.staticfiles.storage.StaticFilesStorage +AWS_S3_ENDPOINT_URL=http://minio:9000 +AWS_S3_ACCESS_KEY_ID=meet +AWS_S3_SECRET_ACCESS_KEY=password # OIDC OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/meet/protocol/openid-connect/certs diff --git a/src/backend/.pylintrc b/src/backend/.pylintrc index d7490918..490486d4 100644 --- a/src/backend/.pylintrc +++ b/src/backend/.pylintrc @@ -450,7 +450,7 @@ max-branches=12 max-locals=15 # Maximum number of parents for a class (see R0901). -max-parents=7 +max-parents=10 # Maximum number of public methods for a class (see R0904). max-public-methods=20 diff --git a/src/backend/core/api/permissions.py b/src/backend/core/api/permissions.py index 7e6d4988..ffa458b5 100644 --- a/src/backend/core/api/permissions.py +++ b/src/backend/core/api/permissions.py @@ -39,30 +39,30 @@ def has_object_permission(self, request, view, obj): return obj == request.user -class RoomPermissions(permissions.BasePermission): - """ - Permissions applying to the room API endpoint. - """ +# class RoomPermissions(permissions.BasePermission): +# """ +# Permissions applying to the room API endpoint. +# """ - def has_permission(self, request, view): - """Only allow authenticated users for unsafe methods.""" - if request.method in permissions.SAFE_METHODS: - return True +# def has_permission(self, request, view): +# """Only allow authenticated users for unsafe methods.""" +# if request.method in permissions.SAFE_METHODS: +# return True - return request.user.is_authenticated +# return request.user.is_authenticated - def has_object_permission(self, request, view, obj): - """Object permissions are only given to administrators of the room.""" +# def has_object_permission(self, request, view, obj): +# """Object permissions are only given to administrators of the room.""" - if request.method in permissions.SAFE_METHODS: - return True +# if request.method in permissions.SAFE_METHODS: +# return True - user = request.user +# user = request.user - if request.method == "DELETE": - return obj.is_owner(user) +# if request.method == "DELETE": +# return obj.is_owner(user) - return obj.is_administrator(user) +# return obj.is_administrator(user) class ResourceAccessPermission(permissions.BasePermission): @@ -82,4 +82,15 @@ def has_object_permission(self, request, view, obj): if request.method == "DELETE" and obj.role == RoleChoices.OWNER: return obj.user == user - return obj.resource.is_administrator(user) + return RoleChoices.is_administrator(obj.resource.get_roles(user)) + + +class AccessPermission(permissions.BasePermission): + """Permission class for access objects.""" + + def has_permission(self, request, view): + return request.user.is_authenticated or view.action != "create" + + def has_object_permission(self, request, view, obj): + """Check permission for a given object.""" + return obj.get_abilities(request.user).get(view.action, False) diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index a1af11a2..edd917e0 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -87,6 +87,15 @@ class NestedResourceAccessSerializer(ResourceAccessSerializer): user = UserSerializer(read_only=True) +class ListRoomSerializer(serializers.ModelSerializer): + """Serialize Room model for a list API endpoint.""" + + class Meta: + model = models.Room + fields = ["id", "name", "slug", "is_public"] + read_only_fields = ["id", "slug"] + + class RoomSerializer(serializers.ModelSerializer): """Serialize Room model for the API.""" @@ -106,10 +115,10 @@ def to_representation(self, instance): if not request: return output - role = instance.get_role(request.user) - is_admin = models.RoleChoices.check_administrator_role(role) + roles = instance.get_roles(request.user) + is_administrator = models.RoleChoices.is_administrator(roles) - if role is not None: + if roles: access_serializer = NestedResourceAccessSerializer( instance.accesses.select_related("resource", "user").all(), context=self.context, @@ -117,10 +126,10 @@ def to_representation(self, instance): ) output["accesses"] = access_serializer.data - if not is_admin: + if not is_administrator: del output["configuration"] - if role is not None or instance.is_public: + if roles or instance.is_public: slug = f"{instance.id!s}".replace("-", "") username = request.query_params.get("username", None) @@ -133,6 +142,17 @@ def to_representation(self, instance): ), } - output["is_administrable"] = is_admin + output["is_administrable"] = is_administrator return output + + +class RecordingSerializer(serializers.ModelSerializer): + """Serialize Recording for the API.""" + + room = ListRoomSerializer(read_only=True) + + class Meta: + model = models.Recording + fields = ["id", "room", "created_at", "updated_at", "stopped_at", "status"] + read_only_fields = fields diff --git a/src/backend/core/api/utils.py b/src/backend/core/api/utils.py new file mode 100644 index 00000000..53d84f68 --- /dev/null +++ b/src/backend/core/api/utils.py @@ -0,0 +1,33 @@ +"""Util to generate S3 authorization headers for object storage access control""" + +from django.core.files.storage import default_storage + +import botocore + + +def generate_s3_authorization_headers(key): + """ + Generate authorization headers for an s3 object. + These headers can be used as an alternative to signed urls with many benefits: + - the urls of our files never expire and can be stored in our documents' content + - we don't leak authorized urls that could be shared (file access can only be done + with cookies) + - access control is truly realtime + - the object storage service does not need to be exposed on internet + """ + url = default_storage.unsigned_connection.meta.client.generate_presigned_url( + "get_object", + ExpiresIn=0, + Params={"Bucket": default_storage.bucket_name, "Key": key}, + ) + request = botocore.awsrequest.AWSRequest(method="get", url=url) + + s3_client = default_storage.connection.meta.client + # pylint: disable=protected-access + credentials = s3_client._request_signer._credentials # noqa: SLF001 + frozen_credentials = credentials.get_frozen_credentials() + region = s3_client.meta.region_name + auth = botocore.auth.S3SigV4Auth(frozen_credentials, "s3", region) + auth.add_auth(request) + + return request diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index fa662946..9bc6ecbe 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -1,17 +1,22 @@ """API endpoints""" +import re import uuid +from urllib.parse import urlparse from django.conf import settings from django.db.models import Q from django.http import Http404 from django.shortcuts import get_object_or_404 +from django.utils import timezone from django.utils.text import slugify from rest_framework import ( decorators, + exceptions, mixins, pagination, + status, viewsets, ) from rest_framework import ( @@ -25,6 +30,13 @@ # pylint: disable=too-many-ancestors +UUID_REGEX = ( + r"[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}" +) +RECORDING_URL_PATTERN = re.compile( + f"{settings.MEDIA_URL:s}recordings/({UUID_REGEX:s})/file.mp4/$" +) + class NestedGenericViewSet(viewsets.GenericViewSet): """ @@ -162,7 +174,7 @@ class RoomViewSet( API endpoints to access and perform actions on rooms. """ - permission_classes = [permissions.RoomPermissions] + permission_classes = [permissions.AccessPermission] queryset = models.Room.objects.all() serializer_class = serializers.RoomSerializer @@ -186,16 +198,10 @@ def retrieve(self, request, *args, **kwargs): """ try: instance = self.get_object() - - analytics.track( - user=self.request.user, - event="Get Room", - properties={"slug": instance.slug}, - ) - except Http404: if not settings.ALLOW_UNREGISTERED_ROOMS: raise + slug = slugify(self.kwargs["pk"]) username = request.query_params.get("username", None) data = { @@ -209,6 +215,11 @@ def retrieve(self, request, *args, **kwargs): }, } else: + analytics.track( + user=self.request.user, + event="Get Room", + properties={"slug": instance.slug}, + ) data = self.get_serializer(instance).data return drf_response.Response(data) @@ -249,6 +260,17 @@ def perform_create(self, serializer): }, ) + @decorators.action(detail=True, methods=["post"], url_path="start-recording") + def start_recording(self, request, *args, **kwargs): + """This view is used to start a recording for a room.""" + # Check permission first + room = self.get_object() + recording = room.start_recording() + + return drf_response.Response( + {"recording": recording.id}, status=status.HTTP_201_CREATED + ) + class ResourceAccessListModelMixin: """List mixin for resource access API.""" @@ -293,3 +315,96 @@ class ResourceAccessViewSet( permission_classes = [permissions.ResourceAccessPermission] queryset = models.ResourceAccess.objects.all() serializer_class = serializers.ResourceAccessSerializer + + +class RecordingViewSet( + mixins.DestroyModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet, +): + """ + API endpoints to access and perform actions on recordings. + """ + + pagination_class = Pagination + permission_classes = [permissions.AccessPermission] + queryset = models.Recording.objects.all() + serializer_class = serializers.RecordingSerializer + + def list(self, request, *args, **kwargs): + """Restrict resources returned by the list endpoint to a user's room.""" + queryset = self.filter_queryset(self.get_queryset()) + user = self.request.user + if user.is_authenticated: + queryset = queryset.filter(room__accesses__user=user) + else: + queryset = queryset.none() + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return drf_response.Response(serializer.data) + + @decorators.action(detail=True, methods=["post"], url_path="stop") + def stop(self, request, *args, **kwargs): + """ + This view is used to stop a recording. It can be called anonymously as the + recording ID will not have been communicated anywhere at the time of recording. + """ + recording = self.get_object() + + if recording.stopped_at is not None: + raise exceptions.PermissionDenied() + + recording.stopped_at = timezone.now() + recording.save() + + # TODO: Generate summary and send to note taking app + + serializer = self.get_serializer(recording) + return drf_response.Response(serializer.data, status=status.HTTP_200_OK) + + @decorators.action(detail=False, methods=["get"], url_path="retrieve-auth") + def retrieve_auth(self, request, *args, **kwargs): + """ + This view is used by an Nginx subrequest to control access to a recording file. + + The original url is passed by nginx in the "HTTP_X_ORIGINAL_URL" header. + See corresponding ingress configuration in Helm chart and read about the + nginx.ingress.kubernetes.io/auth-url annotation to understand how the Nginx ingress + is configured to do this. + + Based on the original url and the logged in user, we must decide if we authorize Nginx + to let this request go through (by returning a 200 code) or if we block it (by returning + a 403 error). Note that we return 403 errors without any further details for security + reasons. + + When we let the request go through, we compute authorization headers that will be added to + the request going through thanks to the nginx.ingress.kubernetes.io/auth-response-headers + annotation. The request will then be proxied to the object storage backend who will + respond with the file after checking the signature included in headers. + """ + if not request.user.is_authenticated: + raise exceptions.AuthenticationFailed() + + original_url = urlparse(request.META.get("HTTP_X_ORIGINAL_URL")) + match = RECORDING_URL_PATTERN.search(original_url.path) + + try: + (pk,) = match.groups() + except AttributeError as excpt: + raise exceptions.PermissionDenied() from excpt + + # Check permission + if not models.Recording.objects.filter( + pk=pk, room__accesses__user=request.user + ).exists(): + raise exceptions.PermissionDenied() + + # Generate authorization headers and return an authorization to proceed with the request + key = models.Recording(pk=pk).key + request = utils.generate_s3_authorization_headers(key) + return drf_response.Response("authorized", headers=request.headers, status=200) diff --git a/src/backend/core/factories.py b/src/backend/core/factories.py index 7fe300e8..41ab59a4 100644 --- a/src/backend/core/factories.py +++ b/src/backend/core/factories.py @@ -65,3 +65,12 @@ class Meta: name = factory.Faker("catch_phrase") slug = factory.LazyAttribute(lambda o: slugify(o.name)) + + +class RecordingFactory(factory.django.DjangoModelFactory): + """Create fake recordings for testing.""" + + class Meta: + model = models.Recording + + room = factory.SubFactory(RoomFactory) diff --git a/src/backend/core/migrations/0005_alter_user_language_recording.py b/src/backend/core/migrations/0005_alter_user_language_recording.py new file mode 100644 index 00000000..897d6e67 --- /dev/null +++ b/src/backend/core/migrations/0005_alter_user_language_recording.py @@ -0,0 +1,37 @@ +# Generated by Django 5.1.1 on 2024-10-12 11:52 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0004_alter_user_language'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='language', + field=models.CharField(choices="(('en-us', 'English'), ('fr-fr', 'French'))", default='en-us', help_text='The language in which the user wants to see the interface.', max_length=10, verbose_name='language'), + ), + migrations.CreateModel( + name='Recording', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, help_text='primary key for the record as UUID', primary_key=True, serialize=False, verbose_name='id')), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='date and time at which a record was created', verbose_name='created on')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='date and time at which a record was last updated', verbose_name='updated on')), + ('stopped_at', models.DateTimeField(blank=True, null=True, verbose_name='End Time')), + ('status', models.CharField(choices=[('recording', 'Recording'), ('done', 'Done')], default='recording')), + ('room', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='meetings', to='core.room', verbose_name='Room')), + ], + options={ + 'verbose_name': 'Recording', + 'verbose_name_plural': 'Recordings', + 'db_table': 'meet_recording', + 'ordering': ('created_at',), + }, + ), + ] diff --git a/src/backend/core/models.py b/src/backend/core/models.py index f2d4e77b..3c292ea8 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -28,14 +28,16 @@ class RoleChoices(models.TextChoices): OWNER = "owner", _("Owner") @classmethod - def check_administrator_role(cls, role): + def is_administrator(cls, roles): """Check if a role is administrator.""" - return role in [cls.ADMIN, cls.OWNER] + return bool(set(roles).intersection({RoleChoices.OWNER, RoleChoices.ADMIN})) - @classmethod - def check_owner_role(cls, role): - """Check if a role is owner.""" - return role == cls.OWNER + +class RecordingStatusChoices(models.TextChoices): + """Recording status choices.""" + + RECORDING = "recording", _("Recording") + DONE = "done", _("Done") class BaseModel(models.Model): @@ -193,34 +195,44 @@ def __str__(self): except AttributeError: return f"Resource {self.id!s}" - def get_role(self, user): - """ - Determine the role of a given user in this resource. - """ - if not user or not user.is_authenticated: - return None - - role = None - for access in self.accesses.filter(user=user): - if access.role == RoleChoices.OWNER: - return RoleChoices.OWNER - if access.role == RoleChoices.ADMIN: - role = RoleChoices.ADMIN - if access.role == RoleChoices.MEMBER and role != RoleChoices.ADMIN: - role = RoleChoices.MEMBER - return role - def is_administrator(self, user): """ Check if a user is administrator of the resource. - Users carrying the "owner" role are considered as administrators a fortiori. """ - return RoleChoices.check_administrator_role(self.get_role(user)) + return RoleChoices.is_administrator(self.get_roles(user)) def is_owner(self, user): """Check if a user is owner of the resource.""" - return RoleChoices.check_owner_role(self.get_role(user)) + return RoleChoices.OWNER in self.get_roles(user) + + def get_roles(self, user): + """Compute the roles a user has in a room.""" + if not user or not user.is_authenticated: + return self.accesses.none() + + try: + roles = self.user_roles or [] + except AttributeError: + try: + roles = self.accesses.filter(user=user).values_list("role", flat=True) + except (models.ObjectDoesNotExist, IndexError): + roles = self.accesses.none() + return roles + + def get_abilities(self, user): + """Compute and return abilities for a given user on the room.""" + roles = self.get_roles(user) + is_owner_or_admin = RoleChoices.is_administrator(roles) + + return { + "start_recording": is_owner_or_admin, + "destroy": RoleChoices.OWNER in roles, + "manage_accesses": is_owner_or_admin, + "partial_update": is_owner_or_admin, + "retrieve": True, + "update": is_owner_or_admin, + } class ResourceAccess(BaseModel): @@ -325,3 +337,47 @@ def clean_fields(self, exclude=None): else: raise ValidationError({"name": f'Room name "{self.name:s}" is reserved.'}) super().clean_fields(exclude=exclude) + + def start_recording(self): + """Create a new related recording object to which Livekit will be able to save a file.""" + return Recording.objects.create(room=self) + + +class Recording(BaseModel): + """Model for recording meetings that take place in a room""" + + room = models.ForeignKey( + Room, on_delete=models.CASCADE, related_name="meetings", verbose_name=_("Room") + ) + stopped_at = models.DateTimeField(verbose_name=_("End Time"), null=True, blank=True) + status = models.CharField( + choices=RecordingStatusChoices, default=RecordingStatusChoices.RECORDING + ) + + class Meta: + db_table = "meet_recording" + ordering = ("created_at",) + verbose_name = _("Recording") + verbose_name_plural = _("Recordings") + + def __str__(self): + return _( + f"Recording in {self.room.name:s} on {self.created_at:%B %d, %Y at %I:%M %p}" + ) + + @property + def key(self): + """Return the path where the recording file will be stored in object storage.""" + return f"recordings/{self.pk!s}/file.mp4/" + + def get_abilities(self, user): + """Compute and return abilities for a given user on the recording.""" + roles = self.room.get_roles(user) + + return { + "destroy": RoleChoices.OWNER in roles, + "partial_update": False, + "retrieve": False, + "stop": True, + "update": False, + } diff --git a/src/backend/core/tests/recordings/__init__.py b/src/backend/core/tests/recordings/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/core/tests/recordings/test_api_recordings_delete.py b/src/backend/core/tests/recordings/test_api_recordings_delete.py new file mode 100644 index 00000000..72ba9187 --- /dev/null +++ b/src/backend/core/tests/recordings/test_api_recordings_delete.py @@ -0,0 +1,102 @@ +""" +Test recordings API endpoints in the Meet core app: delete. +""" + +import pytest +from rest_framework.test import APIClient + +from ...factories import RecordingFactory, UserFactory +from ...models import Recording + +pytestmark = pytest.mark.django_db + + +def test_api_recordings_delete_anonymous(): + """Anonymous users should not be allowed to destroy a recording.""" + recording = RecordingFactory() + client = APIClient() + + response = client.delete( + f"/api/v1.0/recordings/{recording.id!s}/", + ) + + assert response.status_code == 401 + assert Recording.objects.count() == 1 + + +def test_api_recordings_delete_authenticated(): + """ + Authenticated users should not be allowed to delete a recording from a room to which + they are not related. + """ + recording = RecordingFactory() + user = UserFactory() + + client = APIClient() + client.force_login(user) + + response = client.delete( + f"/api/v1.0/recordings/{recording.id!s}/", + ) + + assert response.status_code == 403 + assert Recording.objects.count() == 1 + + +def test_api_recordings_delete_members(): + """ + Authenticated users should not be allowed to delete a recording from a room of which + they are only a member. + """ + user = UserFactory() + recording = RecordingFactory( + room__users=[(user, "member")] + ) # as user declared in the room but not administrator + + client = APIClient() + client.force_login(user) + + response = client.delete( + f"/api/v1.0/recordings/{recording.id}/", + ) + + assert response.status_code == 403 + assert Recording.objects.count() == 1 + + +def test_api_recordings_delete_administrators(): + """ + Authenticated users should not be allowed to delete a recording from a room in which + they are administrator. + """ + user = UserFactory() + recording = RecordingFactory(room__users=[(user, "administrator")]) + + client = APIClient() + client.force_login(user) + + response = client.delete( + f"/api/v1.0/recordings/{recording.id}/", + ) + + assert response.status_code == 403 + assert Recording.objects.count() == 1 + + +def test_api_recordings_delete_owners(): + """ + Authenticated users should be able to delete a recording from a room in which they are + directly owner. + """ + user = UserFactory() + recording = RecordingFactory(room__users=[(user, "owner")]) + + client = APIClient() + client.force_login(user) + + response = client.delete( + f"/api/v1.0/recordings/{recording.id}/", + ) + + assert response.status_code == 204 + assert Recording.objects.exists() is False diff --git a/src/backend/core/tests/recordings/test_api_recordings_list.py b/src/backend/core/tests/recordings/test_api_recordings_list.py new file mode 100644 index 00000000..d19a5894 --- /dev/null +++ b/src/backend/core/tests/recordings/test_api_recordings_list.py @@ -0,0 +1,132 @@ +""" +Test recordings API endpoints in the Meet core app: list. +""" + +from unittest import mock + +import pytest +from rest_framework.pagination import PageNumberPagination +from rest_framework.test import APIClient + +from core import factories + +pytestmark = pytest.mark.django_db + + +def test_api_recordings_list_anonymous(): + """Anonymous users should not be able to list recordings.""" + factories.RecordingFactory(room__is_public=False) + factories.RecordingFactory(room__is_public=True) + + response = APIClient().get("/api/v1.0/recordings/") + + assert response.status_code == 200 + assert response.json() == { + "count": 0, + "next": None, + "previous": None, + "results": [], + } + + +def test_api_recordings_list_authenticated(): + """ + Authenticated users listing recordings, should only see the recordings + to which they are related. + """ + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + other_user = factories.UserFactory() + + factories.RecordingFactory(room__is_public=False) + factories.RecordingFactory(room__is_public=True) + factories.RecordingFactory(room__is_public=False, room__users=[other_user]) + + recording = factories.RecordingFactory(room__is_public=False, room__users=[user]) + room = recording.room + + response = client.get( + "/api/v1.0/recordings/", + ) + + assert response.status_code == 200 + results = response.json()["results"] + assert len(results) == 1 + expected_ids = { + str(recording.id), + } + result_ids = {result["id"] for result in results} + assert expected_ids == result_ids + assert results[0] == { + "id": str(recording.id), + "created_at": recording.created_at.isoformat().replace("+00:00", "Z"), + "stopped_at": None, + "room": { + "id": str(room.id), + "is_public": room.is_public, + "name": room.name, + "slug": room.slug, + }, + "status": "recording", + "updated_at": recording.updated_at.isoformat().replace("+00:00", "Z"), + } + + +@mock.patch.object(PageNumberPagination, "get_page_size", return_value=2) +def test_api_recordings_list_pagination(_mock_page_size): + """Pagination should work as expected.""" + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + recordings = factories.RecordingFactory.create_batch(3, room__users=[user]) + recording_ids = [str(r.id) for r in recordings] + + response = client.get("/api/v1.0/recordings/") + + assert response.status_code == 200 + content = response.json() + assert content["count"] == 3 + assert content["next"] == "http://testserver/api/v1.0/recordings/?page=2" + assert content["previous"] is None + + assert len(content["results"]) == 2 + for item in content["results"]: + recording_ids.remove(item["id"]) + + # Get page 2 + response = client.get( + "/api/v1.0/recordings/?page=2", + ) + + assert response.status_code == 200 + content = response.json() + + assert content["count"] == 3 + assert content["next"] is None + assert content["previous"], "http://testserver/api/v1.0/recordings/" + + assert len(content["results"]) == 1 + recording_ids.remove(content["results"][0]["id"]) + assert recording_ids == [] + + +def test_api_recordings_list_authenticated_distinct(): + """A recording for a public room with several related users should only be listed once.""" + user = factories.UserFactory() + other_user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + recording = factories.RecordingFactory( + room__is_public=True, room__users=[user, other_user] + ) + + response = client.get("/api/v1.0/recordings/") + + assert response.status_code == 200 + content = response.json() + assert len(content["results"]) == 1 + assert content["results"][0]["id"] == str(recording.id) diff --git a/src/backend/core/tests/recordings/test_api_recordings_retrieve_auth.py b/src/backend/core/tests/recordings/test_api_recordings_retrieve_auth.py new file mode 100644 index 00000000..fc142421 --- /dev/null +++ b/src/backend/core/tests/recordings/test_api_recordings_retrieve_auth.py @@ -0,0 +1,128 @@ +""" +Test file downloads API endpoint for users in Meet's core app. +""" + +from urllib.parse import urlparse + +from django.conf import settings +from django.core.files.storage import default_storage +from django.utils import timezone + +import pytest +import requests +from rest_framework.test import APIClient + +from core import factories, models + +pytestmark = pytest.mark.django_db + + +# This is a minimal MP4 file header, which creates a 1-second empty video +VIDEO_BYTES = ( + b"\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42mp41" + b"\x00\x00\x00\x08free\x00\x00\x02\xeemdat" +) + + +@pytest.mark.parametrize("is_public", [True, False]) +def test_api_recordings_retrieve_auth_anonymous(is_public): + """Anonymous users should not be able to retrieve recordings for a room.""" + room = factories.RoomFactory(is_public=is_public) + recording = room.start_recording() + + default_storage.connection.meta.client.put_object( + Bucket=default_storage.bucket_name, + Key=recording.key, + Body=VIDEO_BYTES, + ContentType="video/mp4", + ) + + original_url = f"http://localhost/media/{recording.key:s}" + response = APIClient().get( + "/api/v1.0/recordings/retrieve-auth/", HTTP_X_ORIGINAL_URL=original_url + ) + + assert response.status_code == 401 + + +@pytest.mark.parametrize("is_public", [True, False]) +def test_api_recordings_retrieve_auth_authenticated(is_public): + """ + Authenticated users who are not related to a room should not be able to + retrieve recordings for this room. + """ + room = factories.RoomFactory(is_public=is_public) + recording = room.start_recording() + + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + default_storage.connection.meta.client.put_object( + Bucket=default_storage.bucket_name, + Key=recording.key, + Body=VIDEO_BYTES, + ContentType="video/mp4", + ) + + original_url = f"http://localhost/media/{recording.key:s}" + response = client.get( + "/api/v1.0/recordings/retrieve-auth/", HTTP_X_ORIGINAL_URL=original_url + ) + + assert response.status_code == 403 + + +@pytest.mark.parametrize("is_public", [True, False]) +@pytest.mark.parametrize("role", models.RoleChoices.values) +def test_api_recordings_retrieve_auth_admin_or_owner(role, is_public): + """ + Users who are administrator or owner of a room, should be able to retrieve recordings. + """ + room = factories.RoomFactory(is_public=is_public) + recording = room.start_recording() + + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + factories.UserResourceAccessFactory(resource=room, user=user, role=role) + + # deposit a recording in S3 + default_storage.connection.meta.client.put_object( + Bucket=default_storage.bucket_name, + Key=recording.key, + Body=VIDEO_BYTES, + ContentType="video/mp4", + ) + + # verify getting object content from url with authorization headers + original_url = f"http://localhost/media/{recording.key:s}" + response = client.get( + "/api/v1.0/recordings/retrieve-auth/", HTTP_X_ORIGINAL_URL=original_url + ) + + assert response.status_code == 200 + + authorization = response["Authorization"] + assert "AWS4-HMAC-SHA256 Credential=" in authorization + assert ( + "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=" + in authorization + ) + assert response["X-Amz-Date"] == timezone.now().strftime("%Y%m%dT%H%M%SZ") + + s3_url = urlparse(settings.AWS_S3_ENDPOINT_URL) + file_url = f"{settings.AWS_S3_ENDPOINT_URL:s}/meet-media-storage/{recording.key:s}" + response = requests.get( + file_url, + headers={ + "authorization": authorization, + "x-amz-date": response["x-amz-date"], + "x-amz-content-sha256": response["x-amz-content-sha256"], + "Host": f"{s3_url.hostname:s}:{s3_url.port:d}", + }, + timeout=1, + ) + + assert response.content == VIDEO_BYTES diff --git a/src/backend/core/tests/recordings/test_api_recordings_stop.py b/src/backend/core/tests/recordings/test_api_recordings_stop.py new file mode 100644 index 00000000..38bca7e9 --- /dev/null +++ b/src/backend/core/tests/recordings/test_api_recordings_stop.py @@ -0,0 +1,56 @@ +""" +Test recordings API endpoints in the Meet core app: stop. +""" + +from datetime import datetime, timezone +from unittest import mock +from uuid import uuid4 + +from django.utils import timezone as django_timezone + +import pytest +from rest_framework.test import APIClient + +from core import factories + +pytestmark = pytest.mark.django_db + + +def test_api_recording_stop_recording_success(): + """Anonymous users can stop a recording just with its UUID.""" + recording = factories.RecordingFactory() + + now = datetime(2010, 1, 1, tzinfo=timezone.utc) + with mock.patch.object(django_timezone, "now", return_value=now): + response = APIClient().post(f"/api/v1.0/recordings/{recording.id!s}/stop/") + + assert response.status_code == 200 + recording.refresh_from_db() + assert recording.stopped_at == now + + +def test_api_recording_stop_recording_unknown(): + """Trying to stop an unknown recording should return a 404.""" + response = APIClient().post(f"/api/v1.0/recordings/{uuid4()!s}/stop/") + + assert response.status_code == 404 + + +def test_api_recording_stop_recording_already_stopped(): + """ + Trying to stop a recording that was already stopped should return a 403 and leave + the recording unmodified. + """ + recording = factories.RecordingFactory() + + response = APIClient().post(f"/api/v1.0/recordings/{recording.id!s}/stop/") + + assert response.status_code == 200 + recording.refresh_from_db() + stopped_at = recording.stopped_at + + response = APIClient().post(f"/api/v1.0/recordings/{recording.id!s}/stop/") + + assert response.status_code == 403 + recording.refresh_from_db() + assert recording.stopped_at == stopped_at diff --git a/src/backend/core/tests/rooms/test_api_rooms_retrieve.py b/src/backend/core/tests/rooms/test_api_rooms_retrieve.py index ffa17eaa..9be0fce8 100644 --- a/src/backend/core/tests/rooms/test_api_rooms_retrieve.py +++ b/src/backend/core/tests/rooms/test_api_rooms_retrieve.py @@ -286,7 +286,7 @@ def test_api_rooms_retrieve_members(mock_token, django_assert_num_queries): client = APIClient() client.force_login(user) - with django_assert_num_queries(4): + with django_assert_num_queries(5): response = client.get( f"/api/v1.0/rooms/{room.id!s}/", ) @@ -360,7 +360,7 @@ def test_api_rooms_retrieve_administrators(mock_token, django_assert_num_queries client = APIClient() client.force_login(user) - with django_assert_num_queries(4): + with django_assert_num_queries(5): response = client.get( f"/api/v1.0/rooms/{room.id!s}/", ) diff --git a/src/backend/core/tests/rooms/test_api_rooms_start_recording.py b/src/backend/core/tests/rooms/test_api_rooms_start_recording.py new file mode 100644 index 00000000..1ed5485d --- /dev/null +++ b/src/backend/core/tests/rooms/test_api_rooms_start_recording.py @@ -0,0 +1,88 @@ +""" +Test rooms API endpoints in the Meet core app: start-recording. +""" + +import pytest +from rest_framework.test import APIClient + +from core import factories, models + +pytestmark = pytest.mark.django_db + + +def test_api_rooms_start_recording_anonymous(): + """Anonymous users should not be allowed to start a recording for a room.""" + room = factories.RoomFactory() + + response = APIClient().post(f"/api/v1.0/rooms/{room.id!s}/start-recording/") + + assert response.status_code == 401 + assert models.Recording.objects.exists() is False + + +def test_api_rooms_start_recording_authenticated(): + """ + Authenticated users should not be allowed to start a recording for a room + to which they are not related. + """ + user = factories.UserFactory() + room = factories.RoomFactory() + + client = APIClient() + client.force_login(user) + + response = client.post(f"/api/v1.0/rooms/{room.id!s}/start-recording/") + + assert response.status_code == 403 + assert models.Recording.objects.exists() is False + + +def test_api_rooms_start_recording_members(): + """ + Users who are members of a room but not administrators should + not be allowed to start a recording in this room. + """ + user = factories.UserFactory() + room = factories.RoomFactory(users=[(user, "member")]) + + client = APIClient() + client.force_login(user) + + response = client.post(f"/api/v1.0/rooms/{room.id!s}/start-recording/") + + assert response.status_code, 403 + assert models.Recording.objects.exists() is False + + +@pytest.mark.parametrize("role", ["administrator", "owner"]) +def test_api_rooms_start_recording_administrators(role): + """Administrators or owners of a room should be allowed to start a recording in this room.""" + user = factories.UserFactory() + room = factories.RoomFactory(users=[(user, role)]) + + client = APIClient() + client.force_login(user) + + response = client.post(f"/api/v1.0/rooms/{room.id!s}/start-recording/") + + assert response.status_code == 201 + assert models.Recording.objects.get().room == room + + +@pytest.mark.parametrize("role", ["administrator", "owner"]) +def test_api_rooms_start_recording_administrators_of_another(role): + """ + Being administrator or owner of a room should not grant authorization + to update another room. + """ + user = factories.UserFactory() + factories.RoomFactory(users=[(user, role)]) + other_room = factories.RoomFactory() + + client = APIClient() + client.force_login(user) + + response = client.post(f"/api/v1.0/rooms/{other_room.id!s}/start-recording/") + + assert response.status_code, 403 + assert models.Recording.objects.exists() is False diff --git a/src/backend/core/tests/test_models_rooms.py b/src/backend/core/tests/test_models_rooms.py index 8879bfca..5b1f65c2 100644 --- a/src/backend/core/tests/test_models_rooms.py +++ b/src/backend/core/tests/test_models_rooms.py @@ -86,81 +86,66 @@ def test_models_rooms_is_public_default(): assert room.is_public is True +def test_models_recordings_key(): + """Check the room key format.""" + room = RoomFactory() + recording = room.start_recording() + assert recording.key == f"recordings/{recording.pk!s}/file.mp4/" + + # Access rights methods def test_models_rooms_access_rights_none(django_assert_num_queries): - """Calling access rights methods with None should return None.""" + """The `get_roles` method should return an empty list when called with None.""" room = RoomFactory() with django_assert_num_queries(0): - assert room.get_role(None) is None - with django_assert_num_queries(0): - assert room.is_administrator(None) is False - with django_assert_num_queries(0): - assert room.is_owner(None) is False + assert not list(room.get_roles(None)) def test_models_rooms_access_rights_anonymous(django_assert_num_queries): - """Check access rights methods on the room object for an anonymous user.""" + """The `get_roles` method should return an empty list for an anonymous user.""" user = AnonymousUser() room = RoomFactory() with django_assert_num_queries(0): - assert room.get_role(user) is None - with django_assert_num_queries(0): - assert room.is_administrator(user) is False - with django_assert_num_queries(0): - assert room.is_owner(user) is False + assert not list(room.get_roles(user)) def test_models_rooms_access_rights_authenticated(django_assert_num_queries): - """Check access rights methods on the room object for an unrelated user.""" + """ + The `get_roles` method should return an empty list for a user not related to the room. + """ user = UserFactory() room = RoomFactory() with django_assert_num_queries(1): - assert room.get_role(user) is None - with django_assert_num_queries(1): - assert room.is_administrator(user) is False - with django_assert_num_queries(1): - assert room.is_owner(user) is False + assert not list(room.get_roles(user)) def test_models_rooms_access_rights_member_direct(django_assert_num_queries): - """Check access rights methods on the room object for a direct member.""" + """Check `get_roles` method on the room object for a direct member.""" user = UserFactory() room = RoomFactory(users=[(user, "member")]) with django_assert_num_queries(1): - assert room.get_role(user) == "member" - with django_assert_num_queries(1): - assert room.is_administrator(user) is False - with django_assert_num_queries(1): - assert room.is_owner(user) is False + assert list(room.get_roles(user)) == ["member"] def test_models_rooms_access_rights_administrator_direct(django_assert_num_queries): - """The is_administrator method should return True for a direct administrator.""" + """Check `get_roles` method on the room object for a direct administrator.""" user = UserFactory() room = RoomFactory(users=[(user, "administrator")]) with django_assert_num_queries(1): - assert room.get_role(user) == "administrator" - with django_assert_num_queries(1): - assert room.is_administrator(user) is True - with django_assert_num_queries(1): - assert room.is_owner(user) is False + assert list(room.get_roles(user)) == ["administrator"] def test_models_rooms_access_rights_owner_direct(django_assert_num_queries): - """Check access rights methods on the room object for an owner.""" + """Check `get_roles` method on the room object for an owner.""" user = UserFactory() room = RoomFactory(users=[(user, "owner")]) with django_assert_num_queries(1): - assert room.get_role(user) == "owner" - with django_assert_num_queries(1): - assert room.is_administrator(user) is True - with django_assert_num_queries(1): - assert room.is_owner(user) is True + assert list(room.get_roles(user)) == ["owner"] diff --git a/src/backend/core/urls.py b/src/backend/core/urls.py index f7f0c5a9..937ec482 100644 --- a/src/backend/core/urls.py +++ b/src/backend/core/urls.py @@ -11,6 +11,7 @@ # - Main endpoints router = DefaultRouter() router.register("users", viewsets.UserViewSet, basename="users") +router.register("recordings", viewsets.RecordingViewSet, basename="recordings") router.register("rooms", viewsets.RoomViewSet, basename="rooms") router.register( "resource-accesses", viewsets.ResourceAccessViewSet, basename="resource_accesses" diff --git a/src/backend/core/utils.py b/src/backend/core/utils.py index 304b5de7..572e3c93 100644 --- a/src/backend/core/utils.py +++ b/src/backend/core/utils.py @@ -11,7 +11,9 @@ from uuid import uuid4 from django.conf import settings +from django.core.files.storage import default_storage +import botocore from livekit.api import AccessToken, VideoGrants @@ -81,3 +83,31 @@ def generate_token(room: str, user, username: Optional[str] = None) -> str: ) return token.to_jwt() + + +def generate_s3_authorization_headers(key): + """ + Generate authorization headers for an s3 object. + These headers can be used as an alternative to signed urls with many benefits: + - the urls of our files never expire and can be stored in our documents' content + - we don't leak authorized urls that could be shared (file access can only be done + with cookies) + - access control is truly realtime + - the object storage service does not need to be exposed on internet + """ + url = default_storage.unsigned_connection.meta.client.generate_presigned_url( + "get_object", + ExpiresIn=0, + Params={"Bucket": default_storage.bucket_name, "Key": key}, + ) + request = botocore.awsrequest.AWSRequest(method="get", url=url) + + s3_client = default_storage.connection.meta.client + # pylint: disable=protected-access + credentials = s3_client._request_signer._credentials # noqa: SLF001 + frozen_credentials = credentials.get_frozen_credentials() + region = s3_client.meta.region_name + auth = botocore.auth.S3SigV4Auth(frozen_credentials, "s3", region) + auth.add_auth(request) + + return request diff --git a/src/backend/locale/en_US/LC_MESSAGES/django.mo b/src/backend/locale/en_US/LC_MESSAGES/django.mo new file mode 100644 index 00000000..6c5906d1 Binary files /dev/null and b/src/backend/locale/en_US/LC_MESSAGES/django.mo differ diff --git a/src/backend/locale/fr_FR/LC_MESSAGES/django.mo b/src/backend/locale/fr_FR/LC_MESSAGES/django.mo new file mode 100644 index 00000000..6c5906d1 Binary files /dev/null and b/src/backend/locale/fr_FR/LC_MESSAGES/django.mo differ diff --git a/src/backend/meet/settings.py b/src/backend/meet/settings.py index bf2deaa5..8b9534ff 100755 --- a/src/backend/meet/settings.py +++ b/src/backend/meet/settings.py @@ -119,6 +119,25 @@ class Base(Configuration): }, } + # Media + AWS_S3_ENDPOINT_URL = values.Value( + environ_name="AWS_S3_ENDPOINT_URL", environ_prefix=None + ) + AWS_S3_ACCESS_KEY_ID = values.Value( + environ_name="AWS_S3_ACCESS_KEY_ID", environ_prefix=None + ) + AWS_S3_SECRET_ACCESS_KEY = values.Value( + environ_name="AWS_S3_SECRET_ACCESS_KEY", environ_prefix=None + ) + AWS_S3_REGION_NAME = values.Value( + environ_name="AWS_S3_REGION_NAME", environ_prefix=None + ) + AWS_STORAGE_BUCKET_NAME = values.Value( + "meet-media-storage", + environ_name="AWS_STORAGE_BUCKET_NAME", + environ_prefix=None, + ) + # Internationalization # https://docs.djangoproject.com/en/3.1/topics/i18n/ diff --git a/src/backend/meet/urls.py b/src/backend/meet/urls.py index 2bb20d06..d60fe47d 100644 --- a/src/backend/meet/urls.py +++ b/src/backend/meet/urls.py @@ -1,7 +1,6 @@ """URL configuration for the Meet project""" from django.conf import settings -from django.conf.urls.static import static from django.contrib import admin from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.urls import include, path, re_path @@ -18,11 +17,7 @@ ] if settings.DEBUG: - urlpatterns = ( - urlpatterns - + staticfiles_urlpatterns() - + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) - ) + urlpatterns = urlpatterns + staticfiles_urlpatterns() if settings.USE_SWAGGER or settings.DEBUG: diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index c71c5e78..62eb9a30 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -25,14 +25,14 @@ license = { file = "LICENSE" } readme = "README.md" requires-python = ">=3.10" dependencies = [ - "boto3==1.35.19", + "boto3==1.35.34", "Brotli==1.1.0", "celery[redis]==5.4.0", "django-configurations==2.5.1", "django-cors-headers==4.4.0", "django-countries==7.6.1", "django-parler==2.3", - "redis==5.0.8", + "redis==5.1.1", "django-redis==5.4.0", "django-storages[s3]==1.14.4", "django-timezone-field>=5.1", @@ -42,17 +42,16 @@ dependencies = [ "dockerflow==2024.4.2", "easy_thumbnails==2.10", "factory_boy==3.3.1", - "freezegun==1.5.1", "gunicorn==23.0.0", "jsonschema==4.23.0", "june-analytics-python==2.3.0", "markdown==3.7", "nested-multipart-parser==1.5.0", - "psycopg[binary]==3.2.2", + "psycopg[binary]==3.2.3", "PyJWT==2.9.0", "python-frontmatter==1.1.0", "requests==2.32.3", - "sentry-sdk==2.14.0", + "sentry-sdk==2.15.0", "url-normalize==1.4.3", "WeasyPrint>=60.2", "whitenoise==6.7.0", @@ -70,6 +69,7 @@ dependencies = [ dev = [ "django-extensions==3.2.3", "drf-spectacular-sidecar==2024.7.1", + "freezegun==1.5.1", "ipdb==0.13.13", "ipython==8.27.0", "pyfakefs==5.6.0", diff --git a/src/helm/env.d/dev/values.meet.yaml.gotmpl b/src/helm/env.d/dev/values.meet.yaml.gotmpl index ab10c245..87b41b2f 100644 --- a/src/helm/env.d/dev/values.meet.yaml.gotmpl +++ b/src/helm/env.d/dev/values.meet.yaml.gotmpl @@ -40,6 +40,10 @@ backend: POSTGRES_PASSWORD: pass REDIS_URL: redis://default:pass@redis-master:6379/1 STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage + AWS_S3_ENDPOINT_URL: http://minio.meet.svc.cluster.local:9000 + AWS_S3_ACCESS_KEY_ID: meet + AWS_S3_SECRET_ACCESS_KEY: password + AWS_STORAGE_BUCKET_NAME: meet-media-storage {{- with .Values.livekit.keys }} {{- range $key, $value := . }} LIVEKIT_API_SECRET: {{ $value }} @@ -98,10 +102,23 @@ ingressAdmin: enabled: true host: meet.127.0.0.1.nip.io +ingressMedia: + enabled: true + host: meet.127.0.0.1.nip.io + + annotations: + nginx.ingress.kubernetes.io/auth-url: https://meet.127.0.0.1.nip.io/api/v1.0/recordings/retrieve-auth/ + nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256" + nginx.ingress.kubernetes.io/upstream-vhost: minio.meet.svc.cluster.local:9000 + nginx.ingress.kubernetes.io/rewrite-target: /meet-media-storage/$1 + +serviceMedia: + host: minio.meet.svc.cluster.local + port: 9000 + posthog: ingress: enabled: false ingressAssets: - enabled: false - + enabled: false \ No newline at end of file diff --git a/src/helm/env.d/production/values.meet.yaml.gotmpl b/src/helm/env.d/production/values.meet.yaml.gotmpl index 8844ddbd..7aac462c 100644 --- a/src/helm/env.d/production/values.meet.yaml.gotmpl +++ b/src/helm/env.d/production/values.meet.yaml.gotmpl @@ -85,6 +85,23 @@ backend: name: redis.redis.libre.sh key: url STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage + AWS_S3_ENDPOINT_URL: + secretKeyRef: + name: meet-media-storage.bucket.libre.sh + key: url + AWS_S3_ACCESS_KEY_ID: + secretKeyRef: + name: meet-media-storage.bucket.libre.sh + key: accessKey + AWS_S3_SECRET_ACCESS_KEY: + secretKeyRef: + name: meet-media-storage.bucket.libre.sh + key: secretKey + AWS_STORAGE_BUCKET_NAME: + secretKeyRef: + name: meet-media-storage.bucket.libre.sh + key: bucket + AWS_S3_REGION_NAME: local LIVEKIT_API_SECRET: secretKeyRef: name: backend @@ -130,6 +147,24 @@ ingressAdmin: nginx.ingress.kubernetes.io/auth-signin: https://oauth2-proxy.beta.numerique.gouv.fr/oauth2/start nginx.ingress.kubernetes.io/auth-url: https://oauth2-proxy.beta.numerique.gouv.fr/oauth2/auth +ingressMedia: + enabled: true + host: visio.numerique.gouv.fr + + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256" + nginx.ingress.kubernetes.io/auth-url: https://visio.numerique.gouv.fr/api/v1.0/recordings/retrieve-auth/ + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/rewrite-target: /impress-production-media-storage/$1 + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/upstream-vhost: s3.hedy-lamarr.indiehosters.net + +serviceMedia: + host: s3.hedy-lamarr.indiehosters.net + port: 443 + posthog: ingress: enabled: true diff --git a/src/helm/env.d/staging/values.meet.yaml.gotmpl b/src/helm/env.d/staging/values.meet.yaml.gotmpl index fa090c39..e502dcb4 100644 --- a/src/helm/env.d/staging/values.meet.yaml.gotmpl +++ b/src/helm/env.d/staging/values.meet.yaml.gotmpl @@ -84,6 +84,23 @@ backend: name: redis.redis.libre.sh key: url STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage + AWS_S3_ENDPOINT_URL: + secretKeyRef: + name: meet-media-storage.bucket.libre.sh + key: url + AWS_S3_ACCESS_KEY_ID: + secretKeyRef: + name: meet-media-storage.bucket.libre.sh + key: accessKey + AWS_S3_SECRET_ACCESS_KEY: + secretKeyRef: + name: meet-media-storage.bucket.libre.sh + key: secretKey + AWS_STORAGE_BUCKET_NAME: + secretKeyRef: + name: meet-media-storage.bucket.libre.sh + key: bucket + AWS_S3_REGION_NAME: local LIVEKIT_API_SECRET: secretKeyRef: name: backend @@ -140,6 +157,24 @@ ingressAdmin: hosts: - {{ .Values.newDomain }} +ingressMedia: + enabled: true + host: meet-staging.beta.numerique.gouv.fr + + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256" + nginx.ingress.kubernetes.io/auth-url: https://meet-staging.beta.numerique.gouv.fr/api/v1.0/recordings/retrieve-auth/ + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/rewrite-target: /meet-staging-media-storage/$1 + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/upstream-vhost: s3.margaret-hamilton.indiehosters.net + +serviceMedia: + host: s3.margaret-hamilton.indiehosters.net + port: 443 + posthog: ingress: enabled: true diff --git a/src/helm/helmfile.yaml b/src/helm/helmfile.yaml index d6d08ca8..e688d5aa 100644 --- a/src/helm/helmfile.yaml +++ b/src/helm/helmfile.yaml @@ -45,6 +45,20 @@ releases: enabled: true autoGenerated: true + - name: minio + installed: {{ eq .Environment.Name "dev" | toYaml }} + namespace: {{ .Namespace }} + chart: bitnami/minio + version: 12.10.10 + values: + - auth: + rootUser: meet + rootPassword: password + - provisioning: + enabled: true + buckets: + - name: meet-media-storage + - name: redis installed: {{ eq .Environment.Name "dev" | toYaml }} namespace: {{ .Namespace }} diff --git a/src/helm/meet/templates/ingress_media.yaml b/src/helm/meet/templates/ingress_media.yaml new file mode 100644 index 00000000..73a11be9 --- /dev/null +++ b/src/helm/meet/templates/ingress_media.yaml @@ -0,0 +1,83 @@ +{{- if .Values.ingressMedia.enabled -}} +{{- $fullName := include "meet.fullname" . -}} +{{- if and .Values.ingressMedia.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingressMedia.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingressMedia.annotations "kubernetes.io/ingress.class" .Values.ingressMedia.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }}-media + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "meet.labels" . | nindent 4 }} + {{- with .Values.ingressMedia.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingressMedia.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingressMedia.className }} + {{- end }} + {{- if .Values.ingressMedia.tls.enabled }} + tls: + {{- if .Values.ingressMedia.host }} + - secretName: {{ $fullName }}-tls + hosts: + - {{ .Values.ingressMedia.host | quote }} + {{- end }} + {{- range .Values.ingressMedia.tls.additional }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.ingressMedia.host }} + - host: {{ .Values.ingressMedia.host | quote }} + http: + paths: + - path: {{ .Values.ingressMedia.path | quote }} + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }}-media + port: + number: {{ .Values.serviceMedia.port }} + {{- else }} + serviceName: {{ $fullName }}-media + servicePort: {{ .Values.serviceMedia.port }} + {{- end }} + {{- end }} + {{- range .Values.ingressMedia.hosts }} + - host: {{ . | quote }} + http: + paths: + - path: {{ $.Values.ingressMedia.path | quote }} + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }}-media + port: + number: {{ .Values.serviceMedia.port }} + {{- else }} + serviceName: {{ $fullName }}-media + servicePort: {{ .Values.serviceMedia.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/src/helm/meet/templates/media_svc.yaml b/src/helm/meet/templates/media_svc.yaml new file mode 100644 index 00000000..d1d3b843 --- /dev/null +++ b/src/helm/meet/templates/media_svc.yaml @@ -0,0 +1,14 @@ +{{- $fullName := include "meet.fullname" . -}} +{{- $component := "media" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $fullName }}-media + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "meet.common.labels" (list . $component) | nindent 4 }} + annotations: + {{- toYaml $.Values.serviceMedia.annotations | nindent 4 }} +spec: + type: ExternalName + externalName: {{ $.Values.serviceMedia.host }} diff --git a/src/helm/meet/values.yaml b/src/helm/meet/values.yaml index 5d077c1c..99049503 100644 --- a/src/helm/meet/values.yaml +++ b/src/helm/meet/values.yaml @@ -69,6 +69,36 @@ ingressAdmin: enabled: true additional: [] +## @param ingressMedia.enabled whether to enable the Ingress or not +## @param ingressMedia.className IngressClass to use for the Ingress +## @param ingressMedia.host Host for the Ingress +## @param ingressMedia.path Path to use for the Ingress +ingressMedia: + enabled: false + className: null + host: meet.example.com + path: /media/(.*) + ## @param ingressMedia.hosts Additional host to configure for the Ingress + hosts: [ ] + # - chart-example.local + ## @param ingressMedia.tls.enabled Wether to enable TLS for the Ingress + ## @skip ingressMedia.tls.additional + ## @extra ingressMedia.tls.additional[].secretName Secret name for additional TLS config + ## @extra ingressMedia.tls.additional[].hosts[] Hosts for additional TLS config + tls: + enabled: true + additional: [] + + annotations: + nginx.ingress.kubernetes.io/auth-url: https://impress.example.com/api/v1.0/documents/retrieve-auth/ + nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256" + nginx.ingress.kubernetes.io/upstream-vhost: minio.impress.svc.cluster.local:9000 + +serviceMedia: + host: minio.impress.svc.cluster.local + port: 9000 + annotations: {} + ## @section backend