Merge pull request #97 from knmlprz/Endpoint_for_documents_and_chunks…

…_CRUD

Creating endpoint with CRUD for documents and chunks

.env.example

@@ -3,5 +3,5 @@ SECRET_KEY="SECRET KEY"
 POSTGRES_DB="postgres"
 POSTGRES_USER="postgres"
 POSTGRES_PASSWORD="postgres"
-POSTGRES_HOST="localhost"
+POSTGRES_HOST="db"
 POSTGRES_PORT=5432

api/Dockerfile

@@ -23,6 +23,8 @@ COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 
 COPY . ./
 
-RUN manage.py collectstatic
+RUN pip install django_stubs_ext
+
+RUN python manage.py collectstatic
 
 ENTRYPOINT ["uvicorn", "api.asgi:application", "--host", "0.0.0.0"]

api/api/api.py

@@ -0,0 +1,11 @@
+from ninja import NinjaAPI
+from django.urls import path
+
+api = NinjaAPI()
+
+api.add_router("/", "documents.views.router")
+api.add_router("/", "chunks.views.router")
+
+urlpatterns = [
+    path("api/", api.urls),
+]

api/api/settings.py

@@ -1,18 +1,18 @@
 from pathlib import Path
 import os
-import environ
+import django_stubs_ext
 
-env = environ.Env()
+django_stubs_ext.monkeypatch()
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
-BASE_DIR = Path(__file__).resolve().parent.parent
+BASE_DIR = Path(__file__).resolve().parent
 
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = env("SECRET_KEY")
+SECRET_KEY = os.environ.get("SECRET_KEY")
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
@@ -29,7 +29,8 @@
     "django.contrib.sessions",
     "django.contrib.messages",
     "django.contrib.staticfiles",
-    "documents",
+    "chunks.apps.ChunksConfig",
+    "documents.apps.DocumentsConfig",
 ]
 
 MIDDLEWARE = [
@@ -69,12 +70,12 @@
 DATABASES = {
     "default": {
         "ENGINE": "django.db.backends.postgresql",
-        "NAME": env("POSTGRES_DB"),
-        "USER": env("POSTGRES_USER"),
-        "PASSWORD": env("POSTGRES_PASSWORD"),
-        "HOST": env("POSTGRES_HOST"),
-        "PORT": env("POSTGRES_PORT"),
-    }
+        "NAME": os.environ.get("POSTGRES_DB"),
+        "USER": os.environ.get("POSTGRES_USER"),
+        "PASSWORD": os.environ.get("POSTGRES_PASSWORD"),
+        "HOST": os.environ.get("POSTGRES_HOST"),
+        "PORT": os.environ.get("POSTGRES_PORT"),
+    },
 }
 
 

api/api/urls.py

@@ -14,12 +14,13 @@
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
+
 from django.contrib import admin
 from django.conf import settings
 from django.urls import path, include
 from django.conf.urls.static import static
 
 urlpatterns = [
     path("admin/", admin.site.urls),
-    path("documents/", include("documents.urls")),
+    path("", include("api.api")),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

api/chunks/admin.py

@@ -0,0 +1,4 @@
+from django.contrib import admin
+from .models import Chunk
+
+admin.site.register(Chunk)

api/chunks/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ChunksConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "chunks"

api/chunks/controllers.py

@@ -0,0 +1,34 @@
+from http import HTTPStatus
+
+from chunks.models import Chunk
+from chunks.schemas import ChunkIn, ChunkOut
+
+
+def create_chunk_controller(payload: ChunkIn) -> tuple[HTTPStatus, ChunkOut]:
+    chunk = Chunk(**payload.dict())
+    chunk.full_clean()
+    chunk.save()
+    return HTTPStatus.CREATED, chunk
+
+
+def list_chunks_controller() -> list[ChunkOut]:
+    return Chunk.objects.all()
+
+
+def retrieve_chunk_controller(id: int) -> ChunkOut:
+    chunk = Chunk.objects.get(id=id)
+    return chunk
+
+
+def update_chunk_controller(payload: ChunkIn, id: int) -> ChunkOut:
+    chunk = Chunk.objects.get(id=id)
+    for attr, value in payload.dict().items():
+        setattr(chunk, attr, value)
+    chunk.full_clean()
+    chunk.save()
+    return chunk
+
+
+def delete_chunk_controller(id: int) -> HTTPStatus:
+    Chunk.objects.get(id=id).delete()
+    return HTTPStatus.OK

api/chunks/migrations/0001_initial.py

@@ -0,0 +1,32 @@
+# Generated by Django 5.0.2 on 2024-02-25 15:13
+
+import pgvector.django
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Chunk",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("text", models.CharField(max_length=100)),
+                ("embedding", pgvector.django.VectorField(dimensions=10)),
+                ("chunk_idx", models.IntegerField()),
+                ("start_char", models.IntegerField()),
+                ("end_char", models.IntegerField()),
+            ],
+        ),
+    ]

api/chunks/models.py

@@ -0,0 +1,10 @@
+from django.db import models
+from pgvector.django import VectorField
+
+
+class Chunk(models.Model):
+    text = models.CharField[str, str](max_length=100)
+    embedding = VectorField[list[float], VectorField](dimensions=10)
+    chunk_idx = models.IntegerField[int, int]()
+    start_char = models.IntegerField[int, int]()
+    end_char = models.IntegerField[int, int]()

api/chunks/schemas.py

@@ -0,0 +1,18 @@
+from ninja import Schema
+
+
+class ChunkIn(Schema):
+    text: str
+    embedding: list[float]
+    chunk_idx: int
+    start_char: int
+    end_char: int
+
+
+class ChunkOut(Schema):
+    id: int
+    text: str
+    embedding: list[float]
+    chunk_idx: int
+    start_char: int
+    end_char: int

api/chunks/tests.py

@@ -0,0 +1,51 @@
+import pytest
+from django.test import TestCase
+from .models import Chunk
+
+
+def create_chunk():
+    return Chunk.objects.create(
+        text="example",
+        embedding=list(range(1, 11)),
+        chunk_idx=1,
+        start_char=1,
+        end_char=2,
+    )
+
+
+class ChunkModelTests(TestCase):
+    @pytest.mark.django_db
+    def test_get_method(self):
+        response = self.client.get("/api/chunk/")
+        assert (
+            response.status_code == 200
+        ), "The request could not be received, status code should be 200"
+
+    @pytest.mark.django_db
+    def test_post_method(self):
+        create_chunk()
+        assert Chunk.objects.filter(
+            text="example"
+        ).exists(), "Can't create chunk object"
+
+    @pytest.mark.django_db
+    def test_put_method(self):
+        create_chunk()
+        chunk = Chunk.objects.get(text="example")
+        assert Chunk.objects.filter(
+            text="example"
+        ).exists(), "Can't create chunk object"
+        chunk.text = "notexample"
+        chunk.save()
+        assert all(
+            a == b for a, b in zip(chunk.embedding, range(1, 11))
+        ), "Can't update chunk object"
+
+    @pytest.mark.django_db
+    def test_delete_method(self):
+        create_chunk()
+        chunk = Chunk.objects.get(text="example")
+        chunk.delete()
+        assert not Chunk.objects.filter(
+            text="example"
+        ).exists(), "Can't delete chunk object"

api/chunks/views.py

@@ -0,0 +1,42 @@
+from http import HTTPStatus
+from django.http import HttpRequest
+
+from chunks.schemas import ChunkIn, ChunkOut
+from chunks.controllers import (
+    create_chunk_controller,
+    list_chunks_controller,
+    retrieve_chunk_controller,
+    update_chunk_controller,
+    delete_chunk_controller,
+)
+
+from ninja import Router
+from ninja.pagination import LimitOffsetPagination, paginate
+
+router = Router(tags=["Chunks"])
+
+
+@router.post("/chunk/", response={HTTPStatus.CREATED: ChunkOut})
+def create_chunk(request: HttpRequest, payload: ChunkIn):
+    return create_chunk_controller(payload)
+
+
+@router.get("/chunk/", response={HTTPStatus.OK: list[ChunkOut]})
+@paginate(LimitOffsetPagination)
+def list_chunks(request: HttpRequest):
+    return list_chunks_controller()
+
+
+@router.get("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut})
+def retrieve_chunk(request: HttpRequest, id: int):
+    return retrieve_chunk_controller(id)
+
+
+@router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut})
+def update_chunk(request: HttpRequest, data: ChunkIn, id: int):
+    return update_chunk_controller(data, id)
+
+
+@router.delete("/chunk/{id}/", response={HTTPStatus.OK: None})
+def delete_chunk(request: HttpRequest, id: int):
+    return delete_chunk_controller(id)

api/documents/admin.py

@@ -1,5 +1,4 @@
 from django.contrib import admin
-from .models import Document, Chunk
+from .models import Document
 
-admin.site.register(Chunk)
 admin.site.register(Document)

api/documents/apps.py

@@ -2,5 +2,5 @@
 
 
 class DocumentsConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'documents'
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "documents"

api/documents/controllers.py

@@ -0,0 +1,35 @@
+from http import HTTPStatus
+
+from documents.models import Document
+from documents.schemas import DocumentIn, DocumentOut
+
+
+def create_document_controller(payload: DocumentIn) -> tuple[HTTPStatus, DocumentOut]:
+    document = Document(**payload.dict())
+    document.full_clean()
+    document.save()
+    return HTTPStatus.CREATED, document
+
+
+def list_documents_controller() -> list[DocumentOut]:
+    return Document.objects.all()
+
+
+def retrieve_document_controller(id: int) -> DocumentOut:
+    document = Document.objects.get(id=id)
+    return document
+
+
+def update_document_controller(payload: DocumentIn, id: int) -> DocumentOut:
+    document = Document.objects.get(id=id)
+    for attr, value in payload.dict().items():
+        setattr(document, attr, value)
+    document.full_clean()
+    document.save()
+    return document
+
+
+def delete_document_controller(id: int) -> HTTPStatus:
+    document = Document.objects.get(id=id)
+    document.delete()
+    return HTTPStatus.OK