From dd41b75ac97c3520a59ba35af03a8b53c1ddcd6a Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Wed, 14 Feb 2024 23:44:46 +0100 Subject: [PATCH 01/18] Creating CRUD endpoint for documents and chunks I added file views.py that have declared types of response for client's CRUD request like( GET,POST,PUT,DEL) for chunks and documents, other files exclude .env.example have declared model shemas, url and routing. I think document's response for CRUD is incorect, because i didn't use foreignkey. --- .env.example | 4 +-- api/api/urls.py | 2 +- api/documents/api.py | 7 ++++ api/documents/schemas.py | 26 ++++++++++++++ api/documents/urls.py | 7 +++- api/documents/views.py | 75 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 api/documents/api.py create mode 100644 api/documents/schemas.py create mode 100644 api/documents/views.py diff --git a/.env.example b/.env.example index 75a62cf..0e0ae51 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,7 @@ TOKEN="DISCORD_TOKEN" SECRET_KEY="SECRET KEY" -POSTGRES_NAME="postgres" +POSTGRES_DB="postgres" POSTGRES_USER="postgres" POSTGRES_PASSWORD="postgres" -POSTGRES_HOST="localhost" +POSTGRES_HOST="db" POSTGRES_PORT=5432 diff --git a/api/api/urls.py b/api/api/urls.py index 8f8da57..1e52cf4 100644 --- a/api/api/urls.py +++ b/api/api/urls.py @@ -21,5 +21,5 @@ urlpatterns = [ path("admin/", admin.site.urls), - path("documents/", include("documents.urls")), + path("", include("documents.urls")), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/api/documents/api.py b/api/documents/api.py new file mode 100644 index 0000000..8dd6c02 --- /dev/null +++ b/api/documents/api.py @@ -0,0 +1,7 @@ +from ninja import NinjaAPI +from .views import router as documents_router + +api = NinjaAPI() + + +api.add_router("documents", documents_router, tags=["Documents"]) \ No newline at end of file diff --git a/api/documents/schemas.py b/api/documents/schemas.py new file mode 100644 index 0000000..7b29544 --- /dev/null +++ b/api/documents/schemas.py @@ -0,0 +1,26 @@ +from ninja import Schema +from typing import List + +class ChunkIn(Schema): + text: str + embedding: List[float] + chunk_idx: int + start_char: int + end_char: int + +class ChunkOut(Schema): + id: int + text: str + embedding: List[float] + chunk_idx: int + start_char: int + end_char: int + +class DocumentIn(Schema): + text: str + embedding: List[float] + +class DocumentOut(Schema): + id: int + text: str + embedding: List[float] diff --git a/api/documents/urls.py b/api/documents/urls.py index 637600f..dabd147 100644 --- a/api/documents/urls.py +++ b/api/documents/urls.py @@ -1 +1,6 @@ -urlpatterns = [] +from django.urls import path +from .api import api + +urlpatterns = [ + path("api/", api.urls), +] diff --git a/api/documents/views.py b/api/documents/views.py new file mode 100644 index 0000000..d35d198 --- /dev/null +++ b/api/documents/views.py @@ -0,0 +1,75 @@ +from http import HTTPStatus +from typing import List + +from django.http import HttpRequest +from ninja import Router +from ninja.pagination import LimitOffsetPagination, paginate + +from documents.models import Chunk, Document +from documents.schemas import ChunkIn, ChunkOut, DocumentIn, DocumentOut + +router = Router() + +@router.post("/chunk/", response={HTTPStatus.CREATED: ChunkOut}) +def create_chunk(request: HttpRequest, payload: ChunkIn): + chunk = Chunk(**payload.dict()) + chunk.full_clean() + chunk.save() + return HTTPStatus.CREATED, chunk + +@router.get("/chunk/", response={HTTPStatus.OK: List[ChunkOut]}) +@paginate(LimitOffsetPagination) +def list_chunks(request: HttpRequest): + return Chunk.objects.all() + +@router.get("/chunk/{id}", response={HTTPStatus.OK: ChunkOut}) +def retrieve_chunk(request: HttpRequest, id: int): + chunk = Chunk.objects.get(id=id) + return chunk + +@router.put("/chunk/{id}", response={HTTPStatus.OK: ChunkOut}) +def update_chunk(request: HttpRequest, id: int, payload: ChunkIn): + chunk = Chunk.objects.get(id=id) + for attr, value in payload.dict(exclude_unset=True).items(): + setattr(chunk, attr, value) + chunk.full_clean() + chunk.save() + return chunk + +@router.delete("/chunk/{id}", response={HTTPStatus.NO_CONTENT: None}) +def delete_chunk(request: HttpRequest, id: int): + chunk = Chunk.objects.get(id=id) + chunk.delete() + return HTTPStatus.NO_CONTENT, None + +@router.post("/document/", response={HTTPStatus.CREATED: DocumentOut}) +def create_document(request: HttpRequest, payload: DocumentIn): + document = Document(**payload.dict()) + document.full_clean() + document.save() + return HTTPStatus.CREATED, document + +@router.get("/document/", response={HTTPStatus.OK: List[DocumentOut]}) +@paginate(LimitOffsetPagination) +def list_documents(request: HttpRequest): + return Document.objects.all() + +@router.get("/document/{id}", response={HTTPStatus.OK: DocumentOut}) +def retrieve_document(request: HttpRequest, id: int): + document = Document.objects.get(id=id) + return document + +@router.put("/document/{id}", response={HTTPStatus.OK: DocumentOut}) +def update_document(request: HttpRequest, id: int, payload: DocumentIn): + document = Document.objects.get(id=id) + for attr, value in payload.dict(exclude_unset=True).items(): + setattr(document, attr, value) + document.full_clean() + document.save() + return document + +@router.delete("/document/{id}", response={HTTPStatus.NO_CONTENT: None}) +def delete_document(request: HttpRequest, id: int): + document = Document.objects.get(id=id) + document.delete() + return HTTPStatus.NO_CONTENT, None \ No newline at end of file From 91884fb891cb0b9ecbb10a79736e5580a040d780 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Thu, 15 Feb 2024 22:31:37 +0100 Subject: [PATCH 02/18] added second router and small changes --- api/documents/api.py | 5 ++--- api/documents/schemas.py | 9 ++++----- api/documents/views.py | 28 +++++++++++++++------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/api/documents/api.py b/api/documents/api.py index 8dd6c02..8665e1e 100644 --- a/api/documents/api.py +++ b/api/documents/api.py @@ -1,7 +1,6 @@ from ninja import NinjaAPI -from .views import router as documents_router api = NinjaAPI() - -api.add_router("documents", documents_router, tags=["Documents"]) \ No newline at end of file +api.add_router("/", "documents.views.document_router", tags=["Documents"]) +api.add_router("/", "documents.views.chunk_router", tags=["Chunks"]) diff --git a/api/documents/schemas.py b/api/documents/schemas.py index 7b29544..1a75c57 100644 --- a/api/documents/schemas.py +++ b/api/documents/schemas.py @@ -1,9 +1,8 @@ from ninja import Schema -from typing import List class ChunkIn(Schema): text: str - embedding: List[float] + embedding: list[float] chunk_idx: int start_char: int end_char: int @@ -11,16 +10,16 @@ class ChunkIn(Schema): class ChunkOut(Schema): id: int text: str - embedding: List[float] + embedding: list[float] chunk_idx: int start_char: int end_char: int class DocumentIn(Schema): text: str - embedding: List[float] + embedding: list[float] class DocumentOut(Schema): id: int text: str - embedding: List[float] + embedding: list[float] diff --git a/api/documents/views.py b/api/documents/views.py index d35d198..e5fdaf3 100644 --- a/api/documents/views.py +++ b/api/documents/views.py @@ -8,26 +8,27 @@ from documents.models import Chunk, Document from documents.schemas import ChunkIn, ChunkOut, DocumentIn, DocumentOut -router = Router() +chunk_router = Router() +document_router = Router() -@router.post("/chunk/", response={HTTPStatus.CREATED: ChunkOut}) +@chunk_router.post("/chunk/", response={HTTPStatus.CREATED: ChunkOut}) def create_chunk(request: HttpRequest, payload: ChunkIn): chunk = Chunk(**payload.dict()) chunk.full_clean() chunk.save() return HTTPStatus.CREATED, chunk -@router.get("/chunk/", response={HTTPStatus.OK: List[ChunkOut]}) +@chunk_router.get("/chunk/", response={HTTPStatus.OK: List[ChunkOut]}) @paginate(LimitOffsetPagination) def list_chunks(request: HttpRequest): return Chunk.objects.all() -@router.get("/chunk/{id}", response={HTTPStatus.OK: ChunkOut}) +@chunk_router.get("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) def retrieve_chunk(request: HttpRequest, id: int): chunk = Chunk.objects.get(id=id) return chunk -@router.put("/chunk/{id}", response={HTTPStatus.OK: ChunkOut}) +@chunk_router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) def update_chunk(request: HttpRequest, id: int, payload: ChunkIn): chunk = Chunk.objects.get(id=id) for attr, value in payload.dict(exclude_unset=True).items(): @@ -36,30 +37,30 @@ def update_chunk(request: HttpRequest, id: int, payload: ChunkIn): chunk.save() return chunk -@router.delete("/chunk/{id}", response={HTTPStatus.NO_CONTENT: None}) +@chunk_router.delete("/chunk/{id}/", response={HTTPStatus.OK: None}) def delete_chunk(request: HttpRequest, id: int): chunk = Chunk.objects.get(id=id) chunk.delete() - return HTTPStatus.NO_CONTENT, None + return HTTPStatus.OK -@router.post("/document/", response={HTTPStatus.CREATED: DocumentOut}) +@document_router.post("/document/", response={HTTPStatus.CREATED: DocumentOut}) def create_document(request: HttpRequest, payload: DocumentIn): document = Document(**payload.dict()) document.full_clean() document.save() return HTTPStatus.CREATED, document -@router.get("/document/", response={HTTPStatus.OK: List[DocumentOut]}) +@document_router.get("/document/", response={HTTPStatus.OK: List[DocumentOut]}) @paginate(LimitOffsetPagination) def list_documents(request: HttpRequest): return Document.objects.all() -@router.get("/document/{id}", response={HTTPStatus.OK: DocumentOut}) +@document_router.get("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) def retrieve_document(request: HttpRequest, id: int): document = Document.objects.get(id=id) return document -@router.put("/document/{id}", response={HTTPStatus.OK: DocumentOut}) +@document_router.put("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) def update_document(request: HttpRequest, id: int, payload: DocumentIn): document = Document.objects.get(id=id) for attr, value in payload.dict(exclude_unset=True).items(): @@ -68,8 +69,9 @@ def update_document(request: HttpRequest, id: int, payload: DocumentIn): document.save() return document -@router.delete("/document/{id}", response={HTTPStatus.NO_CONTENT: None}) +@document_router.delete("/document/{id}/", response={HTTPStatus.OK: None}) def delete_document(request: HttpRequest, id: int): document = Document.objects.get(id=id) document.delete() - return HTTPStatus.NO_CONTENT, None \ No newline at end of file + return HTTPStatus.OK + \ No newline at end of file From 07986bb65760cd595191534ecbe086e82c116601 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Sun, 18 Feb 2024 23:37:44 +0100 Subject: [PATCH 03/18] separating documents and chunks I additionally renamed folder containing models for code readability from documents into MyModels --- api/{documents => MyModels}/__init__.py | 0 api/{documents => MyModels}/admin.py | 0 api/MyModels/api.py | 6 +++ api/{documents => MyModels}/apps.py | 2 +- .../chunks}/__init__.py | 0 api/{documents => MyModels/chunks}/schemas.py | 9 ---- api/MyModels/chunks/views.py | 44 +++++++++++++++++++ api/MyModels/documents/__init__.py | 0 api/MyModels/documents/schemas.py | 10 +++++ api/{ => MyModels}/documents/views.py | 37 +--------------- .../migrations/0001_initial.py | 2 +- api/MyModels/migrations/__init__.py | 0 api/{documents => MyModels}/models.py | 0 api/{documents => MyModels}/tests.py | 0 api/{documents => MyModels}/urls.py | 2 +- api/api/settings.py | 2 +- api/api/urls.py | 2 +- api/documents/api.py | 6 --- 18 files changed, 67 insertions(+), 55 deletions(-) rename api/{documents => MyModels}/__init__.py (100%) rename api/{documents => MyModels}/admin.py (100%) create mode 100644 api/MyModels/api.py rename api/{documents => MyModels}/apps.py (84%) rename api/{documents/migrations => MyModels/chunks}/__init__.py (100%) rename api/{documents => MyModels/chunks}/schemas.py (65%) create mode 100644 api/MyModels/chunks/views.py create mode 100644 api/MyModels/documents/__init__.py create mode 100644 api/MyModels/documents/schemas.py rename api/{ => MyModels}/documents/views.py (53%) rename api/{documents => MyModels}/migrations/0001_initial.py (94%) create mode 100644 api/MyModels/migrations/__init__.py rename api/{documents => MyModels}/models.py (100%) rename api/{documents => MyModels}/tests.py (100%) rename api/{documents => MyModels}/urls.py (71%) delete mode 100644 api/documents/api.py diff --git a/api/documents/__init__.py b/api/MyModels/__init__.py similarity index 100% rename from api/documents/__init__.py rename to api/MyModels/__init__.py diff --git a/api/documents/admin.py b/api/MyModels/admin.py similarity index 100% rename from api/documents/admin.py rename to api/MyModels/admin.py diff --git a/api/MyModels/api.py b/api/MyModels/api.py new file mode 100644 index 0000000..5b3ae3c --- /dev/null +++ b/api/MyModels/api.py @@ -0,0 +1,6 @@ +from ninja import NinjaAPI + +api = NinjaAPI() + +api.add_router("/", "MyModels.documents.views.document_router", tags=["Documents"]) +api.add_router("/", "MyModels.chunks.views.chunk_router", tags=["Chunks"]) diff --git a/api/documents/apps.py b/api/MyModels/apps.py similarity index 84% rename from api/documents/apps.py rename to api/MyModels/apps.py index ab8a093..3554922 100644 --- a/api/documents/apps.py +++ b/api/MyModels/apps.py @@ -3,4 +3,4 @@ class DocumentsConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' - name = 'documents' + name = 'MyModels' diff --git a/api/documents/migrations/__init__.py b/api/MyModels/chunks/__init__.py similarity index 100% rename from api/documents/migrations/__init__.py rename to api/MyModels/chunks/__init__.py diff --git a/api/documents/schemas.py b/api/MyModels/chunks/schemas.py similarity index 65% rename from api/documents/schemas.py rename to api/MyModels/chunks/schemas.py index 1a75c57..a43a475 100644 --- a/api/documents/schemas.py +++ b/api/MyModels/chunks/schemas.py @@ -14,12 +14,3 @@ class ChunkOut(Schema): chunk_idx: int start_char: int end_char: int - -class DocumentIn(Schema): - text: str - embedding: list[float] - -class DocumentOut(Schema): - id: int - text: str - embedding: list[float] diff --git a/api/MyModels/chunks/views.py b/api/MyModels/chunks/views.py new file mode 100644 index 0000000..8e73b68 --- /dev/null +++ b/api/MyModels/chunks/views.py @@ -0,0 +1,44 @@ +from http import HTTPStatus +from typing import List + +from django.http import HttpRequest +from ninja import Router +from ninja.pagination import LimitOffsetPagination, paginate +from django.shortcuts import get_object_or_404 + +from MyModels.models import Chunk +from MyModels.chunks.schemas import ChunkIn, ChunkOut + +chunk_router = Router() + +@chunk_router.post("/chunk/", response={HTTPStatus.CREATED: ChunkOut}) +def create_chunk(request: HttpRequest, payload: ChunkIn): + chunk = Chunk(**payload.dict()) + chunk.full_clean() + chunk.save() + return HTTPStatus.CREATED, chunk + +@chunk_router.get("/chunk/", response={HTTPStatus.OK: List[ChunkOut]}) +@paginate(LimitOffsetPagination) +def list_chunks(request: HttpRequest): + return Chunk.objects.all() + +@chunk_router.get("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) +def retrieve_chunk(request: HttpRequest, id: int): + chunk = Chunk.objects.get(id=id) + return chunk + +@chunk_router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) +def update_chunk(request: HttpRequest, id: int, payload: ChunkIn): + chunk = get_object_or_404(Chunk, id=id) + for attr, value in payload.dict(exclude_unset=True).items(): + setattr(chunk, attr, value) + chunk.full_clean() + chunk.save() + return chunk + +@chunk_router.delete("/chunk/{id}/", response={HTTPStatus.OK: None}) +def delete_chunk(request: HttpRequest, id: int): + chunk = Chunk.objects.get(id=id) + chunk.delete() + return HTTPStatus.OK diff --git a/api/MyModels/documents/__init__.py b/api/MyModels/documents/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/MyModels/documents/schemas.py b/api/MyModels/documents/schemas.py new file mode 100644 index 0000000..af23725 --- /dev/null +++ b/api/MyModels/documents/schemas.py @@ -0,0 +1,10 @@ +from ninja import Schema + +class DocumentIn(Schema): + text: str + embedding: list[float] + +class DocumentOut(Schema): + id: int + text: str + embedding: list[float] diff --git a/api/documents/views.py b/api/MyModels/documents/views.py similarity index 53% rename from api/documents/views.py rename to api/MyModels/documents/views.py index e5fdaf3..0f1fb03 100644 --- a/api/documents/views.py +++ b/api/MyModels/documents/views.py @@ -5,44 +5,11 @@ from ninja import Router from ninja.pagination import LimitOffsetPagination, paginate -from documents.models import Chunk, Document -from documents.schemas import ChunkIn, ChunkOut, DocumentIn, DocumentOut +from MyModels.models import Document +from MyModels.documents.schemas import DocumentIn, DocumentOut -chunk_router = Router() document_router = Router() -@chunk_router.post("/chunk/", response={HTTPStatus.CREATED: ChunkOut}) -def create_chunk(request: HttpRequest, payload: ChunkIn): - chunk = Chunk(**payload.dict()) - chunk.full_clean() - chunk.save() - return HTTPStatus.CREATED, chunk - -@chunk_router.get("/chunk/", response={HTTPStatus.OK: List[ChunkOut]}) -@paginate(LimitOffsetPagination) -def list_chunks(request: HttpRequest): - return Chunk.objects.all() - -@chunk_router.get("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) -def retrieve_chunk(request: HttpRequest, id: int): - chunk = Chunk.objects.get(id=id) - return chunk - -@chunk_router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) -def update_chunk(request: HttpRequest, id: int, payload: ChunkIn): - chunk = Chunk.objects.get(id=id) - for attr, value in payload.dict(exclude_unset=True).items(): - setattr(chunk, attr, value) - chunk.full_clean() - chunk.save() - return chunk - -@chunk_router.delete("/chunk/{id}/", response={HTTPStatus.OK: None}) -def delete_chunk(request: HttpRequest, id: int): - chunk = Chunk.objects.get(id=id) - chunk.delete() - return HTTPStatus.OK - @document_router.post("/document/", response={HTTPStatus.CREATED: DocumentOut}) def create_document(request: HttpRequest, payload: DocumentIn): document = Document(**payload.dict()) diff --git a/api/documents/migrations/0001_initial.py b/api/MyModels/migrations/0001_initial.py similarity index 94% rename from api/documents/migrations/0001_initial.py rename to api/MyModels/migrations/0001_initial.py index 0896ad5..0af4ed3 100644 --- a/api/documents/migrations/0001_initial.py +++ b/api/MyModels/migrations/0001_initial.py @@ -30,7 +30,7 @@ class Migration(migrations.Migration): ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('text', models.TextField()), ('embedding', pgvector.django.VectorField(dimensions=10)), - ('chunks', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='documents.chunk')), + ('chunks', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='MyModels.chunk')), ], ), ] diff --git a/api/MyModels/migrations/__init__.py b/api/MyModels/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/documents/models.py b/api/MyModels/models.py similarity index 100% rename from api/documents/models.py rename to api/MyModels/models.py diff --git a/api/documents/tests.py b/api/MyModels/tests.py similarity index 100% rename from api/documents/tests.py rename to api/MyModels/tests.py diff --git a/api/documents/urls.py b/api/MyModels/urls.py similarity index 71% rename from api/documents/urls.py rename to api/MyModels/urls.py index dabd147..6a2855b 100644 --- a/api/documents/urls.py +++ b/api/MyModels/urls.py @@ -2,5 +2,5 @@ from .api import api urlpatterns = [ - path("api/", api.urls), + path("", api.urls), ] diff --git a/api/api/settings.py b/api/api/settings.py index 69343f9..f7f8866 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -29,7 +29,7 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", - "documents", + "MyModels.apps.DocumentsConfig", ] MIDDLEWARE = [ diff --git a/api/api/urls.py b/api/api/urls.py index 1e52cf4..e702596 100644 --- a/api/api/urls.py +++ b/api/api/urls.py @@ -21,5 +21,5 @@ urlpatterns = [ path("admin/", admin.site.urls), - path("", include("documents.urls")), + path("api/", include("MyModels.urls")), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/api/documents/api.py b/api/documents/api.py deleted file mode 100644 index 8665e1e..0000000 --- a/api/documents/api.py +++ /dev/null @@ -1,6 +0,0 @@ -from ninja import NinjaAPI - -api = NinjaAPI() - -api.add_router("/", "documents.views.document_router", tags=["Documents"]) -api.add_router("/", "documents.views.chunk_router", tags=["Chunks"]) From d09b09e6f27a5712c0dac8eb29ea45a9f5893972 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Sun, 18 Feb 2024 23:44:59 +0100 Subject: [PATCH 04/18] changing document get type --- api/MyModels/documents/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/MyModels/documents/views.py b/api/MyModels/documents/views.py index 0f1fb03..7b879c3 100644 --- a/api/MyModels/documents/views.py +++ b/api/MyModels/documents/views.py @@ -4,6 +4,7 @@ from django.http import HttpRequest from ninja import Router from ninja.pagination import LimitOffsetPagination, paginate +from django.shortcuts import get_object_or_404 from MyModels.models import Document from MyModels.documents.schemas import DocumentIn, DocumentOut @@ -29,7 +30,7 @@ def retrieve_document(request: HttpRequest, id: int): @document_router.put("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) def update_document(request: HttpRequest, id: int, payload: DocumentIn): - document = Document.objects.get(id=id) + document = get_object_or_404(Document, id=id) for attr, value in payload.dict(exclude_unset=True).items(): setattr(document, attr, value) document.full_clean() From f2f95cd9fb1cbe58bd02005d8862fff009fd989f Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Mon, 19 Feb 2024 14:28:27 +0100 Subject: [PATCH 05/18] PUT method changes and minor adjustments In new version of PUT method, client can update objects declaring in request only variables that he want change (no all variables needed). --- api/MyModels/chunks/views.py | 10 ++++++---- api/MyModels/documents/views.py | 8 +++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/api/MyModels/chunks/views.py b/api/MyModels/chunks/views.py index 8e73b68..811bf15 100644 --- a/api/MyModels/chunks/views.py +++ b/api/MyModels/chunks/views.py @@ -8,6 +8,7 @@ from MyModels.models import Chunk from MyModels.chunks.schemas import ChunkIn, ChunkOut +import json chunk_router = Router() @@ -25,13 +26,14 @@ def list_chunks(request: HttpRequest): @chunk_router.get("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) def retrieve_chunk(request: HttpRequest, id: int): - chunk = Chunk.objects.get(id=id) + chunk = Chunk.objects.get(id=id) return chunk @chunk_router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) -def update_chunk(request: HttpRequest, id: int, payload: ChunkIn): - chunk = get_object_or_404(Chunk, id=id) - for attr, value in payload.dict(exclude_unset=True).items(): +def update_chunk(request: HttpRequest, id: int): + chunk = Chunk.objects.get(id=id) + request_data = json.loads(request.body.decode("utf-8")) + for attr, value in request_data.items(): setattr(chunk, attr, value) chunk.full_clean() chunk.save() diff --git a/api/MyModels/documents/views.py b/api/MyModels/documents/views.py index 7b879c3..c47b286 100644 --- a/api/MyModels/documents/views.py +++ b/api/MyModels/documents/views.py @@ -8,6 +8,7 @@ from MyModels.models import Document from MyModels.documents.schemas import DocumentIn, DocumentOut +import json document_router = Router() @@ -29,9 +30,10 @@ def retrieve_document(request: HttpRequest, id: int): return document @document_router.put("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) -def update_document(request: HttpRequest, id: int, payload: DocumentIn): - document = get_object_or_404(Document, id=id) - for attr, value in payload.dict(exclude_unset=True).items(): +def update_document(request: HttpRequest, id: int): + document = Document.objects.get(id=id) + request_data = json.loads(request.body.decode("utf-8")) + for attr, value in request_data.items(): setattr(document, attr, value) document.full_clean() document.save() From 0ac6cdb1b8e8f4e56652c21442a5e2b7b147cfe0 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Fri, 23 Feb 2024 21:33:22 +0100 Subject: [PATCH 06/18] separating models into apps I separated models creating similar chunks folder to documents folder and connect them using routers in api.api file --- api/MyModels/api.py | 6 --- api/MyModels/migrations/0001_initial.py | 36 ---------------- api/MyModels/urls.py | 6 --- api/api/api.py | 11 +++++ api/api/settings.py | 4 +- api/api/urls.py | 2 +- api/{MyModels => chunks}/__init__.py | 0 api/chunks/admin.py | 4 ++ api/chunks/apps.py | 6 +++ api/chunks/migrations/0001_initial.py | 32 +++++++++++++++ .../chunks => chunks/migrations}/__init__.py | 0 api/{MyModels => chunks}/models.py | 7 ---- api/{MyModels => }/chunks/schemas.py | 1 + api/{MyModels => chunks}/tests.py | 0 api/{MyModels => }/chunks/views.py | 15 +++---- api/{MyModels => }/documents/__init__.py | 0 api/{MyModels => documents}/admin.py | 3 +- api/{MyModels => documents}/apps.py | 2 +- api/documents/migrations/0001_initial.py | 41 +++++++++++++++++++ .../migrations/__init__.py | 0 api/documents/models.py | 8 ++++ api/{MyModels => }/documents/schemas.py | 0 api/documents/tests.py | 3 ++ api/{MyModels => }/documents/views.py | 10 +++-- 24 files changed, 126 insertions(+), 71 deletions(-) delete mode 100644 api/MyModels/api.py delete mode 100644 api/MyModels/migrations/0001_initial.py delete mode 100644 api/MyModels/urls.py create mode 100644 api/api/api.py rename api/{MyModels => chunks}/__init__.py (100%) create mode 100644 api/chunks/admin.py create mode 100644 api/chunks/apps.py create mode 100644 api/chunks/migrations/0001_initial.py rename api/{MyModels/chunks => chunks/migrations}/__init__.py (100%) rename api/{MyModels => chunks}/models.py (60%) rename api/{MyModels => }/chunks/schemas.py (98%) rename api/{MyModels => chunks}/tests.py (100%) rename api/{MyModels => }/chunks/views.py (78%) rename api/{MyModels => }/documents/__init__.py (100%) rename api/{MyModels => documents}/admin.py (50%) rename api/{MyModels => documents}/apps.py (84%) create mode 100644 api/documents/migrations/0001_initial.py rename api/{MyModels => documents}/migrations/__init__.py (100%) create mode 100644 api/documents/models.py rename api/{MyModels => }/documents/schemas.py (100%) create mode 100644 api/documents/tests.py rename api/{MyModels => }/documents/views.py (83%) diff --git a/api/MyModels/api.py b/api/MyModels/api.py deleted file mode 100644 index 5b3ae3c..0000000 --- a/api/MyModels/api.py +++ /dev/null @@ -1,6 +0,0 @@ -from ninja import NinjaAPI - -api = NinjaAPI() - -api.add_router("/", "MyModels.documents.views.document_router", tags=["Documents"]) -api.add_router("/", "MyModels.chunks.views.chunk_router", tags=["Chunks"]) diff --git a/api/MyModels/migrations/0001_initial.py b/api/MyModels/migrations/0001_initial.py deleted file mode 100644 index 0af4ed3..0000000 --- a/api/MyModels/migrations/0001_initial.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-12 20:31 - -from django.db import migrations, models -import django.db.models.deletion -import pgvector.django - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Chunk', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('text', models.CharField(max_length=100)), - ('embedding', pgvector.django.VectorField(dimensions=10)), - ('chunk_idx', models.IntegerField()), - ('start_char', models.IntegerField()), - ('end_char', models.IntegerField()), - ], - ), - migrations.CreateModel( - name='Document', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('text', models.TextField()), - ('embedding', pgvector.django.VectorField(dimensions=10)), - ('chunks', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='MyModels.chunk')), - ], - ), - ] diff --git a/api/MyModels/urls.py b/api/MyModels/urls.py deleted file mode 100644 index 6a2855b..0000000 --- a/api/MyModels/urls.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.urls import path -from .api import api - -urlpatterns = [ - path("", api.urls), -] diff --git a/api/api/api.py b/api/api/api.py new file mode 100644 index 0000000..43f1672 --- /dev/null +++ b/api/api/api.py @@ -0,0 +1,11 @@ +from ninja import NinjaAPI +from django.urls import path + +api = NinjaAPI() + +api.add_router("/", "documents.views.document_router", tags=["Documents"]) +api.add_router("/", "chunks.views.chunk_router", tags=["Chunks"]) + +urlpatterns = [ + path("api/", api.urls), +] \ No newline at end of file diff --git a/api/api/settings.py b/api/api/settings.py index f7f8866..e452613 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -29,7 +29,9 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", - "MyModels.apps.DocumentsConfig", + "chunks.apps.ChunksConfig", + "documents.apps.DocumentsConfig", + ] MIDDLEWARE = [ diff --git a/api/api/urls.py b/api/api/urls.py index e702596..ef4fbe1 100644 --- a/api/api/urls.py +++ b/api/api/urls.py @@ -21,5 +21,5 @@ urlpatterns = [ path("admin/", admin.site.urls), - path("api/", include("MyModels.urls")), + path("", include("api.api")), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/api/MyModels/__init__.py b/api/chunks/__init__.py similarity index 100% rename from api/MyModels/__init__.py rename to api/chunks/__init__.py diff --git a/api/chunks/admin.py b/api/chunks/admin.py new file mode 100644 index 0000000..d82e7f1 --- /dev/null +++ b/api/chunks/admin.py @@ -0,0 +1,4 @@ +from django.contrib import admin +from .models import Chunk + +admin.site.register(Chunk) diff --git a/api/chunks/apps.py b/api/chunks/apps.py new file mode 100644 index 0000000..58457d7 --- /dev/null +++ b/api/chunks/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ChunksConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "chunks" diff --git a/api/chunks/migrations/0001_initial.py b/api/chunks/migrations/0001_initial.py new file mode 100644 index 0000000..8030279 --- /dev/null +++ b/api/chunks/migrations/0001_initial.py @@ -0,0 +1,32 @@ +# Generated by Django 5.0.2 on 2024-02-21 23:59 + +import pgvector.django +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Chunk", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("text", models.CharField(max_length=100)), + ("embedding", pgvector.django.VectorField(dimensions=10)), + ("chunk_idx", models.IntegerField()), + ("start_char", models.IntegerField()), + ("end_char", models.IntegerField()), + ], + ), + ] diff --git a/api/MyModels/chunks/__init__.py b/api/chunks/migrations/__init__.py similarity index 100% rename from api/MyModels/chunks/__init__.py rename to api/chunks/migrations/__init__.py diff --git a/api/MyModels/models.py b/api/chunks/models.py similarity index 60% rename from api/MyModels/models.py rename to api/chunks/models.py index a272a18..68c309a 100644 --- a/api/MyModels/models.py +++ b/api/chunks/models.py @@ -1,16 +1,9 @@ from django.db import models from pgvector.django import VectorField - class Chunk(models.Model): text = models.CharField(max_length=100) embedding = VectorField(dimensions=10) chunk_idx = models.IntegerField() start_char = models.IntegerField() end_char = models.IntegerField() - - -class Document(models.Model): - text = models.TextField() - embedding = VectorField(dimensions=10) - chunks = models.ForeignKey(Chunk, on_delete=models.CASCADE, null=True, blank=True) diff --git a/api/MyModels/chunks/schemas.py b/api/chunks/schemas.py similarity index 98% rename from api/MyModels/chunks/schemas.py rename to api/chunks/schemas.py index a43a475..02e8478 100644 --- a/api/MyModels/chunks/schemas.py +++ b/api/chunks/schemas.py @@ -14,3 +14,4 @@ class ChunkOut(Schema): chunk_idx: int start_char: int end_char: int + \ No newline at end of file diff --git a/api/MyModels/tests.py b/api/chunks/tests.py similarity index 100% rename from api/MyModels/tests.py rename to api/chunks/tests.py diff --git a/api/MyModels/chunks/views.py b/api/chunks/views.py similarity index 78% rename from api/MyModels/chunks/views.py rename to api/chunks/views.py index 8e73b68..d240fa6 100644 --- a/api/MyModels/chunks/views.py +++ b/api/chunks/views.py @@ -4,10 +4,10 @@ from django.http import HttpRequest from ninja import Router from ninja.pagination import LimitOffsetPagination, paginate -from django.shortcuts import get_object_or_404 -from MyModels.models import Chunk -from MyModels.chunks.schemas import ChunkIn, ChunkOut +from chunks.models import Chunk +from chunks.schemas import ChunkIn, ChunkOut +import json chunk_router = Router() @@ -23,15 +23,16 @@ def create_chunk(request: HttpRequest, payload: ChunkIn): def list_chunks(request: HttpRequest): return Chunk.objects.all() -@chunk_router.get("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) +@chunk_router.get("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) def retrieve_chunk(request: HttpRequest, id: int): chunk = Chunk.objects.get(id=id) return chunk @chunk_router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) -def update_chunk(request: HttpRequest, id: int, payload: ChunkIn): - chunk = get_object_or_404(Chunk, id=id) - for attr, value in payload.dict(exclude_unset=True).items(): +def update_chunk(request: HttpRequest, id: int): + chunk = Chunk.objects.get(id=id) + request_data = json.loads(request.body.decode("utf-8")) + for attr, value in request_data.items(): setattr(chunk, attr, value) chunk.full_clean() chunk.save() diff --git a/api/MyModels/documents/__init__.py b/api/documents/__init__.py similarity index 100% rename from api/MyModels/documents/__init__.py rename to api/documents/__init__.py diff --git a/api/MyModels/admin.py b/api/documents/admin.py similarity index 50% rename from api/MyModels/admin.py rename to api/documents/admin.py index 8914ed5..926b7ef 100644 --- a/api/MyModels/admin.py +++ b/api/documents/admin.py @@ -1,5 +1,4 @@ from django.contrib import admin -from .models import Document, Chunk +from .models import Document -admin.site.register(Chunk) admin.site.register(Document) diff --git a/api/MyModels/apps.py b/api/documents/apps.py similarity index 84% rename from api/MyModels/apps.py rename to api/documents/apps.py index 3554922..ab8a093 100644 --- a/api/MyModels/apps.py +++ b/api/documents/apps.py @@ -3,4 +3,4 @@ class DocumentsConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' - name = 'MyModels' + name = 'documents' diff --git a/api/documents/migrations/0001_initial.py b/api/documents/migrations/0001_initial.py new file mode 100644 index 0000000..47ccfb3 --- /dev/null +++ b/api/documents/migrations/0001_initial.py @@ -0,0 +1,41 @@ +# Generated by Django 5.0.2 on 2024-02-21 23:59 + +import django.db.models.deletion +import pgvector.django +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + ("chunks", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="Document", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("text", models.TextField()), + ("embedding", pgvector.django.VectorField(dimensions=10)), + ( + "chunks", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="chunks.chunk", + ), + ), + ], + ), + ] diff --git a/api/MyModels/migrations/__init__.py b/api/documents/migrations/__init__.py similarity index 100% rename from api/MyModels/migrations/__init__.py rename to api/documents/migrations/__init__.py diff --git a/api/documents/models.py b/api/documents/models.py new file mode 100644 index 0000000..ec27fec --- /dev/null +++ b/api/documents/models.py @@ -0,0 +1,8 @@ +from django.db import models +from pgvector.django import VectorField +from chunks.models import Chunk + +class Document(models.Model): + text = models.TextField() + embedding = VectorField(dimensions=10) + chunks = models.ForeignKey(Chunk, on_delete=models.CASCADE, null=True, blank=True) diff --git a/api/MyModels/documents/schemas.py b/api/documents/schemas.py similarity index 100% rename from api/MyModels/documents/schemas.py rename to api/documents/schemas.py diff --git a/api/documents/tests.py b/api/documents/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/api/documents/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/api/MyModels/documents/views.py b/api/documents/views.py similarity index 83% rename from api/MyModels/documents/views.py rename to api/documents/views.py index 0f1fb03..a031c87 100644 --- a/api/MyModels/documents/views.py +++ b/api/documents/views.py @@ -5,8 +5,9 @@ from ninja import Router from ninja.pagination import LimitOffsetPagination, paginate -from MyModels.models import Document -from MyModels.documents.schemas import DocumentIn, DocumentOut +from documents.models import Document +from documents.schemas import DocumentIn, DocumentOut +import json document_router = Router() @@ -28,9 +29,10 @@ def retrieve_document(request: HttpRequest, id: int): return document @document_router.put("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) -def update_document(request: HttpRequest, id: int, payload: DocumentIn): +def update_document(request: HttpRequest, id: int): document = Document.objects.get(id=id) - for attr, value in payload.dict(exclude_unset=True).items(): + request_data = json.loads(request.body.decode("utf-8")) + for attr, value in request_data.items(): setattr(document, attr, value) document.full_clean() document.save() From 8298ee67703fb703ace247c5f27fe5d99958ebb8 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Fri, 23 Feb 2024 21:39:44 +0100 Subject: [PATCH 07/18] something to update --- README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b9f7631..49bea93 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,59 @@ # ChatKNML +[![KNML](https://i.imgur.com/GLpXodx.png)](https://knml.edu.pl/) + +[![Python](https://img.shields.io/badge/Python-3.11%2B-blue)](https://www.python.org/) +[![Docker](https://img.shields.io/badge/Docker-24.0%2B-blue)](https://www.docker.com/) +[![Poetry](https://img.shields.io/badge/Poetry-1.6%2B-blue)](https://python-poetry.org/) +[![Django](https://img.shields.io/badge/Django-3.2%2B-green)](https://www.djangoproject.com/) +[![Discord.py](https://img.shields.io/badge/Discord.py-2.3.2%2B-blue)](https://discordpy.readthedocs.io/en/stable/) +[![PostgreSQL](https://img.shields.io/badge/PostgreSQL-16%2B-blue)](https://www.postgresql.org/) +[![Git](https://img.shields.io/badge/Git-2.40%2B-red)](https://git-scm.com/) +[![MIT License](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + ## Getting started +### First setup + +1. Clone the repository: + + ```sh + git clone git@github.com:knmlprz/ChatKNML.git + ``` + +2. Navigate to the project directory: + + ```sh + cd ChatKNML + ``` + +3. Create a new branch for your development work: + + ```sh + git checkout -b {your_branch_name} + ``` + +4. Make sure you are working on the correct branch: + + ```sh + git status + ``` + ### Starting app development -Launching services using the "dev" profile: +1. Copy the `.env.example` file: -```sh -docker compose --profile dev up -``` + ```sh + cp .env.example .env + ``` + + Modify the environment variables to suit your requirements. + +2. Launching services using the "dev" profile: + + ```sh + docker compose --profile dev up + ``` ### Starting app production @@ -21,9 +66,10 @@ cd models git clone git@hf.co:intfloat/e5-large-v2 ``` -After starting the app, OpenAI-compatible embedding API will be available at: +Upon app startup, OpenAI-compatible embedding API will be available at: + - Check the docs here: +Check the docs here: #### llamacpp From 5765ae3fb75df9afa0a8d0c0c2224b389603954f Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Thu, 29 Feb 2024 23:12:00 +0100 Subject: [PATCH 08/18] separating logic from the views, tests, annotations In this commit i add: - annotations (i don't know if this is correct) - simple tests (i know that i can't do tests on main database, but i don't know how to do it work with test database (how to add extension with pgvector to test database)) P.S. I type following command to run tests: docker compose exec api python manage.py test --keepdb - i create files: documents/controllers.py and chunks/controllers.py and defined logic in them that is used by views.py. --- api/api/api.py | 6 ++-- api/api/settings.py | 5 ++- api/chunks/controllers.py | 35 +++++++++++++++++++++ api/chunks/migrations/0001_initial.py | 2 +- api/chunks/models.py | 11 ++++--- api/chunks/schemas.py | 1 - api/chunks/tests.py | 30 +++++++++++++++++- api/chunks/views.py | 36 +++++++++------------- api/documents/controllers.py | 35 +++++++++++++++++++++ api/documents/migrations/0001_initial.py | 2 +- api/documents/models.py | 7 +++-- api/documents/tests.py | 30 +++++++++++++++++- api/documents/views.py | 39 +++++++++--------------- 13 files changed, 176 insertions(+), 63 deletions(-) create mode 100644 api/chunks/controllers.py create mode 100644 api/documents/controllers.py diff --git a/api/api/api.py b/api/api/api.py index 43f1672..ca7ddcf 100644 --- a/api/api/api.py +++ b/api/api/api.py @@ -3,9 +3,9 @@ api = NinjaAPI() -api.add_router("/", "documents.views.document_router", tags=["Documents"]) -api.add_router("/", "chunks.views.chunk_router", tags=["Chunks"]) +api.add_router("/", "documents.views.document_router") +api.add_router("/", "chunks.views.chunk_router") urlpatterns = [ path("api/", api.urls), -] \ No newline at end of file +] diff --git a/api/api/settings.py b/api/api/settings.py index e452613..c0993ee 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -76,7 +76,10 @@ "PASSWORD": env("POSTGRES_PASSWORD"), "HOST": env("POSTGRES_HOST"), "PORT": env("POSTGRES_PORT"), - } + "TEST": { + "NAME": env("POSTGRES_DB"), + }, + }, } diff --git a/api/chunks/controllers.py b/api/chunks/controllers.py new file mode 100644 index 0000000..590b455 --- /dev/null +++ b/api/chunks/controllers.py @@ -0,0 +1,35 @@ +from http import HTTPStatus +from django.http import HttpRequest + +from chunks.models import Chunk +from chunks.schemas import ChunkIn, ChunkOut + +import json +from typing import Tuple + +def create_chunk_controller(payload: ChunkIn) -> Tuple[HTTPStatus, ChunkOut]: + chunk = Chunk(**payload.dict()) + chunk.full_clean() + chunk.save() + return HTTPStatus.CREATED, chunk + +def list_chunks_controller() -> (list[ChunkOut]): + return Chunk.objects.all() + +def retrieve_chunk_controller(id: int) -> (ChunkOut): + chunk = Chunk.objects.get(id=id) + return chunk + +def update_chunk_controller(request: HttpRequest, id: int) -> (ChunkOut): + chunk = Chunk.objects.get(id=id) + request_data = json.loads(request.body.decode("utf-8")) + for attr, value in request_data.items(): + setattr(chunk, attr, value) + chunk.full_clean() + chunk.save() + return chunk + +def delete_chunk_controller(id: int) -> (HTTPStatus): + chunk = Chunk.objects.get(id=id) + chunk.delete() + return HTTPStatus.OK diff --git a/api/chunks/migrations/0001_initial.py b/api/chunks/migrations/0001_initial.py index 8030279..2e91487 100644 --- a/api/chunks/migrations/0001_initial.py +++ b/api/chunks/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.2 on 2024-02-21 23:59 +# Generated by Django 5.0.2 on 2024-02-25 15:13 import pgvector.django from django.db import migrations, models diff --git a/api/chunks/models.py b/api/chunks/models.py index 68c309a..f730f6e 100644 --- a/api/chunks/models.py +++ b/api/chunks/models.py @@ -1,9 +1,10 @@ from django.db import models from pgvector.django import VectorField +from typing import Tuple class Chunk(models.Model): - text = models.CharField(max_length=100) - embedding = VectorField(dimensions=10) - chunk_idx = models.IntegerField() - start_char = models.IntegerField() - end_char = models.IntegerField() + text: Tuple[str, str] = models.CharField(max_length=100) + embedding: Tuple[list[float], VectorField] = VectorField(dimensions=10) + chunk_idx: Tuple[int, int] = models.IntegerField() + start_char: Tuple[int, int] = models.IntegerField() + end_char: Tuple[int, int] = models.IntegerField() diff --git a/api/chunks/schemas.py b/api/chunks/schemas.py index 02e8478..a43a475 100644 --- a/api/chunks/schemas.py +++ b/api/chunks/schemas.py @@ -14,4 +14,3 @@ class ChunkOut(Schema): chunk_idx: int start_char: int end_char: int - \ No newline at end of file diff --git a/api/chunks/tests.py b/api/chunks/tests.py index 7ce503c..5b71120 100644 --- a/api/chunks/tests.py +++ b/api/chunks/tests.py @@ -1,3 +1,31 @@ from django.test import TestCase +from .models import Chunk -# Create your tests here. +def create_chunk(): + return Chunk.objects.create(text="example", embedding=[1,2,3,4,5,6,7,8,9,10],chunk_idx=1,start_char=1,end_char=2) + +class ChunkModelTests(TestCase): + def test_get_method(self): + response = self.client.get("/api/chunk/") + self.assertEqual(response.status_code, 200) + + def test_post_method(self): + create_chunk() + chunk = Chunk.objects.get(text="example") + self.assertEqual(chunk.text, "example") + self.assertTrue(Chunk.objects.filter(text="example").exists()) + + def test_put_method(self): + create_chunk() + chunk = Chunk.objects.get(text="example") + chunk.text = "notexample" + chunk.save() + self.assertEqual(chunk.text, "notexample") + self.assertTrue(Chunk.objects.filter(text="notexample").exists()) + self.assertTrue(all(a == b for a, b in zip(chunk.embedding, [1,2,3,4,5,6,7,8,9,10]))) + + def test_delete_method(self): + create_chunk() + chunk = Chunk.objects.get(text="example") + chunk.delete() + self.assertFalse(Chunk.objects.filter(text="example").exists()) diff --git a/api/chunks/views.py b/api/chunks/views.py index d240fa6..ea62192 100644 --- a/api/chunks/views.py +++ b/api/chunks/views.py @@ -1,45 +1,37 @@ from http import HTTPStatus -from typing import List - from django.http import HttpRequest -from ninja import Router -from ninja.pagination import LimitOffsetPagination, paginate from chunks.models import Chunk from chunks.schemas import ChunkIn, ChunkOut -import json +from chunks.controllers import * -chunk_router = Router() +from ninja import Router +from ninja.pagination import LimitOffsetPagination, paginate + +chunk_router = Router(tags=["Chunks"]) @chunk_router.post("/chunk/", response={HTTPStatus.CREATED: ChunkOut}) def create_chunk(request: HttpRequest, payload: ChunkIn): - chunk = Chunk(**payload.dict()) - chunk.full_clean() - chunk.save() - return HTTPStatus.CREATED, chunk + http_status, chunk = create_chunk_controller(payload) + return http_status, chunk -@chunk_router.get("/chunk/", response={HTTPStatus.OK: List[ChunkOut]}) +@chunk_router.get("/chunk/", response={HTTPStatus.OK: list[ChunkOut]}) @paginate(LimitOffsetPagination) def list_chunks(request: HttpRequest): - return Chunk.objects.all() + chunk = list_chunks_controller() + return chunk @chunk_router.get("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) def retrieve_chunk(request: HttpRequest, id: int): - chunk = Chunk.objects.get(id=id) + chunk = retrieve_chunk_controller(id) return chunk @chunk_router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) def update_chunk(request: HttpRequest, id: int): - chunk = Chunk.objects.get(id=id) - request_data = json.loads(request.body.decode("utf-8")) - for attr, value in request_data.items(): - setattr(chunk, attr, value) - chunk.full_clean() - chunk.save() + chunk = update_chunk_controller(request,id) return chunk @chunk_router.delete("/chunk/{id}/", response={HTTPStatus.OK: None}) def delete_chunk(request: HttpRequest, id: int): - chunk = Chunk.objects.get(id=id) - chunk.delete() - return HTTPStatus.OK + http_status = delete_chunk_controller(id) + return http_status diff --git a/api/documents/controllers.py b/api/documents/controllers.py new file mode 100644 index 0000000..9de4a4f --- /dev/null +++ b/api/documents/controllers.py @@ -0,0 +1,35 @@ +from http import HTTPStatus +from django.http import HttpRequest + +from documents.models import Document +from documents.schemas import DocumentIn, DocumentOut + +import json +from typing import Tuple + +def create_document_controller(payload: DocumentIn) -> Tuple[HTTPStatus, DocumentOut]: + document = Document(**payload.dict()) + document.full_clean() + document.save() + return HTTPStatus.CREATED, document + +def list_documents_controller() -> (list[DocumentOut]): + return Document.objects.all() + +def retrieve_document_controller(id: int) -> (DocumentOut): + document = Document.objects.get(id=id) + return document + +def update_document_controller(request: HttpRequest, id: int) -> (DocumentOut): + document = Document.objects.get(id=id) + request_data = json.loads(request.body.decode("utf-8")) + for attr, value in request_data.items(): + setattr(document, attr, value) + document.full_clean() + document.save() + return document + +def delete_document_controller(id: int) -> (HTTPStatus): + document = Document.objects.get(id=id) + document.delete() + return HTTPStatus.OK diff --git a/api/documents/migrations/0001_initial.py b/api/documents/migrations/0001_initial.py index 47ccfb3..f1b6d5d 100644 --- a/api/documents/migrations/0001_initial.py +++ b/api/documents/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.2 on 2024-02-21 23:59 +# Generated by Django 5.0.2 on 2024-02-25 15:13 import django.db.models.deletion import pgvector.django diff --git a/api/documents/models.py b/api/documents/models.py index ec27fec..f638562 100644 --- a/api/documents/models.py +++ b/api/documents/models.py @@ -1,8 +1,9 @@ from django.db import models from pgvector.django import VectorField from chunks.models import Chunk +from typing import Tuple class Document(models.Model): - text = models.TextField() - embedding = VectorField(dimensions=10) - chunks = models.ForeignKey(Chunk, on_delete=models.CASCADE, null=True, blank=True) + text: Tuple[str, str] = models.TextField() + embedding: Tuple[list[float], VectorField] = VectorField(dimensions=10) + chunks: Tuple[int, int] = models.ForeignKey(Chunk, on_delete=models.CASCADE, null=True, blank=True) diff --git a/api/documents/tests.py b/api/documents/tests.py index 7ce503c..e0aa1a3 100644 --- a/api/documents/tests.py +++ b/api/documents/tests.py @@ -1,3 +1,31 @@ from django.test import TestCase +from .models import Document -# Create your tests here. +def create_document(): + return Document.objects.create(text="example", embedding=[1,2,3,4,5,6,7,8,9,10]) + +class DocumentModelTests(TestCase): + def test_get_method(self): + response = self.client.get("/api/document/") + self.assertEqual(response.status_code, 200) + + def test_post_method(self): + create_document() + document = Document.objects.get(text="example") + self.assertEqual(document.text, "example") + self.assertTrue(Document.objects.filter(text="example").exists()) + + def test_put_method(self): + create_document() + document = Document.objects.get(text="example") + document.text = "notexample" + document.save() + self.assertEqual(document.text, "notexample") + self.assertTrue(Document.objects.filter(text="notexample").exists()) + self.assertTrue(all(a == b for a, b in zip(document.embedding, [1,2,3,4,5,6,7,8,9,10]))) + + def test_delete_method(self): + create_document() + document = Document.objects.get(text="example") + document.delete() + self.assertFalse(Document.objects.filter(text="example").exists()) diff --git a/api/documents/views.py b/api/documents/views.py index a031c87..780a981 100644 --- a/api/documents/views.py +++ b/api/documents/views.py @@ -1,46 +1,37 @@ from http import HTTPStatus -from typing import List - from django.http import HttpRequest -from ninja import Router -from ninja.pagination import LimitOffsetPagination, paginate from documents.models import Document from documents.schemas import DocumentIn, DocumentOut -import json +from documents.controllers import * + +from ninja import Router +from ninja.pagination import LimitOffsetPagination, paginate -document_router = Router() +document_router = Router(tags=["Documents"]) @document_router.post("/document/", response={HTTPStatus.CREATED: DocumentOut}) def create_document(request: HttpRequest, payload: DocumentIn): - document = Document(**payload.dict()) - document.full_clean() - document.save() - return HTTPStatus.CREATED, document - -@document_router.get("/document/", response={HTTPStatus.OK: List[DocumentOut]}) + http_status, document = create_document_controller(payload) + return http_status, document + +@document_router.get("/document/", response={HTTPStatus.OK: list[DocumentOut]}) @paginate(LimitOffsetPagination) def list_documents(request: HttpRequest): - return Document.objects.all() + document = list_documents_controller() + return document @document_router.get("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) def retrieve_document(request: HttpRequest, id: int): - document = Document.objects.get(id=id) + document = retrieve_document_controller(id) return document @document_router.put("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) def update_document(request: HttpRequest, id: int): - document = Document.objects.get(id=id) - request_data = json.loads(request.body.decode("utf-8")) - for attr, value in request_data.items(): - setattr(document, attr, value) - document.full_clean() - document.save() + document = update_document_controller(request,id) return document @document_router.delete("/document/{id}/", response={HTTPStatus.OK: None}) def delete_document(request: HttpRequest, id: int): - document = Document.objects.get(id=id) - document.delete() - return HTTPStatus.OK - \ No newline at end of file + http_status = delete_document_controller(id) + return http_status From 92818a739e7a2df2d5b97050aed8d816ecd3ba9f Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Mon, 4 Mar 2024 13:47:14 +0100 Subject: [PATCH 09/18] lint and small changes --- api/api/api.py | 4 ++-- api/api/settings.py | 1 - api/chunks/controllers.py | 13 +++++++---- api/chunks/models.py | 12 +++++------ api/chunks/schemas.py | 2 ++ api/chunks/tests.py | 38 ++++++++++++++++++++++++-------- api/chunks/views.py | 40 +++++++++++++++++++--------------- api/documents/apps.py | 4 ++-- api/documents/controllers.py | 13 +++++++---- api/documents/models.py | 10 +++++---- api/documents/schemas.py | 2 ++ api/documents/tests.py | 34 +++++++++++++++++++++-------- api/documents/views.py | 42 ++++++++++++++++++++---------------- 13 files changed, 139 insertions(+), 76 deletions(-) diff --git a/api/api/api.py b/api/api/api.py index ca7ddcf..8ac62d8 100644 --- a/api/api/api.py +++ b/api/api/api.py @@ -3,8 +3,8 @@ api = NinjaAPI() -api.add_router("/", "documents.views.document_router") -api.add_router("/", "chunks.views.chunk_router") +api.add_router("/", "documents.views.router") +api.add_router("/", "chunks.views.router") urlpatterns = [ path("api/", api.urls), diff --git a/api/api/settings.py b/api/api/settings.py index c0993ee..aa9e6d8 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -31,7 +31,6 @@ "django.contrib.staticfiles", "chunks.apps.ChunksConfig", "documents.apps.DocumentsConfig", - ] MIDDLEWARE = [ diff --git a/api/chunks/controllers.py b/api/chunks/controllers.py index 590b455..8bacf02 100644 --- a/api/chunks/controllers.py +++ b/api/chunks/controllers.py @@ -7,20 +7,24 @@ import json from typing import Tuple + def create_chunk_controller(payload: ChunkIn) -> Tuple[HTTPStatus, ChunkOut]: chunk = Chunk(**payload.dict()) chunk.full_clean() chunk.save() return HTTPStatus.CREATED, chunk -def list_chunks_controller() -> (list[ChunkOut]): + +def list_chunks_controller() -> list[ChunkOut]: return Chunk.objects.all() -def retrieve_chunk_controller(id: int) -> (ChunkOut): + +def retrieve_chunk_controller(id: int) -> ChunkOut: chunk = Chunk.objects.get(id=id) return chunk -def update_chunk_controller(request: HttpRequest, id: int) -> (ChunkOut): + +def update_chunk_controller(request: HttpRequest, id: int) -> ChunkOut: chunk = Chunk.objects.get(id=id) request_data = json.loads(request.body.decode("utf-8")) for attr, value in request_data.items(): @@ -29,7 +33,8 @@ def update_chunk_controller(request: HttpRequest, id: int) -> (ChunkOut): chunk.save() return chunk -def delete_chunk_controller(id: int) -> (HTTPStatus): + +def delete_chunk_controller(id: int) -> HTTPStatus: chunk = Chunk.objects.get(id=id) chunk.delete() return HTTPStatus.OK diff --git a/api/chunks/models.py b/api/chunks/models.py index f730f6e..20551f1 100644 --- a/api/chunks/models.py +++ b/api/chunks/models.py @@ -1,10 +1,10 @@ from django.db import models from pgvector.django import VectorField -from typing import Tuple + class Chunk(models.Model): - text: Tuple[str, str] = models.CharField(max_length=100) - embedding: Tuple[list[float], VectorField] = VectorField(dimensions=10) - chunk_idx: Tuple[int, int] = models.IntegerField() - start_char: Tuple[int, int] = models.IntegerField() - end_char: Tuple[int, int] = models.IntegerField() + text: str = models.CharField(max_length=100) + embedding: list[float] = VectorField(dimensions=10) + chunk_idx: int = models.IntegerField() + start_char: int = models.IntegerField() + end_char: int = models.IntegerField() diff --git a/api/chunks/schemas.py b/api/chunks/schemas.py index a43a475..d84caf7 100644 --- a/api/chunks/schemas.py +++ b/api/chunks/schemas.py @@ -1,5 +1,6 @@ from ninja import Schema + class ChunkIn(Schema): text: str embedding: list[float] @@ -7,6 +8,7 @@ class ChunkIn(Schema): start_char: int end_char: int + class ChunkOut(Schema): id: int text: str diff --git a/api/chunks/tests.py b/api/chunks/tests.py index 5b71120..d3657fd 100644 --- a/api/chunks/tests.py +++ b/api/chunks/tests.py @@ -1,31 +1,51 @@ +import pytest from django.test import TestCase from .models import Chunk + def create_chunk(): - return Chunk.objects.create(text="example", embedding=[1,2,3,4,5,6,7,8,9,10],chunk_idx=1,start_char=1,end_char=2) + return Chunk.objects.create( + text="example", + embedding=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + chunk_idx=1, + start_char=1, + end_char=2, + ) + class ChunkModelTests(TestCase): + @pytest.mark.django_db def test_get_method(self): response = self.client.get("/api/chunk/") - self.assertEqual(response.status_code, 200) + assert ( + response.status_code == 200 + ), "The request could not be received, status code should be 200" + @pytest.mark.django_db def test_post_method(self): create_chunk() - chunk = Chunk.objects.get(text="example") - self.assertEqual(chunk.text, "example") - self.assertTrue(Chunk.objects.filter(text="example").exists()) + assert ( + Chunk.objects.filter(text="example").exists() == True + ), "Can't create chunk object" + @pytest.mark.django_db def test_put_method(self): create_chunk() chunk = Chunk.objects.get(text="example") + assert ( + Chunk.objects.filter(text="example").exists() == True + ), "Can't create chunk object" chunk.text = "notexample" chunk.save() - self.assertEqual(chunk.text, "notexample") - self.assertTrue(Chunk.objects.filter(text="notexample").exists()) - self.assertTrue(all(a == b for a, b in zip(chunk.embedding, [1,2,3,4,5,6,7,8,9,10]))) + assert all( + a == b for a, b in zip(chunk.embedding, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + ), "Can't update chunk object" + @pytest.mark.django_db def test_delete_method(self): create_chunk() chunk = Chunk.objects.get(text="example") chunk.delete() - self.assertFalse(Chunk.objects.filter(text="example").exists()) + assert ( + Chunk.objects.filter(text="example").exists() == False + ), "Can't delete chunk object" diff --git a/api/chunks/views.py b/api/chunks/views.py index ea62192..54444d7 100644 --- a/api/chunks/views.py +++ b/api/chunks/views.py @@ -3,35 +3,41 @@ from chunks.models import Chunk from chunks.schemas import ChunkIn, ChunkOut -from chunks.controllers import * +from chunks.controllers import ( + create_chunk_controller, + list_chunks_controller, + retrieve_chunk_controller, + update_chunk_controller, + delete_chunk_controller, +) from ninja import Router from ninja.pagination import LimitOffsetPagination, paginate -chunk_router = Router(tags=["Chunks"]) +router = Router(tags=["Chunks"]) -@chunk_router.post("/chunk/", response={HTTPStatus.CREATED: ChunkOut}) + +@router.post("/chunk/", response={HTTPStatus.CREATED: ChunkOut}) def create_chunk(request: HttpRequest, payload: ChunkIn): - http_status, chunk = create_chunk_controller(payload) - return http_status, chunk + return create_chunk_controller(payload) + -@chunk_router.get("/chunk/", response={HTTPStatus.OK: list[ChunkOut]}) +@router.get("/chunk/", response={HTTPStatus.OK: list[ChunkOut]}) @paginate(LimitOffsetPagination) def list_chunks(request: HttpRequest): - chunk = list_chunks_controller() - return chunk + return list_chunks_controller() + -@chunk_router.get("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) +@router.get("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) def retrieve_chunk(request: HttpRequest, id: int): - chunk = retrieve_chunk_controller(id) - return chunk + return retrieve_chunk_controller(id) -@chunk_router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) + +@router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) def update_chunk(request: HttpRequest, id: int): - chunk = update_chunk_controller(request,id) - return chunk + return update_chunk_controller(request, id) + -@chunk_router.delete("/chunk/{id}/", response={HTTPStatus.OK: None}) +@router.delete("/chunk/{id}/", response={HTTPStatus.OK: None}) def delete_chunk(request: HttpRequest, id: int): - http_status = delete_chunk_controller(id) - return http_status + return delete_chunk_controller(id) diff --git a/api/documents/apps.py b/api/documents/apps.py index ab8a093..37ce729 100644 --- a/api/documents/apps.py +++ b/api/documents/apps.py @@ -2,5 +2,5 @@ class DocumentsConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'documents' + default_auto_field = "django.db.models.BigAutoField" + name = "documents" diff --git a/api/documents/controllers.py b/api/documents/controllers.py index 9de4a4f..8c94f83 100644 --- a/api/documents/controllers.py +++ b/api/documents/controllers.py @@ -7,20 +7,24 @@ import json from typing import Tuple + def create_document_controller(payload: DocumentIn) -> Tuple[HTTPStatus, DocumentOut]: document = Document(**payload.dict()) document.full_clean() document.save() return HTTPStatus.CREATED, document -def list_documents_controller() -> (list[DocumentOut]): + +def list_documents_controller() -> list[DocumentOut]: return Document.objects.all() -def retrieve_document_controller(id: int) -> (DocumentOut): + +def retrieve_document_controller(id: int) -> DocumentOut: document = Document.objects.get(id=id) return document -def update_document_controller(request: HttpRequest, id: int) -> (DocumentOut): + +def update_document_controller(request: HttpRequest, id: int) -> DocumentOut: document = Document.objects.get(id=id) request_data = json.loads(request.body.decode("utf-8")) for attr, value in request_data.items(): @@ -29,7 +33,8 @@ def update_document_controller(request: HttpRequest, id: int) -> (DocumentOut): document.save() return document -def delete_document_controller(id: int) -> (HTTPStatus): + +def delete_document_controller(id: int) -> HTTPStatus: document = Document.objects.get(id=id) document.delete() return HTTPStatus.OK diff --git a/api/documents/models.py b/api/documents/models.py index f638562..ff69760 100644 --- a/api/documents/models.py +++ b/api/documents/models.py @@ -1,9 +1,11 @@ from django.db import models from pgvector.django import VectorField from chunks.models import Chunk -from typing import Tuple + class Document(models.Model): - text: Tuple[str, str] = models.TextField() - embedding: Tuple[list[float], VectorField] = VectorField(dimensions=10) - chunks: Tuple[int, int] = models.ForeignKey(Chunk, on_delete=models.CASCADE, null=True, blank=True) + text: str = models.TextField() + embedding: list[float] = VectorField(dimensions=10) + chunks: int = models.ForeignKey( + Chunk, on_delete=models.CASCADE, null=True, blank=True + ) diff --git a/api/documents/schemas.py b/api/documents/schemas.py index af23725..3781caf 100644 --- a/api/documents/schemas.py +++ b/api/documents/schemas.py @@ -1,9 +1,11 @@ from ninja import Schema + class DocumentIn(Schema): text: str embedding: list[float] + class DocumentOut(Schema): id: int text: str diff --git a/api/documents/tests.py b/api/documents/tests.py index e0aa1a3..6df8a21 100644 --- a/api/documents/tests.py +++ b/api/documents/tests.py @@ -1,31 +1,47 @@ +import pytest from django.test import TestCase from .models import Document + def create_document(): - return Document.objects.create(text="example", embedding=[1,2,3,4,5,6,7,8,9,10]) + return Document.objects.create( + text="example", embedding=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + ) + class DocumentModelTests(TestCase): + @pytest.mark.django_db def test_get_method(self): response = self.client.get("/api/document/") - self.assertEqual(response.status_code, 200) + assert ( + response.status_code == 200 + ), "The request could not be received, status code should be 200" + @pytest.mark.django_db def test_post_method(self): create_document() - document = Document.objects.get(text="example") - self.assertEqual(document.text, "example") - self.assertTrue(Document.objects.filter(text="example").exists()) + assert ( + Document.objects.filter(text="example").exists() == True + ), "Can't create document object" + @pytest.mark.django_db def test_put_method(self): create_document() document = Document.objects.get(text="example") + assert ( + Document.objects.filter(text="example").exists() == True + ), "Can't create document object" document.text = "notexample" document.save() - self.assertEqual(document.text, "notexample") - self.assertTrue(Document.objects.filter(text="notexample").exists()) - self.assertTrue(all(a == b for a, b in zip(document.embedding, [1,2,3,4,5,6,7,8,9,10]))) + assert all( + a == b for a, b in zip(document.embedding, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + ), "Can't update document object" + @pytest.mark.django_db def test_delete_method(self): create_document() document = Document.objects.get(text="example") document.delete() - self.assertFalse(Document.objects.filter(text="example").exists()) + assert ( + Document.objects.filter(text="example").exists() == False + ), "Can't delete document object" diff --git a/api/documents/views.py b/api/documents/views.py index 780a981..eb67869 100644 --- a/api/documents/views.py +++ b/api/documents/views.py @@ -3,35 +3,41 @@ from documents.models import Document from documents.schemas import DocumentIn, DocumentOut -from documents.controllers import * +from documents.controllers import ( + create_document_controller, + list_documents_controller, + retrieve_document_controller, + update_document_controller, + delete_document_controller, +) from ninja import Router from ninja.pagination import LimitOffsetPagination, paginate -document_router = Router(tags=["Documents"]) +router = Router(tags=["Documents"]) -@document_router.post("/document/", response={HTTPStatus.CREATED: DocumentOut}) + +@router.post("/document/", response={HTTPStatus.CREATED: DocumentOut}) def create_document(request: HttpRequest, payload: DocumentIn): - http_status, document = create_document_controller(payload) - return http_status, document - -@document_router.get("/document/", response={HTTPStatus.OK: list[DocumentOut]}) + return create_document_controller(payload) + + +@router.get("/document/", response={HTTPStatus.OK: list[DocumentOut]}) @paginate(LimitOffsetPagination) def list_documents(request: HttpRequest): - document = list_documents_controller() - return document + return list_documents_controller() -@document_router.get("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) + +@router.get("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) def retrieve_document(request: HttpRequest, id: int): - document = retrieve_document_controller(id) - return document + return retrieve_document_controller(id) + -@document_router.put("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) +@router.put("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) def update_document(request: HttpRequest, id: int): - document = update_document_controller(request,id) - return document + return update_document_controller(request, id) + -@document_router.delete("/document/{id}/", response={HTTPStatus.OK: None}) +@router.delete("/document/{id}/", response={HTTPStatus.OK: None}) def delete_document(request: HttpRequest, id: int): - http_status = delete_document_controller(id) - return http_status + return delete_document_controller(id) From 765f8ec8dd96a1ca61afe0815061a54d7bd5d5b2 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Mon, 4 Mar 2024 14:53:22 +0100 Subject: [PATCH 10/18] changing put controller --- api/chunks/controllers.py | 8 +++----- api/chunks/views.py | 4 ++-- api/documents/controllers.py | 8 +++----- api/documents/views.py | 4 ++-- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/api/chunks/controllers.py b/api/chunks/controllers.py index 8bacf02..caa5ed9 100644 --- a/api/chunks/controllers.py +++ b/api/chunks/controllers.py @@ -5,10 +5,9 @@ from chunks.schemas import ChunkIn, ChunkOut import json -from typing import Tuple -def create_chunk_controller(payload: ChunkIn) -> Tuple[HTTPStatus, ChunkOut]: +def create_chunk_controller(payload: ChunkIn) -> tuple[HTTPStatus, ChunkOut]: chunk = Chunk(**payload.dict()) chunk.full_clean() chunk.save() @@ -24,10 +23,9 @@ def retrieve_chunk_controller(id: int) -> ChunkOut: return chunk -def update_chunk_controller(request: HttpRequest, id: int) -> ChunkOut: +def update_chunk_controller(payload: ChunkIn, id: int) -> ChunkOut: chunk = Chunk.objects.get(id=id) - request_data = json.loads(request.body.decode("utf-8")) - for attr, value in request_data.items(): + for attr, value in payload.dict().items(): setattr(chunk, attr, value) chunk.full_clean() chunk.save() diff --git a/api/chunks/views.py b/api/chunks/views.py index 54444d7..3ab4787 100644 --- a/api/chunks/views.py +++ b/api/chunks/views.py @@ -34,8 +34,8 @@ def retrieve_chunk(request: HttpRequest, id: int): @router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) -def update_chunk(request: HttpRequest, id: int): - return update_chunk_controller(request, id) +def update_chunk(request: HttpRequest,data: ChunkIn, id: int): + return update_chunk_controller(data, id) @router.delete("/chunk/{id}/", response={HTTPStatus.OK: None}) diff --git a/api/documents/controllers.py b/api/documents/controllers.py index 8c94f83..85f888d 100644 --- a/api/documents/controllers.py +++ b/api/documents/controllers.py @@ -5,10 +5,9 @@ from documents.schemas import DocumentIn, DocumentOut import json -from typing import Tuple -def create_document_controller(payload: DocumentIn) -> Tuple[HTTPStatus, DocumentOut]: +def create_document_controller(payload: DocumentIn) -> tuple[HTTPStatus, DocumentOut]: document = Document(**payload.dict()) document.full_clean() document.save() @@ -24,10 +23,9 @@ def retrieve_document_controller(id: int) -> DocumentOut: return document -def update_document_controller(request: HttpRequest, id: int) -> DocumentOut: +def update_document_controller(payload: DocumentIn, id: int) -> DocumentOut: document = Document.objects.get(id=id) - request_data = json.loads(request.body.decode("utf-8")) - for attr, value in request_data.items(): + for attr, value in payload.dict().items(): setattr(document, attr, value) document.full_clean() document.save() diff --git a/api/documents/views.py b/api/documents/views.py index eb67869..efb987e 100644 --- a/api/documents/views.py +++ b/api/documents/views.py @@ -34,8 +34,8 @@ def retrieve_document(request: HttpRequest, id: int): @router.put("/document/{id}/", response={HTTPStatus.OK: DocumentOut}) -def update_document(request: HttpRequest, id: int): - return update_document_controller(request, id) +def update_document(request: HttpRequest, data: DocumentIn, id: int): + return update_document_controller(data, id) @router.delete("/document/{id}/", response={HTTPStatus.OK: None}) From 22afbdedf10e4b1aab05c47ecddf3cae60675248 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Mon, 4 Mar 2024 14:57:04 +0100 Subject: [PATCH 11/18] deleting test's environment --- api/api/settings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/api/api/settings.py b/api/api/settings.py index aa9e6d8..204d028 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -75,9 +75,6 @@ "PASSWORD": env("POSTGRES_PASSWORD"), "HOST": env("POSTGRES_HOST"), "PORT": env("POSTGRES_PORT"), - "TEST": { - "NAME": env("POSTGRES_DB"), - }, }, } From b64f6b4ad0cfac238232b3f32d7c1d444f92b6ee Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Sat, 9 Mar 2024 19:02:35 +0100 Subject: [PATCH 12/18] fix api dockerfile --- api/Dockerfile | 14 ++++++++++++++ api/api/settings.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/api/Dockerfile b/api/Dockerfile index d2901ae..727ea1d 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -23,4 +23,18 @@ COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY . ./ +ENV SECRET_KEY = $SECRET_KEY + +ENV POSTGRES_DB = $POSTGRES_DB + +ENV POSTGRES_USER = $POSTGRES_USER + +ENV POSTGRES_PASSWORD = $POSTGRES_PASSWORD + +ENV POSTGRES_HOST = $POSTGRES_HOST + +ENV POSTGRES_PORT = $POSTGRES_PORT + +RUN python manage.py collectstatic + ENTRYPOINT ["uvicorn", "api.asgi:application", "--host", "0.0.0.0"] diff --git a/api/api/settings.py b/api/api/settings.py index 204d028..1f9d740 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -5,7 +5,7 @@ env = environ.Env() # Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve().parent.parent +BASE_DIR = Path(__file__).resolve().parent # Quick-start development settings - unsuitable for production From 58079071f09d8a2cc2ba58bbd62a922648abad59 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Sun, 10 Mar 2024 21:59:58 +0100 Subject: [PATCH 13/18] changing annotations --- api/Dockerfile | 2 ++ api/api/settings.py | 3 +++ api/chunks/models.py | 10 +++++----- api/chunks/views.py | 2 +- api/documents/models.py | 6 +++--- llm/test/test_promt.py | 15 +++++++-------- 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 727ea1d..bb60c3a 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -35,6 +35,8 @@ ENV POSTGRES_HOST = $POSTGRES_HOST ENV POSTGRES_PORT = $POSTGRES_PORT +RUN pip install django_stubs_ext + RUN python manage.py collectstatic ENTRYPOINT ["uvicorn", "api.asgi:application", "--host", "0.0.0.0"] diff --git a/api/api/settings.py b/api/api/settings.py index 1f9d740..eb3e820 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -1,6 +1,9 @@ from pathlib import Path import os import environ +import django_stubs_ext + +django_stubs_ext.monkeypatch() env = environ.Env() diff --git a/api/chunks/models.py b/api/chunks/models.py index 20551f1..3bd9acf 100644 --- a/api/chunks/models.py +++ b/api/chunks/models.py @@ -3,8 +3,8 @@ class Chunk(models.Model): - text: str = models.CharField(max_length=100) - embedding: list[float] = VectorField(dimensions=10) - chunk_idx: int = models.IntegerField() - start_char: int = models.IntegerField() - end_char: int = models.IntegerField() + text = models.CharField[str, str](max_length=100) + embedding = VectorField[list[float], VectorField](dimensions=10) + chunk_idx = models.IntegerField[int, int]() + start_char = models.IntegerField[int, int]() + end_char = models.IntegerField[int, int]() diff --git a/api/chunks/views.py b/api/chunks/views.py index 3ab4787..7453b6f 100644 --- a/api/chunks/views.py +++ b/api/chunks/views.py @@ -34,7 +34,7 @@ def retrieve_chunk(request: HttpRequest, id: int): @router.put("/chunk/{id}/", response={HTTPStatus.OK: ChunkOut}) -def update_chunk(request: HttpRequest,data: ChunkIn, id: int): +def update_chunk(request: HttpRequest, data: ChunkIn, id: int): return update_chunk_controller(data, id) diff --git a/api/documents/models.py b/api/documents/models.py index ff69760..966b36c 100644 --- a/api/documents/models.py +++ b/api/documents/models.py @@ -4,8 +4,8 @@ class Document(models.Model): - text: str = models.TextField() - embedding: list[float] = VectorField(dimensions=10) - chunks: int = models.ForeignKey( + text = models.TextField[str, str]() + embedding = VectorField[list[float], VectorField](dimensions=10) + chunks = models.ForeignKey[int, int]( Chunk, on_delete=models.CASCADE, null=True, blank=True ) diff --git a/llm/test/test_promt.py b/llm/test/test_promt.py index dbc2ccb..2e900de 100644 --- a/llm/test/test_promt.py +++ b/llm/test/test_promt.py @@ -1,21 +1,20 @@ import requests import json + def query_local_openai_api(prompt, stop_signs): - url = 'http://127.0.0.1:9000/v1/completions' - headers = {'Content-Type': 'application/json'} - data = { - 'prompt': prompt, - 'stop': stop_signs - } + url = "http://127.0.0.1:9000/v1/completions" + headers = {"Content-Type": "application/json"} + data = {"prompt": prompt, "stop": stop_signs} response = requests.post(url, headers=headers, data=json.dumps(data)) - + if response.status_code == 200: return response.json() else: return response.text + # Prompt and stop_signs prompt = "\n\n### Instructions:\n jaka jest stolica polski\n\n### Response:\n" stop_signs = ["\n", "###"] @@ -23,4 +22,4 @@ def query_local_openai_api(prompt, stop_signs): # Query local API result = query_local_openai_api(prompt, stop_signs) -print(result['choices'][0]['text']) +print(result["choices"][0]["text"]) From 1f2e35c6538340312107e95594888802ad04eb95 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Tue, 12 Mar 2024 00:27:15 +0100 Subject: [PATCH 14/18] changing environment variable get type --- api/Dockerfile | 12 ------------ api/api/settings.py | 15 ++++++--------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index bb60c3a..71d88ab 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -23,18 +23,6 @@ COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY . ./ -ENV SECRET_KEY = $SECRET_KEY - -ENV POSTGRES_DB = $POSTGRES_DB - -ENV POSTGRES_USER = $POSTGRES_USER - -ENV POSTGRES_PASSWORD = $POSTGRES_PASSWORD - -ENV POSTGRES_HOST = $POSTGRES_HOST - -ENV POSTGRES_PORT = $POSTGRES_PORT - RUN pip install django_stubs_ext RUN python manage.py collectstatic diff --git a/api/api/settings.py b/api/api/settings.py index eb3e820..5ebe760 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -1,12 +1,9 @@ from pathlib import Path import os -import environ import django_stubs_ext django_stubs_ext.monkeypatch() -env = environ.Env() - # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent @@ -15,7 +12,7 @@ # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env("SECRET_KEY") +SECRET_KEY = os.environ.get("SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True @@ -73,11 +70,11 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql", - "NAME": env("POSTGRES_DB"), - "USER": env("POSTGRES_USER"), - "PASSWORD": env("POSTGRES_PASSWORD"), - "HOST": env("POSTGRES_HOST"), - "PORT": env("POSTGRES_PORT"), + "NAME": os.environ.get("POSTGRES_DB"), + "USER": os.environ.get("POSTGRES_USER"), + "PASSWORD": os.environ.get("POSTGRES_PASSWORD"), + "HOST": os.environ.get("POSTGRES_HOST"), + "PORT": os.environ.get("POSTGRES_PORT"), }, } From df2309ecb59a5900e54468656a80743819620246 Mon Sep 17 00:00:00 2001 From: bafaurazan <146839542+bafaurazan@users.noreply.github.com> Date: Wed, 13 Mar 2024 19:46:59 +0100 Subject: [PATCH 15/18] Apply suggestions from code review Co-authored-by: Patryk Gronkiewicz <164157@stud.prz.edu.pl> --- api/chunks/controllers.py | 3 +-- api/chunks/tests.py | 10 +++++----- api/documents/tests.py | 10 +++++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/api/chunks/controllers.py b/api/chunks/controllers.py index caa5ed9..5ec6679 100644 --- a/api/chunks/controllers.py +++ b/api/chunks/controllers.py @@ -33,6 +33,5 @@ def update_chunk_controller(payload: ChunkIn, id: int) -> ChunkOut: def delete_chunk_controller(id: int) -> HTTPStatus: - chunk = Chunk.objects.get(id=id) - chunk.delete() +Chunk.objects.get(id=id).delete() return HTTPStatus.OK diff --git a/api/chunks/tests.py b/api/chunks/tests.py index d3657fd..86542bf 100644 --- a/api/chunks/tests.py +++ b/api/chunks/tests.py @@ -6,7 +6,7 @@ def create_chunk(): return Chunk.objects.create( text="example", - embedding=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + embedding=list(range(1, 11)), chunk_idx=1, start_char=1, end_char=2, @@ -25,7 +25,7 @@ def test_get_method(self): def test_post_method(self): create_chunk() assert ( - Chunk.objects.filter(text="example").exists() == True + Chunk.objects.filter(text="example").exists() ), "Can't create chunk object" @pytest.mark.django_db @@ -33,12 +33,12 @@ def test_put_method(self): create_chunk() chunk = Chunk.objects.get(text="example") assert ( - Chunk.objects.filter(text="example").exists() == True + Chunk.objects.filter(text="example").exists() ), "Can't create chunk object" chunk.text = "notexample" chunk.save() assert all( - a == b for a, b in zip(chunk.embedding, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + a == b for a, b in zip(chunk.embedding, range(1, 11)) ), "Can't update chunk object" @pytest.mark.django_db @@ -47,5 +47,5 @@ def test_delete_method(self): chunk = Chunk.objects.get(text="example") chunk.delete() assert ( - Chunk.objects.filter(text="example").exists() == False + not Chunk.objects.filter(text="example").exists() ), "Can't delete chunk object" diff --git a/api/documents/tests.py b/api/documents/tests.py index 6df8a21..b29b94b 100644 --- a/api/documents/tests.py +++ b/api/documents/tests.py @@ -5,7 +5,7 @@ def create_document(): return Document.objects.create( - text="example", embedding=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + text="example", embedding=list(range(1, 11)) ) @@ -21,7 +21,7 @@ def test_get_method(self): def test_post_method(self): create_document() assert ( - Document.objects.filter(text="example").exists() == True + Document.objects.filter(text="example").exists() ), "Can't create document object" @pytest.mark.django_db @@ -29,12 +29,12 @@ def test_put_method(self): create_document() document = Document.objects.get(text="example") assert ( - Document.objects.filter(text="example").exists() == True + Document.objects.filter(text="example").exists() ), "Can't create document object" document.text = "notexample" document.save() assert all( - a == b for a, b in zip(document.embedding, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + a == b for a, b in zip(document.embedding, range(1, 10)) ), "Can't update document object" @pytest.mark.django_db @@ -43,5 +43,5 @@ def test_delete_method(self): document = Document.objects.get(text="example") document.delete() assert ( - Document.objects.filter(text="example").exists() == False + not Document.objects.filter(text="example").exists() ), "Can't delete document object" From 7638db85148228b55c835cfa9191c4b492b402c2 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Wed, 13 Mar 2024 20:00:29 +0100 Subject: [PATCH 16/18] fix --- api/chunks/controllers.py | 2 +- api/chunks/tests.py | 18 +++--- api/documents/tests.py | 22 ++++--- api/poetry.lock | 122 +++++++++++++++++++++++++++++++++++++- api/pyproject.toml | 4 ++ 5 files changed, 144 insertions(+), 24 deletions(-) diff --git a/api/chunks/controllers.py b/api/chunks/controllers.py index 5ec6679..76913b4 100644 --- a/api/chunks/controllers.py +++ b/api/chunks/controllers.py @@ -33,5 +33,5 @@ def update_chunk_controller(payload: ChunkIn, id: int) -> ChunkOut: def delete_chunk_controller(id: int) -> HTTPStatus: -Chunk.objects.get(id=id).delete() + Chunk.objects.get(id=id).delete() return HTTPStatus.OK diff --git a/api/chunks/tests.py b/api/chunks/tests.py index 86542bf..d5bc151 100644 --- a/api/chunks/tests.py +++ b/api/chunks/tests.py @@ -24,17 +24,17 @@ def test_get_method(self): @pytest.mark.django_db def test_post_method(self): create_chunk() - assert ( - Chunk.objects.filter(text="example").exists() - ), "Can't create chunk object" + assert Chunk.objects.filter( + text="example" + ).exists(), "Can't create chunk object" @pytest.mark.django_db def test_put_method(self): create_chunk() chunk = Chunk.objects.get(text="example") - assert ( - Chunk.objects.filter(text="example").exists() - ), "Can't create chunk object" + assert Chunk.objects.filter( + text="example" + ).exists(), "Can't create chunk object" chunk.text = "notexample" chunk.save() assert all( @@ -46,6 +46,6 @@ def test_delete_method(self): create_chunk() chunk = Chunk.objects.get(text="example") chunk.delete() - assert ( - not Chunk.objects.filter(text="example").exists() - ), "Can't delete chunk object" + assert not Chunk.objects.filter( + text="example" + ).exists(), "Can't delete chunk object" diff --git a/api/documents/tests.py b/api/documents/tests.py index b29b94b..99be3a8 100644 --- a/api/documents/tests.py +++ b/api/documents/tests.py @@ -4,9 +4,7 @@ def create_document(): - return Document.objects.create( - text="example", embedding=list(range(1, 11)) - ) + return Document.objects.create(text="example", embedding=list(range(1, 11))) class DocumentModelTests(TestCase): @@ -20,17 +18,17 @@ def test_get_method(self): @pytest.mark.django_db def test_post_method(self): create_document() - assert ( - Document.objects.filter(text="example").exists() - ), "Can't create document object" + assert Document.objects.filter( + text="example" + ).exists(), "Can't create document object" @pytest.mark.django_db def test_put_method(self): create_document() document = Document.objects.get(text="example") - assert ( - Document.objects.filter(text="example").exists() - ), "Can't create document object" + assert Document.objects.filter( + text="example" + ).exists(), "Can't create document object" document.text = "notexample" document.save() assert all( @@ -42,6 +40,6 @@ def test_delete_method(self): create_document() document = Document.objects.get(text="example") document.delete() - assert ( - not Document.objects.filter(text="example").exists() - ), "Can't delete document object" + assert not Document.objects.filter( + text="example" + ).exists(), "Can't delete document object" diff --git a/api/poetry.lock b/api/poetry.lock index 3d5fbb8..0e09bff 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "annotated-types" @@ -25,6 +25,50 @@ files = [ [package.extras] tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] +[[package]] +name = "black" +version = "24.2.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-24.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29"}, + {file = "black-24.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430"}, + {file = "black-24.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f"}, + {file = "black-24.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a"}, + {file = "black-24.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd"}, + {file = "black-24.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2"}, + {file = "black-24.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92"}, + {file = "black-24.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23"}, + {file = "black-24.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b"}, + {file = "black-24.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9"}, + {file = "black-24.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693"}, + {file = "black-24.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982"}, + {file = "black-24.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7e53a8c630f71db01b28cd9602a1ada68c937cbf2c333e6ed041390d6968faf4"}, + {file = "black-24.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93601c2deb321b4bad8f95df408e3fb3943d85012dddb6121336b8e24a0d1218"}, + {file = "black-24.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0057f800de6acc4407fe75bb147b0c2b5cbb7c3ed110d3e5999cd01184d53b0"}, + {file = "black-24.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d"}, + {file = "black-24.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8"}, + {file = "black-24.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8"}, + {file = "black-24.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540"}, + {file = "black-24.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31"}, + {file = "black-24.2.0-py3-none-any.whl", hash = "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6"}, + {file = "black-24.2.0.tar.gz", hash = "sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "channels" version = "4.0.0" @@ -136,6 +180,17 @@ files = [ {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, ] +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + [[package]] name = "numpy" version = "1.26.3" @@ -181,6 +236,28 @@ files = [ {file = "numpy-1.26.3.tar.gz", hash = "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4"}, ] +[[package]] +name = "packaging" +version = "24.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + [[package]] name = "pgvector" version = "0.2.4" @@ -194,6 +271,21 @@ files = [ [package.dependencies] numpy = "*" +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + [[package]] name = "psycopg2-binary" version = "2.9.9" @@ -411,6 +503,32 @@ files = [ [package.dependencies] typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +[[package]] +name = "ruff" +version = "0.3.2" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.3.2-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77f2612752e25f730da7421ca5e3147b213dca4f9a0f7e0b534e9562c5441f01"}, + {file = "ruff-0.3.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9966b964b2dd1107797be9ca7195002b874424d1d5472097701ae8f43eadef5d"}, + {file = "ruff-0.3.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b83d17ff166aa0659d1e1deaf9f2f14cbe387293a906de09bc4860717eb2e2da"}, + {file = "ruff-0.3.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb875c6cc87b3703aeda85f01c9aebdce3d217aeaca3c2e52e38077383f7268a"}, + {file = "ruff-0.3.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be75e468a6a86426430373d81c041b7605137a28f7014a72d2fc749e47f572aa"}, + {file = "ruff-0.3.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:967978ac2d4506255e2f52afe70dda023fc602b283e97685c8447d036863a302"}, + {file = "ruff-0.3.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1231eacd4510f73222940727ac927bc5d07667a86b0cbe822024dd00343e77e9"}, + {file = "ruff-0.3.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c6d613b19e9a8021be2ee1d0e27710208d1603b56f47203d0abbde906929a9b"}, + {file = "ruff-0.3.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8439338a6303585d27b66b4626cbde89bb3e50fa3cae86ce52c1db7449330a7"}, + {file = "ruff-0.3.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:de8b480d8379620cbb5ea466a9e53bb467d2fb07c7eca54a4aa8576483c35d36"}, + {file = "ruff-0.3.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b74c3de9103bd35df2bb05d8b2899bf2dbe4efda6474ea9681280648ec4d237d"}, + {file = "ruff-0.3.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f380be9fc15a99765c9cf316b40b9da1f6ad2ab9639e551703e581a5e6da6745"}, + {file = "ruff-0.3.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:0ac06a3759c3ab9ef86bbeca665d31ad3aa9a4b1c17684aadb7e61c10baa0df4"}, + {file = "ruff-0.3.2-py3-none-win32.whl", hash = "sha256:9bd640a8f7dd07a0b6901fcebccedadeb1a705a50350fb86b4003b805c81385a"}, + {file = "ruff-0.3.2-py3-none-win_amd64.whl", hash = "sha256:0c1bdd9920cab5707c26c8b3bf33a064a4ca7842d91a99ec0634fec68f9f4037"}, + {file = "ruff-0.3.2-py3-none-win_arm64.whl", hash = "sha256:5f65103b1d76e0d600cabd577b04179ff592064eaa451a70a81085930e907d0b"}, + {file = "ruff-0.3.2.tar.gz", hash = "sha256:fa78ec9418eb1ca3db392811df3376b46471ae93792a81af2d1cbb0e5dcb5142"}, +] + [[package]] name = "sqlparse" version = "0.4.4" @@ -470,4 +588,4 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "a4511216d4e4030e26af61a18ba4ed1b2b18bfdba4ff6af0ae4259bf5c282a5a" +content-hash = "597a5bea6ac5910888e0ce16d25e5e423f6fa9c8b1a2329b4e457958346b1203" diff --git a/api/pyproject.toml b/api/pyproject.toml index b5a8e37..6987783 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -16,6 +16,10 @@ django-ninja = "^1.0.1" pgvector = "^0.2.4" +[tool.poetry.group.dev.dependencies] +black = "^24.2.0" +ruff = "^0.3.2" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From 59f3866d7a087dc65d951265b9e0b6dcd375c72e Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Wed, 13 Mar 2024 20:04:51 +0100 Subject: [PATCH 17/18] fix2 --- api/api/urls.py | 1 + discord_bot/discord_bot/bot.py | 1 + discord_bot/discord_bot/config.py | 1 + 3 files changed, 3 insertions(+) diff --git a/api/api/urls.py b/api/api/urls.py index ef4fbe1..d8e809e 100644 --- a/api/api/urls.py +++ b/api/api/urls.py @@ -14,6 +14,7 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ + from django.contrib import admin from django.conf import settings from django.urls import path, include diff --git a/discord_bot/discord_bot/bot.py b/discord_bot/discord_bot/bot.py index 0531fdc..24bf24c 100644 --- a/discord_bot/discord_bot/bot.py +++ b/discord_bot/discord_bot/bot.py @@ -1,4 +1,5 @@ """Sets up discord bot.""" + import os from collections import defaultdict from typing import Self diff --git a/discord_bot/discord_bot/config.py b/discord_bot/discord_bot/config.py index f6b2c04..fb668bd 100644 --- a/discord_bot/discord_bot/config.py +++ b/discord_bot/discord_bot/config.py @@ -1,3 +1,4 @@ """All configurations.""" + PREFIX = "!" BOT_NAME = "" From 6eca621a706d5bfaf63615bb64944c8713ec8324 Mon Sep 17 00:00:00 2001 From: bafaurazan Date: Wed, 13 Mar 2024 20:10:17 +0100 Subject: [PATCH 18/18] fix3 --- api/chunks/controllers.py | 3 --- api/chunks/views.py | 1 - api/documents/controllers.py | 3 --- api/documents/views.py | 1 - 4 files changed, 8 deletions(-) diff --git a/api/chunks/controllers.py b/api/chunks/controllers.py index 76913b4..e8b6347 100644 --- a/api/chunks/controllers.py +++ b/api/chunks/controllers.py @@ -1,11 +1,8 @@ from http import HTTPStatus -from django.http import HttpRequest from chunks.models import Chunk from chunks.schemas import ChunkIn, ChunkOut -import json - def create_chunk_controller(payload: ChunkIn) -> tuple[HTTPStatus, ChunkOut]: chunk = Chunk(**payload.dict()) diff --git a/api/chunks/views.py b/api/chunks/views.py index 7453b6f..df363bb 100644 --- a/api/chunks/views.py +++ b/api/chunks/views.py @@ -1,7 +1,6 @@ from http import HTTPStatus from django.http import HttpRequest -from chunks.models import Chunk from chunks.schemas import ChunkIn, ChunkOut from chunks.controllers import ( create_chunk_controller, diff --git a/api/documents/controllers.py b/api/documents/controllers.py index 85f888d..9c56107 100644 --- a/api/documents/controllers.py +++ b/api/documents/controllers.py @@ -1,11 +1,8 @@ from http import HTTPStatus -from django.http import HttpRequest from documents.models import Document from documents.schemas import DocumentIn, DocumentOut -import json - def create_document_controller(payload: DocumentIn) -> tuple[HTTPStatus, DocumentOut]: document = Document(**payload.dict()) diff --git a/api/documents/views.py b/api/documents/views.py index efb987e..d873c5c 100644 --- a/api/documents/views.py +++ b/api/documents/views.py @@ -1,7 +1,6 @@ from http import HTTPStatus from django.http import HttpRequest -from documents.models import Document from documents.schemas import DocumentIn, DocumentOut from documents.controllers import ( create_document_controller,