Merge pull request #18 from dfe-analytical-services/update_pydantic

chore: update settings to use pydantic v2

.env.example

@@ -1,11 +1,11 @@
+CHAT_URL_API=http://localhost:8010/api/chat
+CHAT_URL_PUBLIC_UI=http://localhost:3002
+EES_URL_API_CONTENT=http://localhost:5010/api
+EES_URL_API_DATA=http://localhost:5000/api
+EES_URL_PUBLIC_UI=http://localhost:3000
 OPENAI_API_KEY=
 OPENAI_API_MODEL=gpt-4
-OPENAI_API_EMBEDDING_MODEL=text-embedding-ada-002
+OPENAI_EMBEDDING_MODEL=text-embedding-ada-002
 QDRANT_COLLECTION=ees
 QDRANT_HOST=localhost
-QDRANT_PORT=6333
-URL_API_DATA_ENDPOINT=http://localhost:8010/api/chat
-URL_API_CONTENT=http://localhost:5010/api
-URL_API_DATA=http://localhost:5000/api
-URL_CHAT_UI=http://localhost:3002
-URL_PUBLIC_SITE=http://localhost:3000
+QDRANT_PORT=6333

Pipfile

@@ -9,7 +9,7 @@ openai = "==0.28.1"
 fastapi = "==0.103.1"
 uvicorn = "==0.23.2"
 asyncio = "==3.4.3"
-pydantic = "==1.10.12"
+pydantic-settings = "2.0.3"
 python-dotenv = "==1.0.0"
 langchain = "==0.0.267"
 requests = "==2.31.0"

data_ingestion/config.py

@@ -1,21 +1,17 @@
-from pydantic import BaseSettings
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
 class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    ees_url_api_content: str = "http://localhost:5010/api"
+    ees_url_api_data: str = "http://localhost:5000/api"
+    ees_url_public_ui: str = "http://localhost:3000"
     openai_api_key: str
-    embedding_model: str = "text-embedding-ada-002"
+    openai_embedding_model: str = "text-embedding-ada-002"
     qdrant_collection: str = "ees"
     qdrant_host: str = "localhost"
     qdrant_port: int = 6333
-    url_api_content: str = "http://localhost:5010/api"
-    url_api_data: str = "http://localhost:5000/api"
-    url_public_site: str = "http://localhost:3000"
-
-    # TODO update to use SettingsConfigDict from pydantic-settings once langchain is updated to use pydantic v2
-    # model_config = SettingsConfigDict(env_file=".env")
-
-    class Config:
-        env_file = "../../.env"
 
 
 LOGGING_CONFIG = {

data_ingestion/services/methodology_service.py

@@ -11,7 +11,7 @@
 
 
 def delete_methodology(slug: str):
-    delete_url(url=f"{settings.url_api_content}/methodology{slug}")
+    delete_url(url=f"{settings.ees_url_api_content}/methodology{slug}")
 
 
 def extract_methodologies(slugs):
@@ -27,8 +27,8 @@ def extract_methodologies(slugs):
 
 def fetch_methodology(slug: str):
     methodology_content = {}
-    methodology_content["link"] = f"{settings.url_public_site}/methodology/{slug}"
-    res = requests.get(f"{settings.url_api_content}/methodologies/{slug}")
+    methodology_content["link"] = f"{settings.ees_url_public_ui}/methodology/{slug}"
+    res = requests.get(f"{settings.ees_url_api_content}/methodologies/{slug}")
     text = json.loads(res.text)
     try:
         methodology_content["data"] = "Headlines Section: "
@@ -51,7 +51,7 @@ def fetch_methodology(slug: str):
 
 
 def fetch_methodology_slugs():
-    data = requests.get(f"{settings.url_api_content}/methodology-themes").json()
+    data = requests.get(f"{settings.ees_url_api_content}/methodology-themes").json()
     slugs = []
     for item in data:
         for topic in item["topics"]:

data_ingestion/services/publication_service.py

@@ -10,13 +10,13 @@
 
 
 def delete_publication(slug: str):
-    delete_url(url=f"{settings.url_public_site}/find-statistics/{slug}")
+    delete_url(url=f"{settings.ees_url_public_ui}/find-statistics/{slug}")
 
 
 def fetch_publication_slugs():
     try:
         response = requests.get(
-            f"{settings.url_api_content}/publications?page=1&pageSize=9999&sort=published&order=asc"
+            f"{settings.ees_url_api_content}/publications?page=1&pageSize=9999&sort=published&order=asc"
         )
         response.raise_for_status()
         publications = json.loads(response.text)["results"]

data_ingestion/services/release_service.py

@@ -14,7 +14,7 @@ def extract_releases(slugs: str) -> List[Dict]:
     texts = []
     for slug in slugs:
         slug_info = {}
-        res = requests.get(f"{settings.url_api_content}/publications/{slug}/releases/latest")
+        res = requests.get(f"{settings.ees_url_api_content}/publications/{slug}/releases/latest")
         key_stats = {}
         response_json = res.json()
         release_id = response_json["publication"]["releases"][0]["id"]
@@ -43,7 +43,7 @@ def extract_releases(slugs: str) -> List[Dict]:
 
 def fetch_release(slug: str, res: dict) -> dict:
     slug_content = {}
-    slug_content["link"] = f"{settings.url_public_site}/find-statistics/{slug}"
+    slug_content["link"] = f"{settings.ees_url_public_ui}/find-statistics/{slug}"
     try:
         slug_content["data"] = "Headlines Section: "
         slug_content["data"] += BeautifulSoup(res["headlinesSection"]["content"][0]["body"], "html.parser").get_text()

data_ingestion/services/tablebuilder_service.py

@@ -10,7 +10,7 @@
 def fetch_key_stat(statistic: dict, release_id: str, i: int) -> str:
     try:
         data_block_id = statistic["dataBlockId"]
-        res = requests.get(f"{settings.url_api_data}/tablebuilder/release/{release_id}/data-block/{data_block_id}")
+        res = requests.get(f"{settings.ees_url_api_data}/tablebuilder/release/{release_id}/data-block/{data_block_id}")
         response_json = res.json()
         label = response_json["subjectMeta"]["indicators"][0]["label"]
         measure = list(response_json["results"][0]["measures"].values())[0]