-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
291 lines (246 loc) · 11 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
import os.path
from enum import Enum
from typing import List
from fastapi import FastAPI, UploadFile, File, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from database_functions import *
from io_processing import *
from query_with_gptindex import *
from query_with_langchain import *
from cloud_storage import *
import uuid
import shutil
from zipfile import ZipFile
from query_with_tfidf import querying_with_tfidf
api_description = """
Jugalbandi.ai has a vector datastore that allows you to get factual Q&A over a document set.
API is currently available in it's alpha version. We are currently gaining test data to improve our systems and
predictions. 🚀
## Factual Q&A over large documents
You will be able to:
* **Upload documents** (_implemented_).
Allows you to upload documents and create a vector space for semantic similarity search. Basically a better search
than Ctrl+F
* **Factual Q&A** (_implemented_).
Allows you to pass uuid for a document set and ask a question for factual response over it.
"""
app = FastAPI(title="Jugalbandi.ai",
description=api_description,
version="0.0.1",
terms_of_service="http://example.com/terms/",
contact={
"name": "Karanraj - Major contributor in Jugalbandi.ai",
"email": "[email protected]",
},
license_info={
"name": "MIT License",
"url": "https://www.jugalbandi.ai/",
}, )
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
class Response(BaseModel):
query: str = None
answer: str = None
source_text: str = None
class ResponseForAudio(BaseModel):
query: str = None
query_in_english: str = None
answer: str = None
answer_in_english: str = None
audio_output_url: str = None
source_text: str = None
class DropdownOutputFormat(str, Enum):
TEXT = "Text"
VOICE = "Voice"
class DropDownInputLanguage(str, Enum):
en = "English"
hi = "Hindi"
kn = "Kannada"
@app.get("/")
async def root():
return {"message": "Welcome to Jugalbandi API"}
@app.get("/query-with-gptindex", tags=["Q&A over Document Store"])
async def query_using_gptindex(uuid_number: str, query_string: str) -> Response:
load_dotenv()
answer, source_text, error_message, status_code = querying_with_gptindex(uuid_number, query_string)
engine = await create_engine()
await insert_qa_logs(engine=engine, model_name="gpt-index", uuid_number=uuid_number, query=query_string,
paraphrased_query=None, response=answer, source_text=source_text, error_message=error_message)
await engine.close()
if status_code != 200:
raise HTTPException(status_code=status_code, detail=error_message)
response = Response()
response.query = query_string
response.answer = answer
response.source_text = source_text
return response
@app.get("/query-with-langchain", tags=["Q&A over Document Store"])
async def query_using_langchain(uuid_number: str, query_string: str) -> Response:
load_dotenv()
answer, source_text, paraphrased_query, error_message, status_code = querying_with_langchain(uuid_number,
query_string)
engine = await create_engine()
await insert_qa_logs(engine=engine, model_name="langchain", uuid_number=uuid_number, query=query_string,
paraphrased_query=paraphrased_query, response=answer, source_text=source_text,
error_message=error_message)
await engine.close()
if status_code != 200:
raise HTTPException(status_code=status_code, detail=error_message)
response = Response()
response.query = query_string
response.answer = answer
response.source_text = source_text
return response
@app.post("/upload-files", tags=["Document Store"])
async def upload_files(description: str, files: List[UploadFile] = File(...)):
load_dotenv()
uuid_number = str(uuid.uuid1())
os.makedirs(uuid_number)
files_list = []
for file in files:
try:
contents = file.file.read()
with open(file.filename, 'wb') as f:
f.write(contents)
except OSError:
return "There was an error uploading the file(s)"
finally:
if ".zip" in file.filename:
os.makedirs("temp_archive")
with ZipFile(file.filename, 'r') as zip_ref:
zip_ref.extractall("temp_archive")
bad_zip_folder = "temp_archive/__MACOSX"
if os.path.exists(bad_zip_folder):
shutil.rmtree(bad_zip_folder)
archived_files = os.listdir("temp_archive")
files_list.extend(archived_files)
for archived_file in archived_files:
shutil.move("temp_archive/" + archived_file, archived_file)
upload_file(uuid_number, archived_file)
shutil.move(archived_file, uuid_number + "/" + archived_file)
shutil.rmtree("temp_archive")
os.remove(file.filename)
else:
files_list.append(file.filename)
upload_file(uuid_number, file.filename)
file.file.close()
shutil.move(file.filename, uuid_number + "/" + file.filename)
error_message, status_code = gpt_indexing(uuid_number)
if status_code == 200:
error_message, status_code = langchain_indexing(uuid_number)
engine = await create_engine()
await insert_document_store_logs(engine=engine, description=description, uuid_number=uuid_number,
documents_list=files_list, error_message=error_message)
await engine.close()
if status_code != 200:
raise HTTPException(status_code=status_code, detail=error_message)
index_files = ["index.json", "index.faiss", "index.pkl"]
for index_file in index_files:
upload_file(uuid_number, index_file)
os.remove(index_file)
shutil.rmtree(uuid_number)
return {"uuid_number": str(uuid_number), "message": "Files uploading is successful"}
@app.get("/query-using-voice", tags=["Q&A over Document Store"])
async def query_with_voice_input(uuid_number: str, input_language: DropDownInputLanguage,
output_format: DropdownOutputFormat, query_text: str = "",
audio_url: str = "") -> ResponseForAudio:
load_dotenv()
language = input_language.name
output_medium = output_format.name
is_audio = False
text = None
paraphrased_query = None
regional_answer = None
answer = None
audio_output_url = None
source_text = None
if query_text == "" and audio_url == "":
query_text = None
error_message = "Either 'Query Text' or 'Audio URL' should be present"
status_code = 422
else:
if query_text != "":
text, error_message = process_incoming_text(query_text, language)
if output_format.name == "VOICE":
is_audio = True
else:
query_text, text, error_message = process_incoming_voice(audio_url, language)
output_medium = "VOICE"
is_audio = True
if text is not None:
print(text)
answer, source_text, paraphrased_query, error_message, status_code = querying_with_langchain(uuid_number,
text)
if answer is not None:
regional_answer, error_message = process_outgoing_text(answer, language)
if regional_answer is not None:
if is_audio:
output_file, error_message = process_outgoing_voice(regional_answer, language)
if output_file is not None:
upload_file("output_audio_files", output_file.name)
audio_output_url = give_public_url(output_file.name)
output_file.close()
os.remove(output_file.name)
else:
status_code = 503
else:
audio_output_url = ""
else:
status_code = 503
else:
status_code = 503
engine = await create_engine()
await insert_qa_voice_logs(engine=engine, uuid_number=uuid_number, input_language=input_language.value,
output_format=output_medium, query=query_text, query_in_english=text,
paraphrased_query=paraphrased_query, response=regional_answer,
response_in_english=answer,
audio_output_link=audio_output_url, source_text=source_text, error_message=error_message)
await engine.close()
if status_code != 200:
raise HTTPException(status_code=status_code, detail=error_message)
response = ResponseForAudio()
response.query = query_text
response.query_in_english = text
response.answer = regional_answer
response.answer_in_english = answer
response.audio_output_url = audio_output_url
response.source_text = source_text
return response
@app.get("/rephrased-query")
async def get_rephrased_query(query_string: str):
load_dotenv()
answer = rephrased_question(query_string)
return {"given_query": query_string, "rephrased_query": answer}
@app.post("/source-document", tags=["Source Document over Document Store"])
async def get_source_document(query_string: str = "", input_language: DropDownInputLanguage = DropDownInputLanguage.en,
audio_file: UploadFile = File(None)):
load_dotenv()
filename = ""
if audio_file is not None:
try:
contents = audio_file.file.read()
with open(audio_file.filename, 'wb') as f:
f.write(contents)
filename = audio_file.filename
except OSError:
return "There was an parsing the audio file"
answer = querying_with_tfidf(query_string, input_language.name, filename)
return answer
@app.get("/query-with-langchain-gpt4", tags=["Q&A over Document Store"])
async def query_using_langchain_with_gpt4(uuid_number: str, query_string: str) -> Response:
load_dotenv()
answer, source_text, paraphrased_query, error_message, status_code = querying_with_langchain_gpt4(uuid_number,
query_string)
if status_code != 200:
raise HTTPException(status_code=status_code, detail=error_message)
response = Response()
response.query = query_string
response.answer = answer
response.source_text = source_text
return response