diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e7e0f3d1..c22a4400a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, 3.10, 3.11] + python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 @@ -30,14 +30,15 @@ jobs: - name: Lint run: | - pip install flake8 + pip install flake8 ruff flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # stop the build if there are Python syntax errors or undefined names - #ruff --format=github --select=E9,F63,F7,F82 --target-version=py37 . + ruff --format=github --select=E9,F63,F7,F82 --target-version=py37 . # default set of ruff rules with GitHub Annotations - #ruff --format=github --target-version=py37 . + #ruff --format=github --target-version=py37 --ignore=F401,E501 . - name: Run tests run: | pip install pytest - pytest + cd pilot + PYTHONPATH=. pytest diff --git a/pilot/const/llm.py b/pilot/const/llm.py index 8ae8ab85a..f1f2d6f0d 100644 --- a/pilot/const/llm.py +++ b/pilot/const/llm.py @@ -1,5 +1,5 @@ import os -MAX_GPT_MODEL_TOKENS = int(os.getenv('MAX_TOKENS')) +MAX_GPT_MODEL_TOKENS = int(os.getenv('MAX_TOKENS', 8192)) MIN_TOKENS_FOR_GPT_RESPONSE = 600 MAX_QUESTIONS = 5 END_RESPONSE = "EVERYTHING_CLEAR" \ No newline at end of file diff --git a/pilot/database/database.py b/pilot/database/database.py index 03b131a28..770c9e884 100644 --- a/pilot/database/database.py +++ b/pilot/database/database.py @@ -4,10 +4,10 @@ from functools import reduce import operator import psycopg2 -from const.common import PROMPT_DATA_TO_IGNORE -from logger.logger import logger from psycopg2.extensions import quote_ident +from const.common import PROMPT_DATA_TO_IGNORE +from logger.logger import logger from utils.utils import hash_data from database.config import DB_NAME, DB_HOST, DB_PORT, DB_USER, DB_PASSWORD, DATABASE_TYPE from database.models.components.base_models import database diff --git a/pilot/database/models/architecture.py b/pilot/database/models/architecture.py index 261acb907..e80613079 100644 --- a/pilot/database/models/architecture.py +++ b/pilot/database/models/architecture.py @@ -12,4 +12,4 @@ class Architecture(ProgressStep): architecture = JSONField() # Custom JSON field for SQLite class Meta: - db_table = 'architecture' + table_name = 'architecture' diff --git a/pilot/database/models/command_runs.py b/pilot/database/models/command_runs.py index b6c34c494..a6b55fbce 100644 --- a/pilot/database/models/command_runs.py +++ b/pilot/database/models/command_runs.py @@ -13,7 +13,7 @@ class CommandRuns(BaseModel): previous_step = ForeignKeyField('self', null=True, column_name='previous_step') class Meta: - db_table = 'command_runs' + table_name = 'command_runs' indexes = ( (('app', 'hash_id'), True), ) \ No newline at end of file diff --git a/pilot/database/models/development.py b/pilot/database/models/development.py index dea5a7e63..2d3faaa60 100644 --- a/pilot/database/models/development.py +++ b/pilot/database/models/development.py @@ -5,4 +5,4 @@ class Development(ProgressStep): class Meta: - db_table = 'development' + table_name = 'development' diff --git a/pilot/database/models/development_planning.py b/pilot/database/models/development_planning.py index 8fe7a5595..c5831a994 100644 --- a/pilot/database/models/development_planning.py +++ b/pilot/database/models/development_planning.py @@ -12,4 +12,4 @@ class DevelopmentPlanning(ProgressStep): development_plan = JSONField() # Custom JSON field for SQLite class Meta: - db_table = 'development_planning' + table_name = 'development_planning' diff --git a/pilot/database/models/development_steps.py b/pilot/database/models/development_steps.py index 6492a4de9..aaf6b663e 100644 --- a/pilot/database/models/development_steps.py +++ b/pilot/database/models/development_steps.py @@ -20,7 +20,7 @@ class DevelopmentSteps(BaseModel): previous_step = ForeignKeyField('self', null=True, column_name='previous_step') class Meta: - db_table = 'development_steps' + table_name = 'development_steps' indexes = ( (('app', 'hash_id'), True), ) diff --git a/pilot/database/models/environment_setup.py b/pilot/database/models/environment_setup.py index e34d60901..1fe22b706 100644 --- a/pilot/database/models/environment_setup.py +++ b/pilot/database/models/environment_setup.py @@ -3,4 +3,4 @@ class EnvironmentSetup(ProgressStep): class Meta: - db_table = 'environment_setup' + table_name = 'environment_setup' diff --git a/pilot/database/models/file_snapshot.py b/pilot/database/models/file_snapshot.py index 9138a5ff4..68659f240 100644 --- a/pilot/database/models/file_snapshot.py +++ b/pilot/database/models/file_snapshot.py @@ -12,7 +12,7 @@ class FileSnapshot(BaseModel): content = TextField() class Meta: - db_table = 'file_snapshot' + table_name = 'file_snapshot' indexes = ( (('development_step', 'file'), True), ) \ No newline at end of file diff --git a/pilot/database/models/project_description.py b/pilot/database/models/project_description.py index 462c1a266..bb4b5ac20 100644 --- a/pilot/database/models/project_description.py +++ b/pilot/database/models/project_description.py @@ -7,4 +7,4 @@ class ProjectDescription(ProgressStep): summary = TextField() class Meta: - db_table = 'project_description' + table_name = 'project_description' diff --git a/pilot/database/models/user_apps.py b/pilot/database/models/user_apps.py index d70672f16..38aee23dd 100644 --- a/pilot/database/models/user_apps.py +++ b/pilot/database/models/user_apps.py @@ -12,7 +12,7 @@ class UserApps(BaseModel): workspace = CharField(null=True) class Meta: - db_table = 'user_apps' + table_name = 'user_apps' indexes = ( (('app', 'user'), True), ) diff --git a/pilot/database/models/user_inputs.py b/pilot/database/models/user_inputs.py index 7d2451c01..8c8399376 100644 --- a/pilot/database/models/user_inputs.py +++ b/pilot/database/models/user_inputs.py @@ -13,7 +13,7 @@ class UserInputs(BaseModel): previous_step = ForeignKeyField('self', null=True, column_name='previous_step') class Meta: - db_table = 'user_inputs' + table_name = 'user_inputs' indexes = ( (('app', 'hash_id'), True), ) \ No newline at end of file diff --git a/pilot/database/models/user_stories.py b/pilot/database/models/user_stories.py index 025e2555d..dd7e06a4e 100644 --- a/pilot/database/models/user_stories.py +++ b/pilot/database/models/user_stories.py @@ -11,4 +11,4 @@ class UserStories(ProgressStep): else: user_stories = JSONField() # Custom JSON field for SQLite class Meta: - db_table = 'user_stories' + table_name = 'user_stories' diff --git a/pilot/database/models/user_tasks.py b/pilot/database/models/user_tasks.py index 533340a4d..261de2d9f 100644 --- a/pilot/database/models/user_tasks.py +++ b/pilot/database/models/user_tasks.py @@ -12,4 +12,4 @@ class UserTasks(ProgressStep): user_tasks = JSONField() # Custom JSON field for SQLite class Meta: - db_table = 'user_tasks' + table_name = 'user_tasks' diff --git a/pilot/helpers/agents/Developer.py b/pilot/helpers/agents/Developer.py index 47e41756e..9ff9e607f 100644 --- a/pilot/helpers/agents/Developer.py +++ b/pilot/helpers/agents/Developer.py @@ -259,7 +259,7 @@ def implement_step(self, convo, step_index, type, description): if type == 'COMMAND': for cmd in step_details: run_command_until_success(cmd['command'], cmd['timeout'], convo) - elif type == 'CODE_CHANGE': - code_changes_details = get_step_code_changes() + # elif type == 'CODE_CHANGE': + # code_changes_details = get_step_code_changes() # TODO: give to code monkey for implementation pass diff --git a/pilot/logger/__init__.py b/pilot/logger/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pilot/main.py b/pilot/main.py index 751ea9aa6..d4f9a8484 100644 --- a/pilot/main.py +++ b/pilot/main.py @@ -2,13 +2,10 @@ from __future__ import print_function, unicode_literals import sys - from dotenv import load_dotenv -from termcolor import colored load_dotenv() - +from termcolor import colored from helpers.Project import Project - from utils.arguments import get_arguments from utils.exit import exit_gpt_pilot from logger.logger import logger diff --git a/pilot/prompts/dev_ops/debug.prompt b/pilot/prompts/dev_ops/debug.prompt index 30de7cf95..bd0ddf9ca 100644 --- a/pilot/prompts/dev_ops/debug.prompt +++ b/pilot/prompts/dev_ops/debug.prompt @@ -7,7 +7,7 @@ You wanted me to check this - `{{ issue_description }}` but there was a problem{ `run_command` function will run a command on the machine and will return the CLI output to you so you can see what to do next. -`implement_code_changes` function will change the code where you just need to thoroughly describe what needs to be implmemented, I will implement the requested changes and let you know. +`implement_code_changes` function will change the code where you just need to thoroughly describe what needs to be implemented, I will implement the requested changes and let you know. Return a list of steps that are needed to debug this issue. By the time we execute the last step, the issue should be fixed completely. Also, make sure that at least the last step has `check_if_fixed` set to TRUE. diff --git a/pilot/utils/arguments.py b/pilot/utils/arguments.py index 0a0714e71..e4409ebd5 100644 --- a/pilot/utils/arguments.py +++ b/pilot/utils/arguments.py @@ -1,10 +1,10 @@ -import getpass import hashlib +import os +import re import sys import uuid - +from getpass import getuser from termcolor import colored - from database.database import get_app, get_app_by_user_workspace @@ -25,7 +25,7 @@ def get_arguments(): arguments[arg] = True if 'user_id' not in arguments: - arguments['user_id'] = username_to_uuid(getpass.getuser()) + arguments['user_id'] = username_to_uuid(getuser()) app = None if 'workspace' in arguments: @@ -40,7 +40,6 @@ def get_arguments(): if app is None: app = get_app(arguments['app_id']) - # arguments['user_id'] = str(app.user.id) arguments['app_type'] = app.app_type arguments['name'] = app.name # Add any other fields from the App model you wish to include @@ -54,19 +53,12 @@ def get_arguments(): else: arguments['app_id'] = str(uuid.uuid4()) print(colored('\n------------------ STARTING NEW PROJECT ----------------------', 'green', attrs=['bold'])) - print(f"If you wish to continue with this project in future run:") + print("If you wish to continue with this project in future run:") print(colored(f'python {sys.argv[0]} app_id={arguments["app_id"]}', 'green', attrs=['bold'])) print(colored('--------------------------------------------------------------\n', 'green', attrs=['bold'])) - - - if 'user_id' not in arguments: - arguments['user_id'] = username_to_uuid(getpass.getuser()) - if 'email' not in arguments: - # todo change email so its not uuid4 but make sure to fix storing of development steps where - # 1 user can have multiple apps. In that case each app should have its own development steps - arguments['email'] = str(uuid.uuid4()) + arguments['email'] = get_email() if 'password' not in arguments: arguments['password'] = 'password' @@ -77,6 +69,26 @@ def get_arguments(): return arguments +def get_email(): + # Attempt to get email from .gitconfig + gitconfig_path = os.path.expanduser('~/.gitconfig') + + if os.path.exists(gitconfig_path): + with open(gitconfig_path, 'r') as file: + content = file.read() + + # Use regex to search for email address + email_match = re.search(r'email\s*=\s*([\w\.-]+@[\w\.-]+)', content) + + if email_match: + return email_match.group(1) + + # If not found, return a UUID + # todo change email so its not uuid4 but make sure to fix storing of development steps where + # 1 user can have multiple apps. In that case each app should have its own development steps + return str(uuid.uuid4()) + + # TODO can we make BaseModel.id a CharField with default=uuid4? def username_to_uuid(username): sha1 = hashlib.sha1(username.encode()).hexdigest() diff --git a/pilot/utils/llm_connection.py b/pilot/utils/llm_connection.py index d72515e29..72167f289 100644 --- a/pilot/utils/llm_connection.py +++ b/pilot/utils/llm_connection.py @@ -1,6 +1,8 @@ +import re import requests import os import sys +import time import json import tiktoken import questionary @@ -116,7 +118,7 @@ def create_gpt_chat_completion(messages: List[dict], req_type, min_tokens=MIN_TO # Check if the error message is related to token limit if "context_length_exceeded" in error_message.lower(): - raise Exception(f'Too many tokens in the request. Please try to continue the project with some previous development step.') + raise Exception('Too many tokens in the request. Please try to continue the project with some previous development step.') else: print('The request to OpenAI API failed. Here is the error message:') print(e) @@ -147,8 +149,15 @@ def wrapper(*args, **kwargs): # If the specific error "context_length_exceeded" is present, simply return without retry if "context_length_exceeded" in err_str: raise Exception("context_length_exceeded") - - print(colored(f'There was a problem with request to openai API:', 'red')) + if "rate_limit_exceeded" in err_str: + # Extracting the duration from the error string + match = re.search(r"Please try again in (\d+)ms.", err_str) + if match: + wait_duration = int(match.group(1)) / 1000 + time.sleep(wait_duration) + continue + + print(colored('There was a problem with request to openai API:', 'red')) print(err_str) user_message = questionary.text( @@ -187,10 +196,16 @@ def return_result(result_data, lines_printed): if endpoint == 'AZURE': # If yes, get the AZURE_ENDPOINT from .ENV file endpoint_url = os.getenv('AZURE_ENDPOINT') + '/openai/deployments/' + model + '/chat/completions?api-version=2023-05-15' - headers = {'Content-Type': 'application/json', 'api-key': os.getenv('AZURE_API_KEY')} + headers = { + 'Content-Type': 'application/json', + 'api-key': os.getenv('AZURE_API_KEY') + } else: # If not, send the request to the OpenAI endpoint - headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + os.getenv("OPENAI_API_KEY")} + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer ' + os.getenv("OPENAI_API_KEY") + } endpoint_url = 'https://api.openai.com/v1/chat/completions' response = requests.post( @@ -230,7 +245,7 @@ def return_result(result_data, lines_printed): if json_line['choices'][0]['finish_reason'] == 'function_call': function_calls['arguments'] = load_data_to_json(function_calls['arguments']) - return return_result({'function_calls': function_calls}, lines_printed); + return return_result({'function_calls': function_calls}, lines_printed) json_line = json_line['choices'][0]['delta'] diff --git a/pilot/utils/test_arguments.py b/pilot/utils/test_arguments.py new file mode 100644 index 000000000..25998e3aa --- /dev/null +++ b/pilot/utils/test_arguments.py @@ -0,0 +1,40 @@ +import pytest +from unittest.mock import patch, mock_open +import uuid +from .arguments import get_email, username_to_uuid + + +def test_email_found_in_gitconfig(): + mock_file_content = """ + [user] + name = test_user + email = test@example.com + """ + with patch('os.path.exists', return_value=True): + with patch('builtins.open', mock_open(read_data=mock_file_content)): + assert get_email() == "test@example.com" + + +def test_email_not_found_in_gitconfig(): + mock_file_content = """ + [user] + name = test_user + """ + mock_uuid = "12345678-1234-5678-1234-567812345678" + + with patch('os.path.exists', return_value=True): + with patch('builtins.open', mock_open(read_data=mock_file_content)): + with patch.object(uuid, "uuid4", return_value=mock_uuid): + assert get_email() == mock_uuid + + +def test_gitconfig_not_present(): + mock_uuid = "12345678-1234-5678-1234-567812345678" + + with patch('os.path.exists', return_value=False): + with patch.object(uuid, "uuid4", return_value=mock_uuid): + assert get_email() == mock_uuid + + +def test_username_to_uuid(): + assert username_to_uuid("test_user") == "31676025-316f-b555-e0bf-a12f0bcfd0ea"