From f9ab197f26fe0d15ce1f20d1eba19c938dec9df0 Mon Sep 17 00:00:00 2001 From: Nicholas Albion Date: Wed, 11 Oct 2023 16:50:42 +1100 Subject: [PATCH 1/9] #121 create .gpt-pilot directory, save project info & chat log --- pilot/const/common.py | 1 + pilot/helpers/Project.py | 7 +++ pilot/helpers/agents/ProductOwner.py | 4 +- pilot/helpers/agents/test_CodeMonkey.py | 4 +- pilot/helpers/agents/test_Developer.py | 4 +- pilot/helpers/agents/test_TechLead.py | 4 +- pilot/helpers/test_Project.py | 62 ++++++++++++++++++- .../ux_tests/run_command_until_success.py | 4 +- pilot/utils/dot_gpt_pilot.py | 54 ++++++++++++++++ pilot/utils/llm_connection.py | 3 +- 10 files changed, 133 insertions(+), 14 deletions(-) create mode 100644 pilot/utils/dot_gpt_pilot.py diff --git a/pilot/const/common.py b/pilot/const/common.py index 5a4eb373b..1e9f559d8 100644 --- a/pilot/const/common.py +++ b/pilot/const/common.py @@ -21,6 +21,7 @@ IGNORE_FOLDERS = [ '.git', + '.gpt-pilot', '.idea', '.vscode', '__pycache__', diff --git a/pilot/helpers/Project.py b/pilot/helpers/Project.py index a416e3bbc..390a8ec0f 100644 --- a/pilot/helpers/Project.py +++ b/pilot/helpers/Project.py @@ -19,6 +19,7 @@ from database.models.file_snapshot import FileSnapshot from database.models.files import File from logger.logger import logger +from utils.dot_gpt_pilot import DotGptPilot class Project: @@ -69,6 +70,11 @@ def __init__(self, args, name=None, description=None, user_stories=None, user_ta self.architecture = architecture # if development_plan is not None: # self.development_plan = development_plan + self.dot_pilot_gpt = DotGptPilot() + + def set_root_path(self, root_path: str): + self.root_path = root_path + self.dot_pilot_gpt.with_root_path(root_path) def start(self): """ @@ -128,6 +134,7 @@ def start(self): break # TODO END + self.dot_pilot_gpt.write_project(self) print(json.dumps({ "project_stage": "coding" }), type='info') diff --git a/pilot/helpers/agents/ProductOwner.py b/pilot/helpers/agents/ProductOwner.py index c8d0d43cd..069bb5846 100644 --- a/pilot/helpers/agents/ProductOwner.py +++ b/pilot/helpers/agents/ProductOwner.py @@ -27,7 +27,7 @@ def get_project_description(self): step = get_progress_steps(self.project.args['app_id'], PROJECT_DESCRIPTION_STEP) if step and not should_execute_step(self.project.args['step'], PROJECT_DESCRIPTION_STEP): step_already_finished(self.project.args, step) - self.project.root_path = setup_workspace(self.project.args) + self.project.set_oot_path(setup_workspace(self.project.args)) self.project.project_description = step['summary'] self.project.project_description_messages = step['messages'] return @@ -39,7 +39,7 @@ def get_project_description(self): if 'name' not in self.project.args: self.project.args['name'] = clean_filename(ask_user(self.project, 'What is the project name?')) - self.project.root_path = setup_workspace(self.project.args) + self.project.set_root_path(setup_workspace(self.project.args)) self.project.app = save_app(self.project) diff --git a/pilot/helpers/agents/test_CodeMonkey.py b/pilot/helpers/agents/test_CodeMonkey.py index 7a0bc62c4..e3f6c5804 100644 --- a/pilot/helpers/agents/test_CodeMonkey.py +++ b/pilot/helpers/agents/test_CodeMonkey.py @@ -30,8 +30,8 @@ def setup_method(self): current_step='coding', ) - self.project.root_path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), - '../../../workspace/TestDeveloper')) + self.project.set_root_path(os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), + '../../../workspace/TestDeveloper'))) self.project.technologies = [] last_step = DevelopmentSteps() last_step.id = 1 diff --git a/pilot/helpers/agents/test_Developer.py b/pilot/helpers/agents/test_Developer.py index b356f4e6e..84f599326 100644 --- a/pilot/helpers/agents/test_Developer.py +++ b/pilot/helpers/agents/test_Developer.py @@ -31,8 +31,8 @@ def setup_method(self): user_stories=[] ) - self.project.root_path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), - '../../../workspace/TestDeveloper')) + self.project.set_root_path(os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), + '../../../workspace/TestDeveloper'))) self.project.technologies = [] self.project.current_step = ENVIRONMENT_SETUP_STEP self.developer = Developer(self.project) diff --git a/pilot/helpers/agents/test_TechLead.py b/pilot/helpers/agents/test_TechLead.py index d738a4584..158b45c8c 100644 --- a/pilot/helpers/agents/test_TechLead.py +++ b/pilot/helpers/agents/test_TechLead.py @@ -27,8 +27,8 @@ def setup_method(self): user_stories=[] ) - self.project.root_path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), - '../../../workspace/TestTechLead')) + self.project.set_root_path(os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), + '../../../workspace/TestTechLead'))) self.project.technologies = [] self.project.project_description = ''' The project entails creating a web-based chat application, tentatively named "chat_app." diff --git a/pilot/helpers/test_Project.py b/pilot/helpers/test_Project.py index 4ebc8c121..68bf93a18 100644 --- a/pilot/helpers/test_Project.py +++ b/pilot/helpers/test_Project.py @@ -1,5 +1,7 @@ +import os +import json import pytest -from unittest.mock import patch +from unittest.mock import patch, MagicMock from helpers.Project import Project from database.models.files import File @@ -14,7 +16,7 @@ def create_project(): architecture=[], user_stories=[] ) - project.root_path = "/temp/gpt-pilot-test" + project.set_root_path('/temp/gpt-pilot-test') project.app = 'test' return project @@ -100,7 +102,6 @@ def test_get_full_path_absolute(file_path, file_name, expected): # Then assert absolute_path == expected - # This is known to fail and should be avoided # def test_get_full_file_path_error(): # # Given @@ -112,3 +113,58 @@ def test_get_full_path_absolute(file_path, file_name, expected): # # # Then # assert full_path == '/temp/gpt-pilot-test/path/to/file/' + + +class TestProjectFileLists: + def setup_method(self): + # Given a project + project = create_project() + self.project = project + project.set_root_path(os.path.join(os.path.dirname(__file__), '../../workspace/directory_tree')) + project.project_description = 'Test Project' + project.development_plan = [{ + 'description': 'Test User Story', + 'programmatic_goal': 'Test Programmatic Goal', + 'user_review_goal': 'Test User Review Goal', + }] + + # with directories including common.IGNORE_FOLDERS + src = os.path.join(project.root_path, 'src') + os.makedirs(src, exist_ok=True) + for dir in ['.git', '.idea', '.vscode', '__pycache__', 'node_modules', 'venv', 'dist', 'build']: + os.makedirs(os.path.join(project.root_path, dir), exist_ok=True) + + # ...and files + with open(os.path.join(project.root_path, 'package.json'), 'w') as file: + json.dump({'name': 'test app'}, file, indent=2) + with open(os.path.join(src, 'main.js'), 'w') as file: + file.write('console.log("Hello World!");') + + # and a non-empty .gpt-pilot directory + project.dot_pilot_gpt.write_project(project) + + def test_get_directory_tree(self): + # When + tree = self.project.get_directory_tree() + + # Then we should not be including the .gpt-pilot directory or other ignored directories + assert tree == ''' +|-- / +| |-- package.json +| |-- src/ +| | |-- main.js +'''.lstrip() + + @patch('helpers.Project.DevelopmentSteps.get_or_create', return_value=('test', True)) + @patch('helpers.Project.File.get_or_create', return_value=('test', True)) + @patch('helpers.Project.FileSnapshot.get_or_create', return_value=(MagicMock(), True)) + def test_save_files_snapshot(self, mock_snap, mock_file, mock_step): + # Given a snapshot of the files in the project + + # When we save the file snapshot + self.project.save_files_snapshot('test') + + # Then the files should be saved to the project, but nothing from `.gpt-pilot/` + assert mock_file.call_count == 2 + assert mock_file.call_args_list[0][1]['name'] == 'package.json' + assert mock_file.call_args_list[1][1]['name'] == 'main.js' diff --git a/pilot/test/ux_tests/run_command_until_success.py b/pilot/test/ux_tests/run_command_until_success.py index 8b676129d..76b379d1c 100644 --- a/pilot/test/ux_tests/run_command_until_success.py +++ b/pilot/test/ux_tests/run_command_until_success.py @@ -20,8 +20,8 @@ def run_command_until_success(): user_stories=[] ) - project.root_path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), - '../../../workspace/TestDeveloper')) + project.set_root_path(os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), + '../../../workspace/TestDeveloper'))) project.technologies = [] project.current_step = ENVIRONMENT_SETUP_STEP project.app = save_app(project) diff --git a/pilot/utils/dot_gpt_pilot.py b/pilot/utils/dot_gpt_pilot.py new file mode 100644 index 000000000..983f440c3 --- /dev/null +++ b/pilot/utils/dot_gpt_pilot.py @@ -0,0 +1,54 @@ +import os +import yaml +from datetime import datetime + + +# TODO: Parse files from the `.gpt-pilot` directory to resume a project - `user_stories` may have changed - include checksums for sections which may need to be reprocessed. +# TODO: Save a summary at the end of each task/sprint. +class DotGptPilot: + """ + Manages the `.gpt-pilot` directory. + """ + def __init__(self, log_chat_completions: bool = True): + self.log_chat_completions = log_chat_completions + self.dot_gpt_pilot_path = self.with_root_path('~', create=False) + + def with_root_path(self, root_path: str, create=True): + print(f'--------------------with_root_path: {root_path}, create: {create}') + dot_gpt_pilot_path = os.path.join(root_path, '.gpt-pilot') + + # Create the `.gpt-pilot` directory if required. + print(f'create and self.log_chat_completions: {create}, {self.log_chat_completions}') + if create and self.log_chat_completions: # (... or ...): + print('creating dirs: ' + os.path.join(dot_gpt_pilot_path, 'chat_log')) + os.makedirs(os.path.join(dot_gpt_pilot_path, 'chat_log'), exist_ok=True) + else: + print('not creating dirs') + + self.dot_gpt_pilot_path = dot_gpt_pilot_path + return dot_gpt_pilot_path + + def log_chat_completion(self, endpoint: str, model: str, req_type: str, messages: list[dict], response: str): + if self.log_chat_completions: + time = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + with open(os.path.join(self.dot_gpt_pilot_path, 'chat_log', f'{time}-{req_type}.yaml'), 'w') as file: + data = { + 'endpoint': endpoint, + 'model': model, + 'messages': messages, + 'response': response, + } + + yaml.safe_dump(data, file, width=120, indent=2, default_flow_style=False, sort_keys=False) + + def write_project(self, project): + data = { + 'name': project.args['name'], + 'description': project.project_description, + 'user_stories': project.user_stories, + 'architecture': project.architecture, + 'development_plan': project.development_plan, + } + + with open(os.path.join(self.dot_gpt_pilot_path, 'project.yaml'), 'w') as file: + yaml.safe_dump(data, file, width=120, indent=2, default_flow_style=False, sort_keys=False) diff --git a/pilot/utils/llm_connection.py b/pilot/utils/llm_connection.py index 593b98043..7590319e5 100644 --- a/pilot/utils/llm_connection.py +++ b/pilot/utils/llm_connection.py @@ -17,7 +17,6 @@ from utils.function_calling import add_function_calls_to_request, FunctionCallSet, FunctionType from utils.questionary import styled_text - def get_tokens_in_messages(messages: List[str]) -> int: tokenizer = tiktoken.get_encoding("cl100k_base") # GPT-4 tokenizer tokenized_messages = [tokenizer.encode(message['content']) for message in messages] @@ -332,6 +331,7 @@ def return_result(result_data, lines_printed): logger.debug(f'Response status code: {response.status_code}') if response.status_code != 200: + project.dot_pilot_gpt.log_chat_completion(endpoint, model, req_type, data['messages'], response.text) logger.info(f'problem with request: {response.text}') raise Exception(f"API responded with status code: {response.status_code}. Response text: {response.text}") @@ -405,6 +405,7 @@ def return_result(result_data, lines_printed): # function_calls['arguments'] = load_data_to_json(function_calls['arguments']) # return return_result({'function_calls': function_calls}, lines_printed) logger.info('<<<<<<<<<< LLM Response <<<<<<<<<<\n%s\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<', gpt_response) + project.dot_pilot_gpt.log_chat_completion(endpoint, model, req_type, data['messages'], gpt_response) if expecting_json: gpt_response = clean_json_response(gpt_response) From b4817bb534ea6571fd5179665f1d2822ca58d838 Mon Sep 17 00:00:00 2001 From: Nicholas Albion Date: Wed, 11 Oct 2023 17:02:14 +1100 Subject: [PATCH 2/9] add pyyaml 6.0.1 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2da398d51..23110be6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ psycopg2-binary==2.9.6 python-dotenv==1.0.0 python-editor==1.0.4 pytest==7.4.2 +pyyaml==6.0.1 questionary==1.10.0 readchar==4.0.5 regex==2023.6.3 From 55f5580078a17b4f4a40062101b1d3c26a2a42ba Mon Sep 17 00:00:00 2001 From: Nicholas Albion Date: Wed, 11 Oct 2023 18:17:52 +1100 Subject: [PATCH 3/9] Log JSON request/responses for readabilty --- pilot/utils/dot_gpt_pilot.py | 16 +++++++++++++++- pilot/utils/llm_connection.py | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pilot/utils/dot_gpt_pilot.py b/pilot/utils/dot_gpt_pilot.py index 983f440c3..dd7b89345 100644 --- a/pilot/utils/dot_gpt_pilot.py +++ b/pilot/utils/dot_gpt_pilot.py @@ -1,3 +1,4 @@ +import json import os import yaml from datetime import datetime @@ -30,7 +31,7 @@ def with_root_path(self, root_path: str, create=True): def log_chat_completion(self, endpoint: str, model: str, req_type: str, messages: list[dict], response: str): if self.log_chat_completions: - time = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + time = datetime.now().strftime('%Y-%m-%d_%H:%M:%S') with open(os.path.join(self.dot_gpt_pilot_path, 'chat_log', f'{time}-{req_type}.yaml'), 'w') as file: data = { 'endpoint': endpoint, @@ -41,6 +42,19 @@ def log_chat_completion(self, endpoint: str, model: str, req_type: str, messages yaml.safe_dump(data, file, width=120, indent=2, default_flow_style=False, sort_keys=False) + def log_chat_completion_json(self, endpoint: str, model: str, req_type: str, functions: dict, json_response: str): + if self.log_chat_completions: + time = datetime.now().strftime('%Y-%m-%d_%H:%M:%S') + with open(os.path.join(self.dot_gpt_pilot_path, 'chat_log', f'{time}-{req_type}.json'), 'w') as file: + data = { + 'endpoint': endpoint, + 'model': model, + 'functions': functions, + 'response': json.loads(json_response), + } + + json.dump(data, file, indent=2) + def write_project(self, project): data = { 'name': project.args['name'], diff --git a/pilot/utils/llm_connection.py b/pilot/utils/llm_connection.py index 7590319e5..531bf4805 100644 --- a/pilot/utils/llm_connection.py +++ b/pilot/utils/llm_connection.py @@ -410,6 +410,8 @@ def return_result(result_data, lines_printed): if expecting_json: gpt_response = clean_json_response(gpt_response) assert_json_schema(gpt_response, expecting_json) + # Note, we log JSON separately from the YAML log above incase the JSON is invalid and an error is raised + project.dot_pilot_gpt.log_chat_completion_json(endpoint, model, req_type, expecting_json, gpt_response) new_code = postprocessing(gpt_response, req_type) # TODO add type dynamically return return_result({'text': new_code}, lines_printed) From 81b7fc12f6f8ac3160dd677a2dcd6b5a49a98c4e Mon Sep 17 00:00:00 2001 From: Nicholas Albion Date: Wed, 11 Oct 2023 18:42:38 +1100 Subject: [PATCH 4/9] fixed tests --- pilot/helpers/test_Project.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/pilot/helpers/test_Project.py b/pilot/helpers/test_Project.py index 68bf93a18..ac94d3480 100644 --- a/pilot/helpers/test_Project.py +++ b/pilot/helpers/test_Project.py @@ -5,6 +5,8 @@ from helpers.Project import Project from database.models.files import File +test_root = os.path.join(os.path.dirname(__file__), '../../workspace/gpt-pilot-test').replace('\\', '/') + def create_project(): project = Project({ @@ -16,17 +18,17 @@ def create_project(): architecture=[], user_stories=[] ) - project.set_root_path('/temp/gpt-pilot-test') + project.set_root_path(test_root) project.app = 'test' return project @pytest.mark.parametrize('test_data', [ - {'name': 'package.json', 'path': 'package.json', 'saved_to': '/temp/gpt-pilot-test/package.json'}, - {'name': 'package.json', 'path': '', 'saved_to': '/temp/gpt-pilot-test/package.json'}, - # {'name': 'Dockerfile', 'path': None, 'saved_to': '/temp/gpt-pilot-test/Dockerfile'}, - {'name': None, 'path': 'public/index.html', 'saved_to': '/temp/gpt-pilot-test/public/index.html'}, - {'name': '', 'path': 'public/index.html', 'saved_to': '/temp/gpt-pilot-test/public/index.html'}, + {'name': 'package.json', 'path': 'package.json', 'saved_to': f'{test_root}/package.json'}, + {'name': 'package.json', 'path': '', 'saved_to': f'{test_root}/package.json'}, + # {'name': 'Dockerfile', 'path': None, 'saved_to': f'{test_root}/Dockerfile'}, + {'name': None, 'path': 'public/index.html', 'saved_to': f'{test_root}/public/index.html'}, + {'name': '', 'path': 'public/index.html', 'saved_to': f'{test_root}/public/index.html'}, # TODO: Treatment of paths outside of the project workspace - https://github.com/Pythagora-io/gpt-pilot/issues/129 # {'name': '/etc/hosts', 'path': None, 'saved_to': '/etc/hosts'}, @@ -67,12 +69,12 @@ def test_save_file(mock_file_insert, mock_update_file, test_data): @pytest.mark.parametrize('file_path, file_name, expected', [ - ('file.txt', 'file.txt', '/temp/gpt-pilot-test/file.txt'), - ('', 'file.txt', '/temp/gpt-pilot-test/file.txt'), - ('path/', 'file.txt', '/temp/gpt-pilot-test/path/file.txt'), - ('path/to/', 'file.txt', '/temp/gpt-pilot-test/path/to/file.txt'), - ('path/to/file.txt', 'file.txt', '/temp/gpt-pilot-test/path/to/file.txt'), - ('./path/to/file.txt', 'file.txt', '/temp/gpt-pilot-test/./path/to/file.txt'), # ideally result would not have `./` + ('file.txt', 'file.txt', f'{test_root}/file.txt'), + ('', 'file.txt', f'{test_root}/file.txt'), + ('path/', 'file.txt', f'{test_root}/path/file.txt'), + ('path/to/', 'file.txt', f'{test_root}/path/to/file.txt'), + ('path/to/file.txt', 'file.txt', f'{test_root}/path/to/file.txt'), + ('./path/to/file.txt', 'file.txt', f'{test_root}/./path/to/file.txt'), # ideally result would not have `./` ]) def test_get_full_path(file_path, file_name, expected): # Given @@ -112,7 +114,7 @@ def test_get_full_path_absolute(file_path, file_name, expected): # full_path = project.get_full_file_path(file_path, file_name) # # # Then -# assert full_path == '/temp/gpt-pilot-test/path/to/file/' +# assert full_path == f'{test_root}/path/to/file/' class TestProjectFileLists: From fe5416cfb42e05738842d924d2dd1aa8a334292a Mon Sep 17 00:00:00 2001 From: Nicholas Albion Date: Wed, 11 Oct 2023 18:57:11 +1100 Subject: [PATCH 5/9] disable `dot_pilot_gpt` for tests --- pilot/helpers/Project.py | 4 ++-- pilot/utils/dot_gpt_pilot.py | 4 ---- pilot/utils/test_llm_connection.py | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pilot/helpers/Project.py b/pilot/helpers/Project.py index 390a8ec0f..58047d9d3 100644 --- a/pilot/helpers/Project.py +++ b/pilot/helpers/Project.py @@ -24,7 +24,7 @@ class Project: def __init__(self, args, name=None, description=None, user_stories=None, user_tasks=None, architecture=None, - development_plan=None, current_step=None, ipc_client_instance=None): + development_plan=None, current_step=None, ipc_client_instance=None, enable_dot_pilot_gpt=True): """ Initialize a project. @@ -70,7 +70,7 @@ def __init__(self, args, name=None, description=None, user_stories=None, user_ta self.architecture = architecture # if development_plan is not None: # self.development_plan = development_plan - self.dot_pilot_gpt = DotGptPilot() + self.dot_pilot_gpt = DotGptPilot(log_chat_completions=enable_dot_pilot_gpt) def set_root_path(self, root_path: str): self.root_path = root_path diff --git a/pilot/utils/dot_gpt_pilot.py b/pilot/utils/dot_gpt_pilot.py index dd7b89345..dba694793 100644 --- a/pilot/utils/dot_gpt_pilot.py +++ b/pilot/utils/dot_gpt_pilot.py @@ -15,16 +15,12 @@ def __init__(self, log_chat_completions: bool = True): self.dot_gpt_pilot_path = self.with_root_path('~', create=False) def with_root_path(self, root_path: str, create=True): - print(f'--------------------with_root_path: {root_path}, create: {create}') dot_gpt_pilot_path = os.path.join(root_path, '.gpt-pilot') # Create the `.gpt-pilot` directory if required. - print(f'create and self.log_chat_completions: {create}, {self.log_chat_completions}') if create and self.log_chat_completions: # (... or ...): print('creating dirs: ' + os.path.join(dot_gpt_pilot_path, 'chat_log')) os.makedirs(os.path.join(dot_gpt_pilot_path, 'chat_log'), exist_ok=True) - else: - print('not creating dirs') self.dot_gpt_pilot_path = dot_gpt_pilot_path return dot_gpt_pilot_path diff --git a/pilot/utils/test_llm_connection.py b/pilot/utils/test_llm_connection.py index 0d9e79b87..609b693b7 100644 --- a/pilot/utils/test_llm_connection.py +++ b/pilot/utils/test_llm_connection.py @@ -19,7 +19,7 @@ load_dotenv() -project = Project({'app_id': 'test-app'}, current_step='test') +project = Project({'app_id': 'test-app'}, current_step='test', enable_dot_pilot_gpt=False) def test_clean_json_response_True_False(): From 6aaa5938192de5f63573e2cf537ad1d93434c8ff Mon Sep 17 00:00:00 2001 From: Nicholas Albion Date: Wed, 11 Oct 2023 20:45:43 +1100 Subject: [PATCH 6/9] log task chat messages in a task folder --- pilot/helpers/agents/Developer.py | 3 ++- pilot/utils/dot_gpt_pilot.py | 24 +++++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pilot/helpers/agents/Developer.py b/pilot/helpers/agents/Developer.py index 7c88fa9ba..4ae59d661 100644 --- a/pilot/helpers/agents/Developer.py +++ b/pilot/helpers/agents/Developer.py @@ -40,11 +40,12 @@ def start_coding(self): self.implement_task(i, dev_task) # DEVELOPMENT END - + self.project.dot_pilot_gpt.chat_log_folder(None) logger.info('The app is DONE!!! Yay...you can use it now.') def implement_task(self, i, development_task=None): print(green_bold(f'Implementing task #{i + 1}: ') + green(f' {development_task["description"]}\n')) + self.project.dot_pilot_gpt.chat_log_folder(i + 1) convo_dev_task = AgentConvo(self) task_description = convo_dev_task.send_message('development/task/breakdown.prompt', { diff --git a/pilot/utils/dot_gpt_pilot.py b/pilot/utils/dot_gpt_pilot.py index dba694793..8fec0f50b 100644 --- a/pilot/utils/dot_gpt_pilot.py +++ b/pilot/utils/dot_gpt_pilot.py @@ -13,22 +13,31 @@ class DotGptPilot: def __init__(self, log_chat_completions: bool = True): self.log_chat_completions = log_chat_completions self.dot_gpt_pilot_path = self.with_root_path('~', create=False) + self.chat_log_path = self.chat_log_folder(None) def with_root_path(self, root_path: str, create=True): dot_gpt_pilot_path = os.path.join(root_path, '.gpt-pilot') + self.dot_gpt_pilot_path = dot_gpt_pilot_path # Create the `.gpt-pilot` directory if required. if create and self.log_chat_completions: # (... or ...): - print('creating dirs: ' + os.path.join(dot_gpt_pilot_path, 'chat_log')) - os.makedirs(os.path.join(dot_gpt_pilot_path, 'chat_log'), exist_ok=True) + self.chat_log_folder(None) - self.dot_gpt_pilot_path = dot_gpt_pilot_path return dot_gpt_pilot_path + def chat_log_folder(self, task): + chat_log_path = os.path.join(self.dot_gpt_pilot_path, 'chat_log') + if task is not None: + chat_log_path = os.path.join(chat_log_path, 'task_' + task) + + os.makedirs(chat_log_path, exist_ok=True) + self.chat_log_path = chat_log_path + return chat_log_path + def log_chat_completion(self, endpoint: str, model: str, req_type: str, messages: list[dict], response: str): if self.log_chat_completions: - time = datetime.now().strftime('%Y-%m-%d_%H:%M:%S') - with open(os.path.join(self.dot_gpt_pilot_path, 'chat_log', f'{time}-{req_type}.yaml'), 'w') as file: + time = datetime.now().strftime('%Y-%m-%d_%H_%M_%S') + with open(os.path.join(self.chat_log_path, f'{time}-{req_type}.yaml'), 'w') as file: data = { 'endpoint': endpoint, 'model': model, @@ -40,8 +49,9 @@ def log_chat_completion(self, endpoint: str, model: str, req_type: str, messages def log_chat_completion_json(self, endpoint: str, model: str, req_type: str, functions: dict, json_response: str): if self.log_chat_completions: - time = datetime.now().strftime('%Y-%m-%d_%H:%M:%S') - with open(os.path.join(self.dot_gpt_pilot_path, 'chat_log', f'{time}-{req_type}.json'), 'w') as file: + time = datetime.now().strftime('%Y-%m-%d_%H_%M_%S') + + with open(os.path.join(self.chat_log_path, f'{time}-{req_type}.json'), 'w') as file: data = { 'endpoint': endpoint, 'model': model, From ba36717ed4f6bcc34ad041e5295f06e9a4cad477 Mon Sep 17 00:00:00 2001 From: Nicholas Albion Date: Thu, 12 Oct 2023 10:34:36 +1100 Subject: [PATCH 7/9] fixed typo --- pilot/helpers/agents/ProductOwner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pilot/helpers/agents/ProductOwner.py b/pilot/helpers/agents/ProductOwner.py index 069bb5846..0187bfd84 100644 --- a/pilot/helpers/agents/ProductOwner.py +++ b/pilot/helpers/agents/ProductOwner.py @@ -27,7 +27,7 @@ def get_project_description(self): step = get_progress_steps(self.project.args['app_id'], PROJECT_DESCRIPTION_STEP) if step and not should_execute_step(self.project.args['step'], PROJECT_DESCRIPTION_STEP): step_already_finished(self.project.args, step) - self.project.set_oot_path(setup_workspace(self.project.args)) + self.project.set_root_path(setup_workspace(self.project.args)) self.project.project_description = step['summary'] self.project.project_description_messages = step['messages'] return From 1f0ed3e36b851ecdd43845d7bc0588605af97781 Mon Sep 17 00:00:00 2001 From: Nicholas Albion Date: Thu, 12 Oct 2023 10:35:23 +1100 Subject: [PATCH 8/9] disable `.gpt-pilot/` by default --- pilot/utils/dot_gpt_pilot.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pilot/utils/dot_gpt_pilot.py b/pilot/utils/dot_gpt_pilot.py index 8fec0f50b..c658d6a72 100644 --- a/pilot/utils/dot_gpt_pilot.py +++ b/pilot/utils/dot_gpt_pilot.py @@ -2,6 +2,11 @@ import os import yaml from datetime import datetime +from dotenv import load_dotenv + +load_dotenv() + +USE_GPTPILOT_FOLDER = os.getenv('USE_GPTPILOT_FOLDER') == 'true' # TODO: Parse files from the `.gpt-pilot` directory to resume a project - `user_stories` may have changed - include checksums for sections which may need to be reprocessed. @@ -11,11 +16,15 @@ class DotGptPilot: Manages the `.gpt-pilot` directory. """ def __init__(self, log_chat_completions: bool = True): + if not USE_GPTPILOT_FOLDER: + return self.log_chat_completions = log_chat_completions self.dot_gpt_pilot_path = self.with_root_path('~', create=False) self.chat_log_path = self.chat_log_folder(None) def with_root_path(self, root_path: str, create=True): + if not USE_GPTPILOT_FOLDER: + return dot_gpt_pilot_path = os.path.join(root_path, '.gpt-pilot') self.dot_gpt_pilot_path = dot_gpt_pilot_path @@ -26,6 +35,8 @@ def with_root_path(self, root_path: str, create=True): return dot_gpt_pilot_path def chat_log_folder(self, task): + if not USE_GPTPILOT_FOLDER: + return chat_log_path = os.path.join(self.dot_gpt_pilot_path, 'chat_log') if task is not None: chat_log_path = os.path.join(chat_log_path, 'task_' + task) @@ -35,6 +46,8 @@ def chat_log_folder(self, task): return chat_log_path def log_chat_completion(self, endpoint: str, model: str, req_type: str, messages: list[dict], response: str): + if not USE_GPTPILOT_FOLDER: + return if self.log_chat_completions: time = datetime.now().strftime('%Y-%m-%d_%H_%M_%S') with open(os.path.join(self.chat_log_path, f'{time}-{req_type}.yaml'), 'w') as file: @@ -48,6 +61,8 @@ def log_chat_completion(self, endpoint: str, model: str, req_type: str, messages yaml.safe_dump(data, file, width=120, indent=2, default_flow_style=False, sort_keys=False) def log_chat_completion_json(self, endpoint: str, model: str, req_type: str, functions: dict, json_response: str): + if not USE_GPTPILOT_FOLDER: + return if self.log_chat_completions: time = datetime.now().strftime('%Y-%m-%d_%H_%M_%S') @@ -62,6 +77,8 @@ def log_chat_completion_json(self, endpoint: str, model: str, req_type: str, fun json.dump(data, file, indent=2) def write_project(self, project): + if not USE_GPTPILOT_FOLDER: + return data = { 'name': project.args['name'], 'description': project.project_description, From b90d22de0ee17fb93874045f690937c2bc9010ca Mon Sep 17 00:00:00 2001 From: Nicholas Albion Date: Thu, 12 Oct 2023 14:29:20 +1100 Subject: [PATCH 9/9] convert `task` int to str --- pilot/utils/dot_gpt_pilot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pilot/utils/dot_gpt_pilot.py b/pilot/utils/dot_gpt_pilot.py index c658d6a72..d182e6b75 100644 --- a/pilot/utils/dot_gpt_pilot.py +++ b/pilot/utils/dot_gpt_pilot.py @@ -39,7 +39,7 @@ def chat_log_folder(self, task): return chat_log_path = os.path.join(self.dot_gpt_pilot_path, 'chat_log') if task is not None: - chat_log_path = os.path.join(chat_log_path, 'task_' + task) + chat_log_path = os.path.join(chat_log_path, 'task_' + str(task)) os.makedirs(chat_log_path, exist_ok=True) self.chat_log_path = chat_log_path