Process awareness and termination

.github/workflows/ci.yml

@@ -7,16 +7,34 @@ on:
   pull_request:
     branches:
       - main
-      - debugging_ipc
 
 jobs:
-  build:
+  Docker:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build the Docker image
+        run: docker compose build
+      - name: Run the Docker image
+        run: docker compose up gpt-pilot -d
+      - name: Wait for the Docker image to start
+        run: docker ps
+      - name: Stop the Docker image
+        run: docker compose down
+
+  Test:
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         # 3.10 - 04 Oct 2021
         # 3.11 - 24 Oct 2022
         python-version: ['3.9', '3.10', '3.11', '3.12']
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        exclude:
+          # LINK : fatal error LNK1181: cannot open input file 'libpq.lib'
+          # Maybe related: https://github.com/psycopg/psycopg2/issues/1628
+          - os: windows-latest
+            python-version: '3.12'
 
     steps:
     - uses: actions/checkout@v4
@@ -42,7 +60,9 @@ jobs:
         #ruff --format=github --target-version=py37 --ignore=F401,E501 .
 
     - name: Run tests
+      env:
+        PYTHONPATH: .
       run: |
         pip install pytest
         cd pilot
-        PYTHONPATH=. pytest -m "not slow and not uses_tokens and not ux_test"
+        pytest -m "not slow and not uses_tokens and not ux_test"

pilot/const/function_calls.py

@@ -44,7 +44,7 @@ def command_definition(description_command=f'A single command that needs to be e
                        description_timeout=
                        'Timeout in milliseconds that represent the approximate time this command takes to finish. '
                        'If you need to run a command that doesnt\'t finish by itself (eg. a command to run an app), '
-                       'set the timeout to -1 and provide a process_name. '
+                       'set the timeout to to a value long enough to determine that it has started successfully and provide a process_name. '
                        'If you need to create a directory that doesn\'t exist and is not the root project directory, '
                        'always create it by running a command `mkdir`'):
     return {
@@ -59,6 +59,10 @@ def command_definition(description_command=f'A single command that needs to be e
                 'type': 'number',
                 'description': description_timeout,
             },
+            'success_message': {
+                'type': 'string',
+                'description': 'A message to look for in the output of the command to determine if successful or not.',
+            },
             'process_name': {
                 'type': 'string',
                 'description': 'If the process needs to continue running after the command is executed provide '
@@ -187,6 +191,10 @@ def command_definition(description_command=f'A single command that needs to be e
                                     'description': 'Type of the development step that needs to be done to complete the entire task.',
                                 },
                                 'command': command_definition(),
+                                'kill_process': {
+                                    'type': 'string',
+                                    'description': 'To kill a process that was left running by a previous `command` provide the `process_name` in this field.'
+                                },
                                 'code_change': {
                                     'type': 'object',
                                     'description': 'A code change that needs to be implemented. This should be used only if the task is of a type "code_change".',
@@ -456,7 +464,7 @@ def command_definition(description_command=f'A single command that needs to be e
                             },
                             'path': {
                                 'type': 'string',
-                                'description': 'Path of the file that needs to be saved on the disk.',
+                                'description': 'Full path of the file with the file name that needs to be saved.',
                             },
                             'content': {
                                 'type': 'string',

pilot/helpers/AgentConvo.py

@@ -82,6 +82,8 @@ def send_message(self, prompt_path=None, prompt_data=None, function_calls: Funct
                 development_step = save_development_step(self.agent.project, prompt_path, prompt_data, self.messages, response)
 
         # TODO handle errors from OpenAI
+        # It's complicated because calling functions are expecting different types of responses - string or tuple
+        # https://github.com/Pythagora-io/gpt-pilot/issues/165 & #91
         if response == {}:
             logger.error(f'Aborting with "OpenAI API error happened"')
             raise Exception("OpenAI API error happened.")

pilot/helpers/Project.py

@@ -1,5 +1,6 @@
 import json
 import os
+import re
 from typing import Tuple
 from utils.style import  yellow_bold, cyan, white_bold
 from const.common import IGNORE_FOLDERS, STEPS
@@ -191,16 +192,17 @@ def get_files(self, files):
             list: A list of files with content.
         """
         files_with_content = []
-        for file in files:
+        for file_path in files:
             # TODO this is a hack, fix it
             try:
-                relative_path, full_path = self.get_full_file_path('', file)
+                name = os.path.basename(file_path)
+                relative_path, full_path = self.get_full_file_path(file_path, name)
                 file_content = open(full_path, 'r').read()
             except:
                 file_content = ''
 
             files_with_content.append({
-                "path": file,
+                "path": file_path,
                 "content": file_content
             })
         return files_with_content
@@ -212,88 +214,44 @@ def save_file(self, data):
         Args:
             data: { name: 'hello.py', path: 'path/to/hello.py', content: 'print("Hello!")' }
         """
-        # TODO fix this in prompts
-        if 'path' not in data:
-            data['path'] = data['name']
-
-        if 'name' not in data or data['name'] == '':
-            data['name'] = os.path.basename(data['path'])
-        elif not data['path'].endswith(data['name']):
-            if data['path'] == '':
-                data['path'] = data['name']
-            else:
-                data['path'] = data['path'] + '/' + data['name']
-        # TODO END
+        name = data['name'] if 'name' in data and data['name'] != '' else os.path.basename(data['path'])
+        path = data['path'] if 'path' in data else name
 
-        data['path'], data['full_path'] = self.get_full_file_path(data['path'], data['name'])
-        update_file(data['full_path'], data['content'])
+        path, full_path = self.get_full_file_path(path, name)
+        update_file(full_path, data['content'])
 
-        (File.insert(app=self.app, path=data['path'], name=data['name'], full_path=data['full_path'])
+        (File.insert(app=self.app, path=path, name=name, full_path=full_path)
          .on_conflict(
             conflict_target=[File.app, File.name, File.path],
             preserve=[],
-            update={'name': data['name'], 'path': data['path'], 'full_path': data['full_path']})
+            update={'name': name, 'path': path, 'full_path': full_path})
          .execute())
 
     def get_full_file_path(self, file_path: str, file_name: str) -> Tuple[str, str]:
+        file_name = os.path.basename(file_name)
+
+        if file_path.startswith(self.root_path):
+            file_path = file_path.replace(self.root_path, '')
 
-        # WINDOWS
         are_windows_paths = '\\' in file_path or '\\' in file_name or '\\' in self.root_path
         if are_windows_paths:
-            file_name = file_name.replace('\\', '/')
             file_path = file_path.replace('\\', '/')
-        # END WINDOWS
-
-        # Universal modifications
-        file_path = file_path.replace('~', '')
-        file_name = file_name.replace('~', '')
-
-        file_path = file_path.replace(self.root_path, '')
-        file_name = file_name.replace(self.root_path, '')
-
-        if '.' not in file_path and not file_path.endswith('/'):
-            file_path += '/'
-        if '.' not in file_name and not file_name.endswith('/'):
-            file_name += '/'
-
-        if '/' in file_path and not file_path.startswith('/'):
-            file_path = '/' + file_path
-        if '/' in file_name and not file_name.startswith('/'):
-            file_name = '/' + file_name
-        # END Universal modifications
-
-        head_path, tail_path = os.path.split(file_path)
-        head_name, tail_name = os.path.split(file_name)
-
-        final_file_path = head_path if head_path != '' else head_name
-        final_file_name = tail_name if tail_name != '' else tail_path
-
-        if head_path in head_name:
-            final_file_path = head_name
-        elif final_file_path != head_name:
-            if head_name not in head_path and head_path not in head_name:
-                if '.' in file_path:
-                    final_file_path = head_name + head_path
-                else:
-                    final_file_path = head_path + head_name
-
-        if final_file_path == '':
-            final_file_path = '/'
 
-        final_absolute_path = self.root_path + final_file_path + '/' + final_file_name
+        # Force all paths to be relative to the workspace
+        file_path = re.sub(r'^(\w+:/|[/~.]+)', '', file_path, 1)
 
-        if '//' in final_absolute_path:
-            final_absolute_path = final_absolute_path.replace('//', '/')
-        if '//' in final_file_path:
-            final_file_path = final_file_path.replace('//', '/')
+        # file_path should not include the file name
+        if file_path == file_name:
+            file_path = ''
+        elif file_path.endswith('/' + file_name):
+            file_path = file_path.replace('/' + file_name, '')
+        elif file_path.endswith('/'):
+            file_path = file_path[:-1]
 
-        # WINDOWS
-        if are_windows_paths:
-            final_file_path = final_file_path.replace('/', '\\')
-            final_absolute_path = final_absolute_path.replace('/', '\\')
-        # END WINDOWS
+        absolute_path = self.root_path + '/' + file_name if file_path == '' \
+            else self.root_path + '/' + file_path + '/' + file_name
 
-        return final_file_path, final_absolute_path
+        return file_path, absolute_path
 
     def save_files_snapshot(self, development_step_id):
         files = get_files_content(self.root_path, ignore=IGNORE_FOLDERS)

pilot/helpers/agents/Developer.py

@@ -1,3 +1,4 @@
+import platform
 import uuid
 from utils.style import green, red, green_bold, yellow_bold, red_bold, blue_bold, white_bold
 from helpers.exceptions.TokenLimitError import TokenLimitError
@@ -11,7 +12,7 @@
 from helpers.Agent import Agent
 from helpers.AgentConvo import AgentConvo
 from utils.utils import should_execute_step, array_of_objects_to_string, generate_app_data
-from helpers.cli import run_command_until_success, execute_command_and_check_cli_response
+from helpers.cli import run_command_until_success, execute_command_and_check_cli_response, running_processes
 from const.function_calls import FILTER_OS_TECHNOLOGIES, EXECUTE_COMMANDS, GET_TEST_TYPE, IMPLEMENT_TASK
 from database.database import save_progress, get_progress_steps, update_app_status
 from utils.utils import get_os_info
@@ -97,10 +98,12 @@ def step_command_run(self, convo, step, i):
         additional_message = 'Let\'s start with the step #0:\n\n' if i == 0 else f'So far, steps { ", ".join(f"#{j}" for j in range(i)) } are finished so let\'s do step #{i + 1} now.\n\n'
 
         process_name = data['process_name'] if 'process_name' in data else None
+        success_message = data['success_message'] if 'success_message' in data else None
 
         return run_command_until_success(convo, data['command'],
                                          timeout=data['timeout'],
                                          process_name=process_name,
+                                         success_message=success_message,
                                          additional_message=additional_message)
 
     def step_human_intervention(self, convo, step: dict):
@@ -171,7 +174,9 @@ def task_postprocessing(self, convo, development_task, continue_development, tas
 
         if development_task is not None:
             convo.remove_last_x_messages(2)
-            detailed_user_review_goal = convo.send_message('development/define_user_review_goal.prompt', {})
+            detailed_user_review_goal = convo.send_message('development/define_user_review_goal.prompt', {
+                'os': platform.system()
+            })
             convo.remove_last_x_messages(2)
 
         try:
@@ -340,7 +345,9 @@ def continue_development(self, iteration_convo, last_branch_name, continue_descr
 
                 # self.debugger.debug(iteration_convo, user_input=user_feedback)
 
-                task_steps = iteration_convo.send_message('development/parse_task.prompt', {}, IMPLEMENT_TASK)
+                task_steps = iteration_convo.send_message('development/parse_task.prompt', {
+                    'running_processes': running_processes
+                }, IMPLEMENT_TASK)
                 iteration_convo.remove_last_x_messages(2)
 
                 return self.execute_task(iteration_convo, task_steps, is_root_task=True)