diff --git a/wooey/migrations/0051_add_virtual_env.py b/wooey/migrations/0051_add_virtual_env.py index 5b8be51c..dfd477c1 100644 --- a/wooey/migrations/0051_add_virtual_env.py +++ b/wooey/migrations/0051_add_virtual_env.py @@ -25,7 +25,7 @@ class Migration(migrations.Migration): ), ("name", models.CharField(max_length=25)), ("python_binary", models.CharField(max_length=1024)), - ("requirements", models.TextField()), + ("requirements", models.TextField(null=True, blank=True)), ("venv_directory", models.CharField(max_length=1024)), ], options={ diff --git a/wooey/models/core.py b/wooey/models/core.py index d624158d..7adb8b9d 100644 --- a/wooey/models/core.py +++ b/wooey/models/core.py @@ -734,7 +734,7 @@ def __str__(self): class VirtualEnvironment(models.Model): name = models.CharField(max_length=25) python_binary = models.CharField(max_length=1024) - requirements = models.TextField() + requirements = models.TextField(null=True, blank=True) venv_directory = models.CharField(max_length=1024) class Meta: @@ -742,5 +742,22 @@ class Meta: verbose_name = _("virtual environment") verbose_name_plural = _("virtual environments") + def get_venv_python_binary(self): + return os.path.join( + self.get_install_path(), + "bin", + "python", + ) + + def get_install_path(self, ensure_exists=False): + path = os.path.join( + self.venv_directory, + "".join(x for x in self.python_binary if x.isalnum()), + self.name, + ) + if ensure_exists: + os.makedirs(path, exist_ok=True) + return path + def __str__(self): return self.name diff --git a/wooey/tasks.py b/wooey/tasks.py index 10578b1b..36c8140a 100644 --- a/wooey/tasks.py +++ b/wooey/tasks.py @@ -117,7 +117,7 @@ def check_output(job, stdout, stderr, prev_std): stderr = update_from_output_queue(qerr, stderr) # If there are changes, update the db - if (stdout, stderr) != prev_std: + if job is not None and (stdout, stderr) != prev_std: job.update_realtime(stdout=stdout, stderr=stderr) prev_std = (stdout, stderr) @@ -137,14 +137,11 @@ def check_output(job, stdout, stderr, prev_std): return (stdout, stderr, return_code) -def setup_venv(virtual_environment, job, stdout, stderr): - venv_binary_namespace = os.path.join( - virtual_environment.venv_directory, - "".join(x for x in virtual_environment.python_binary if x.isalnum()), - ) - venv_path = os.path.join(venv_binary_namespace, virtual_environment.name) - os.makedirs(venv_binary_namespace, exist_ok=True) - venv_executable = os.path.join(venv_path, "bin", "python") +def setup_venv(virtual_environment, job=None, stdout="", stderr=""): + venv_path = virtual_environment.get_install_path() + venv_executable = virtual_environment.get_venv_python_binary() + return_code = 0 + if not os.path.exists(venv_path): venv_command = [ virtual_environment.python_binary, @@ -157,31 +154,34 @@ def setup_venv(virtual_environment, job, stdout, stderr): (stdout, stderr, return_code) = run_and_stream_command( venv_command, cwd=None, job=job, stdout=stdout, stderr=stderr ) + if return_code: raise Exception("VirtualEnv setup failed.\n{}\n{}".format(stdout, stderr)) - pip_setup = [venv_executable, "-m", "pip", "-I", "pip"] + pip_setup = [venv_executable, "-m", "pip", "install", "-I", "pip"] (stdout, stderr, return_code) = run_and_stream_command( pip_setup, cwd=None, job=job, stdout=stdout, stderr=stderr ) if return_code: raise Exception("Pip setup failed.\n{}\n{}".format(stdout, stderr)) - with tempfile.NamedTemporaryFile( - mode="w", prefix="requirements", suffix=".txt" - ) as reqs_txt: - reqs_txt.write(virtual_environment.requirements) - reqs_txt.flush() - os.fsync(reqs_txt.fileno()) - venv_command = [ - venv_executable, - "-m", - "pip", - "install", - "-r", - reqs_txt.name, - ] - (stdout, stderr, return_code) = run_and_stream_command( - venv_command, cwd=None, job=job, stdout=stdout, stderr=stderr - ) + requirements = virtual_environment.requirements + if requirements: + with tempfile.NamedTemporaryFile( + mode="w", prefix="requirements", suffix=".txt" + ) as reqs_txt: + reqs_txt.write(requirements) + reqs_txt.flush() + os.fsync(reqs_txt.fileno()) + venv_command = [ + venv_executable, + "-m", + "pip", + "install", + "-r", + reqs_txt.name, + ] + (stdout, stderr, return_code) = run_and_stream_command( + venv_command, cwd=None, job=job, stdout=stdout, stderr=stderr + ) return (venv_executable, stdout, stderr, return_code) diff --git a/wooey/tests/factories.py b/wooey/tests/factories.py index 493c2148..f6b71f16 100644 --- a/wooey/tests/factories.py +++ b/wooey/tests/factories.py @@ -1,7 +1,18 @@ +import sys +import tempfile + import factory from django.contrib.auth import get_user_model -from ..models import APIKey, Script, ScriptGroup, WooeyJob, WooeyProfile, WooeyWidget +from ..models import ( + APIKey, + Script, + ScriptGroup, + VirtualEnvironment, + WooeyJob, + WooeyProfile, + WooeyWidget, +) from . import utils as test_utils @@ -85,6 +96,15 @@ class Meta: name = "test widget" +class VirtualEnvFactory(factory.DjangoModelFactory): + class Meta: + model = VirtualEnvironment + + name = factory.Sequence(lambda n: "venv_%d" % n) + python_binary = sys.executable + venv_directory = tempfile.gettempdir() + + def generate_script(script_path, script_name=None): new_file = test_utils.save_script_path(script_path) from ..backend import utils diff --git a/wooey/tests/test_virtual_envs.py b/wooey/tests/test_virtual_envs.py new file mode 100644 index 00000000..f778fe97 --- /dev/null +++ b/wooey/tests/test_virtual_envs.py @@ -0,0 +1,52 @@ +import os +import shutil +import subprocess +from unittest import mock + +from django.test import TestCase + +from wooey.tasks import setup_venv + +from .factories import VirtualEnvFactory + + +class TestVirtualEnvironments(TestCase): + def setUp(self): + super().setUp() + self.venv = VirtualEnvFactory() + install_path = self.venv.get_install_path() + if os.path.exists(install_path): + shutil.rmtree(install_path) + + def test_sets_up_virtual_env(self): + venv = self.venv + (venv_executable, stdout, stderr, return_code) = setup_venv(venv) + self.assertTrue(os.path.exists(venv_executable)) + + def test_reuses_virtual_env(self): + venv = self.venv + (venv_executable, stdout, stderr, return_code) = setup_venv(venv) + self.assertTrue(os.path.exists(venv_executable)) + with mock.patch("wooey.tasks.run_and_stream_command") as command_runner: + command_runner.return_value = ("stdout", "stderr", 0) + setup_venv(venv) + self.assertFalse(command_runner.called) + + def test_installs_pip(self): + venv = self.venv + setup_venv(venv) + self.assertTrue( + os.path.exists(os.path.join(venv.get_install_path(), "bin", "pip")) + ) + + def test_installs_requirements(self): + venv = self.venv + venv.requirements = "flask" + venv.save() + setup_venv(venv) + binary = venv.get_venv_python_binary() + results = subprocess.run( + [binary, "-m" "pip", "freeze", "--local"], capture_output=True + ) + packages = results.stdout.decode().lower() + self.assertIn("flask", packages)